summaryrefslogtreecommitdiff
path: root/searx
diff options
context:
space:
mode:
Diffstat (limited to 'searx')
-rw-r--r--searx/engines/nyaa.py89
-rw-r--r--searx/engines/tokyotoshokan.py5
-rw-r--r--searx/templates/courgette/result_templates/code.html22
-rw-r--r--searx/templates/legacy/result_templates/code.html22
-rw-r--r--searx/templates/oscar/macros.html175
-rw-r--r--searx/templates/simple/macros.html3
-rw-r--r--searx/utils.py18
-rw-r--r--searx/webapp.py10
8 files changed, 176 insertions, 168 deletions
diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py
index 272c712c4..6a8e598c4 100644
--- a/searx/engines/nyaa.py
+++ b/searx/engines/nyaa.py
@@ -1,7 +1,7 @@
"""
- Nyaa.se (Anime Bittorrent tracker)
+ Nyaa.si (Anime Bittorrent tracker)
- @website http://www.nyaa.se/
+ @website http://www.nyaa.si/
@provide-api no
@using-api no
@results HTML
@@ -12,50 +12,25 @@
from lxml import html
from searx.engines.xpath import extract_text
from searx.url_utils import urlencode
+from searx.utils import get_torrent_size, int_or_zero
# engine dependent config
categories = ['files', 'images', 'videos', 'music']
paging = True
# search-url
-base_url = 'http://www.nyaa.se/'
+base_url = 'http://www.nyaa.si/'
search_url = base_url + '?page=search&{query}&offset={offset}'
# xpath queries
-xpath_results = '//table[@class="tlist"]//tr[contains(@class, "tlistrow")]'
-xpath_category = './/td[@class="tlisticon"]/a'
-xpath_title = './/td[@class="tlistname"]/a'
-xpath_torrent_file = './/td[@class="tlistdownload"]/a'
-xpath_filesize = './/td[@class="tlistsize"]/text()'
-xpath_seeds = './/td[@class="tlistsn"]/text()'
-xpath_leeches = './/td[@class="tlistln"]/text()'
-xpath_downloads = './/td[@class="tlistdn"]/text()'
-
-
-# convert a variable to integer or return 0 if it's not a number
-def int_or_zero(num):
- if isinstance(num, list):
- if len(num) < 1:
- return 0
- num = num[0]
- if num.isdigit():
- return int(num)
- return 0
-
-
-# get multiplier to convert torrent size to bytes
-def get_filesize_mul(suffix):
- return {
- 'KB': 1024,
- 'MB': 1024 ** 2,
- 'GB': 1024 ** 3,
- 'TB': 1024 ** 4,
-
- 'KIB': 1024,
- 'MIB': 1024 ** 2,
- 'GIB': 1024 ** 3,
- 'TIB': 1024 ** 4
- }[str(suffix).upper()]
+xpath_results = '//table[contains(@class, "torrent-list")]//tr[not(th)]'
+xpath_category = './/td[1]/a[1]'
+xpath_title = './/td[2]/a[last()]'
+xpath_torrent_links = './/td[3]/a'
+xpath_filesize = './/td[4]/text()'
+xpath_seeds = './/td[6]/text()'
+xpath_leeches = './/td[7]/text()'
+xpath_downloads = './/td[8]/text()'
# do search-request
@@ -72,25 +47,32 @@ def response(resp):
dom = html.fromstring(resp.text)
for result in dom.xpath(xpath_results):
+ # defaults
+ filesize = 0
+ magnet_link = ""
+ torrent_link = ""
+
# category in which our torrent belongs
- category = result.xpath(xpath_category)[0].attrib.get('title')
+ try:
+ category = result.xpath(xpath_category)[0].attrib.get('title')
+ except:
+ pass
# torrent title
page_a = result.xpath(xpath_title)[0]
title = extract_text(page_a)
# link to the page
- href = page_a.attrib.get('href')
+ href = base_url + page_a.attrib.get('href')
- # link to the torrent file
- torrent_link = result.xpath(xpath_torrent_file)[0].attrib.get('href')
-
- # torrent size
- try:
- file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')
- file_size = int(float(file_size) * get_filesize_mul(suffix))
- except:
- file_size = None
+ for link in result.xpath(xpath_torrent_links):
+ url = link.attrib.get('href')
+ if 'magnet' in url:
+ # link to the magnet
+ magnet_link = url
+ else:
+ # link to the torrent file
+ torrent_link = url
# seed count
seed = int_or_zero(result.xpath(xpath_seeds))
@@ -101,6 +83,14 @@ def response(resp):
# torrent downloads count
downloads = int_or_zero(result.xpath(xpath_downloads))
+ # let's try to calculate the torrent size
+ try:
+ filesize_info = result.xpath(xpath_filesize)[0]
+ filesize, filesize_multiplier = filesize_info.split()
+ filesize = get_torrent_size(filesize, filesize_multiplier)
+ except:
+ pass
+
# content string contains all information not included into template
content = 'Category: "{category}". Downloaded {downloads} times.'
content = content.format(category=category, downloads=downloads)
@@ -110,8 +100,9 @@ def response(resp):
'content': content,
'seed': seed,
'leech': leech,
- 'filesize': file_size,
+ 'filesize': filesize,
'torrentfile': torrent_link,
+ 'magnetlink': magnet_link,
'template': 'torrent.html'})
return results
diff --git a/searx/engines/tokyotoshokan.py b/searx/engines/tokyotoshokan.py
index 9a6b5e57d..773212043 100644
--- a/searx/engines/tokyotoshokan.py
+++ b/searx/engines/tokyotoshokan.py
@@ -14,8 +14,8 @@ import re
from lxml import html
from searx.engines.xpath import extract_text
from datetime import datetime
-from searx.engines.nyaa import int_or_zero, get_filesize_mul
from searx.url_utils import urlencode
+from searx.utils import get_torrent_size, int_or_zero
# engine dependent config
categories = ['files', 'videos', 'music']
@@ -76,8 +76,7 @@ def response(resp):
try:
# ('1.228', 'GB')
groups = size_re.match(item).groups()
- multiplier = get_filesize_mul(groups[1])
- params['filesize'] = int(multiplier * float(groups[0]))
+ params['filesize'] = get_torrent_size(groups[0], groups[1])
except:
pass
elif item.startswith('Date:'):
diff --git a/searx/templates/courgette/result_templates/code.html b/searx/templates/courgette/result_templates/code.html
index 953617e84..d6dbd5ed7 100644
--- a/searx/templates/courgette/result_templates/code.html
+++ b/searx/templates/courgette/result_templates/code.html
@@ -1,11 +1,11 @@
-<div class="result {{ result.class }}">
- <h3 class="result_title">{% if result['favicon'] %}<img width="14" height="14" class="favicon" src="static/{{theme}}/img/icon_{{result['favicon']}}.ico" alt="{{result['favicon']}}" />{% endif %}<a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>
- {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span>{% endif %}
- <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>
- {% if result.repository %}<p class="content"><a href="{{ result.repository|safe }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.repository }}</a></p>{% endif %}
- <div dir="ltr">
- {{ result.codelines|code_highlighter(result.code_language)|safe }}
- </div>
-
- <p class="url">{{ result.pretty_url }}&lrm;</p>
-</div>
+<div class="result {{ result.class }}">
+ <h3 class="result_title"><a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>
+ {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span>{% endif %}
+ <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>
+ {% if result.repository %}<p class="content"><a href="{{ result.repository|safe }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.repository }}</a></p>{% endif %}
+ <div dir="ltr">
+ {{ result.codelines|code_highlighter(result.code_language)|safe }}
+ </div>
+
+ <p class="url">{{ result.pretty_url }}&lrm;</p>
+</div>
diff --git a/searx/templates/legacy/result_templates/code.html b/searx/templates/legacy/result_templates/code.html
index 9e3ed20af..fee6c8221 100644
--- a/searx/templates/legacy/result_templates/code.html
+++ b/searx/templates/legacy/result_templates/code.html
@@ -1,11 +1,11 @@
-<div class="result {{ result.class }}">
- <h3 class="result_title"> {% if result['favicon'] %}<img width="14" height="14" class="favicon" src="static/{{theme}}/img/icon_{{result['favicon']}}.ico" alt="{{result['favicon']}}" />{% endif %}<a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>
- <p class="url">{{ result.pretty_url }}&lrm; <a class="cache_link" href="https://web.archive.org/web/{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('cached') }}</a></p>
- {% if result.publishedDate %}<p class="published_date">{{ result.publishedDate }}</p>{% endif %}
- <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>
- {% if result.repository %}<p class="result-content"><a href="{{ result.repository|safe }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.repository }}</a></p>{% endif %}
-
- <div dir="ltr">
- {{ result.codelines|code_highlighter(result.code_language)|safe }}
- </div>
-</div>
+<div class="result {{ result.class }}">
+ <h3 class="result_title"><a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>
+ <p class="url">{{ result.pretty_url }}&lrm; <a class="cache_link" href="https://web.archive.org/web/{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('cached') }}</a></p>
+ {% if result.publishedDate %}<p class="published_date">{{ result.publishedDate }}</p>{% endif %}
+ <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>
+ {% if result.repository %}<p class="result-content"><a href="{{ result.repository|safe }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.repository }}</a></p>{% endif %}
+
+ <div dir="ltr">
+ {{ result.codelines|code_highlighter(result.code_language)|safe }}
+ </div>
+</div>
diff --git a/searx/templates/oscar/macros.html b/searx/templates/oscar/macros.html
index e71091e70..c35a19158 100644
--- a/searx/templates/oscar/macros.html
+++ b/searx/templates/oscar/macros.html
@@ -1,88 +1,87 @@
-<!-- Draw glyphicon icon from bootstrap-theme -->
-{% macro icon(action) -%}
- <span class="glyphicon glyphicon-{{ action }}"></span>
-{%- endmacro %}
-
-<!-- Draw favicon -->
-<!-- TODO: using url_for methode -->
-{% macro draw_favicon(favicon) -%}
- <img width="32" height="32" class="favicon" src="static/themes/oscar/img/icons/{{ favicon }}.png" alt="{{ favicon }}" />
-{%- endmacro %}
-
-{%- macro result_link(url, title, classes='') -%}
-<a href="{{ url }}" {% if classes %}class="{{ classes }}" {% endif %}{% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ title }}</a>
-{%- endmacro -%}
-
-<!-- Draw result header -->
-{% macro result_header(result, favicons) -%}
-<h4 class="result_header">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{{ result_link(result.url, result.title|safe) }}</h4>
-{%- endmacro %}
-
-<!-- Draw result sub header -->
-{% macro result_sub_header(result) -%}
- {% if result.publishedDate %}<time class="text-muted" datetime="{{ result.pubdate }}" >{{ result.publishedDate }}</time>{% endif %}
- {% if result.magnetlink %}<small> &bull; {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }}</small>{% endif %}
- {% if result.torrentfile %}<small> &bull; {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }}</small>{% endif %}
-{%- endmacro %}
-
-<!-- Draw result footer -->
-{% macro result_footer(result) -%}
- <div class="clearfix"></div>
- <div class="pull-right">
- {% for engine in result.engines %}
- <span class="label label-default">{{ engine }}</span>
- {% endfor %}
- <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
- {% if proxify %}
- <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
- {% endif %}
-</div>
-<div class="external-link">{{ result.pretty_url }}</div>
-{%- endmacro %}
-
-<!-- Draw result footer -->
-{% macro result_footer_rtl(result) -%}
- <div class="clearfix"></div>
- {% for engine in result.engines %}
- <span class="label label-default">{{ engine }}</span>
- {% endfor %}
- <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
- {% if proxify %}
- <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
- {% endif %}
- <div class="external-link">{{ result.pretty_url }}</div>
-{%- endmacro %}
-
-{% macro preferences_item_header(info, label, rtl) -%}
- {% if rtl %}
- <div class="row form-group">
- <label class="col-sm-3 col-md-2 pull-right">{{ label }}</label>
- <span class="col-sm-5 col-md-6 help-block pull-left">{{ info }}</span>
- <div class="col-sm-4 col-md-4">
- {% else %}
- <div class="row form-group">
- <label class="col-sm-3 col-md-2">{{ label }}</label>
- <div class="col-sm-4 col-md-4">
- {% endif %}
-{%- endmacro %}
-
-{% macro preferences_item_footer(info, label, rtl) -%}
- {% if rtl %}
- </div>
- </div>
- {% else %}
- </div>
- <span class="col-sm-5 col-md-6 help-block">{{ info }}</span>
- </div>
- {% endif %}
-{%- endmacro %}
-
-{% macro checkbox_toggle(id, blocked) -%}
- <div class="onoffswitch">
- <input type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} class="onoffswitch-checkbox">
- <label class="onoffswitch-label" for="{{ id }}">
- <span class="onoffswitch-inner"></span>
- <span class="onoffswitch-switch"></span>
- </label>
- </div>
-{%- endmacro %}
+<!-- Draw glyphicon icon from bootstrap-theme -->
+{% macro icon(action) -%}
+ <span class="glyphicon glyphicon-{{ action }}"></span>
+{%- endmacro %}
+
+<!-- Draw favicon -->
+{% macro draw_favicon(favicon) -%}
+ <img width="32" height="32" class="favicon" src="{{ url_for('static', filename='/themes/oscar/img/icons/' + favicon + '.png') }}" alt="{{ favicon }}" />
+{%- endmacro %}
+
+{%- macro result_link(url, title, classes='') -%}
+<a href="{{ url }}" {% if classes %}class="{{ classes }}" {% endif %}{% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ title }}</a>
+{%- endmacro -%}
+
+<!-- Draw result header -->
+{% macro result_header(result, favicons) -%}
+<h4 class="result_header">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{{ result_link(result.url, result.title|safe) }}</h4>
+{%- endmacro %}
+
+<!-- Draw result sub header -->
+{% macro result_sub_header(result) -%}
+ {% if result.publishedDate %}<time class="text-muted" datetime="{{ result.pubdate }}" >{{ result.publishedDate }}</time>{% endif %}
+ {% if result.magnetlink %}<small> &bull; {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }}</small>{% endif %}
+ {% if result.torrentfile %}<small> &bull; {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }}</small>{% endif %}
+{%- endmacro %}
+
+<!-- Draw result footer -->
+{% macro result_footer(result) -%}
+ <div class="clearfix"></div>
+ <div class="pull-right">
+ {% for engine in result.engines %}
+ <span class="label label-default">{{ engine }}</span>
+ {% endfor %}
+ <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+ {% if proxify %}
+ <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+ {% endif %}
+</div>
+<div class="external-link">{{ result.pretty_url }}</div>
+{%- endmacro %}
+
+<!-- Draw result footer -->
+{% macro result_footer_rtl(result) -%}
+ <div class="clearfix"></div>
+ {% for engine in result.engines %}
+ <span class="label label-default">{{ engine }}</span>
+ {% endfor %}
+ <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+ {% if proxify %}
+ <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+ {% endif %}
+ <div class="external-link">{{ result.pretty_url }}</div>
+{%- endmacro %}
+
+{% macro preferences_item_header(info, label, rtl) -%}
+ {% if rtl %}
+ <div class="row form-group">
+ <label class="col-sm-3 col-md-2 pull-right">{{ label }}</label>
+ <span class="col-sm-5 col-md-6 help-block pull-left">{{ info }}</span>
+ <div class="col-sm-4 col-md-4">
+ {% else %}
+ <div class="row form-group">
+ <label class="col-sm-3 col-md-2">{{ label }}</label>
+ <div class="col-sm-4 col-md-4">
+ {% endif %}
+{%- endmacro %}
+
+{% macro preferences_item_footer(info, label, rtl) -%}
+ {% if rtl %}
+ </div>
+ </div>
+ {% else %}
+ </div>
+ <span class="col-sm-5 col-md-6 help-block">{{ info }}</span>
+ </div>
+ {% endif %}
+{%- endmacro %}
+
+{% macro checkbox_toggle(id, blocked) -%}
+ <div class="onoffswitch">
+ <input type="checkbox" id="{{ id }}" name="{{ id }}"{% if blocked %} checked="checked"{% endif %} class="onoffswitch-checkbox">
+ <label class="onoffswitch-label" for="{{ id }}">
+ <span class="onoffswitch-inner"></span>
+ <span class="onoffswitch-switch"></span>
+ </label>
+ </div>
+{%- endmacro %}
diff --git a/searx/templates/simple/macros.html b/searx/templates/simple/macros.html
index 76c53a635..883503683 100644
--- a/searx/templates/simple/macros.html
+++ b/searx/templates/simple/macros.html
@@ -8,9 +8,8 @@
{%- endmacro %}
<!-- Draw favicon -->
-<!-- TODO: using url_for methode -->
{% macro draw_favicon(favicon) -%}
- <img width="14" height="14" class="favicon" src="static/themes/simple/img/icons/{{ favicon }}.png" alt="{{ favicon }}" />
+ <img width="14" height="14" class="favicon" src="{{ url_for('static', filename='/themes/simple/img/icons/' + favicon + '.png') }}" alt="{{ favicon }}" />
{%- endmacro %}
{% macro result_open_link(url, classes='') -%}
diff --git a/searx/utils.py b/searx/utils.py
index 3df571160..8be7beb62 100644
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -1,4 +1,6 @@
import csv
+import hashlib
+import hmac
import os
import re
@@ -290,6 +292,15 @@ def convert_str_to_int(number_str):
return 0
+# convert a variable to integer or return 0 if it's not a number
+def int_or_zero(num):
+ if isinstance(num, list):
+ if len(num) < 1:
+ return 0
+ num = num[0]
+ return convert_str_to_int(num)
+
+
def is_valid_lang(lang):
is_abbr = (len(lang) == 2)
if is_abbr:
@@ -312,3 +323,10 @@ def load_module(filename, module_dir):
module = load_source(modname, filepath)
module.name = modname
return module
+
+
+def new_hmac(secret_key, url):
+ if sys.version_info[0] == 2:
+ return hmac.new(bytes(secret_key), url, hashlib.sha256).hexdigest()
+ else:
+ return hmac.new(bytes(secret_key, 'utf-8'), url, hashlib.sha256).hexdigest()
diff --git a/searx/webapp.py b/searx/webapp.py
index 3afdbefd8..f90299b2a 100644
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -69,6 +69,7 @@ from searx.plugins import plugins
from searx.preferences import Preferences, ValidationException
from searx.answerers import answerers
from searx.url_utils import urlencode, urlparse, urljoin
+from searx.utils import new_hmac
# check if the pyopenssl package is installed.
# It is needed for SSL connection without trouble, see #298
@@ -290,7 +291,7 @@ def image_proxify(url):
if settings.get('result_proxy'):
return proxify(url)
- h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
+ h = new_hmac(settings['server']['secret_key'], url.encode('utf-8'))
return '{0}?{1}'.format(url_for('image_proxy'),
urlencode(dict(url=url.encode('utf-8'), h=h)))
@@ -704,7 +705,7 @@ def image_proxy():
if not url:
return '', 400
- h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest()
+ h = new_hmac(settings['server']['secret_key'], url)
if h != request.args.get('h'):
return '', 400
@@ -731,7 +732,7 @@ def image_proxy():
logger.debug('image-proxy: wrong content-type: {0}'.format(resp.headers.get('content-type')))
return '', 400
- img = ''
+ img = b''
chunk_counter = 0
for chunk in resp.iter_content(1024 * 1024):
@@ -792,7 +793,8 @@ def opensearch():
@app.route('/favicon.ico')
def favicon():
return send_from_directory(os.path.join(app.root_path,
- 'static/themes',
+ static_path,
+ 'themes',
get_current_theme_name(),
'img'),
'favicon.png',
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-29 14:34:50 +0000