summaryrefslogtreecommitdiff
path: root/searx
diff options
context:
space:
mode:
Diffstat (limited to 'searx')
-rw-r--r--searx/autocomplete.py5
-rw-r--r--searx/engines/duckduckgo_definitions.py4
-rw-r--r--searx/engines/filecrop.py85
-rw-r--r--searx/engines/seedpeer.py78
-rw-r--r--searx/engines/soundcloud.py2
-rw-r--r--searx/engines/wikipedia.py2
-rw-r--r--searx/engines/www1x.py24
-rw-r--r--searx/engines/youtube_noapi.py2
-rw-r--r--searx/settings.yml7
9 files changed, 19 insertions, 190 deletions
diff --git a/searx/autocomplete.py b/searx/autocomplete.py
index 420b8a461..fbe634a5b 100644
--- a/searx/autocomplete.py
+++ b/searx/autocomplete.py
@@ -113,7 +113,7 @@ def searx_bang(full_query):
def dbpedia(query, lang):
# dbpedia autocompleter, no HTTPS
- autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'
+ autocomplete_url = 'https://lookup.dbpedia.org/api/search.asmx/KeywordSearch?'
response = get(autocomplete_url + urlencode(dict(QueryString=query)))
@@ -121,8 +121,7 @@ def dbpedia(query, lang):
if response.ok:
dom = etree.fromstring(response.content)
- results = dom.xpath('//a:Result/a:Label//text()',
- namespaces={'a': 'http://lookup.dbpedia.org/'})
+ results = dom.xpath('//Result/Label//text()')
return results
diff --git a/searx/engines/duckduckgo_definitions.py b/searx/engines/duckduckgo_definitions.py
index 5a7649173..1d1c84b4b 100644
--- a/searx/engines/duckduckgo_definitions.py
+++ b/searx/engines/duckduckgo_definitions.py
@@ -10,7 +10,7 @@ DuckDuckGo (definitions)
"""
import json
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urlparse, urljoin
from lxml import html
from searx import logger
@@ -102,6 +102,8 @@ def response(resp):
# image
image = search_res.get('Image')
image = None if image == '' else image
+ if image is not None and urlparse(image).netloc == '':
+ image = urljoin('https://duckduckgo.com', image)
# urls
# Official website, Wikipedia page
diff --git a/searx/engines/filecrop.py b/searx/engines/filecrop.py
deleted file mode 100644
index 0331e7b19..000000000
--- a/searx/engines/filecrop.py
+++ /dev/null
@@ -1,85 +0,0 @@
-from html.parser import HTMLParser
-from urllib.parse import urlencode
-
-
-url = 'http://www.filecrop.com/'
-search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}' # noqa
-
-paging = True
-
-
-class FilecropResultParser(HTMLParser): # pylint: disable=W0223 # (see https://bugs.python.org/issue31844)
-
- def __init__(self):
- HTMLParser.__init__(self)
- self.__start_processing = False
-
- self.results = []
- self.result = {}
-
- self.tr_counter = 0
- self.data_counter = 0
-
- def handle_starttag(self, tag, attrs):
-
- if tag == 'tr':
- if ('bgcolor', '#edeff5') in attrs or\
- ('bgcolor', '#ffffff') in attrs:
- self.__start_processing = True
-
- if not self.__start_processing:
- return
-
- if tag == 'label':
- self.result['title'] = [attr[1] for attr in attrs
- if attr[0] == 'title'][0]
- elif tag == 'a' and ('rel', 'nofollow') in attrs\
- and ('class', 'sourcelink') in attrs:
- if 'content' in self.result:
- self.result['content'] += [attr[1] for attr in attrs
- if attr[0] == 'title'][0]
- else:
- self.result['content'] = [attr[1] for attr in attrs
- if attr[0] == 'title'][0]
- self.result['content'] += ' '
- elif tag == 'a':
- self.result['url'] = url + [attr[1] for attr in attrs
- if attr[0] == 'href'][0]
-
- def handle_endtag(self, tag):
- if self.__start_processing is False:
- return
-
- if tag == 'tr':
- self.tr_counter += 1
-
- if self.tr_counter == 2:
- self.__start_processing = False
- self.tr_counter = 0
- self.data_counter = 0
- self.results.append(self.result)
- self.result = {}
-
- def handle_data(self, data):
- if not self.__start_processing:
- return
-
- if 'content' in self.result:
- self.result['content'] += data + ' '
- else:
- self.result['content'] = data + ' '
-
- self.data_counter += 1
-
-
-def request(query, params):
- index = 1 + (params['pageno'] - 1) * 30
- params['url'] = search_url.format(query=urlencode({'w': query}), index=index)
- return params
-
-
-def response(resp):
- parser = FilecropResultParser()
- parser.feed(resp.text)
-
- return parser.results
diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py
deleted file mode 100644
index 39916da6e..000000000
--- a/searx/engines/seedpeer.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# Seedpeer (Videos, Music, Files)
-#
-# @website https://seedpeer.me
-# @provide-api no (nothing found)
-#
-# @using-api no
-# @results HTML (using search portal)
-# @stable yes (HTML can change)
-# @parse url, title, content, seed, leech, magnetlink
-
-from lxml import html
-from json import loads
-from operator import itemgetter
-from urllib.parse import quote, urljoin
-from searx.utils import extract_text
-
-
-url = 'https://seedpeer.me/'
-search_url = url + 'search/{search_term}?page={page_no}'
-torrent_file_url = url + 'torrent/{torrent_hash}'
-
-# specific xpath variables
-script_xpath = '//script[@type="text/javascript"][not(@src)]'
-torrent_xpath = '(//table)[2]/tbody/tr'
-link_xpath = '(./td)[1]/a/@href'
-age_xpath = '(./td)[2]'
-size_xpath = '(./td)[3]'
-
-
-# do search-request
-def request(query, params):
- params['url'] = search_url.format(search_term=quote(query),
- page_no=params['pageno'])
- return params
-
-
-# get response from search-request
-def response(resp):
- results = []
- dom = html.fromstring(resp.text)
- result_rows = dom.xpath(torrent_xpath)
-
- try:
- script_element = dom.xpath(script_xpath)[0]
- json_string = script_element.text[script_element.text.find('{'):]
- torrents_json = loads(json_string)
- except:
- return []
-
- # parse results
- for torrent_row, torrent_json in zip(result_rows, torrents_json['data']['list']):
- title = torrent_json['name']
- seed = int(torrent_json['seeds'])
- leech = int(torrent_json['peers'])
- size = int(torrent_json['size'])
- torrent_hash = torrent_json['hash']
-
- torrentfile = torrent_file_url.format(torrent_hash=torrent_hash)
- magnetlink = 'magnet:?xt=urn:btih:{}'.format(torrent_hash)
-
- age = extract_text(torrent_row.xpath(age_xpath))
- link = torrent_row.xpath(link_xpath)[0]
-
- href = urljoin(url, link)
-
- # append result
- results.append({'url': href,
- 'title': title,
- 'content': age,
- 'seed': seed,
- 'leech': leech,
- 'filesize': size,
- 'torrentfile': torrentfile,
- 'magnetlink': magnetlink,
- 'template': 'torrent.html'})
-
- # return results sorted by seeder
- return sorted(results, key=itemgetter('seed'), reverse=True)
diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py
index b1e01759f..84ff21a88 100644
--- a/searx/engines/soundcloud.py
+++ b/searx/engines/soundcloud.py
@@ -91,7 +91,7 @@ def response(resp):
for result in search_res.get('collection', []):
if result['kind'] in ('track', 'playlist'):
title = result['title']
- content = result['description']
+ content = result['description'] or ''
publishedDate = parser.parse(result['last_modified'])
uri = quote_plus(result['uri'])
embedded = embedded_url.format(uri=uri)
diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
index 9fce170eb..000e1af76 100644
--- a/searx/engines/wikipedia.py
+++ b/searx/engines/wikipedia.py
@@ -52,7 +52,7 @@ def response(resp):
api_result = loads(resp.text)
# skip disambiguation pages
- if api_result['type'] != 'standard':
+ if api_result.get('type') != 'standard':
return []
title = api_result['title']
diff --git a/searx/engines/www1x.py b/searx/engines/www1x.py
index 8d691c852..b8f111a50 100644
--- a/searx/engines/www1x.py
+++ b/searx/engines/www1x.py
@@ -7,12 +7,12 @@
@using-api no
@results HTML
@stable no (HTML can change)
- @parse url, title, thumbnail, img_src, content
+ @parse url, title, thumbnail
"""
-from lxml import html
+from lxml import html, etree
from urllib.parse import urlencode, urljoin
-from searx.utils import extract_text
+from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
# engine dependent config
categories = ['images']
@@ -21,6 +21,7 @@ paging = False
# search-url
base_url = 'https://1x.com'
search_url = base_url + '/backend/search.php?{query}'
+gallery_url = 'https://gallery.1x.com/'
# do search-request
@@ -33,23 +34,18 @@ def request(query, params):
# get response from search-request
def response(resp):
results = []
-
- dom = html.fromstring(resp.text)
- for res in dom.xpath('//div[@class="List-item MainListing"]'):
- # processed start and end of link
- link = res.xpath('//a')[0]
-
+ xmldom = etree.fromstring(resp.content)
+ xmlsearchresult = eval_xpath_getindex(xmldom, '//searchresult', 0)
+ dom = html.fragment_fromstring(xmlsearchresult.text, create_parent='div')
+ for link in eval_xpath_list(dom, '/div/table/tr/td/div[2]//a'):
url = urljoin(base_url, link.attrib.get('href'))
title = extract_text(link)
-
- thumbnail_src = urljoin(base_url, res.xpath('.//img')[0].attrib['src'])
- # TODO: get image with higher resolution
- img_src = thumbnail_src
+ thumbnail_src = urljoin(gallery_url, eval_xpath_getindex(link, './/img', 0).attrib['src'])
# append result
results.append({'url': url,
'title': title,
- 'img_src': img_src,
+ 'img_src': thumbnail_src,
'content': '',
'thumbnail_src': thumbnail_src,
'template': 'images.html'})
diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py
index 5f7d2ceab..36fc72e36 100644
--- a/searx/engines/youtube_noapi.py
+++ b/searx/engines/youtube_noapi.py
@@ -49,7 +49,7 @@ def response(resp):
results = []
results_data = resp.text[resp.text.find('ytInitialData'):]
- results_data = results_data[results_data.find('{'):results_data.find(';\n')]
+ results_data = results_data[results_data.find('{'):results_data.find(';</script>')]
results_json = loads(results_data) if results_data else {}
sections = results_json.get('contents', {})\
diff --git a/searx/settings.yml b/searx/settings.yml
index 04b658e0a..412d0fd1f 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -289,6 +289,7 @@ engines:
- name : 1x
engine : www1x
shortcut : 1x
+ timeout : 3.0
disabled : True
- name : fdroid
@@ -966,12 +967,6 @@ engines:
page_size : 10
disabled : True
- - name : seedpeer
- shortcut : speu
- engine : seedpeer
- categories: files, music, videos
-
-
- name : naver
shortcut: nvr
engine: xpath
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-11 04:50:12 +0000