diff options
Diffstat (limited to 'searx/engines')
38 files changed, 135 insertions, 132 deletions
diff --git a/searx/engines/1337x.py b/searx/engines/1337x.py index 76a7a1634..9e045bc51 100644 --- a/searx/engines/1337x.py +++ b/searx/engines/1337x.py @@ -1,7 +1,6 @@ from urllib.parse import quote, urljoin from lxml import html -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size +from searx.utils import extract_text, get_torrent_size url = 'https://1337x.to/' diff --git a/searx/engines/acgsou.py b/searx/engines/acgsou.py index d5d3e3178..7bab64f5b 100644 --- a/searx/engines/acgsou.py +++ b/searx/engines/acgsou.py @@ -11,8 +11,7 @@ from urllib.parse import urlencode from lxml import html -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size, int_or_zero +from searx.utils import extract_text, get_torrent_size, int_or_zero # engine dependent config categories = ['files', 'images', 'videos', 'music'] diff --git a/searx/engines/apkmirror.py b/searx/engines/apkmirror.py index 4e6dcd486..a8ff499af 100644 --- a/searx/engines/apkmirror.py +++ b/searx/engines/apkmirror.py @@ -11,7 +11,7 @@ from urllib.parse import urlencode from lxml import html -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py index e2f44b0f5..8f93f4f38 100644 --- a/searx/engines/archlinux.py +++ b/searx/engines/archlinux.py @@ -13,7 +13,7 @@ from urllib.parse import urlencode, urljoin from lxml import html -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config categories = ['it'] diff --git a/searx/engines/bing.py b/searx/engines/bing.py index c7b619369..a4fa1db78 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -17,8 +17,7 @@ import re from urllib.parse import urlencode from lxml import html from searx import logger, utils -from searx.engines.xpath import extract_text -from searx.utils import match_language, gen_useragent, eval_xpath +from searx.utils import extract_text, match_language, gen_useragent, eval_xpath logger = logger.getChild('bing engine') diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py index 2faade3e2..d73ee23a6 100644 --- a/searx/engines/btdigg.py +++ b/searx/engines/btdigg.py @@ -13,8 +13,7 @@ from lxml import html from operator import itemgetter from urllib.parse import quote, urljoin -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size +from searx.utils import extract_text, get_torrent_size # engine dependent config categories = ['videos', 'music', 'files'] diff --git a/searx/engines/deviantart.py b/searx/engines/deviantart.py index 2bd21fa5d..b91816daa 100644 --- a/searx/engines/deviantart.py +++ b/searx/engines/deviantart.py @@ -15,7 +15,7 @@ from lxml import html import re from urllib.parse import urlencode -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config diff --git a/searx/engines/digbt.py b/searx/engines/digbt.py index e2c0389c6..b1a90fb2f 100644 --- a/searx/engines/digbt.py +++ b/searx/engines/digbt.py @@ -12,8 +12,7 @@ from urllib.parse import urljoin from lxml import html -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size +from searx.utils import extract_text, get_torrent_size categories = ['videos', 'music', 'files'] diff --git a/searx/engines/doku.py b/searx/engines/doku.py index 513ffda89..e1b10d664 100644 --- a/searx/engines/doku.py +++ b/searx/engines/doku.py @@ -11,8 +11,7 @@ from urllib.parse import urlencode from lxml.html import fromstring -from searx.engines.xpath import extract_text -from searx.utils import eval_xpath +from searx.utils import extract_text, eval_xpath # engine dependent config categories = ['general'] # TODO , 'images', 'music', 'videos', 'files' diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index fb1ea2b2d..4a42fe7f1 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -16,9 +16,8 @@ from lxml.html import fromstring from json import loads from urllib.parse import urlencode -from searx.engines.xpath import extract_text from searx.poolrequests import get -from searx.utils import match_language, eval_xpath +from searx.utils import extract_text, match_language, eval_xpath # engine dependent config categories = ['general'] diff --git a/searx/engines/duckduckgo_definitions.py b/searx/engines/duckduckgo_definitions.py index 73154a525..7ce54f056 100644 --- a/searx/engines/duckduckgo_definitions.py +++ b/searx/engines/duckduckgo_definitions.py @@ -13,9 +13,8 @@ import json from urllib.parse import urlencode from lxml import html from re import compile -from searx.engines.xpath import extract_text from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, language_aliases -from searx.utils import html_to_text, match_language +from searx.utils import extract_text, html_to_text, match_language url = 'https://api.duckduckgo.com/'\ + '?{query}&format=json&pretty=0&no_redirect=1&d=1' diff --git a/searx/engines/duckduckgo_images.py b/searx/engines/duckduckgo_images.py index 38e141f8b..27ef7442e 100644 --- a/searx/engines/duckduckgo_images.py +++ b/searx/engines/duckduckgo_images.py @@ -15,12 +15,12 @@ from json import loads from urllib.parse import urlencode -from searx.engines.xpath import extract_text from searx.engines.duckduckgo import ( _fetch_supported_languages, supported_languages_url, get_region_code, language_aliases ) from searx.poolrequests import get +from searx.utils import extract_text # engine dependent config categories = ['images'] diff --git a/searx/engines/duden.py b/searx/engines/duden.py index a711f422e..bbe11de9b 100644 --- a/searx/engines/duden.py +++ b/searx/engines/duden.py @@ -11,8 +11,7 @@ from lxml import html, etree import re from urllib.parse import quote, urljoin -from searx.engines.xpath import extract_text -from searx.utils import eval_xpath +from searx.utils import extract_text, eval_xpath from searx import logger categories = ['general'] diff --git a/searx/engines/etools.py b/searx/engines/etools.py index efc102ef6..a0762d1c7 100644 --- a/searx/engines/etools.py +++ b/searx/engines/etools.py @@ -11,8 +11,7 @@ from lxml import html from urllib.parse import quote -from searx.engines.xpath import extract_text -from searx.utils import eval_xpath +from searx.utils import extract_text, eval_xpath categories = ['general'] paging = False diff --git a/searx/engines/fdroid.py b/searx/engines/fdroid.py index a2a5114df..3d37db44e 100644 --- a/searx/engines/fdroid.py +++ b/searx/engines/fdroid.py @@ -11,7 +11,7 @@ from urllib.parse import urlencode from lxml import html -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config categories = ['files'] diff --git a/searx/engines/framalibre.py b/searx/engines/framalibre.py index 14b659b5f..e3d056425 100644 --- a/searx/engines/framalibre.py +++ b/searx/engines/framalibre.py @@ -13,7 +13,7 @@ from html import escape from urllib.parse import urljoin, urlencode from lxml import html -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config categories = ['it'] diff --git a/searx/engines/gentoo.py b/searx/engines/gentoo.py index b6bc99fab..16b3e692d 100644 --- a/searx/engines/gentoo.py +++ b/searx/engines/gentoo.py @@ -13,7 +13,7 @@ from urllib.parse import urlencode, urljoin from lxml import html -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config categories = ['it'] diff --git a/searx/engines/google.py b/searx/engines/google.py index c9faadb6e..83b18a9a0 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -21,9 +21,8 @@ Definitions`_. from urllib.parse import urlencode, urlparse from lxml import html from flask_babel import gettext -from searx.engines.xpath import extract_text from searx import logger -from searx.utils import match_language, eval_xpath +from searx.utils import match_language, extract_text, eval_xpath logger = logger.getChild('google engine') diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 9dd5fad2c..8cfb1e17f 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -28,8 +28,7 @@ from urllib.parse import urlencode, urlparse, unquote from lxml import html from flask_babel import gettext from searx import logger -from searx.utils import eval_xpath -from searx.engines.xpath import extract_text +from searx.utils import extract_text, eval_xpath # pylint: disable=unused-import from searx.engines.google import ( diff --git a/searx/engines/google_videos.py b/searx/engines/google_videos.py index 08af55902..78e1eb1cb 100644 --- a/searx/engines/google_videos.py +++ b/searx/engines/google_videos.py @@ -14,7 +14,7 @@ from datetime import date, timedelta from json import loads from urllib.parse import urlencode from lxml import html -from searx.engines.xpath import extract_text +from searx.utils import extract_text import re # engine dependent config diff --git a/searx/engines/ina.py b/searx/engines/ina.py index cce580273..52c939498 100644 --- a/searx/engines/ina.py +++ b/searx/engines/ina.py @@ -16,7 +16,7 @@ from urllib.parse import urlencode from lxml import html from dateutil import parser from html.parser import HTMLParser -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py index af48d990b..90bd33063 100644 --- a/searx/engines/kickass.py +++ b/searx/engines/kickass.py @@ -13,8 +13,7 @@ from lxml import html from operator import itemgetter from urllib.parse import quote, urljoin -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size, convert_str_to_int +from searx.utils import extract_text, get_torrent_size, convert_str_to_int # engine dependent config categories = ['videos', 'music', 'files'] diff --git a/searx/engines/nyaa.py b/searx/engines/nyaa.py index ed8897ddc..e0a91494f 100644 --- a/searx/engines/nyaa.py +++ b/searx/engines/nyaa.py @@ -11,8 +11,7 @@ from lxml import html from urllib.parse import urlencode -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size, int_or_zero +from searx.utils import extract_text, get_torrent_size, int_or_zero # engine dependent config categories = ['files', 'images', 'videos', 'music'] diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py index 42866d058..b56b92a24 100644 --- a/searx/engines/piratebay.py +++ b/searx/engines/piratebay.py @@ -13,8 +13,7 @@ from datetime import datetime from operator import itemgetter from urllib.parse import quote, urljoin -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size +from searx.utils import extract_text, get_torrent_size # engine dependent config categories = ["videos", "music", "files"] diff --git a/searx/engines/seedpeer.py b/searx/engines/seedpeer.py index 3778abe7b..39916da6e 100644 --- a/searx/engines/seedpeer.py +++ b/searx/engines/seedpeer.py @@ -12,7 +12,7 @@ from lxml import html from json import loads from operator import itemgetter from urllib.parse import quote, urljoin -from searx.engines.xpath import extract_text +from searx.utils import extract_text url = 'https://seedpeer.me/' diff --git a/searx/engines/sepiasearch.py b/searx/engines/sepiasearch.py new file mode 100644 index 000000000..0b7c1ba6e --- /dev/null +++ b/searx/engines/sepiasearch.py @@ -0,0 +1,97 @@ +# SepiaSearch (Videos) +# +# @website https://sepiasearch.org +# @provide-api https://framagit.org/framasoft/peertube/search-index/-/tree/master/server/controllers/api +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, content, publishedDate, thumbnail + +from json import loads +from dateutil import parser, relativedelta +from urllib.parse import urlencode +from datetime import datetime + +categories = ['videos'] +paging = True +language_support = True +time_range_support = True +safesearch = True +supported_languages = [ + 'en', 'fr', 'ja', 'eu', 'ca', 'cs', 'eo', 'el', + 'de', 'it', 'nl', 'es', 'oc', 'gd', 'zh', 'pt', + 'sv', 'pl', 'fi', 'ru' +] +base_url = 'https://sepiasearch.org/api/v1/search/videos' + +safesearch_table = { + 0: 'both', + 1: 'false', + 2: 'false' +} + +time_range_table = { + 'day': relativedelta.relativedelta(), + 'week': relativedelta.relativedelta(weeks=-1), + 'month': relativedelta.relativedelta(months=-1), + 'year': relativedelta.relativedelta(years=-1) +} + + +embedded_url = '<iframe width="540" height="304" src="{url}" frameborder="0" allowfullscreen></iframe>' + + +def minute_to_hm(minute): + if isinstance(minute, int): + return "%d:%02d" % (divmod(minute, 60)) + return None + + +def request(query, params): + params['url'] = base_url + '?' + urlencode({ + 'search': query, + 'start': (params['pageno'] - 1) * 10, + 'count': 10, + 'sort': '-match', + 'nsfw': safesearch_table[params['safesearch']] + }) + + language = params['language'].split('-')[0] + if language in supported_languages: + params['url'] += '&languageOneOf[]=' + language + if params['time_range'] in time_range_table: + time = datetime.now().date() + time_range_table[params['time_range']] + params['url'] += '&startDate=' + time.isoformat() + + return params + + +def response(resp): + results = [] + + search_results = loads(resp.text) + + if 'data' not in search_results: + return [] + + for result in search_results['data']: + title = result['name'] + content = result['description'] + thumbnail = result['thumbnailUrl'] + publishedDate = parser.parse(result['publishedAt']) + embedded = embedded_url.format(url=result.get('embedUrl')) + author = result.get('account', {}).get('displayName') + length = minute_to_hm(result.get('duration')) + url = result['url'] + + results.append({'url': url, + 'title': title, + 'content': content, + 'author': author, + 'length': length, + 'template': 'videos.html', + 'publishedDate': publishedDate, + 'embedded': embedded, + 'thumbnail': thumbnail}) + + return results diff --git a/searx/engines/stackoverflow.py b/searx/engines/stackoverflow.py index 90e4543d7..c6d58de65 100644 --- a/searx/engines/stackoverflow.py +++ b/searx/engines/stackoverflow.py @@ -12,7 +12,7 @@ from urllib.parse import urlencode, urljoin from lxml import html -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config categories = ['it'] diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index b6d0c24f7..d26c2d105 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -17,9 +17,8 @@ import re from unicodedata import normalize, combining from babel import Locale from babel.localedata import locale_identifiers -from searx.engines.xpath import extract_text from searx.languages import language_codes -from searx.utils import eval_xpath, match_language +from searx.utils import extract_text, eval_xpath, match_language # engine dependent config categories = ['general'] diff --git a/searx/engines/tokyotoshokan.py b/searx/engines/tokyotoshokan.py index 9c8774d7c..9fffba8a6 100644 --- a/searx/engines/tokyotoshokan.py +++ b/searx/engines/tokyotoshokan.py @@ -13,9 +13,8 @@ import re from urllib.parse import urlencode from lxml import html -from searx.engines.xpath import extract_text from datetime import datetime -from searx.utils import get_torrent_size, int_or_zero +from searx.utils import extract_text, get_torrent_size, int_or_zero # engine dependent config categories = ['files', 'videos', 'music'] diff --git a/searx/engines/torrentz.py b/searx/engines/torrentz.py index fcc8c042c..4d3e6fdd7 100644 --- a/searx/engines/torrentz.py +++ b/searx/engines/torrentz.py @@ -15,8 +15,7 @@ import re from urllib.parse import urlencode from lxml import html from datetime import datetime -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size +from searx.utils import extract_text, get_torrent_size # engine dependent config categories = ['files', 'videos', 'music'] diff --git a/searx/engines/twitter.py b/searx/engines/twitter.py index 549b14e96..6d9bdbb5c 100644 --- a/searx/engines/twitter.py +++ b/searx/engines/twitter.py @@ -15,7 +15,7 @@ from urllib.parse import urlencode, urljoin from lxml import html from datetime import datetime -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config categories = ['social media'] diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py index ffa3724fd..c557f4e59 100644 --- a/searx/engines/wikidata.py +++ b/searx/engines/wikidata.py @@ -13,9 +13,8 @@ from searx import logger from searx.poolrequests import get -from searx.engines.xpath import extract_text from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url -from searx.utils import match_language, eval_xpath +from searx.utils import extract_text, match_language, eval_xpath from urllib.parse import urlencode from json import loads diff --git a/searx/engines/www1x.py b/searx/engines/www1x.py index 1cb74dbad..8d691c852 100644 --- a/searx/engines/www1x.py +++ b/searx/engines/www1x.py @@ -12,7 +12,7 @@ from lxml import html from urllib.parse import urlencode, urljoin -from searx.engines.xpath import extract_text +from searx.utils import extract_text # engine dependent config categories = ['images'] diff --git a/searx/engines/xpath.py b/searx/engines/xpath.py index bd97a93a5..a269253d7 100644 --- a/searx/engines/xpath.py +++ b/searx/engines/xpath.py @@ -1,7 +1,6 @@ -from urllib.parse import unquote, urlencode, urljoin, urlparse from lxml import html -from lxml.etree import _ElementStringResult, _ElementUnicodeResult -from searx.utils import html_to_text, eval_xpath +from urllib.parse import urlencode +from searx.utils import extract_text, extract_url, eval_xpath search_url = None url_xpath = None @@ -21,76 +20,6 @@ page_size = 1 first_page_num = 1 -''' -if xpath_results is list, extract the text from each result and concat the list -if xpath_results is a xml element, extract all the text node from it - ( text_content() method from lxml ) -if xpath_results is a string element, then it's already done -''' - - -def extract_text(xpath_results): - if type(xpath_results) == list: - # it's list of result : concat everything using recursive call - result = '' - for e in xpath_results: - result = result + extract_text(e) - return result.strip() - elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]: - # it's a string - return ''.join(xpath_results) - else: - # it's a element - text = html.tostring( - xpath_results, encoding='unicode', method='text', with_tail=False - ) - text = text.strip().replace('\n', ' ') - return ' '.join(text.split()) - - -def extract_url(xpath_results, search_url): - if xpath_results == []: - raise Exception('Empty url resultset') - url = extract_text(xpath_results) - - if url.startswith('//'): - # add http or https to this kind of url //example.com/ - parsed_search_url = urlparse(search_url) - url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url) - elif url.startswith('/'): - # fix relative url to the search engine - url = urljoin(search_url, url) - - # fix relative urls that fall through the crack - if '://' not in url: - url = urljoin(search_url, url) - - # normalize url - url = normalize_url(url) - - return url - - -def normalize_url(url): - parsed_url = urlparse(url) - - # add a / at this end of the url if there is no path - if not parsed_url.netloc: - raise Exception('Cannot parse url') - if not parsed_url.path: - url += '/' - - # FIXME : hack for yahoo - if parsed_url.hostname == 'search.yahoo.com'\ - and parsed_url.path.startswith('/r'): - p = parsed_url.path - mark = p.find('/**') - if mark != -1: - return unquote(p[mark + 3:]).decode() - - return url - - def request(query, params): query = urlencode({'q': query})[2:] diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 0133b57b5..3420aa6d5 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -13,8 +13,7 @@ from urllib.parse import unquote, urlencode from lxml import html -from searx.engines.xpath import extract_text, extract_url -from searx.utils import match_language, eval_xpath +from searx.utils import extract_text, extract_url, match_language, eval_xpath # engine dependent config categories = ['general'] diff --git a/searx/engines/yahoo_news.py b/searx/engines/yahoo_news.py index 345e4d91f..e9dd5d6db 100644 --- a/searx/engines/yahoo_news.py +++ b/searx/engines/yahoo_news.py @@ -13,12 +13,11 @@ import re from datetime import datetime, timedelta from urllib.parse import urlencode from lxml import html -from searx.engines.xpath import extract_text, extract_url from searx.engines.yahoo import ( parse_url, _fetch_supported_languages, supported_languages_url, language_aliases ) from dateutil import parser -from searx.utils import match_language +from searx.utils import extract_text, extract_url, match_language # engine dependent config categories = ['news'] diff --git a/searx/engines/yggtorrent.py b/searx/engines/yggtorrent.py index 37bf3b1d9..ec84d2c6b 100644 --- a/searx/engines/yggtorrent.py +++ b/searx/engines/yggtorrent.py @@ -12,8 +12,7 @@ from lxml import html from operator import itemgetter from datetime import datetime from urllib.parse import quote -from searx.engines.xpath import extract_text -from searx.utils import get_torrent_size +from searx.utils import extract_text, get_torrent_size from searx.poolrequests import get as http_get # engine dependent config diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py index fef501458..16c0eddeb 100644 --- a/searx/engines/youtube_noapi.py +++ b/searx/engines/youtube_noapi.py @@ -11,8 +11,7 @@ from functools import reduce from json import loads from urllib.parse import quote_plus -from searx.engines.xpath import extract_text -from searx.utils import list_get +from searx.utils import extract_text, list_get # engine dependent config categories = ['videos', 'music'] |