diff options
38 files changed, 892 insertions, 210 deletions
diff --git a/AUTHORS.rst b/AUTHORS.rst index dacb45923..ef1ae7809 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -28,3 +28,4 @@ generally made searx better: - @courgette - @kernc - @Cqoicebordel +- @Reventl0v diff --git a/searx/engines/500px.py b/searx/engines/500px.py new file mode 100644 index 000000000..5d53af32c --- /dev/null +++ b/searx/engines/500px.py @@ -0,0 +1,57 @@ +## 500px (Images) +# +# @website https://500px.com +# @provide-api yes (https://developers.500px.com/) +# +# @using-api no +# @results HTML +# @stable no (HTML can change) +# @parse url, title, thumbnail, img_src, content +# +# @todo rewrite to api + + +from urllib import urlencode +from urlparse import urljoin +from lxml import html + +# engine dependent config +categories = ['images'] +paging = True + +# search-url +base_url = 'https://500px.com' +search_url = base_url+'/search?search?page={pageno}&type=photos&{query}' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(pageno=params['pageno'], + query=urlencode({'q': query})) + + return params + + +# get response from search-request +def response(resp): + results = [] + + dom = html.fromstring(resp.text) + + # parse results + for result in dom.xpath('//div[@class="photo"]'): + link = result.xpath('.//a')[0] + url = urljoin(base_url, link.attrib.get('href')) + title = result.xpath('.//div[@class="title"]//text()')[0] + img_src = link.xpath('.//img')[0].attrib['src'] + content = result.xpath('.//div[@class="info"]//text()')[0] + + # append result + results.append({'url': url, + 'title': title, + 'img_src': img_src, + 'content': content, + 'template': 'images.html'}) + + # return results + return results diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index 80356a8cd..d42339af8 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -41,11 +41,8 @@ def load_module(filename): module.name = modname return module -if 'engines' not in settings or not settings['engines']: - print '[E] Error no engines found. Edit your settings.yml' - exit(2) -for engine_data in settings['engines']: +def load_engine(engine_data): engine_name = engine_data['engine'] engine = load_module(engine_name + '.py') @@ -84,10 +81,10 @@ for engine_data in settings['engines']: if engine_attr.startswith('_'): continue if getattr(engine, engine_attr) is None: - print '[E] Engine config error: Missing attribute "{0}.{1}"'.format(engine.name, engine_attr) # noqa + print('[E] Engine config error: Missing attribute "{0}.{1}"'\ + .format(engine.name, engine_attr)) sys.exit(1) - engines[engine.name] = engine engine.stats = { 'result_count': 0, 'search_count': 0, @@ -104,7 +101,12 @@ for engine_data in settings['engines']: if engine.shortcut: # TODO check duplications + if engine.shortcut in engine_shortcuts: + print('[E] Engine config error: ambigious shortcut: {0}'\ + .format(engine.shortcut)) + sys.exit(1) engine_shortcuts[engine.shortcut] = engine.name + return engine def get_engines_stats(): @@ -194,3 +196,12 @@ def get_engines_stats(): sorted(errors, key=itemgetter('avg'), reverse=True) ), ] + + +if 'engines' not in settings or not settings['engines']: + print '[E] Error no engines found. Edit your settings.yml' + exit(2) + +for engine_data in settings['engines']: + engine = load_engine(engine_data) + engines[engine.name] = engine diff --git a/searx/engines/bing_news.py b/searx/engines/bing_news.py index 5dce4a2b2..3dda04cbb 100644 --- a/searx/engines/bing_news.py +++ b/searx/engines/bing_news.py @@ -57,12 +57,16 @@ def response(resp): link = result.xpath('.//div[@class="newstitle"]/a')[0] url = link.attrib.get('href') title = ' '.join(link.xpath('.//text()')) - contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()') + contentXPath = result.xpath('.//div[@class="sn_txt"]/div' + '//span[@class="sn_snip"]//text()') if contentXPath is not None: content = escape(' '.join(contentXPath)) # parse publishedDate - publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()') + publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div' + '//span[contains(@class,"sn_ST")]' + '//span[contains(@class,"sn_tm")]' + '//text()') if publishedDateXPath is not None: publishedDate = escape(' '.join(publishedDateXPath)) @@ -74,7 +78,8 @@ def response(resp): timeNumbers = re.findall(r'\d+', publishedDate) publishedDate = datetime.now()\ - timedelta(hours=int(timeNumbers[0])) - elif re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$", publishedDate): + elif re.match("^[0-9]+ hour(s|)," + " [0-9]+ minute(s|) ago$", publishedDate): timeNumbers = re.findall(r'\d+', publishedDate) publishedDate = datetime.now()\ - timedelta(hours=int(timeNumbers[0]))\ diff --git a/searx/engines/faroo.py b/searx/engines/faroo.py index dada4758d..5360ea156 100644 --- a/searx/engines/faroo.py +++ b/searx/engines/faroo.py @@ -22,10 +22,17 @@ api_key = None # search-url url = 'http://www.faroo.com/' -search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}' +search_url = url + 'api?{query}'\ + '&start={offset}'\ + '&length={number_of_results}'\ + '&l={language}'\ + '&src={categorie}'\ + '&i=false'\ + '&f=json'\ + '&key={api_key}' # noqa search_category = {'general': 'web', - 'news': 'news'} + 'news': 'news'} # do search-request @@ -80,8 +87,8 @@ def response(resp): # parse results for result in search_res['results']: if result['news']: - # timestamp (how many milliseconds have passed between now and the beginning of 1970) - publishedDate = datetime.datetime.fromtimestamp(result['date']/1000.0) + # timestamp (milliseconds since 1970) + publishedDate = datetime.datetime.fromtimestamp(result['date']/1000.0) # noqa # append news result results.append({'url': result['url'], diff --git a/searx/engines/flickr-noapi.py b/searx/engines/flickr-noapi.py new file mode 100644 index 000000000..522503b53 --- /dev/null +++ b/searx/engines/flickr-noapi.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +# Flickr (Images) +# +# @website https://www.flickr.com +# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html) +# +# @using-api no +# @results HTML +# @stable no +# @parse url, title, thumbnail, img_src + +from urllib import urlencode +from json import loads +import re + +categories = ['images'] + +url = 'https://secure.flickr.com/' +search_url = url+'search/?{query}&page={page}' +photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' +regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) +image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') + +paging = True + + +def build_flickr_url(user_id, photo_id): + return photo_url.format(userid=user_id, photoid=photo_id) + + +def request(query, params): + params['url'] = search_url.format(query=urlencode({'text': query}), + page=params['pageno']) + return params + + +def response(resp): + results = [] + + matches = regex.search(resp.text) + + if matches is None: + return results + + match = matches.group(1) + search_results = loads(match) + + if '_data' not in search_results: + return [] + + photos = search_results['_data'] + + for photo in photos: + + # In paged configuration, the first pages' photos are represented by a None object + if photo is None: + continue + + img_src = None + # From the biggest to the lowest format + for image_size in image_sizes: + if image_size in photo['sizes']: + img_src = photo['sizes'][image_size]['displayUrl'] + break + + if not img_src: + continue + + if 'id' not in photo['owner']: + continue + + url = build_flickr_url(photo['owner']['id'], photo['id']) + + title = photo['title'] + + content = '<span class="photo-author">' + photo['owner']['username'] + '</span><br />' + + if 'description' in photo: + content = content + '<span class="description">' + photo['description'] + '</span>' + + # append result + results.append({'url': url, + 'title': title, + 'img_src': img_src, + 'content': content, + 'template': 'images.html'}) + + return results diff --git a/searx/engines/flickr.py b/searx/engines/flickr.py index 4ec2841dd..2fa5ed7ec 100644 --- a/searx/engines/flickr.py +++ b/searx/engines/flickr.py @@ -1,54 +1,80 @@ #!/usr/bin/env python +## Flickr (Images) +# +# @website https://www.flickr.com +# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, thumbnail, img_src +#More info on api-key : https://www.flickr.com/services/apps/create/ + from urllib import urlencode -#from json import loads -from urlparse import urljoin -from lxml import html -from time import time +from json import loads categories = ['images'] -url = 'https://secure.flickr.com/' -search_url = url+'search/?{query}&page={page}' -results_xpath = '//div[@class="view display-item-tile"]/figure/div' +nb_per_page = 15 +paging = True +api_key= None + + +url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key={api_key}&{text}&sort=relevance&extras=description%2C+owner_name%2C+url_o%2C+url_z&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}' +photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' paging = True +def build_flickr_url(user_id, photo_id): + return photo_url.format(userid=user_id,photoid=photo_id) + def request(query, params): - params['url'] = search_url.format(query=urlencode({'text': query}), - page=params['pageno']) - time_string = str(int(time())-3) - params['cookies']['BX'] = '3oqjr6d9nmpgl&b=3&s=dh' - params['cookies']['xb'] = '421409' - params['cookies']['localization'] = 'en-us' - params['cookies']['flrbp'] = time_string +\ - '-3a8cdb85a427a33efda421fbda347b2eaf765a54' - params['cookies']['flrbs'] = time_string +\ - '-ed142ae8765ee62c9ec92a9513665e0ee1ba6776' - params['cookies']['flrb'] = '9' + params['url'] = url.format(text=urlencode({'text': query}), + api_key=api_key, + nb_per_page=nb_per_page, + page=params['pageno']) return params def response(resp): results = [] - dom = html.fromstring(resp.text) - for result in dom.xpath(results_xpath): - img = result.xpath('.//img') + + search_results = loads(resp.text) - if not img: - continue + # return empty array if there are no results + if not 'photos' in search_results: + return [] + + if not 'photo' in search_results['photos']: + return [] - img = img[0] - img_src = 'https:'+img.attrib.get('src') + photos = search_results['photos']['photo'] - if not img_src: + # parse results + for photo in photos: + if 'url_o' in photo: + img_src = photo['url_o'] + elif 'url_z' in photo: + img_src = photo['url_z'] + else: continue - href = urljoin(url, result.xpath('.//a')[0].attrib.get('href')) - title = img.attrib.get('alt', '') - results.append({'url': href, + url = build_flickr_url(photo['owner'], photo['id']) + + title = photo['title'] + + content = '<span class="photo-author">'+ photo['ownername'] +'</span><br />' + + content = content + '<span class="description">' + photo['description']['_content'] + '</span>' + + # append result + results.append({'url': url, 'title': title, 'img_src': img_src, + 'content': content, 'template': 'images.html'}) + + # return results return results diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 491f5c2c2..79fac3fb0 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -9,7 +9,7 @@ # @stable yes (but deprecated) # @parse url, title, img_src -from urllib import urlencode,unquote +from urllib import urlencode, unquote from json import loads # engine dependent config diff --git a/searx/engines/kickass.py b/searx/engines/kickass.py new file mode 100644 index 000000000..f1fcd9e1a --- /dev/null +++ b/searx/engines/kickass.py @@ -0,0 +1,87 @@ +## Kickass Torrent (Videos, Music, Files) +# +# @website https://kickass.so +# @provide-api no (nothing found) +# +# @using-api no +# @results HTML (using search portal) +# @stable yes (HTML can change) +# @parse url, title, content, seed, leech, magnetlink + +from urlparse import urljoin +from cgi import escape +from urllib import quote +from lxml import html +from operator import itemgetter + +# engine dependent config +categories = ['videos', 'music', 'files'] +paging = True + +# search-url +url = 'https://kickass.so/' +search_url = url + 'search/{search_term}/{pageno}/' + +# specific xpath variables +magnet_xpath = './/a[@title="Torrent magnet link"]' +#content_xpath = './/font[@class="detDesc"]//text()' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(search_term=quote(query), + pageno=params['pageno']) + + # FIX: SSLError: hostname 'kickass.so' + # doesn't match either of '*.kickass.to', 'kickass.to' + params['verify'] = False + + return params + + +# get response from search-request +def response(resp): + results = [] + + dom = html.fromstring(resp.text) + + search_res = dom.xpath('//table[@class="data"]//tr') + + # return empty array if nothing is found + if not search_res: + return [] + + # parse results + for result in search_res[1:]: + link = result.xpath('.//a[@class="cellMainLink"]')[0] + href = urljoin(url, link.attrib['href']) + title = ' '.join(link.xpath('.//text()')) + content = escape(html.tostring(result.xpath('.//span[@class="font11px lightgrey block"]')[0], method="text")) + seed = result.xpath('.//td[contains(@class, "green")]/text()')[0] + leech = result.xpath('.//td[contains(@class, "red")]/text()')[0] + + # convert seed to int if possible + if seed.isdigit(): + seed = int(seed) + else: + seed = 0 + + # convert leech to int if possible + if leech.isdigit(): + leech = int(leech) + else: + leech = 0 + + magnetlink = result.xpath(magnet_xpath)[0].attrib['href'] + + # append result + results.append({'url': href, + 'title': title, + 'content': content, + 'seed': seed, + 'leech': leech, + 'magnetlink': magnetlink, + 'template': 'torrent.html'}) + + # return results sorted by seeder + return sorted(results, key=itemgetter('seed'), reverse=True) diff --git a/searx/engines/mediawiki.py b/searx/engines/mediawiki.py index 4a8b0e8b8..8ca32c62a 100644 --- a/searx/engines/mediawiki.py +++ b/searx/engines/mediawiki.py @@ -28,15 +28,17 @@ search_url = base_url + 'w/api.php?action=query'\ '&srprop=timestamp'\ '&format=json'\ '&sroffset={offset}'\ - '&srlimit={limit}' + '&srlimit={limit}' # noqa # do search-request def request(query, params): offset = (params['pageno'] - 1) * number_of_results + string_args = dict(query=urlencode({'srsearch': query}), - offset=offset, - limit=number_of_results) + offset=offset, + limit=number_of_results) + format_strings = list(Formatter().parse(base_url)) if params['language'] == 'all': @@ -67,7 +69,8 @@ def response(resp): # parse results for result in search_results['query']['search']: - url = base_url.format(language=resp.search_params['language']) + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')) + url = base_url.format(language=resp.search_params['language']) +\ + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')) # append result results.append({'url': url, diff --git a/searx/engines/openstreetmap.py b/searx/engines/openstreetmap.py index 36b6011e7..68446ef5f 100644 --- a/searx/engines/openstreetmap.py +++ b/searx/engines/openstreetmap.py @@ -9,20 +9,24 @@ # @parse url, title from json import loads +from searx.utils import searx_useragent # engine dependent config categories = ['map'] paging = False # search-url -url = 'https://nominatim.openstreetmap.org/search/{query}?format=json&polygon_geojson=1&addressdetails=1' - +base_url = 'https://nominatim.openstreetmap.org/' +search_string = 'search/{query}?format=json&polygon_geojson=1&addressdetails=1' result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}' # do search-request def request(query, params): - params['url'] = url.format(query=query) + params['url'] = base_url + search_string.format(query=query) + + # using searx User-Agent + params['headers']['User-Agent'] = searx_useragent() return params @@ -68,8 +72,8 @@ def response(resp): address.update({'house_number': address_raw.get('house_number'), 'road': address_raw.get('road'), 'locality': address_raw.get('city', - address_raw.get('town', - address_raw.get('village'))), + address_raw.get('town', # noqa + address_raw.get('village'))), # noqa 'postcode': address_raw.get('postcode'), 'country': address_raw.get('country'), 'country_code': address_raw.get('country_code')}) diff --git a/searx/engines/photon.py b/searx/engines/photon.py new file mode 100644 index 000000000..16340d24a --- /dev/null +++ b/searx/engines/photon.py @@ -0,0 +1,132 @@ +## Photon (Map) +# +# @website https://photon.komoot.de +# @provide-api yes (https://photon.komoot.de/) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title + +from urllib import urlencode +from json import loads +from searx.utils import searx_useragent + +# engine dependent config +categories = ['map'] +paging = False +language_support = True +number_of_results = 10 + +# search-url +base_url = 'https://photon.komoot.de/' +search_string = 'api/?{query}&limit={limit}' +result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}' + +# list of supported languages +allowed_languages = ['de', 'en', 'fr', 'it'] + + +# do search-request +def request(query, params): + params['url'] = base_url +\ + search_string.format(query=urlencode({'q': query}), + limit=number_of_results) + + if params['language'] != 'all': + language = params['language'].split('_')[0] + if language in allowed_languages: + params['url'] = params['url'] + "&lang=" + language + + # using searx User-Agent + params['headers']['User-Agent'] = searx_useragent() + + # FIX: SSLError: SSL3_GET_SERVER_CERTIFICATE:certificate verify failed + params['verify'] = False + + return params + + +# get response from search-request +def response(resp): + results = [] + json = loads(resp.text) + + # parse results + for r in json.get('features', {}): + + properties = r.get('properties') + + if not properties: + continue + + # get title + title = properties['name'] + + # get osm-type + if properties.get('osm_type') == 'N': + osm_type = 'node' + elif properties.get('osm_type') == 'W': + osm_type = 'way' + elif properties.get('osm_type') == 'R': + osm_type = 'relation' + else: + # continue if invalide osm-type + continue + + url = result_base_url.format(osm_type=osm_type, + osm_id=properties.get('osm_id')) + + osm = {'type': osm_type, + 'id': properties.get('osm_id')} + + geojson = r.get('geometry') + + if properties.get('extent'): + boundingbox = [properties.get('extent')[3], + properties.get('extent')[1], + properties.get('extent')[0], + properties.get('extent')[2]] + else: + # TODO: better boundingbox calculation + boundingbox = [geojson['coordinates'][1], + geojson['coordinates'][1], + geojson['coordinates'][0], + geojson['coordinates'][0]] + + # address calculation + address = {} + + # get name + if properties.get('osm_key') == 'amenity' or\ + properties.get('osm_key') == 'shop' or\ + properties.get('osm_key') == 'tourism' or\ + properties.get('osm_key') == 'leisure': + address = {'name': properties.get('name')} + + # add rest of adressdata, if something is already found + if address.get('name'): + address.update({'house_number': properties.get('housenumber'), + 'road': properties.get('street'), + 'locality': properties.get('city', + properties.get('town', # noqa + properties.get('village'))), # noqa + 'postcode': properties.get('postcode'), + 'country': properties.get('country')}) + else: + address = None + + # append result + results.append({'template': 'map.html', + 'title': title, + 'content': '', + 'longitude': geojson['coordinates'][0], + 'latitude': geojson['coordinates'][1], + 'boundingbox': boundingbox, + 'geojson': geojson, + 'address': address, + 'osm': osm, + 'url': url}) + + # return results + return results diff --git a/searx/engines/piratebay.py b/searx/engines/piratebay.py index 14905dc83..f6144faa2 100644 --- a/searx/engines/piratebay.py +++ b/searx/engines/piratebay.py @@ -19,7 +19,7 @@ categories = ['videos', 'music', 'files'] paging = True # search-url -url = 'https://thepiratebay.se/' +url = 'https://thepiratebay.cr/' search_url = url + 'search/{search_term}/{pageno}/99/{search_type}' # piratebay specific type-definitions diff --git a/searx/engines/searchcode_code.py b/searx/engines/searchcode_code.py new file mode 100644 index 000000000..2ba0e52f1 --- /dev/null +++ b/searx/engines/searchcode_code.py @@ -0,0 +1,65 @@ +## Searchcode (It) +# +# @website https://searchcode.com/ +# @provide-api yes (https://searchcode.com/api/) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, content + +from urllib import urlencode +from json import loads +import cgi +import re + +# engine dependent config +categories = ['it'] +paging = True + +# search-url +url = 'https://searchcode.com/' +search_url = url+'api/codesearch_I/?{query}&p={pageno}' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(query=urlencode({'q': query}), + pageno=params['pageno']-1) + + return params + + +# get response from search-request +def response(resp): + results = [] + + search_results = loads(resp.text) + + # parse results + for result in search_results['results']: + href = result['url'] + title = "" + result['name'] + " - " + result['filename'] + content = result['repo'] + "<br />" + + lines = dict() + for line, code in result['lines'].items(): + lines[int(line)] = code + + content = content + '<pre class="code-formatter"><table class="code">' + for line, code in sorted(lines.items()): + content = content + '<tr><td class="line-number" style="padding-right:5px;">' + content = content + str(line) + '</td><td class="code-snippet">' + # Replace every two spaces with ' &nbps;' to keep formatting while allowing the browser to break the line if necessary + content = content + cgi.escape(code).replace('\t', ' ').replace(' ', ' ').replace(' ', ' ') + content = content + "</td></tr>" + + content = content + "</table></pre>" + + # append result + results.append({'url': href, + 'title': title, + 'content': content}) + + # return results + return results diff --git a/searx/engines/searchcode_doc.py b/searx/engines/searchcode_doc.py new file mode 100644 index 000000000..e07cbeab9 --- /dev/null +++ b/searx/engines/searchcode_doc.py @@ -0,0 +1,49 @@ +## Searchcode (It) +# +# @website https://searchcode.com/ +# @provide-api yes (https://searchcode.com/api/) +# +# @using-api yes +# @results JSON +# @stable yes +# @parse url, title, content + +from urllib import urlencode +from json import loads + +# engine dependent config +categories = ['it'] +paging = True + +# search-url +url = 'https://searchcode.com/' +search_url = url+'api/search_IV/?{query}&p={pageno}' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(query=urlencode({'q': query}), + pageno=params['pageno']-1) + + return params + + +# get response from search-request +def response(resp): + results = [] + + search_results = loads(resp.text) + + # parse results + for result in search_results['results']: + href = result['url'] + title = "[" + result['type'] + "] " + result['namespace'] + " " + result['name'] + content = '<span class="highlight">[' + result['type'] + "] " + result['name'] + " " + result['synopsis'] + "</span><br />" + result['description'] + + # append result + results.append({'url': href, + 'title': title, + 'content': content}) + + # return results + return results diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py index 390e7ca1f..164a569a3 100644 --- a/searx/engines/soundcloud.py +++ b/searx/engines/soundcloud.py @@ -20,7 +20,12 @@ guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28' # search-url url = 'https://api.soundcloud.com/' -search_url = url + 'search?{query}&facet=model&limit=20&offset={offset}&linked_partitioning=1&client_id={client_id}' +search_url = url + 'search?{query}'\ + '&facet=model'\ + '&limit=20'\ + '&offset={offset}'\ + '&linked_partitioning=1'\ + '&client_id={client_id}' # noqa # do search-request diff --git a/searx/engines/subtitleseeker.py b/searx/engines/subtitleseeker.py new file mode 100644 index 000000000..48790a35c --- /dev/null +++ b/searx/engines/subtitleseeker.py @@ -0,0 +1,64 @@ +## Subtitleseeker (Video) +# +# @website http://www.subtitleseeker.com +# @provide-api no +# +# @using-api no +# @results HTML +# @stable no (HTML can change) +# @parse url, title, content + +from cgi import escape +from urllib import quote_plus +from lxml import html + +# engine dependent config +categories = ['videos'] +paging = True +language = "" + +# search-url +url = 'http://www.subtitleseeker.com/' +search_url = url+'search/TITLES/{query}&p={pageno}' + +# specific xpath variables +results_xpath = '//div[@class="boxRows"]' + + +# do search-request +def request(query, params): + params['url'] = search_url.format(query=quote_plus(query), + pageno=params['pageno']) + return params + + +# get response from search-request +def response(resp): + results = [] + + dom = html.fromstring(resp.text) + + # parse results + for result in dom.xpath(results_xpath): + link = result.xpath(".//a")[0] + href = link.attrib.get('href') + + if language is not "": + href = href + language + "/" + + title = escape(link.xpath(".//text()")[0]) + + content = result.xpath('.//div[contains(@class,"red")]//text()')[0] + content = content + " - " + content = content + html.tostring(result.xpath('.//div[contains(@class,"grey-web")]')[0], method='text') + + if result.xpath(".//span") != []: + content = content + " - (" + result.xpath(".//span//text()")[0].strip() + ")" + + # append result + results.append({'url': href, + 'title': title, + 'content': escape(content)}) + + # return results + return results diff --git a/searx/engines/yacy.py b/searx/engines/yacy.py index 3ee0e91c7..4c4fac7df 100644 --- a/searx/engines/yacy.py +++ b/searx/engines/yacy.py @@ -24,7 +24,11 @@ number_of_results = 5 # search-url base_url = 'http://localhost:8090' -search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limit}&contentdom={search_type}&resource=global' +search_url = '/yacysearch.json?{query}'\ + '&startRecord={offset}'\ + '&maximumRecords={limit}'\ + '&contentdom={search_type}'\ + '&resource=global' # noqa # yacy specific type-definitions search_types = {'general': 'text', @@ -39,10 +43,11 @@ def request(query, params): offset = (params['pageno'] - 1) * number_of_results search_type = search_types.get(params['category'], '0') - params['url'] = base_url + search_url.format(query=urlencode({'query': query}), - offset=offset, - limit=number_of_results, - search_type=search_type) + params['url'] = base_url +\ + search_url.format(query=urlencode({'query': query}), + offset=offset, + limit=number_of_results, + search_type=search_type) # add language tag if specified if params['language'] != 'all': @@ -70,19 +75,19 @@ def response(resp): # append result results.append({'url': result['link'], - 'title': result['title'], - 'content': result['description'], - 'publishedDate': publishedDate}) + 'title': result['title'], + 'content': result['description'], + 'publishedDate': publishedDate}) elif resp.search_params['category'] == 'images': # parse image results for result in search_results: # append result results.append({'url': result['url'], - 'title': result['title'], - 'content': '', - 'img_src': result['image'], - 'template': 'images.html'}) + 'title': result['title'], + 'content': '', + 'img_src': result['image'], + 'template': 'images.html'}) #TODO parse video, audio and file results diff --git a/searx/engines/yahoo.py b/searx/engines/yahoo.py index 938540ece..c6c5b0d0d 100644 --- a/searx/engines/yahoo.py +++ b/searx/engines/yahoo.py @@ -20,7 +20,8 @@ paging = True language_support = True # search-url -search_url = 'https://search.yahoo.com/search?{query}&b={offset}&fl=1&vl=lang_{lang}' +base_url = 'https://search.yahoo.com/' +search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}' # specific xpath variables results_xpath = '//div[@class="res"]' @@ -57,9 +58,9 @@ def request(query, params): else: language = params['language'].split('_')[0] - params['url'] = search_url.format(offset=offset, - query=urlencode({'p': query}), - lang=language) + params['url'] = base_url + search_url.format(offset=offset, + query=urlencode({'p': query}), + lang=language) # TODO required? params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\ diff --git a/searx/https_rewrite.py b/searx/https_rewrite.py index 9faf3599d..408474a44 100644 --- a/searx/https_rewrite.py +++ b/searx/https_rewrite.py @@ -16,6 +16,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >. ''' import re +from urlparse import urlparse from lxml import etree from os import listdir from os.path import isfile, isdir, join @@ -86,15 +87,23 @@ def load_single_https_ruleset(filepath): # TODO hack, which convert a javascript regex group # into a valid python regex group - rule_from = ruleset.attrib.get('from').replace('$', '\\') - rule_to = ruleset.attrib.get('to').replace('$', '\\') + rule_from = ruleset.attrib['from'].replace('$', '\\') + if rule_from.endswith('\\'): + rule_from = rule_from[:-1]+'$' + rule_to = ruleset.attrib['to'].replace('$', '\\') + if rule_to.endswith('\\'): + rule_to = rule_to[:-1]+'$' # TODO, not working yet because of the hack above, # currently doing that in webapp.py # rule_from_rgx = re.compile(rule_from, re.I) # append rule - rules.append((rule_from, rule_to)) + try: + rules.append((re.compile(rule_from, re.I | re.U), rule_to)) + except: + # TODO log regex error + continue # this child define an exclusion elif ruleset.tag == 'exclusion': @@ -143,3 +152,56 @@ def load_https_rules(rules_path): https_rules.append(ruleset) print(' * {n} https-rules loaded'.format(n=len(https_rules))) + + + +def https_url_rewrite(result): + skip_https_rewrite = False + # check if HTTPS rewrite is possible + for target, rules, exclusions in https_rules: + + # check if target regex match with url + if target.match(result['parsed_url'].netloc): + # process exclusions + for exclusion in exclusions: + # check if exclusion match with url + if exclusion.match(result['url']): + skip_https_rewrite = True + break + + # skip https rewrite if required + if skip_https_rewrite: + break + + # process rules + for rule in rules: + try: + new_result_url = rule[0].sub(rule[1], result['url']) + except: + break + + # parse new url + new_parsed_url = urlparse(new_result_url) + + # continiue if nothing was rewritten + if result['url'] == new_result_url: + continue + + # get domainname from result + # TODO, does only work correct with TLD's like + # asdf.com, not for asdf.com.de + # TODO, using publicsuffix instead of this rewrite rule + old_result_domainname = '.'.join( + result['parsed_url'].hostname.split('.')[-2:]) + new_result_domainname = '.'.join( + new_parsed_url.hostname.split('.')[-2:]) + + # check if rewritten hostname is the same, + # to protect against wrong or malicious rewrite rules + if old_result_domainname == new_result_domainname: + # set new url + result['url'] = new_result_url + + # target has matched, do not search over the other rules + break + return result diff --git a/searx/https_rules/Soundcloud.xml b/searx/https_rules/Soundcloud.xml index 0baa5832b..6958e8cbc 100644 --- a/searx/https_rules/Soundcloud.xml +++ b/searx/https_rules/Soundcloud.xml @@ -89,7 +89,7 @@ <rule from="^http://([aiw]\d|api|wis)\.sndcdn\.com/" to="https://$1.sndcdn.com/" /> - <rule from="^http://((?:api|backstage|blog|connect|developers|ec-media|eventlogger|help-assets|media|visuals|w|www)\.)?soundcloud\.com/" + <rule from="^http://((?:api|backstage|blog|connect|developers|ec-media|eventlogger|help-assets|media|visuals|w|www)\.|)soundcloud\.com/" to="https://$1soundcloud.com/" /> <rule from="^https?://scbackstage\.wpengine\.netdna-cdn\.com/" diff --git a/searx/search.py b/searx/search.py index 5b5cc6200..d1d03805f 100644 --- a/searx/search.py +++ b/searx/search.py @@ -19,8 +19,9 @@ import requests as requests_lib import threading import re from itertools import izip_longest, chain -from datetime import datetime from operator import itemgetter +from Queue import Queue +from time import time from urlparse import urlparse, unquote from searx.engines import ( categories, engines @@ -33,82 +34,74 @@ from searx.query import Query number_of_searches = 0 +def search_request_wrapper(fn, url, engine_name, **kwargs): + try: + return fn(url, **kwargs) + except Exception, e: + # increase errors stats + engines[engine_name].stats['errors'] += 1 + + # print engine name and specific error message + print('[E] Error with engine "{0}":\n\t{1}'.format( + engine_name, str(e))) + return + + def threaded_requests(requests): - for fn, url, request_args in requests: + timeout_limit = max(r[2]['timeout'] for r in requests) + search_start = time() + for fn, url, request_args, engine_name in requests: + request_args['timeout'] = timeout_limit th = threading.Thread( - target=fn, - args=(url,), + target=search_request_wrapper, + args=(fn, url, engine_name), kwargs=request_args, name='search_request', ) + th._engine_name = engine_name th.start() for th in threading.enumerate(): if th.name == 'search_request': - th.join() + remaining_time = max(0.0, timeout_limit - (time() - search_start)) + th.join(remaining_time) + if th.isAlive(): + print('engine timeout: {0}'.format(th._engine_name)) + # get default reqest parameter def default_request_params(): return { - 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}} + 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}, 'verify': True} # create a callback wrapper for the search engine results -def make_callback(engine_name, - results, - suggestions, - answers, - infoboxes, - callback, - params): +def make_callback(engine_name, results_queue, callback, params): # creating a callback wrapper for the search engine results def process_callback(response, **kwargs): - cb_res = [] response.search_params = params - # callback - try: - search_results = callback(response) - except Exception, e: - # increase errors stats + timeout_overhead = 0.2 # seconds + search_duration = time() - params['started'] + timeout_limit = engines[engine_name].timeout + timeout_overhead + if search_duration > timeout_limit: + engines[engine_name].stats['page_load_time'] += timeout_limit engines[engine_name].stats['errors'] += 1 - results[engine_name] = cb_res - - # print engine name and specific error message - print '[E] Error with engine "{0}":\n\t{1}'.format( - engine_name, str(e)) return + # callback + search_results = callback(response) + # add results for result in search_results: result['engine'] = engine_name - # if it is a suggestion, add it to list of suggestions - if 'suggestion' in result: - # TODO type checks - suggestions.add(result['suggestion']) - continue - - # if it is an answer, add it to list of answers - if 'answer' in result: - answers.add(result['answer']) - continue - - # if it is an infobox, add it to list of infoboxes - if 'infobox' in result: - infoboxes.append(result) - continue - - # append result - cb_res.append(result) - - results[engine_name] = cb_res + results_queue.put_nowait((engine_name, search_results)) # update stats with current page-load-time - engines[engine_name].stats['page_load_time'] += \ - (datetime.now() - params['started']).total_seconds() + engines[engine_name].stats['page_load_time'] += search_duration return process_callback @@ -420,6 +413,7 @@ class Search(object): # init vars requests = [] + results_queue = Queue() results = {} suggestions = set() answers = set() @@ -452,14 +446,13 @@ class Search(object): request_params = default_request_params() request_params['headers']['User-Agent'] = user_agent request_params['category'] = selected_engine['category'] - request_params['started'] = datetime.now() + request_params['started'] = time() request_params['pageno'] = self.pageno request_params['language'] = self.lang # update request parameters dependent on # search-engine (contained in engines folder) - request_params = engine.request(self.query.encode('utf-8'), - request_params) + engine.request(self.query.encode('utf-8'), request_params) if request_params['url'] is None: # TODO add support of offline engines @@ -468,13 +461,9 @@ class Search(object): # create a callback wrapper for the search engine results callback = make_callback( selected_engine['name'], - results, - suggestions, - answers, - infoboxes, + results_queue, engine.response, - request_params - ) + request_params) # create dictionary which contain all # informations about the request @@ -482,7 +471,8 @@ class Search(object): headers=request_params['headers'], hooks=dict(response=callback), cookies=request_params['cookies'], - timeout=engine.timeout + timeout=engine.timeout, + verify=request_params['verify'] ) # specific type of request (GET or POST) @@ -497,11 +487,34 @@ class Search(object): continue # append request to list - requests.append((req, request_params['url'], request_args)) + requests.append((req, request_params['url'], request_args, selected_engine['name'])) + if not requests: + return results, suggestions, answers, infoboxes # send all search-request threaded_requests(requests) + + while not results_queue.empty(): + engine_name, engine_results = results_queue.get_nowait() + + # TODO type checks + [suggestions.add(x['suggestion']) + for x in list(engine_results) + if 'suggestion' in x + and engine_results.remove(x) is None] + + [answers.add(x['answer']) + for x in list(engine_results) + if 'answer' in x + and engine_results.remove(x) is None] + + infoboxes.extend(x for x in list(engine_results) + if 'infobox' in x + and engine_results.remove(x) is None) + + results[engine_name] = engine_results + # update engine-specific stats for engine_name, engine_results in results.items(): engines[engine_name].stats['search_count'] += 1 diff --git a/searx/settings.yml b/searx/settings.yml index d64a23bf1..260b56ca8 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -64,12 +64,20 @@ engines: # engine : filecrop # categories : files # shortcut : fc + + - name : 500px + engine : 500px + shortcut : px - name : flickr - engine : flickr categories : images shortcut : fl - timeout: 3.0 +# You can use the engine using the official stable API, but you need an API key +# See : https://www.flickr.com/services/apps/create/ +# engine : flickr +# api_key: 'apikey' # required! +# Or you can use the html non-stable engine, activated by default + engine : flickr-noapi - name : general-file engine : generalfile @@ -95,9 +103,17 @@ engines: engine : openstreetmap shortcut : osm - - name : piratebay - engine : piratebay - shortcut : tpb + - name : photon + engine : photon + shortcut : ph + +# - name : piratebay +# engine : piratebay +# shortcut : tpb + + - name : kickass + engine : kickass + shortcut : ka - name : soundcloud engine : soundcloud @@ -106,6 +122,21 @@ engines: - name : stackoverflow engine : stackoverflow shortcut : st + + - name : searchcode doc + engine : searchcode_doc + shortcut : scd + + - name : searchcode code + engine : searchcode_code + shortcut : scc + + - name : subtitleseeker + engine : subtitleseeker + shortcut : ss +# The language is an option. You can put any language written in english +# Examples : English, French, German, Hungarian, Chinese... +# language : English - name : startpage engine : startpage diff --git a/searx/static/courgette/img/icon_kickass.ico b/searx/static/courgette/img/icon_kickass.ico Binary files differnew file mode 100644 index 000000000..4aa2c77a5 --- /dev/null +++ b/searx/static/courgette/img/icon_kickass.ico diff --git a/searx/static/default/img/icon_kickass.ico b/searx/static/default/img/icon_kickass.ico Binary files differnew file mode 100644 index 000000000..4aa2c77a5 --- /dev/null +++ b/searx/static/default/img/icon_kickass.ico diff --git a/searx/static/oscar/img/icons/kickass.png b/searx/static/oscar/img/icons/kickass.png Binary files differnew file mode 100644 index 000000000..567d1039f --- /dev/null +++ b/searx/static/oscar/img/icons/kickass.png diff --git a/searx/static/oscar/js/searx.min.js b/searx/static/oscar/js/searx.min.js index 69cb816ae..2fba71063 100644 --- a/searx/static/oscar/js/searx.min.js +++ b/searx/static/oscar/js/searx.min.js @@ -1,2 +1,2 @@ -/*! oscar/searx.min.js | 30-11-2014 | https://github.com/asciimoo/searx */ -requirejs.config({baseUrl:"/static/oscar/js",paths:{app:"../app"}}),searx.autocompleter&&(searx.searchResults=new Bloodhound({datumTokenizer:Bloodhound.tokenizers.obj.whitespace("value"),queryTokenizer:Bloodhound.tokenizers.whitespace,remote:"/autocompleter?q=%QUERY"}),searx.searchResults.initialize()),$(document).ready(function(){searx.autocompleter&&$("#q").typeahead(null,{name:"search-results",displayKey:function(a){return a},source:searx.searchResults.ttAdapter()})}),$(document).ready(function(){$("#q.autofocus").focus(),$(".select-all-on-click").click(function(){$(this).select()}),$(".btn-collapse").click(function(){var a=$(this).data("btn-text-collapsed"),b=$(this).data("btn-text-not-collapsed");""!==a&&""!==b&&(new_html=$(this).hasClass("collapsed")?$(this).html().replace(a,b):$(this).html().replace(b,a),$(this).html(new_html))}),$(".btn-toggle .btn").click(function(){var a="btn-"+$(this).data("btn-class"),b=$(this).data("btn-label-default"),c=$(this).data("btn-label-toggled");""!==c&&(new_html=$(this).hasClass("btn-default")?$(this).html().replace(b,c):$(this).html().replace(c,b),$(this).html(new_html)),$(this).toggleClass(a),$(this).toggleClass("btn-default")})}),$(document).ready(function(){$(".searx_overpass_request").on("click",function(a){var b="https://overpass-api.de/api/interpreter?data=",c=b+"[out:json][timeout:25];(",d=");out meta;",e=$(this).data("osm-id"),f=$(this).data("osm-type"),g=$(this).data("result-table"),h="#"+$(this).data("result-table-loadicon"),i=["addr:city","addr:country","addr:housenumber","addr:postcode","addr:street"];if(e&&f&&g){g="#"+g;var j=null;switch(f){case"node":j=c+"node("+e+");"+d;break;case"way":j=c+"way("+e+");"+d;break;case"relation":j=c+"relation("+e+");"+d}if(j){$.ajax(j).done(function(a){if(a&&a.elements&&a.elements[0]){var b=a.elements[0],c=$(g).html();for(var d in b.tags)if(null===b.tags.name||-1==i.indexOf(d)){switch(c+="<tr><td>"+d+"</td><td>",d){case"phone":case"fax":c+='<a href="tel:'+b.tags[d].replace(/ /g,"")+'">'+b.tags[d]+"</a>";break;case"email":c+='<a href="mailto:'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"website":case"url":c+='<a href="'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"wikidata":c+='<a href="https://www.wikidata.org/wiki/'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"wikipedia":if(-1!=b.tags[d].indexOf(":")){c+='<a href="https://'+b.tags[d].substring(0,b.tags[d].indexOf(":"))+".wikipedia.org/wiki/"+b.tags[d].substring(b.tags[d].indexOf(":")+1)+'">'+b.tags[d]+"</a>";break}default:c+=b.tags[d]}c+="</td></tr>"}$(g).html(c),$(g).removeClass("hidden"),$(h).addClass("hidden")}}).fail(function(){$(h).html($(h).html()+'<p class="text-muted">could not load data!</p>')})}}$(this).off(a)}),$(".searx_init_map").on("click",function(a){var b=$(this).data("leaflet-target"),c=$(this).data("map-lon"),d=$(this).data("map-lat"),e=$(this).data("map-zoom"),f=$(this).data("map-boundingbox"),g=$(this).data("map-geojson");require(["leaflet-0.7.3.min"],function(){f&&(southWest=L.latLng(f[0],f[2]),northEast=L.latLng(f[1],f[3]),map_bounds=L.latLngBounds(southWest,northEast)),L.Icon.Default.imagePath="/static/oscar/img/map";{var a=L.map(b),h="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",i='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors',j=new L.TileLayer(h,{minZoom:1,maxZoom:19,attribution:i}),k="http://otile{s}.mqcdn.com/tiles/1.0.0/map/{z}/{x}/{y}.jpg",l='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors | Tiles Courtesy of <a href="http://www.mapquest.com/" target="_blank">MapQuest</a> <img src="http://developer.mapquest.com/content/osm/mq_logo.png">',m=new L.TileLayer(k,{minZoom:1,maxZoom:18,subdomains:"1234",attribution:l}),n="http://otile{s}.mqcdn.com/tiles/1.0.0/sat/{z}/{x}/{y}.jpg",o='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors | Tiles Courtesy of <a href="http://www.mapquest.com/" target="_blank">MapQuest</a> <img src="https://developer.mapquest.com/content/osm/mq_logo.png"> | Portions Courtesy NASA/JPL-Caltech and U.S. Depart. of Agriculture, Farm Service Agency';new L.TileLayer(n,{minZoom:1,maxZoom:11,subdomains:"1234",attribution:o})}map_bounds?setTimeout(function(){a.fitBounds(map_bounds,{maxZoom:17})},0):c&&d&&(e?a.setView(new L.LatLng(d,c),e):a.setView(new L.LatLng(d,c),8)),a.addLayer(m);var p={"OSM Mapnik":j,MapQuest:m};L.control.layers(p).addTo(a),g&&L.geoJson(g).addTo(a)}),$(this).off(a)})});
\ No newline at end of file +/*! oscar/searx.min.js | 19-12-2014 | https://github.com/asciimoo/searx */ +requirejs.config({baseUrl:"./static/oscar/js",paths:{app:"../app"}}),searx.autocompleter&&(searx.searchResults=new Bloodhound({datumTokenizer:Bloodhound.tokenizers.obj.whitespace("value"),queryTokenizer:Bloodhound.tokenizers.whitespace,remote:"/autocompleter?q=%QUERY"}),searx.searchResults.initialize()),$(document).ready(function(){searx.autocompleter&&$("#q").typeahead(null,{name:"search-results",displayKey:function(a){return a},source:searx.searchResults.ttAdapter()})}),$(document).ready(function(){$("#q.autofocus").focus(),$(".select-all-on-click").click(function(){$(this).select()}),$(".btn-collapse").click(function(){var a=$(this).data("btn-text-collapsed"),b=$(this).data("btn-text-not-collapsed");""!==a&&""!==b&&(new_html=$(this).hasClass("collapsed")?$(this).html().replace(a,b):$(this).html().replace(b,a),$(this).html(new_html))}),$(".btn-toggle .btn").click(function(){var a="btn-"+$(this).data("btn-class"),b=$(this).data("btn-label-default"),c=$(this).data("btn-label-toggled");""!==c&&(new_html=$(this).hasClass("btn-default")?$(this).html().replace(b,c):$(this).html().replace(c,b),$(this).html(new_html)),$(this).toggleClass(a),$(this).toggleClass("btn-default")}),$(".btn-sm").dblclick(function(){var a="btn-"+$(this).data("btn-class");$(this).hasClass("btn-default")?($(".btn-sm > input").attr("checked","checked"),$(".btn-sm > input").prop("checked",!0),$(".btn-sm").addClass(a),$(".btn-sm").addClass("active"),$(".btn-sm").removeClass("btn-default")):($(".btn-sm > input").attr("checked",""),$(".btn-sm > input").removeAttr("checked"),$(".btn-sm > input").checked=!1,$(".btn-sm").removeClass(a),$(".btn-sm").removeClass("active"),$(".btn-sm").addClass("btn-default"))})}),$(document).ready(function(){$(".searx_overpass_request").on("click",function(a){var b="https://overpass-api.de/api/interpreter?data=",c=b+"[out:json][timeout:25];(",d=");out meta;",e=$(this).data("osm-id"),f=$(this).data("osm-type"),g=$(this).data("result-table"),h="#"+$(this).data("result-table-loadicon"),i=["addr:city","addr:country","addr:housenumber","addr:postcode","addr:street"];if(e&&f&&g){g="#"+g;var j=null;switch(f){case"node":j=c+"node("+e+");"+d;break;case"way":j=c+"way("+e+");"+d;break;case"relation":j=c+"relation("+e+");"+d}if(j){$.ajax(j).done(function(a){if(a&&a.elements&&a.elements[0]){var b=a.elements[0],c=$(g).html();for(var d in b.tags)if(null===b.tags.name||-1==i.indexOf(d)){switch(c+="<tr><td>"+d+"</td><td>",d){case"phone":case"fax":c+='<a href="tel:'+b.tags[d].replace(/ /g,"")+'">'+b.tags[d]+"</a>";break;case"email":c+='<a href="mailto:'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"website":case"url":c+='<a href="'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"wikidata":c+='<a href="https://www.wikidata.org/wiki/'+b.tags[d]+'">'+b.tags[d]+"</a>";break;case"wikipedia":if(-1!=b.tags[d].indexOf(":")){c+='<a href="https://'+b.tags[d].substring(0,b.tags[d].indexOf(":"))+".wikipedia.org/wiki/"+b.tags[d].substring(b.tags[d].indexOf(":")+1)+'">'+b.tags[d]+"</a>";break}default:c+=b.tags[d]}c+="</td></tr>"}$(g).html(c),$(g).removeClass("hidden"),$(h).addClass("hidden")}}).fail(function(){$(h).html($(h).html()+'<p class="text-muted">could not load data!</p>')})}}$(this).off(a)}),$(".searx_init_map").on("click",function(a){var b=$(this).data("leaflet-target"),c=$(this).data("map-lon"),d=$(this).data("map-lat"),e=$(this).data("map-zoom"),f=$(this).data("map-boundingbox"),g=$(this).data("map-geojson");require(["leaflet-0.7.3.min"],function(){f&&(southWest=L.latLng(f[0],f[2]),northEast=L.latLng(f[1],f[3]),map_bounds=L.latLngBounds(southWest,northEast)),L.Icon.Default.imagePath="./static/oscar/img/map";{var a=L.map(b),h="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",i='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors',j=new L.TileLayer(h,{minZoom:1,maxZoom:19,attribution:i}),k="http://otile{s}.mqcdn.com/tiles/1.0.0/map/{z}/{x}/{y}.jpg",l='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors | Tiles Courtesy of <a href="http://www.mapquest.com/" target="_blank">MapQuest</a> <img src="http://developer.mapquest.com/content/osm/mq_logo.png">',m=new L.TileLayer(k,{minZoom:1,maxZoom:18,subdomains:"1234",attribution:l}),n="http://otile{s}.mqcdn.com/tiles/1.0.0/sat/{z}/{x}/{y}.jpg",o='Map data © <a href="https://openstreetmap.org">OpenStreetMap</a> contributors | Tiles Courtesy of <a href="http://www.mapquest.com/" target="_blank">MapQuest</a> <img src="https://developer.mapquest.com/content/osm/mq_logo.png"> | Portions Courtesy NASA/JPL-Caltech and U.S. Depart. of Agriculture, Farm Service Agency';new L.TileLayer(n,{minZoom:1,maxZoom:11,subdomains:"1234",attribution:o})}map_bounds?setTimeout(function(){a.fitBounds(map_bounds,{maxZoom:17})},0):c&&d&&(e?a.setView(new L.LatLng(d,c),e):a.setView(new L.LatLng(d,c),8)),a.addLayer(m);var p={"OSM Mapnik":j,MapQuest:m};L.control.layers(p).addTo(a),g&&L.geoJson(g).addTo(a)}),$(this).off(a)})});
\ No newline at end of file diff --git a/searx/static/oscar/js/searx_src/00_requirejs_config.js b/searx/static/oscar/js/searx_src/00_requirejs_config.js index 36767843b..99ec4b585 100644 --- a/searx/static/oscar/js/searx_src/00_requirejs_config.js +++ b/searx/static/oscar/js/searx_src/00_requirejs_config.js @@ -16,7 +16,7 @@ */
requirejs.config({
- baseUrl: '/static/oscar/js',
+ baseUrl: './static/oscar/js',
paths: {
app: '../app'
}
diff --git a/searx/static/oscar/js/searx_src/element_modifiers.js b/searx/static/oscar/js/searx_src/element_modifiers.js index 088bd7a46..dd45b77e0 100644 --- a/searx/static/oscar/js/searx_src/element_modifiers.js +++ b/searx/static/oscar/js/searx_src/element_modifiers.js @@ -63,4 +63,25 @@ $(document).ready(function(){ $(this).toggleClass(btnClass);
$(this).toggleClass('btn-default');
});
+
+ /**
+ * Select or deselect every categories on double clic
+ */
+ $(".btn-sm").dblclick(function() {
+ var btnClass = 'btn-' + $(this).data('btn-class'); // primary
+ if($(this).hasClass('btn-default')) {
+ $(".btn-sm > input").attr('checked', 'checked');
+ $(".btn-sm > input").prop("checked", true);
+ $(".btn-sm").addClass(btnClass);
+ $(".btn-sm").addClass('active');
+ $(".btn-sm").removeClass('btn-default');
+ } else {
+ $(".btn-sm > input").attr('checked', '');
+ $(".btn-sm > input").removeAttr('checked');
+ $(".btn-sm > input").checked = false;
+ $(".btn-sm").removeClass(btnClass);
+ $(".btn-sm").removeClass('active');
+ $(".btn-sm").addClass('btn-default');
+ }
+ });
});
diff --git a/searx/static/oscar/js/searx_src/leaflet_map.js b/searx/static/oscar/js/searx_src/leaflet_map.js index 88af1e712..b5112ef5e 100644 --- a/searx/static/oscar/js/searx_src/leaflet_map.js +++ b/searx/static/oscar/js/searx_src/leaflet_map.js @@ -116,7 +116,7 @@ $(document).ready(function(){ // TODO hack
// change default imagePath
- L.Icon.Default.imagePath = "/static/oscar/img/map";
+ L.Icon.Default.imagePath = "./static/oscar/img/map";
// init map
var map = L.map(leaflet_target);
diff --git a/searx/templates/courgette/result_templates/images.html b/searx/templates/courgette/result_templates/images.html index 1f15ff2bb..ebda5380b 100644 --- a/searx/templates/courgette/result_templates/images.html +++ b/searx/templates/courgette/result_templates/images.html @@ -1,6 +1,6 @@ <div class="image_result"> <p> - <a href="{{ result.img_src }}"><img src="{{ result.img_src }}" title={{ result.title }}/></a> + <a href="{{ result.img_src }}"><img src="{{ result.img_src }}" title="{{ result.title|striptags }}" alt="{{ result.title|striptags }}"/></a> <span class="url"><a href="{{ result.url }}" class="small_font">original context</a></span> </p> </div> diff --git a/searx/templates/courgette/results.html b/searx/templates/courgette/results.html index d0b53b48a..62bef8c90 100644 --- a/searx/templates/courgette/results.html +++ b/searx/templates/courgette/results.html @@ -10,7 +10,7 @@ <div id="search_url"> {{ _('Search URL') }}: - <input type="text" value="{{ base_url }}?q={{ q|urlencode }}&pageno={{ pageno }}{% if selected_categories %}&category_{{ selected_categories|join("&category_") }}{% endif %}" readonly="" /> + <input type="text" value="{{ base_url }}?q={{ q|urlencode }}&pageno={{ pageno }}{% if selected_categories %}&category_{{ selected_categories|join("&category_")|replace(' ','+') }}{% endif %}" readonly /> </div> <div id="apis"> {{ _('Download results') }} @@ -43,9 +43,9 @@ {% for result in results %} {% if result['template'] %} - {% include 'default/result_templates/'+result['template'] %} + {% include 'courgette/result_templates/'+result['template'] %} {% else %} - {% include 'default/result_templates/default.html' %} + {% include 'courgette/result_templates/default.html' %} {% endif %} {% endfor %} diff --git a/searx/templates/default/infobox.html b/searx/templates/default/infobox.html index d03b008f9..d3ff8f06d 100644 --- a/searx/templates/default/infobox.html +++ b/searx/templates/default/infobox.html @@ -1,6 +1,6 @@ <div class="infobox"> <h2>{{ infobox.infobox }}</h2> - {% if infobox.img_src %}<img src="{{ infobox.img_src }}" />{% endif %} + {% if infobox.img_src %}<img src="{{ infobox.img_src }}" title="{{ infobox.infobox|striptags }}" alt="{{ infobox.infobox|striptags }}" />{% endif %} <p>{{ infobox.entity }}</p> <p>{{ infobox.content | safe }}</p> {% if infobox.attributes %} diff --git a/searx/templates/default/result_templates/images.html b/searx/templates/default/result_templates/images.html index bead78c48..4c6d59e01 100644 --- a/searx/templates/default/result_templates/images.html +++ b/searx/templates/default/result_templates/images.html @@ -1,6 +1,6 @@ <div class="image_result"> <p> - <a href="{{ result.img_src }}"><img src="{{ result.img_src }}" title="{{ result.title }}"/></a> + <a href="{{ result.img_src }}"><img src="{{ result.img_src }}" title="{{ result.title|striptags }}" alt="{{ result.title|striptags }}" /></a> <span class="url"><a href="{{ result.url }}" class="small_font">original context</a></span> </p> </div> diff --git a/searx/templates/default/result_templates/torrent.html b/searx/templates/default/result_templates/torrent.html index 6c62793a5..5925f1313 100644 --- a/searx/templates/default/result_templates/torrent.html +++ b/searx/templates/default/result_templates/torrent.html @@ -1,7 +1,6 @@ <div class="result torrent_result"> <h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3> - {% if result.content %}<p class="content">{{ result.content|safe }}</p>{% endif %} - <p class="stats">Seed: {{ result.seed }}, Leech: {{ result.leech }}</p> - <p><a href="{{ result.magnetlink }}" class="magnetlink">magnet link</a></p> <p class="url">{{ result.pretty_url }}</p> + {% if result.content %}<p class="content">{{ result.content|safe }}</p>{% endif %} + <p><a href="{{ result.magnetlink }}" class="magnetlink">magnet link</a> - <span class="stats">Seed: {{ result.seed }}, Leech: {{ result.leech }}</span></p> </div> diff --git a/searx/templates/default/results.html b/searx/templates/default/results.html index 541983532..199eb1d96 100644 --- a/searx/templates/default/results.html +++ b/searx/templates/default/results.html @@ -10,7 +10,7 @@ <div id="search_url"> {{ _('Search URL') }}: - <input type="text" value="{{ base_url }}?q={{ q|urlencode }}&pageno={{ pageno }}{% if selected_categories %}&category_{{ selected_categories|join("&category_") }}{% endif %}" readonly="" /> + <input type="text" value="{{ base_url }}?q={{ q|urlencode }}&pageno={{ pageno }}{% if selected_categories %}&category_{{ selected_categories|join("&category_")|replace(' ','+') }}{% endif %}" readonly /> </div> <div id="apis"> {{ _('Download results') }} diff --git a/searx/tests/test_webapp.py b/searx/tests/test_webapp.py index 7771567f0..b922a3675 100644 --- a/searx/tests/test_webapp.py +++ b/searx/tests/test_webapp.py @@ -49,7 +49,7 @@ class ViewsTestCase(SearxTestCase): ) result = self.app.post('/', data={'q': 'test'}) self.assertIn( - '<h3 class="result_title"> <img width="14" height="14" class="favicon" src="static/default/img/icon_youtube.ico" /><a href="http://first.test.xyz">First <span class="highlight">Test</span></a></h3>', # noqa + '<h3 class="result_title"> <img width="14" height="14" class="favicon" src="static/default/img/icon_youtube.ico" alt="youtube" /><a href="http://first.test.xyz">First <span class="highlight">Test</span></a></h3>', # noqa result.data ) self.assertIn( diff --git a/searx/webapp.py b/searx/webapp.py index ca58d8ea3..8d1676808 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -41,15 +41,12 @@ from searx.utils import ( UnicodeWriter, highlight_content, html_to_text, get_themes ) from searx.version import VERSION_STRING -from searx.https_rewrite import https_rules from searx.languages import language_codes +from searx.https_rewrite import https_url_rewrite from searx.search import Search from searx.query import Query from searx.autocomplete import backends as autocomplete_backends -from urlparse import urlparse -import re - static_path, templates_path, themes =\ get_themes(settings['themes_path'] @@ -69,7 +66,7 @@ app.secret_key = settings['server']['secret_key'] babel = Babel(app) global_favicons = [] -for indice,theme in enumerate(themes): +for indice, theme in enumerate(themes): global_favicons.append([]) theme_img_path = searx_dir+"/static/"+theme+"/img/" for (dirpath, dirnames, filenames) in os.walk(theme_img_path): @@ -218,59 +215,7 @@ def index(): if settings['server']['https_rewrite']\ and result['parsed_url'].scheme == 'http': - skip_https_rewrite = False - - # check if HTTPS rewrite is possible - for target, rules, exclusions in https_rules: - - # check if target regex match with url - if target.match(result['url']): - # process exclusions - for exclusion in exclusions: - # check if exclusion match with url - if exclusion.match(result['url']): - skip_https_rewrite = True - break - - # skip https rewrite if required - if skip_https_rewrite: - break - - # process rules - for rule in rules: - try: - # TODO, precompile rule - p = re.compile(rule[0]) - - # rewrite url if possible - new_result_url = p.sub(rule[1], result['url']) - except: - break - - # parse new url - new_parsed_url = urlparse(new_result_url) - - # continiue if nothing was rewritten - if result['url'] == new_result_url: - continue - - # get domainname from result - # TODO, does only work correct with TLD's like - # asdf.com, not for asdf.com.de - # TODO, using publicsuffix instead of this rewrite rule - old_result_domainname = '.'.join( - result['parsed_url'].hostname.split('.')[-2:]) - new_result_domainname = '.'.join( - new_parsed_url.hostname.split('.')[-2:]) - - # check if rewritten hostname is the same, - # to protect against wrong or malicious rewrite rules - if old_result_domainname == new_result_domainname: - # set new url - result['url'] = new_result_url - - # target has matched, do not search over the other rules - break + result = https_url_rewrite(result) if search.request_data.get('format', 'html') == 'html': if 'content' in result: @@ -290,7 +235,7 @@ def index(): result['pretty_url'] = u'{0}[...]{1}'.format(*url_parts) else: result['pretty_url'] = result['url'] - + # TODO, check if timezone is calculated right if 'publishedDate' in result: result['pubdate'] = result['publishedDate'].strftime('%Y-%m-%d %H:%M:%S%z') |
