From 930f724ec639c167d870d716240ac5d4512beba2 Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Tue, 16 Dec 2014 20:40:03 +0100 Subject: Add an No Api Flickr Engine It uses the webpage json infos to build the results Let the user choose the engine in setting.yml. Noapi active by default + little corrections on Flickr engine --- searx/engines/flickr-noapi.py | 102 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 searx/engines/flickr-noapi.py (limited to 'searx/engines/flickr-noapi.py') diff --git a/searx/engines/flickr-noapi.py b/searx/engines/flickr-noapi.py new file mode 100644 index 000000000..b44affec6 --- /dev/null +++ b/searx/engines/flickr-noapi.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +## Flickr (Images) +# +# @website https://www.flickr.com +# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html) +# +# @using-api no +# @results HTML +# @stable no +# @parse url, title, thumbnail, img_src + +from urllib import urlencode +from json import loads +from urlparse import urljoin +from lxml import html +import re + +categories = ['images'] + +url = 'https://secure.flickr.com/' +search_url = url+'search/?{query}&page={page}' +photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' +regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) + +paging = True + +def build_flickr_url(user_id, photo_id): + return photo_url.format(userid=user_id,photoid=photo_id) + + +def request(query, params): + params['url'] = search_url.format(query=urlencode({'text': query}), + page=params['pageno']) + return params + + +def response(resp): + results = [] + + matches = regex.search(resp.text) + + if matches == None: + return results + + match = matches.group(1) + search_results = loads(match) + + if not '_data' in search_results: + return [] + + photos = search_results['_data'] + + for photo in photos: + + # In paged configuration, the first pages' photos are represented by a None object + if photo == None: + continue + + # From the biggest to the lowest format + if 'o' in photo['sizes']: + img_src = photo['sizes']['o']['displayUrl'] + elif 'k' in photo['sizes']: + img_src = photo['sizes']['k']['displayUrl'] + elif 'h' in photo['sizes']: + img_src = photo['sizes']['h']['displayUrl'] + elif 'b' in photo['sizes']: + img_src = photo['sizes']['b']['displayUrl'] + elif 'c' in photo['sizes']: + img_src = photo['sizes']['c']['displayUrl'] + elif 'z' in photo['sizes']: + img_src = photo['sizes']['z']['displayUrl'] + elif 'n' in photo['sizes']: + img_src = photo['sizes']['n']['displayUrl'] + elif 'm' in photo['sizes']: + img_src = photo['sizes']['m']['displayUrl'] + elif 't' in photo['sizes']: + img_src = photo['sizes']['to']['displayUrl'] + elif 'q' in photo['sizes']: + img_src = photo['sizes']['q']['displayUrl'] + elif 's' in photo['sizes']: + img_src = photo['sizes']['s']['displayUrl'] + else: + continue + + url = build_flickr_url(photo['owner']['id'], photo['id']) + + title = photo['title'] + + content = ''+ photo['owner']['username'] +'
' + + if 'description' in photo: + content = content + '' + photo['description'] + '' + + # append result + results.append({'url': url, + 'title': title, + 'img_src': img_src, + 'content': content, + 'template': 'images.html'}) + + return results -- cgit v1.2.3 From b975418e4ce33aef530f7ad88e100d47d73e4761 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Mon, 22 Dec 2014 14:15:59 +0100 Subject: [fix] flickr engine code cleanup ++ handle missing owner --- searx/engines/flickr-noapi.py | 77 ++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 45 deletions(-) (limited to 'searx/engines/flickr-noapi.py') diff --git a/searx/engines/flickr-noapi.py b/searx/engines/flickr-noapi.py index b44affec6..522503b53 100644 --- a/searx/engines/flickr-noapi.py +++ b/searx/engines/flickr-noapi.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -## Flickr (Images) -# +# Flickr (Images) +# # @website https://www.flickr.com -# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html) -# +# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html) +# # @using-api no # @results HTML # @stable no @@ -12,8 +12,6 @@ from urllib import urlencode from json import loads -from urlparse import urljoin -from lxml import html import re categories = ['images'] @@ -22,11 +20,13 @@ url = 'https://secure.flickr.com/' search_url = url+'search/?{query}&page={page}' photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}' regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL) +image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's') paging = True + def build_flickr_url(user_id, photo_id): - return photo_url.format(userid=user_id,photoid=photo_id) + return photo_url.format(userid=user_id, photoid=photo_id) def request(query, params): @@ -37,58 +37,45 @@ def request(query, params): def response(resp): results = [] - + matches = regex.search(resp.text) - - if matches == None: + + if matches is None: return results match = matches.group(1) search_results = loads(match) - - if not '_data' in search_results: + + if '_data' not in search_results: return [] - + photos = search_results['_data'] - + for photo in photos: - + # In paged configuration, the first pages' photos are represented by a None object - if photo == None: + if photo is None: continue - + + img_src = None # From the biggest to the lowest format - if 'o' in photo['sizes']: - img_src = photo['sizes']['o']['displayUrl'] - elif 'k' in photo['sizes']: - img_src = photo['sizes']['k']['displayUrl'] - elif 'h' in photo['sizes']: - img_src = photo['sizes']['h']['displayUrl'] - elif 'b' in photo['sizes']: - img_src = photo['sizes']['b']['displayUrl'] - elif 'c' in photo['sizes']: - img_src = photo['sizes']['c']['displayUrl'] - elif 'z' in photo['sizes']: - img_src = photo['sizes']['z']['displayUrl'] - elif 'n' in photo['sizes']: - img_src = photo['sizes']['n']['displayUrl'] - elif 'm' in photo['sizes']: - img_src = photo['sizes']['m']['displayUrl'] - elif 't' in photo['sizes']: - img_src = photo['sizes']['to']['displayUrl'] - elif 'q' in photo['sizes']: - img_src = photo['sizes']['q']['displayUrl'] - elif 's' in photo['sizes']: - img_src = photo['sizes']['s']['displayUrl'] - else: + for image_size in image_sizes: + if image_size in photo['sizes']: + img_src = photo['sizes'][image_size]['displayUrl'] + break + + if not img_src: + continue + + if 'id' not in photo['owner']: continue - + url = build_flickr_url(photo['owner']['id'], photo['id']) title = photo['title'] - - content = ''+ photo['owner']['username'] +'
' - + + content = '' + photo['owner']['username'] + '
' + if 'description' in photo: content = content + '' + photo['description'] + '' @@ -98,5 +85,5 @@ def response(resp): 'img_src': img_src, 'content': content, 'template': 'images.html'}) - + return results -- cgit v1.2.3 From 5d977056f7aa216eae09a22c3baaff73546f6ff1 Mon Sep 17 00:00:00 2001 From: Cqoicebordel Date: Mon, 29 Dec 2014 21:31:04 +0100 Subject: Flake8 and Twitter corrections Lots of Flake8 corrections Maybe we should change the rule to allow lines of 120 chars. It seems more usable. Big twitter correction : now it outputs the words in right order... --- searx/engines/flickr-noapi.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'searx/engines/flickr-noapi.py') diff --git a/searx/engines/flickr-noapi.py b/searx/engines/flickr-noapi.py index 522503b53..f90903647 100644 --- a/searx/engines/flickr-noapi.py +++ b/searx/engines/flickr-noapi.py @@ -53,7 +53,8 @@ def response(resp): for photo in photos: - # In paged configuration, the first pages' photos are represented by a None object + # In paged configuration, the first pages' photos + # are represented by a None object if photo is None: continue @@ -74,10 +75,15 @@ def response(resp): title = photo['title'] - content = '' + photo['owner']['username'] + '
' + content = '' +\ + photo['owner']['username'] +\ + '
' if 'description' in photo: - content = content + '' + photo['description'] + '' + content = content +\ + '' +\ + photo['description'] +\ + '' # append result results.append({'url': url, -- cgit v1.2.3 From cc4e17b6686dbefe0d57862e045f98f72a4e58fc Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Fri, 2 Jan 2015 12:33:40 +0100 Subject: [fix] pep8 --- searx/engines/flickr-noapi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'searx/engines/flickr-noapi.py') diff --git a/searx/engines/flickr-noapi.py b/searx/engines/flickr-noapi.py index f90903647..aa2fa5d3b 100644 --- a/searx/engines/flickr-noapi.py +++ b/searx/engines/flickr-noapi.py @@ -81,9 +81,9 @@ def response(resp): if 'description' in photo: content = content +\ - '' +\ - photo['description'] +\ - '' + '' +\ + photo['description'] +\ + '' # append result results.append({'url': url, -- cgit v1.2.3