diff options
Diffstat (limited to 'searx/engines')
| -rw-r--r-- | searx/engines/bing_images.py | 2 | ||||
| -rw-r--r-- | searx/engines/currency_convert.py | 6 | ||||
| -rw-r--r-- | searx/engines/google.py | 147 | ||||
| -rw-r--r-- | searx/engines/vimeo.py | 12 | ||||
| -rw-r--r-- | searx/engines/www1x.py | 4 |
5 files changed, 143 insertions, 28 deletions
diff --git a/searx/engines/bing_images.py b/searx/engines/bing_images.py index b06a57edc..839b8e5be 100644 --- a/searx/engines/bing_images.py +++ b/searx/engines/bing_images.py @@ -28,7 +28,7 @@ safesearch = True # search-url base_url = 'https://www.bing.com/' search_string = 'images/search?{query}&count=10&first={offset}' -thumb_url = "http://ts1.mm.bing.net/th?id={ihk}" # no https, bad certificate +thumb_url = "https://www.bing.com/th?id={ihk}" # safesearch definitions safesearch_types = {2: 'STRICT', diff --git a/searx/engines/currency_convert.py b/searx/engines/currency_convert.py index 1ba4575c5..26830a167 100644 --- a/searx/engines/currency_convert.py +++ b/searx/engines/currency_convert.py @@ -9,7 +9,7 @@ categories = [] url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X' weight = 100 -parser_re = re.compile(r'^\W*(\d+(?:\.\d+)?)\W*([^.0-9].+)\W*in?\W*([^\.]+)\W*$', re.I) # noqa +parser_re = re.compile(u'^\W*(\d+(?:\.\d+)?)\W*([^.0-9].+)\W+in?\W+([^\.]+)\W*$', re.I) # noqa db = 1 @@ -17,7 +17,7 @@ db = 1 def normalize_name(name): name = name.lower().replace('-', ' ') name = re.sub(' +', ' ', name) - return unicodedata.normalize('NFKD', u"" + name).lower() + return unicodedata.normalize('NFKD', name).lower() def name_to_iso4217(name): @@ -35,7 +35,7 @@ def iso4217_to_name(iso4217, language): def request(query, params): - m = parser_re.match(query) + m = parser_re.match(unicode(query, 'utf8')) if not m: # wrong query return params diff --git a/searx/engines/google.py b/searx/engines/google.py index 785cd5e66..0e78a9e2c 100644 --- a/searx/engines/google.py +++ b/searx/engines/google.py @@ -8,6 +8,7 @@ # @stable no (HTML can change) # @parse url, title, content, suggestion +import re from urllib import urlencode from urlparse import urlparse, parse_qsl from lxml import html @@ -78,15 +79,22 @@ country_to_hostname = { 'TW': 'www.google.com.tw' # Taiwan } +# osm +url_map = 'https://www.openstreetmap.org/'\ + + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M' + # search-url search_path = '/search' -maps_path = '/maps/' -redirect_path = '/url' -images_path = '/images' search_url = ('https://{hostname}' + search_path + '?{query}&start={offset}&gbv=1') +# other URLs +map_hostname_start = 'maps.google.' +maps_path = '/maps' +redirect_path = '/url' +images_path = '/images' + # specific xpath variables results_xpath = '//li[@class="g"]' url_xpath = './/h3/a/@href' @@ -95,10 +103,29 @@ content_xpath = './/span[@class="st"]' content_misc_xpath = './/div[@class="f slp"]' suggestion_xpath = '//p[@class="_Bmc"]' +# map : detail location +map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()' +map_phone_xpath = './/div[@class="s"]//table//td[2]/span/span' +map_website_url_xpath = 'h3[2]/a/@href' +map_website_title_xpath = 'h3[2]' + +# map : near the location +map_near = 'table[@class="ts"]//tr' +map_near_title = './/h4' +map_near_url = './/h4/a/@href' +map_near_phone = './/span[@class="nobr"]' + +# images images_xpath = './/div/a' image_url_xpath = './@href' image_img_src_xpath = './img/@src' +# property names +# FIXME : no translation +property_address = "Address" +property_phone = "Phone number" + +# cookies pref_cookie = '' nid_cookie = {} @@ -122,6 +149,11 @@ def get_google_nid_cookie(google_hostname): # remove google-specific tracking-url def parse_url(url_string, google_hostname): + # sanity check + if url_string is None: + return url_string + + # normal case parsed_url = urlparse(url_string) if (parsed_url.netloc in [google_hostname, ''] and parsed_url.path == redirect_path): @@ -151,7 +183,7 @@ def request(query, params): if len(language_array) == 2: country = language_array[1] else: - country = ' ' + country = 'US' language = language_array[0] + ',' + language_array[0] + '-' + country if use_locale_domain: @@ -196,21 +228,32 @@ def response(resp): try: url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname) parsed_url = urlparse(url, google_hostname) - if (parsed_url.netloc == google_hostname - and (parsed_url.path == search_path - or parsed_url.path.startswith(maps_path))): - # remove the link to google news and google maps - # FIXME : sometimes the URL is https://maps.google.*/maps - # no consequence, the result trigger an exception after which is ignored - continue + + # map result + if ((parsed_url.netloc == google_hostname and parsed_url.path.startswith(maps_path)) + or (parsed_url.netloc.startswith(map_hostname_start))): + x = result.xpath(map_near) + if len(x) > 0: + # map : near the location + results = results + parse_map_near(parsed_url, x, google_hostname) + else: + # map : detail about a location + results = results + parse_map_detail(parsed_url, result, google_hostname) + + # google news + elif (parsed_url.netloc == google_hostname + and parsed_url.path == search_path): + # skipping news results + pass # images result - if (parsed_url.netloc == google_hostname - and parsed_url.path == images_path): + elif (parsed_url.netloc == google_hostname + and parsed_url.path == images_path): # only thumbnail image provided, # so skipping image results # results = results + parse_images(result, google_hostname) pass + else: # normal result content = extract_text_from_dom(result, content_xpath) @@ -222,8 +265,9 @@ def response(resp): # append result results.append({'url': url, 'title': title, - 'content': content}) - except Exception: + 'content': content + }) + except: continue # parse suggestion @@ -246,6 +290,77 @@ def parse_images(result, google_hostname): 'title': '', 'content': '', 'img_src': img_src, - 'template': 'images.html'}) + 'template': 'images.html' + }) + + return results + + +def parse_map_near(parsed_url, x, google_hostname): + results = [] + + for result in x: + title = extract_text_from_dom(result, map_near_title) + url = parse_url(extract_text_from_dom(result, map_near_url), google_hostname) + attributes = [] + phone = extract_text_from_dom(result, map_near_phone) + add_attributes(attributes, property_phone, phone, 'tel:' + phone) + results.append({'title': title, + 'url': url, + 'content': attributes_to_html(attributes) + }) return results + + +def parse_map_detail(parsed_url, result, google_hostname): + results = [] + + # try to parse the geoloc + m = re.search('@([0-9\.]+),([0-9\.]+),([0-9]+)', parsed_url.path) + if m is None: + m = re.search('ll\=([0-9\.]+),([0-9\.]+)\&z\=([0-9]+)', parsed_url.query) + + if m is not None: + # geoloc found (ignored) + lon = float(m.group(2)) # noqa + lat = float(m.group(1)) # noqa + zoom = int(m.group(3)) # noqa + + # attributes + attributes = [] + address = extract_text_from_dom(result, map_address_xpath) + phone = extract_text_from_dom(result, map_phone_xpath) + add_attributes(attributes, property_address, address, 'geo:' + str(lat) + ',' + str(lon)) + add_attributes(attributes, property_phone, phone, 'tel:' + phone) + + # title / content / url + website_title = extract_text_from_dom(result, map_website_title_xpath) + content = extract_text_from_dom(result, content_xpath) + website_url = parse_url(extract_text_from_dom(result, map_website_url_xpath), google_hostname) + + # add a result if there is a website + if website_url is not None: + results.append({'title': website_title, + 'content': (content + '<br />' if content is not None else '') + + attributes_to_html(attributes), + 'url': website_url + }) + + return results + + +def add_attributes(attributes, name, value, url): + if value is not None and len(value) > 0: + attributes.append({'label': name, 'value': value, 'url': url}) + + +def attributes_to_html(attributes): + retval = '<table class="table table-striped">' + for a in attributes: + value = a.get('value') + if 'url' in a: + value = '<a href="' + a.get('url') + '">' + value + '</a>' + retval = retval + '<tr><th>' + a.get('label') + '</th><td>' + value + '</td></tr>' + retval = retval + '</table>' + return retval diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py index 0dcc65b7c..517ac1c44 100644 --- a/searx/engines/vimeo.py +++ b/searx/engines/vimeo.py @@ -27,11 +27,11 @@ base_url = 'https://vimeo.com' search_url = base_url + '/search/page:{pageno}?{query}' # specific xpath variables -results_xpath = '//div[@id="browse_content"]/ol/li' -url_xpath = './a/@href' -title_xpath = './a/div[@class="data"]/p[@class="title"]' -content_xpath = './a/img/@src' -publishedDate_xpath = './/p[@class="meta"]//attribute::datetime' +results_xpath = '//div[contains(@class,"results_grid")]/ul/li' +url_xpath = './/a/@href' +title_xpath = './/span[@class="title"]' +thumbnail_xpath = './/img[@class="js-clip_thumbnail_image"]/@src' +publishedDate_xpath = './/time/attribute::datetime' embedded_url = '<iframe data-src="//player.vimeo.com/video{videoid}" ' +\ 'width="540" height="304" frameborder="0" ' +\ @@ -58,7 +58,7 @@ def response(resp): videoid = result.xpath(url_xpath)[0] url = base_url + videoid title = p.unescape(extract_text(result.xpath(title_xpath))) - thumbnail = extract_text(result.xpath(content_xpath)[0]) + thumbnail = extract_text(result.xpath(thumbnail_xpath)[0]) publishedDate = parser.parse(extract_text(result.xpath(publishedDate_xpath)[0])) embedded = embedded_url.format(videoid=videoid) diff --git a/searx/engines/www1x.py b/searx/engines/www1x.py index 12868ad22..ddb79bfea 100644 --- a/searx/engines/www1x.py +++ b/searx/engines/www1x.py @@ -20,8 +20,8 @@ import re categories = ['images'] paging = False -# search-url, no HTTPS (there is a valid certificate for https://api2.1x.com/ ) -base_url = 'http://1x.com' +# search-url +base_url = 'https://1x.com' search_url = base_url+'/backend/search.php?{query}' |