diff options
Diffstat (limited to 'tests')
41 files changed, 1245 insertions, 1549 deletions
diff --git a/tests/unit/engines/test_archlinux.py b/tests/unit/engines/test_archlinux.py index f2ba483c1..062f023bd 100644 --- a/tests/unit/engines/test_archlinux.py +++ b/tests/unit/engines/test_archlinux.py @@ -4,7 +4,7 @@ from searx.engines import archlinux from searx.testing import SearxTestCase domains = { - 'en': 'https://wiki.archlinux.org', + 'all': 'https://wiki.archlinux.org', 'de': 'https://wiki.archlinux.de', 'fr': 'https://wiki.archlinux.fr', 'ja': 'https://wiki.archlinuxjp.org', diff --git a/tests/unit/engines/test_arxiv.py b/tests/unit/engines/test_arxiv.py index b32c0e605..83c4f8595 100644 --- a/tests/unit/engines/test_arxiv.py +++ b/tests/unit/engines/test_arxiv.py @@ -8,7 +8,7 @@ from searx.testing import SearxTestCase class TestBaseEngine(SearxTestCase): def test_request(self): - query = 'test_query' + query = 'test_query'.encode('utf-8') dicto = defaultdict(dict) dicto['pageno'] = 1 params = arxiv.request(query, dicto) diff --git a/tests/unit/engines/test_bing.py b/tests/unit/engines/test_bing.py index 48a5e744a..387034735 100644 --- a/tests/unit/engines/test_bing.py +++ b/tests/unit/engines/test_bing.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from collections import defaultdict import mock from searx.engines import bing @@ -10,7 +11,7 @@ class TestBingEngine(SearxTestCase): bing.supported_languages = ['en', 'fr', 'zh-CHS', 'zh-CHT', 'pt-PT', 'pt-BR'] query = u'test_query' dicto = defaultdict(dict) - dicto['pageno'] = 0 + dicto['pageno'] = 1 dicto['language'] = 'fr-FR' params = bing.request(query.encode('utf-8'), dicto) self.assertTrue('url' in params) @@ -18,71 +19,131 @@ class TestBingEngine(SearxTestCase): self.assertTrue('language%3AFR' in params['url']) self.assertTrue('bing.com' in params['url']) + dicto['language'] = 'all' + params = bing.request(query.encode('utf-8'), dicto) + self.assertTrue('language' in params['url']) + def test_response(self): + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr-FR' self.assertRaises(AttributeError, bing.response, None) self.assertRaises(AttributeError, bing.response, []) self.assertRaises(AttributeError, bing.response, '') self.assertRaises(AttributeError, bing.response, '[]') response = mock.Mock(text='<html></html>') + response.search_params = dicto self.assertEqual(bing.response(response), []) response = mock.Mock(text='<html></html>') + response.search_params = dicto self.assertEqual(bing.response(response), []) html = """ - <div class="sa_cc" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> - <div Class="sa_mc"> - <div class="sb_tlst"> - <h3> - <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1"> - <strong>This</strong> should be the title</a> - </h3> - </div> - <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite> - <span class="c_tlbxTrg"> - <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1"> - </span> - </span> - </div> - <p><strong>This</strong> should be the content.</p> + <div> + <div id="b_tween"> + <span class="sb_count" data-bm="4">23 900 000 résultats</span> </div> + <ol id="b_results" role="main"> + <div class="sa_cc" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> + <div Class="sa_mc"> + <div class="sb_tlst"> + <h3> + <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1"> + <strong>This</strong> should be the title</a> + </h3> + </div> + <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite> + <span class="c_tlbxTrg"> + <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1"> + </span> + </span> + </div> + <p><strong>This</strong> should be the content.</p> + </div> + </div> + </ol> </div> """ response = mock.Mock(text=html) + response.search_params = dicto results = bing.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') + self.assertEqual(results[-1]['number_of_results'], 23900000) html = """ - <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> - <div Class="sa_mc"> - <div class="sb_tlst"> - <h2> - <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1"> - <strong>This</strong> should be the title</a> - </h2> - </div> - <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite> - <span class="c_tlbxTrg"> - <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1"> - </span> - </span> - </div> - <p><strong>This</strong> should be the content.</p> + <div> + <div id="b_tween"> + <span class="sb_count" data-bm="4">9-18 résultats sur 23 900 000</span> </div> - </li> + <ol id="b_results" role="main"> + <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> + <div Class="sa_mc"> + <div class="sb_tlst"> + <h2> + <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1"> + <strong>This</strong> should be the title</a> + </h2> + </div> + <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite> + <span class="c_tlbxTrg"> + <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1"> + </span> + </span> + </div> + <p><strong>This</strong> should be the content.</p> + </div> + </li> + </ol> + </div> """ + dicto['pageno'] = 2 response = mock.Mock(text=html) + response.search_params = dicto results = bing.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') + self.assertEqual(results[-1]['number_of_results'], 23900000) + + html = """ + <div> + <div id="b_tween"> + <span class="sb_count" data-bm="4">23 900 000 résultats</span> + </div> + <ol id="b_results" role="main"> + <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO"> + <div Class="sa_mc"> + <div class="sb_tlst"> + <h2> + <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1"> + <strong>This</strong> should be the title</a> + </h2> + </div> + <div class="sb_meta"><cite><strong>this</strong>.meta.com</cite> + <span class="c_tlbxTrg"> + <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1"> + </span> + </span> + </div> + <p><strong>This</strong> should be the content.</p> + </div> + </li> + </ol> + </div> + """ + dicto['pageno'] = 33900000 + response = mock.Mock(text=html) + response.search_params = dicto + results = bing.response(response) + self.assertEqual(bing.response(response), []) def test_fetch_supported_languages(self): html = """<html></html>""" diff --git a/tests/unit/engines/test_bing_images.py b/tests/unit/engines/test_bing_images.py index afc4cd6f0..a4efcab58 100644 --- a/tests/unit/engines/test_bing_images.py +++ b/tests/unit/engines/test_bing_images.py @@ -9,6 +9,7 @@ class TestBingImagesEngine(SearxTestCase): def test_request(self): bing_images.supported_languages = ['fr-FR', 'en-US'] + bing_images.language_aliases = {} query = 'test_query' dicto = defaultdict(dict) dicto['pageno'] = 1 @@ -52,17 +53,25 @@ class TestBingImagesEngine(SearxTestCase): <li> <div> <div class="imgpt"> - <a m='{"purl":"page_url","murl":"img_url"}' mad='{"turl":"thumb_url"}'> + <a m='{"purl":"page_url","murl":"img_url","turl":"thumb_url","t":"Page 1 title"}'> <img src="" alt="alt text" /> </a> + <div class="img_info"> + <span>1 x 1 - jpeg</span> + <a>1.example.org</a> + </div> </div> <div></div> </div> <div> <div class="imgpt"> - <a m='{"purl":"page_url2","murl":"img_url2"}' mad='{"turl":"thumb_url2"}'> + <a m='{"purl":"page_url2","murl":"img_url2","turl":"thumb_url2","t":"Page 2 title"}'> <img src="" alt="alt text 2" /> </a> + <div class="img_info"> + <span>2 x 2 - jpeg</span> + <a>2.example.org</a> + </div> </div> </div> </li> @@ -71,9 +80,13 @@ class TestBingImagesEngine(SearxTestCase): <li> <div> <div class="imgpt"> - <a m='{"purl":"page_url3","murl":"img_url3"}' mad='{"turl":"thumb_url3"}'> + <a m='{"purl":"page_url3","murl":"img_url3","turl":"thumb_url3","t":"Page 3 title"}'> <img src="" alt="alt text 3" /> </a> + <div class="img_info"> + <span>3 x 3 - jpeg</span> + <a>3.example.org</a> + </div> </div> </div> </li> @@ -85,11 +98,13 @@ class TestBingImagesEngine(SearxTestCase): results = bing_images.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 3) - self.assertEqual(results[0]['title'], 'alt text') + self.assertEqual(results[0]['title'], 'Page 1 title') self.assertEqual(results[0]['url'], 'page_url') self.assertEqual(results[0]['content'], '') self.assertEqual(results[0]['thumbnail_src'], 'thumb_url') self.assertEqual(results[0]['img_src'], 'img_url') + self.assertEqual(results[0]['img_format'], '1 x 1 - jpeg') + self.assertEqual(results[0]['source'], '1.example.org') def test_fetch_supported_languages(self): html = """ diff --git a/tests/unit/engines/test_bing_news.py b/tests/unit/engines/test_bing_news.py index 8fc26ee32..1155e79c4 100644 --- a/tests/unit/engines/test_bing_news.py +++ b/tests/unit/engines/test_bing_news.py @@ -20,6 +20,10 @@ class TestBingNewsEngine(SearxTestCase): self.assertIn('bing.com', params['url']) self.assertIn('fr', params['url']) + dicto['language'] = 'all' + params = bing_news.request(query, dicto) + self.assertIn('en', params['url']) + def test_no_url_in_request_year_time_range(self): dicto = defaultdict(dict) query = 'test_query' diff --git a/tests/unit/engines/test_bing_videos.py b/tests/unit/engines/test_bing_videos.py index 24387c888..5e171eb53 100644 --- a/tests/unit/engines/test_bing_videos.py +++ b/tests/unit/engines/test_bing_videos.py @@ -9,6 +9,7 @@ class TestBingVideosEngine(SearxTestCase): def test_request(self): bing_videos.supported_languages = ['fr-FR', 'en-US'] + bing_videos.language_aliases = {} query = 'test_query' dicto = defaultdict(dict) dicto['pageno'] = 1 @@ -28,7 +29,7 @@ class TestBingVideosEngine(SearxTestCase): dicto['time_range'] = 'day' dicto['safesearch'] = 2 params = bing_videos.request(query, dicto) - self.assertTrue('first=11' in params['url']) + self.assertTrue('first=29' in params['url']) self.assertTrue('1440' in params['url']) self.assertIn('SRCHHPGUSR', params['cookies']) self.assertTrue('STRICT' in params['cookies']['SRCHHPGUSR']) @@ -47,53 +48,25 @@ class TestBingVideosEngine(SearxTestCase): html = """ <div class="dg_u"> - <div id="mc_vtvc_1" class="mc_vtvc"> - <a class="mc_vtvc_link" href="/video"> - <div class="mc_vtvc_th"> - <div class="cico"> - <img src="thumb_1.jpg" /> - </div> - <div class="mc_vtvc_ban_lo"> - <div class="vtbc"> - <div class="mc_bc_w b_smText"> - <div class="mc_bc pivot bpi_2"> - <span title=""> - <span class="mv_vtvc_play cipg "></span> - </span> - </div> - <div class="mc_bc items">10:06</div> - </div> + <div> + <a> + <div> + <div> + <div class="mc_vtvc_meta_block"> + <div><span>100 views</span><span>1 year ago</span></div><div><span>ExampleTube</span><span>Channel 1<span></div> #noqa </div> </div> - </div> - <div class="mc_vtvc_meta"> - <div class="mc_vtvc_title" title="Title 1"></div> - <div class="mc_vtvc_meta_block_area"> - <div class="mc_vtvc_meta_block"> - <div class="mc_vtvc_meta_row"> - <span>65,696,000+ views</span> - <span>1 year ago</span> - </div> - <div class="mc_vtvc_meta_row mc_vtvc_meta_row_channel">Content 1</div> - <div class="mc_vtvc_meta_row"><span> - <div class="cico mc_vtvc_src_ico"> - <div></div> - </div> - <span>YouTube</span> - </span></div> - </div> - </div> + <div class="vrhdata" vrhm='{"du":"01:11","murl":"https://www.example.com/watch?v=DEADBEEF","thid":"OVP.BINGTHUMB1","vt":"Title 1"}'></div> # noqa </div> - <div class="vrhdata"></div> - </a> - </div> + </a> </div> + </div> """ response = mock.Mock(text=html) results = bing_videos.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 1) self.assertEqual(results[0]['title'], 'Title 1') - self.assertEqual(results[0]['url'], 'https://bing.com/video') - self.assertEqual(results[0]['content'], 'Content 1') - self.assertEqual(results[0]['thumbnail'], 'thumb_1.jpg') + self.assertEqual(results[0]['url'], 'https://www.example.com/watch?v=DEADBEEF') + self.assertEqual(results[0]['content'], '01:11 - 100 views - 1 year ago - ExampleTube - Channel 1') + self.assertEqual(results[0]['thumbnail'], 'https://www.bing.com/th?id=OVP.BINGTHUMB1') diff --git a/tests/unit/engines/test_btdigg.py b/tests/unit/engines/test_btdigg.py index 6a88e3f75..45ddaa6e3 100644 --- a/tests/unit/engines/test_btdigg.py +++ b/tests/unit/engines/test_btdigg.py @@ -14,7 +14,7 @@ class TestBtdiggEngine(SearxTestCase): params = btdigg.request(query, dicto) self.assertIn('url', params) self.assertIn(query, params['url']) - self.assertIn('btdigg.org', params['url']) + self.assertIn('btdig.com', params['url']) def test_response(self): self.assertRaises(AttributeError, btdigg.response, None) @@ -26,359 +26,87 @@ class TestBtdiggEngine(SearxTestCase): self.assertEqual(btdigg.response(response), []) html = u""" - <div id="search_res"> - <table> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">8 B</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">5</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - </table> + <div class="one_result" style="display:table-row;background-color:#e8e8e8"> + <div style="display:table-cell;color:rgb(0, 0, 0)"> + <div style="display:table"> + <div style="display:table-row"> + <div class="torrent_name" style="display:table-cell"> + <a style="color:rgb(0, 0, 204);text-decoration:underline;font-size:150%" + href="http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&p=1&order=0" + >3.9GBdeLibrosByHuasoFromHell(3de4)</a> + </div> + </div> + </div> + <div style="display:table"> + <div style="display:table-row"> + <div style="display:table-cell"> + <span class="torrent_files" style="color:#666;padding-left:10px">4217</span> files <span + class="torrent_size" style="color:#666;padding-left:10px">1 GB</span><span + class="torrent_age" style="color:rgb(0, 102, 0);padding-left:10px;margin: 0px 4px" + >found 3 years ago</span> + </div> + </div> + </div> + <div style="display:table;width:100%;padding:10px"> + <div style="display:table-row"> + <div class="torrent_magnet" style="display:table-cell"> + <div class="fa fa-magnet" style="color:#cc0000"> + <a href="magnet:?xt=urn:btih:a72f35b7ee3a10928f02bb799e40ae5db701ed1c&dn=3.9GBdeLibrosBy..." + title="Download via magnet-link"> magnet:?xt=urn:btih:a72f35b7ee...</a> + </div> + </div> + <div style="display:table-cell;color:rgb(0, 0, 0);text-align:right"> + <span style="color:rgb(136, 136, 136);margin: 0px 0px 0px 4px"></span><span + style="color:rgb(0, 102, 0);margin: 0px 4px">found 3 years ago</span> + </div> + </div> + </div> + <div class="torrent_excerpt" style="display:table;padding:10px;white-space:nowrap"> + <div class="fa fa-folder-open" style="padding-left:0em"> 3.9GBdeLibrosByHuasoFromHell(3de4)</div><br/> + <div class="fa fa-folder-open" style="padding-left:1em"> Libros H-Z</div><br/> + <div class="fa fa-folder-open" style="padding-left:2em"> H</div><br/><div class="fa fa-file-archive-o" + style="padding-left:3em"> H.H. Hollis - El truco de la espada-<b + style="color:red; background-color:yellow">pdf</b>.zip</div><span + style="color:#666;padding-left:10px">17 KB</span><br/> + <div class="fa fa-file-archive-o" style="padding-left:3em"> Hagakure - El Libro del Samurai-<b + style="color:red; background-color:yellow">pdf</b>.zip</div><span + style="color:#666;padding-left:10px">95 KB</span><br/> + <div class="fa fa-folder-open" style="padding-left:3em"> Hamsun, Knut (1859-1952)</div><br/> + <div class="fa fa-file-archive-o" style="padding-left:4em"> Hamsun, Knut - Hambre-<b + style="color:red; background-color:yellow">pdf</b>.zip</div><span + style="color:#666;padding-left:10px">786 KB</span><br/> + <div class="fa fa-plus-circle"><a + href="http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&p=1&order=0" + > 4214 hidden files<span style="color:#666;padding-left:10px">1 GB</span></a></div> + </div> + </div> </div> """ response = mock.Mock(text=html.encode('utf-8')) results = btdigg.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Should be the title') - self.assertEqual(results[0]['url'], 'https://btdigg.org/url') - self.assertEqual(results[0]['content'], 'Content') - self.assertEqual(results[0]['seed'], 5) - self.assertEqual(results[0]['leech'], 0) - self.assertEqual(results[0]['filesize'], 8) - self.assertEqual(results[0]['files'], 710) - self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:magnet&dn=Test') + self.assertEqual(results[0]['title'], '3.9GBdeLibrosByHuasoFromHell(3de4)') + self.assertEqual(results[0]['url'], + 'http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&p=1&order=0') + self.assertEqual(results[0]['content'], + '3.9GBdeLibrosByHuasoFromHell(3de4) | ' + + 'Libros H-Z | ' + + 'H H.H. Hollis - El truco de la espada-pdf.zip17 KB | ' + + 'Hagakure - El Libro del Samurai-pdf.zip95 KB | ' + + 'Hamsun, Knut (1859-1952) | Hamsun, Knut - Hambre-pdf.zip786 KB | ' + + '4214 hidden files1 GB') + self.assertEqual(results[0]['filesize'], 1 * 1024 * 1024 * 1024) + self.assertEqual(results[0]['files'], 4217) + self.assertEqual(results[0]['magnetlink'], + 'magnet:?xt=urn:btih:a72f35b7ee3a10928f02bb799e40ae5db701ed1c&dn=3.9GBdeLibrosBy...') html = """ - <div id="search_res"> - <table> - </table> - </div> - """ - response = mock.Mock(text=html.encode('utf-8')) - results = btdigg.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) + <div style="display:table-row;background-color:#e8e8e8"> - html = u""" - <div id="search_res"> - <table> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">1 KB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">5</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">1 MB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">a</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">4</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">1 GB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">3</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">1 TB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">2</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">a TB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">z</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - </table> </div> """ response = mock.Mock(text=html.encode('utf-8')) results = btdigg.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 5) - self.assertEqual(results[0]['title'], 'Should be the title') - self.assertEqual(results[0]['url'], 'https://btdigg.org/url') - self.assertEqual(results[0]['content'], 'Content') - self.assertEqual(results[0]['seed'], 5) - self.assertEqual(results[0]['leech'], 0) - self.assertEqual(results[0]['files'], 710) - self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:magnet&dn=Test') - self.assertEqual(results[0]['filesize'], 1024) - self.assertEqual(results[1]['filesize'], 1048576) - self.assertEqual(results[2]['filesize'], 1073741824) - self.assertEqual(results[3]['filesize'], 1099511627776) + self.assertEqual(len(results), 0) diff --git a/tests/unit/engines/test_dailymotion.py b/tests/unit/engines/test_dailymotion.py index 803b5c4d2..ad7f3d283 100644 --- a/tests/unit/engines/test_dailymotion.py +++ b/tests/unit/engines/test_dailymotion.py @@ -19,6 +19,10 @@ class TestDailymotionEngine(SearxTestCase): self.assertTrue('dailymotion.com' in params['url']) self.assertTrue('fr' in params['url']) + dicto['language'] = 'all' + params = dailymotion.request(query, dicto) + self.assertTrue('en' in params['url']) + def test_response(self): self.assertRaises(AttributeError, dailymotion.response, None) self.assertRaises(AttributeError, dailymotion.response, []) diff --git a/tests/unit/engines/test_deviantart.py b/tests/unit/engines/test_deviantart.py index bd2cf182f..a31151037 100644 --- a/tests/unit/engines/test_deviantart.py +++ b/tests/unit/engines/test_deviantart.py @@ -22,74 +22,3 @@ class TestDeviantartEngine(SearxTestCase): dicto['time_range'] = 'year' params = deviantart.request(query, dicto) self.assertEqual({}, params['url']) - - def test_response(self): - self.assertRaises(AttributeError, deviantart.response, None) - self.assertRaises(AttributeError, deviantart.response, []) - self.assertRaises(AttributeError, deviantart.response, '') - self.assertRaises(AttributeError, deviantart.response, '[]') - - response = mock.Mock(text='<html></html>') - self.assertEqual(deviantart.response(response), []) - - response = mock.Mock(status_code=302) - self.assertEqual(deviantart.response(response), []) - - html = """ - <div id="page-1-results" class="page-results results-page-thumb torpedo-container"> - <span class="thumb wide" href="http://amai911.deviantart.com/art/Horse-195212845" - data-super-full-width="900" data-super-full-height="600"> - <a class="torpedo-thumb-link" href="https://url.of.image"> - <img data-sigil="torpedo-img" src="https://url.of.thumbnail" /> - </a> - <span class="info"><span class="title-wrap"><span class="title">Title of image</span></span> - </div> - """ - response = mock.Mock(text=html) - results = deviantart.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Title of image') - self.assertEqual(results[0]['url'], 'https://url.of.image') - self.assertNotIn('content', results[0]) - self.assertEqual(results[0]['thumbnail_src'], 'https://url.of.thumbnail') - - html = """ - <span class="tt-fh-tc" style="width: 202px;"> - <span class="tt-bb" style="width: 202px;"> - </span> - <span class="shadow"> - <a class="thumb" href="http://url.of.result/2nd.part.of.url" - title="Behoimi BE Animation Test by test-0, Jan 4, - 2010 in Digital Art > Animation"> <i></i> - <img width="200" height="200" alt="Test" - src="http://url.of.thumbnail" data-src="http://th08.deviantart.net/test.jpg"> - </a> - </span> - <!-- ^TTT --> - </span> - <span class="details"> - <a href="http://test-0.deviantart.com/art/Test" class="t" - title="Behoimi BE Animation Test by test-0, Jan 4, 2010"> - <span class="tt-fh-oe">Title of image</span> </a> - <small> - <span class="category"> - <span class="age"> - 5 years ago - </span> - in <a title="Behoimi BE Animation Test by test-0, Jan 4, 2010" - href="http://www.deviantart.com/browse/all/digitalart/animation/">Animation</a> - </span> - <div class="commentcount"> - <a href="http://test-0.deviantart.com/art/Test#comments"> - <span class="iconcommentsstats"></span>9 Comments</a> - </div> - <a class="mlt-link" href="http://www.deviantart.com/morelikethis/149167425"> - <span class="mlt-icon"></span> <span class="mlt-text">More Like This</span> </a> - </span> - </small> <!-- TTT$ --> - """ - response = mock.Mock(text=html) - results = deviantart.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) diff --git a/tests/unit/engines/test_digg.py b/tests/unit/engines/test_digg.py index 6e7c9cc99..8bc4c67c2 100644 --- a/tests/unit/engines/test_digg.py +++ b/tests/unit/engines/test_digg.py @@ -14,88 +14,3 @@ class TestDiggEngine(SearxTestCase): self.assertIn('url', params) self.assertIn(query, params['url']) self.assertIn('digg.com', params['url']) - - def test_response(self): - self.assertRaises(AttributeError, digg.response, None) - self.assertRaises(AttributeError, digg.response, []) - self.assertRaises(AttributeError, digg.response, '') - self.assertRaises(AttributeError, digg.response, '[]') - - response = mock.Mock(text='{}') - self.assertEqual(digg.response(response), []) - - response = mock.Mock(text='{"data": []}') - self.assertEqual(digg.response(response), []) - - json = """ - { - "status": "ok", - "num": 10, - "next_position": 20, - "html": "<article itemscope itemtype=\\"http://schema.org/Article\\" - class=\\"story-container digg-story-el hentry entry story-1sRANah col-1\\" - data-content-id=\\"1sRANah\\" data-contenturl=\\"http://url.of.link\\" - data-position=\\"0\\" data-diggs=\\"24\\" data-tweets=\\"69\\" - data-digg-score=\\"1190\\"> <div class=\\"story-image story-image-thumb\\"> - <a data-position=\\"0\\" data-content-id=\\"1sRANah\\" - class=\\"story-link\\" href=\\"http://www.thedailybeast.com/\\" - target=\\"_blank\\"><img class=\\"story-image-img\\" - src=\\"http://url.of.image.jpeg\\" width=\\"312\\" height=\\"170\\" - alt=\\"\\" /> </a> </div> <div class=\\"story-content\\"><header - class=\\"story-header\\"> <div itemprop=\\"alternativeHeadline\\" - class=\\"story-kicker\\" >Kicker</div> <h2 itemprop=\\"headline\\" - class=\\"story-title entry-title\\"><a class=\\"story-title-link story-link\\" - rel=\\"bookmark\\" itemprop=\\"url\\" href=\\"http://www.thedailybeast.com/\\" - target=\\"_blank\\">Title of article</h2> <div class=\\"story-meta\\"> - <div class=\\"story-score \\"> - <div class=\\"story-score-diggscore diggscore-1sRANah\\">1190</div> - <div class=\\"story-score-details\\"> <div class=\\"arrow\\"></div> - <ul class=\\"story-score-details-list\\"> <li - class=\\"story-score-detail story-score-diggs\\"><span - class=\\"label\\">Diggs:</span> <span class=\\"count diggs-1sRANah\\">24</span> - </li> <li class=\\"story-score-detail story-score-twitter\\"><span - class=\\"label\\">Tweets:</span> <span class=\\"count tweets-1sRANah\\">69</span> - </li> <li class=\\"story-score-detail story-score-facebook\\"><span - class=\\"label\\">Facebook Shares:</span> <span - class=\\"count fb_shares-1sRANah\\">1097</span></li> </ul> </div> </div> - <span class=\\"story-meta-item story-source\\"> <a - itemprop=\\"publisher copyrightHolder sourceOrganization provider\\" - class=\\"story-meta-item-link story-source-link\\" - href=\\"/source/thedailybeast.com\\">The Daily Beast </a> </span> - <span class=\\"story-meta-item story-tag first-tag\\"> <a - itemprop=\\"keywords\\" rel=\\"tag\\" - class=\\"story-meta-item-link story-tag-link\\" href=\\"/tag/news\\">News</a> - </span> <abbr class=\\"published story-meta-item story-timestamp\\" - title=\\"2014-10-18 14:53:45\\"> <time datetime=\\"2014-10-18 14:53:45\\">18 Oct 2014</time> - </abbr> </div> </header> </div> <ul class=\\"story-actions\\"> <li - class=\\"story-action story-action-digg btn-story-action-container\\"> - <a class=\\"target digg-1sRANah\\" href=\\"#\\">Digg</a></li> <li - class=\\"story-action story-action-save btn-story-action-container\\"> - <a class=\\"target save-1sRANah\\" href=\\"#\\">Save</a></li> <li - class=\\"story-action story-action-share\\"><a - class=\\"target share-facebook\\" href=\\"https://www.facebook.com/\\">Facebook</a></li> - <li class=\\"story-action story-action-share\\"><a class=\\"target share-twitter\\" - href=\\"https://twitter.com/\\">Twitter</a></li> </ul> </article>" - } - """ - json = json.replace('\r\n', '').replace('\n', '').replace('\r', '') - response = mock.Mock(text=json) - results = digg.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Title of article') - self.assertEqual(results[0]['url'], 'http://url.of.link') - self.assertEqual(results[0]['thumbnail'], 'http://url.of.image.jpeg') - self.assertEqual(results[0]['content'], '') - - json = """ - { - "status": "error", - "num": 10, - "next_position": 20 - } - """ - response = mock.Mock(text=json) - results = digg.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) diff --git a/tests/unit/engines/test_duckduckgo_images.py b/tests/unit/engines/test_duckduckgo_images.py index 5301057fd..0d152bec1 100644 --- a/tests/unit/engines/test_duckduckgo_images.py +++ b/tests/unit/engines/test_duckduckgo_images.py @@ -14,7 +14,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase): dicto['is_test'] = True dicto['pageno'] = 1 dicto['safesearch'] = 0 - dicto['language'] = 'en-US' + dicto['language'] = 'all' params = duckduckgo_images.request(query, dicto) self.assertIn('url', params) self.assertIn(query, params['url']) @@ -41,7 +41,7 @@ class TestDuckduckgoImagesEngine(SearxTestCase): self.assertRaises(AttributeError, duckduckgo_images.response, '[]') response = mock.Mock(text='If this error persists, please let us know: ops@duckduckgo.com') - self.assertEqual(duckduckgo_images.response(response), []) + self.assertRaises(Exception, duckduckgo_images.response, response) json = u""" { diff --git a/tests/unit/engines/test_duden.py b/tests/unit/engines/test_duden.py index d9bbfef8b..52fc513d0 100644 --- a/tests/unit/engines/test_duden.py +++ b/tests/unit/engines/test_duden.py @@ -10,24 +10,30 @@ class TestDudenEngine(SearxTestCase): def test_request(self): query = 'Haus' dic = defaultdict(dict) - dic['pageno'] = 1 - params = duden.request(query, dic) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('duden.de' in params['url']) + data = [ + [1, 'https://www.duden.de/suchen/dudenonline/Haus'], + [2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1'] + ] + for page_no, exp_res in data: + dic['pageno'] = page_no + params = duden.request(query, dic) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('duden.de' in params['url']) + self.assertEqual(params['url'], exp_res) def test_response(self): resp = mock.Mock(text='<html></html>') self.assertEqual(duden.response(resp), []) html = """ - <section class="wide"> - <h2><a href="https://this.is.the.url/" class="hidden-link"><strong>This is the title</strong> also here</a></h2> - <p>This is the <strong>content</strong></p> - <a href="https://this.is.the.url/">Zum vollständigen Artikel</a> + <section class="vignette"> + <h2"> <a href="/rechtschreibung/Haus"> + <strong>This is the title also here</strong> + </a> </h2> + <p>This is the content</p> </section> """ - resp = mock.Mock(text=html) results = duden.response(resp) @@ -36,6 +42,6 @@ class TestDudenEngine(SearxTestCase): # testing result (dictionary entry) r = results[0] - self.assertEqual(r['url'], 'https://this.is.the.url/') + self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus') self.assertEqual(r['title'], 'This is the title also here') self.assertEqual(r['content'], 'This is the content') diff --git a/tests/unit/engines/test_faroo.py b/tests/unit/engines/test_faroo.py index 0cc8bc3e0..1bd9f51c3 100644 --- a/tests/unit/engines/test_faroo.py +++ b/tests/unit/engines/test_faroo.py @@ -20,6 +20,10 @@ class TestFarooEngine(SearxTestCase): self.assertIn('en', params['url']) self.assertIn('web', params['url']) + dicto['language'] = 'all' + params = faroo.request(query, dicto) + self.assertIn('en', params['url']) + dicto['language'] = 'de-DE' params = faroo.request(query, dicto) self.assertIn('de', params['url']) diff --git a/tests/unit/engines/test_fdroid.py b/tests/unit/engines/test_fdroid.py index d75f4f0b4..42a0a7148 100644 --- a/tests/unit/engines/test_fdroid.py +++ b/tests/unit/engines/test_fdroid.py @@ -13,29 +13,40 @@ class TestFdroidEngine(SearxTestCase): params = fdroid.request(query, dic) self.assertTrue('url' in params) self.assertTrue(query in params['url']) - self.assertTrue('f-droid.org' in params['url']) + self.assertTrue('search.f-droid.org' in params['url']) - def test_response(self): + def test_response_empty(self): resp = mock.Mock(text='<html></html>') self.assertEqual(fdroid.response(resp), []) + def test_response_oneresult(self): html = """ - <a href="https://google.com/qwerty"> - <div id="appheader"> - <div style="float:left;padding-right:10px;"> - <img src="http://example.com/image.png" - style="width:48px;border:none;"> - </div> - <div style="float:right;"> - <p>Details...</p> - </div> - <p style="color:#000000;"> - <span style="font-size:20px;">Sample title</span> - <br> - Sample content - </p> - </div> - </a> +<!DOCTYPE html> +<html> +<head> + <title>test</title> +</head> +<body> + <div class="site-wrapper"> + <div class="main-content"> + <a class="package-header" href="https://example.com/app.url"> + <img class="package-icon" src="https://example.com/appexample.logo.png" /> + + <div class="package-info"> + <h4 class="package-name"> + App Example 1 + </h4> + + <div class="package-desc"> + <span class="package-summary">Description App Example 1</span> + <span class="package-license">GPL-3.0-only</span> + </div> + </div> + </a> + </div> + </div> +</body> +</html> """ resp = mock.Mock(text=html) @@ -43,7 +54,7 @@ class TestFdroidEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 1) - self.assertEqual(results[0]['url'], 'https://google.com/qwerty') - self.assertEqual(results[0]['title'], 'Sample title') - self.assertEqual(results[0]['content'], 'Sample content') - self.assertEqual(results[0]['img_src'], 'http://example.com/image.png') + self.assertEqual(results[0]['url'], 'https://example.com/app.url') + self.assertEqual(results[0]['title'], 'App Example 1') + self.assertEqual(results[0]['content'], 'Description App Example 1 - GPL-3.0-only') + self.assertEqual(results[0]['img_src'], 'https://example.com/appexample.logo.png') diff --git a/tests/unit/engines/test_flickr_noapi.py b/tests/unit/engines/test_flickr_noapi.py index 5f8b069e3..67699f2f0 100644 --- a/tests/unit/engines/test_flickr_noapi.py +++ b/tests/unit/engines/test_flickr_noapi.py @@ -27,116 +27,132 @@ class TestFlickrNoapiEngine(SearxTestCase): self.assertRaises(AttributeError, flickr_noapi.response, '') self.assertRaises(AttributeError, flickr_noapi.response, '[]') - response = mock.Mock(text='"search-photos-lite-models","photos":{},"totalItems":') + response = mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{}}]}}') self.assertEqual(flickr_noapi.response(response), []) - response = mock.Mock(text='search-photos-lite-models","photos":{"data": []},"totalItems":') + response = \ + mock.Mock(text='"modelExport:{"legend":[],"main":{"search-photos-lite-models":[{"photos":{"_data":[]}}]}}') self.assertEqual(flickr_noapi.response(response), []) # everthing is ok test json = """ - "search-photos-lite-models","photos": - { - "_data": [ - { - "_flickrModelRegistry": "photo-lite-models", - "title": "This is the title", - "username": "Owner", - "pathAlias": "klink692", - "realname": "Owner", - "license": 0, - "ownerNsid": "59729010@N00", - "canComment": false, - "commentCount": 14, - "faveCount": 21, - "id": "14001294434", - "sizes": { - "c": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg", - "width": 541, - "height": 800, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg", - "key": "c" - }, - "h": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg", - "width": 1081, - "height": 1600, - "url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg", - "key": "h" - }, - "k": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg", - "width": 1383, - "height": 2048, - "url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg", - "key": "k" - }, - "l": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg", - "width": 692, - "height": 1024, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg", - "key": "l" - }, - "m": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg", - "width": 338, - "height": 500, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg", - "key": "m" - }, - "n": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg", - "width": 216, - "height": 320, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg", - "key": "n" - }, - "q": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg", - "width": 150, - "height": 150, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg", - "key": "q" - }, - "s": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg", - "width": 162, - "height": 240, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg", - "key": "s" - }, - "sq": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg", - "width": 75, - "height": 75, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg", - "key": "sq" - }, - "t": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg", - "width": 68, - "height": 100, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg", - "key": "t" - }, - "z": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", - "width": 433, - "height": 640, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", - "key": "z" + modelExport: { + "legend": [ + [ + "search-photos-lite-models", + "0", + "photos", + "_data", + "0" + ] + ], + "main": { + "search-photos-lite-models": [ + { + "photos": { + "_data": [ + { + "_flickrModelRegistry": "photo-lite-models", + "title": "This%20is%20the%20title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", + "sizes": { + "c": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_c.jpg", + "width": 541, + "height": 800, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_c.jpg", + "key": "c" + }, + "h": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_761d32237a_h.jpg", + "width": 1081, + "height": 1600, + "url": "//c4.staticflickr.com/8/7246/14001294434_761d32237a_h.jpg", + "key": "h" + }, + "k": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_f145a2c11a_k.jpg", + "width": 1383, + "height": 2048, + "url": "//c4.staticflickr.com/8/7246/14001294434_f145a2c11a_k.jpg", + "key": "k" + }, + "l": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_b.jpg", + "width": 692, + "height": 1024, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_b.jpg", + "key": "l" + }, + "m": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777.jpg", + "width": 338, + "height": 500, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777.jpg", + "key": "m" + }, + "n": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_n.jpg", + "width": 216, + "height": 320, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_n.jpg", + "key": "n" + }, + "q": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_q.jpg", + "width": 150, + "height": 150, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_q.jpg", + "key": "q" + }, + "s": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_m.jpg", + "width": 162, + "height": 240, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_m.jpg", + "key": "s" + }, + "sq": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_s.jpg", + "width": 75, + "height": 75, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_s.jpg", + "key": "sq" + }, + "t": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_t.jpg", + "width": 68, + "height": 100, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_t.jpg", + "key": "t" + }, + "z": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", + "width": 433, + "height": 640, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", + "key": "z" + } + } + } + ] } } - } - ], - "fetchedStart": true, - "fetchedEnd": false, - "totalItems": "4386039" - },"totalItems": + ] + } + } """ - json = json.replace('\r\n', '').replace('\n', '').replace('\r', '') + # Flickr serves search results in a json block named 'modelExport' buried inside a script tag, + # this json is served as a single line terminating with a comma. + json = ''.join(json.split()) + ',\n' response = mock.Mock(text=json) results = flickr_noapi.response(response) self.assertEqual(type(results), list) @@ -149,37 +165,51 @@ class TestFlickrNoapiEngine(SearxTestCase): # no n size, only the z size json = """ - "search-photos-lite-models","photos": - { - "_data": [ - { - "_flickrModelRegistry": "photo-lite-models", - "title": "This is the title", - "username": "Owner", - "pathAlias": "klink692", - "realname": "Owner", - "license": 0, - "ownerNsid": "59729010@N00", - "canComment": false, - "commentCount": 14, - "faveCount": 21, - "id": "14001294434", - "sizes": { - "z": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", - "width": 433, - "height": 640, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", - "key": "z" + modelExport: { + "legend": [ + [ + "search-photos-lite-models", + "0", + "photos", + "_data", + "0" + ] + ], + "main": { + "search-photos-lite-models": [ + { + "photos": { + "_data": [ + { + "_flickrModelRegistry": "photo-lite-models", + "title": "This%20is%20the%20title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", + "sizes": { + "z": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_z.jpg", + "width": 433, + "height": 640, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_z.jpg", + "key": "z" + } + } + } + ] } } - } - ], - "fetchedStart": true, - "fetchedEnd": false, - "totalItems": "4386039" - },"totalItems": + ] + } + } """ + json = ''.join(json.split()) + ',\n' response = mock.Mock(text=json) results = flickr_noapi.response(response) self.assertEqual(type(results), list) @@ -192,37 +222,51 @@ class TestFlickrNoapiEngine(SearxTestCase): # no z or n size json = """ - "search-photos-lite-models","photos": - { - "_data": [ - { - "_flickrModelRegistry": "photo-lite-models", - "title": "This is the title", - "username": "Owner", - "pathAlias": "klink692", - "realname": "Owner", - "license": 0, - "ownerNsid": "59729010@N00", - "canComment": false, - "commentCount": 14, - "faveCount": 21, - "id": "14001294434", - "sizes": { - "o": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", - "width": 433, - "height": 640, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", - "key": "o" + modelExport: { + "legend": [ + [ + "search-photos-lite-models", + "0", + "photos", + "_data", + "0" + ] + ], + "main": { + "search-photos-lite-models": [ + { + "photos": { + "_data": [ + { + "_flickrModelRegistry": "photo-lite-models", + "title": "This%20is%20the%20title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", + "sizes": { + "o": { + "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", + "width": 433, + "height": 640, + "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", + "key": "o" + } + } + } + ] } } - } - ], - "fetchedStart": true, - "fetchedEnd": false, - "totalItems": "4386039" - },"totalItems": + ] + } + } """ + json = ''.join(json.split()) + ',\n' response = mock.Mock(text=json) results = flickr_noapi.response(response) self.assertEqual(type(results), list) @@ -235,30 +279,44 @@ class TestFlickrNoapiEngine(SearxTestCase): # no image test json = """ - "search-photos-lite-models","photos": - { - "_data": [ - { - "_flickrModelRegistry": "photo-lite-models", - "title": "This is the title", - "username": "Owner", - "pathAlias": "klink692", - "realname": "Owner", - "license": 0, - "ownerNsid": "59729010@N00", - "canComment": false, - "commentCount": 14, - "faveCount": 21, - "id": "14001294434", - "sizes": { - } - } + modelExport: { + "legend": [ + [ + "search-photos-lite-models", + "0", + "photos", + "_data", + "0" + ] ], - "fetchedStart": true, - "fetchedEnd": false, - "totalItems": "4386039" - },"totalItems": + "main": { + "search-photos-lite-models": [ + { + "photos": { + "_data": [ + { + "_flickrModelRegistry": "photo-lite-models", + "title": "This is the title", + "username": "Owner", + "pathAlias": "klink692", + "realname": "Owner", + "license": 0, + "ownerNsid": "59729010@N00", + "canComment": false, + "commentCount": 14, + "faveCount": 21, + "id": "14001294434", + "sizes": { + } + } + ] + } + } + ] + } + } """ + json = ''.join(json.split()) + ',\n' response = mock.Mock(text=json) results = flickr_noapi.response(response) self.assertEqual(type(results), list) @@ -266,51 +324,20 @@ class TestFlickrNoapiEngine(SearxTestCase): # null test json = """ - "search-photos-models","photos": - { - "_data": [null], - "fetchedStart": true, - "fetchedEnd": false, - "totalItems": "4386039" - },"totalItems": - """ - response = mock.Mock(text=json) - results = flickr_noapi.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) - - # no ownerNsid test - json = """ - "search-photos-lite-models","photos": - { - "_data": [ - { - "_flickrModelRegistry": "photo-lite-models", - "title": "This is the title", - "username": "Owner", - "pathAlias": "klink692", - "realname": "Owner", - "license": 0, - "canComment": false, - "commentCount": 14, - "faveCount": 21, - "id": "14001294434", - "sizes": { - "o": { - "displayUrl": "//farm8.staticflickr.com/7246/14001294434_410f653777_o.jpg", - "width": 433, - "height": 640, - "url": "//c4.staticflickr.com/8/7246/14001294434_410f653777_o.jpg", - "key": "o" + modelExport: { + "legend": [null], + "main": { + "search-photos-lite-models": [ + { + "photos": { + "_data": [null] } } - } - ], - "fetchedStart": true, - "fetchedEnd": false, - "totalItems": "4386039" - },"totalItems": + ] + } + } """ + json = ''.join(json.split()) + ',\n' response = mock.Mock(text=json) results = flickr_noapi.response(response) self.assertEqual(type(results), list) @@ -323,6 +350,7 @@ class TestFlickrNoapiEngine(SearxTestCase): "link":"http:\/\/www.flickr.com\/artist\/1217","type":"artist"} ]} """ + json = ''.join(json.split()) + ',\n' response = mock.Mock(text=json) results = flickr_noapi.response(response) self.assertEqual(type(results), list) diff --git a/tests/unit/engines/test_gigablast.py b/tests/unit/engines/test_gigablast.py index f0ddb63bc..6b2d26458 100644 --- a/tests/unit/engines/test_gigablast.py +++ b/tests/unit/engines/test_gigablast.py @@ -11,11 +11,16 @@ class TestGigablastEngine(SearxTestCase): dicto = defaultdict(dict) dicto['pageno'] = 0 dicto['safesearch'] = 0 - dicto['language'] = 'en-US' + dicto['language'] = 'all' params = gigablast.request(query, dicto) self.assertTrue('url' in params) self.assertTrue(query in params['url']) self.assertTrue('gigablast.com' in params['url']) + self.assertTrue('xx' in params['url']) + + dicto['language'] = 'en-US' + params = gigablast.request(query, dicto) + self.assertTrue('en' in params['url']) self.assertFalse('en-US' in params['url']) def test_response(self): diff --git a/tests/unit/engines/test_google.py b/tests/unit/engines/test_google.py index 413b67769..9d0edd439 100644 --- a/tests/unit/engines/test_google.py +++ b/tests/unit/engines/test_google.py @@ -15,7 +15,8 @@ class TestGoogleEngine(SearxTestCase): return response def test_request(self): - google.supported_languages = ['en', 'fr', 'zh-CN'] + google.supported_languages = ['en', 'fr', 'zh-CN', 'iw'] + google.language_aliases = {'he': 'iw'} query = 'test_query' dicto = defaultdict(dict) @@ -41,6 +42,12 @@ class TestGoogleEngine(SearxTestCase): self.assertIn('zh-CN', params['url']) self.assertIn('zh-CN', params['headers']['Accept-Language']) + dicto['language'] = 'he' + params = google.request(query, dicto) + self.assertIn('google.com', params['url']) + self.assertIn('iw', params['url']) + self.assertIn('iw', params['headers']['Accept-Language']) + def test_response(self): self.assertRaises(AttributeError, google.response, None) self.assertRaises(AttributeError, google.response, []) @@ -51,93 +58,50 @@ class TestGoogleEngine(SearxTestCase): self.assertEqual(google.response(response), []) html = """ - <div class="g"> - <h3 class="r"> - <a href="http://this.should.be.the.link/"> - <b>This</b> is <b>the</b> title - </a> - </h3> - <div class="s"> - <div class="kv" style="margin-bottom:2px"> - <cite> - <b>test</b>.psychologies.com/ - </cite> - <div class="_nBb"> - <div style="display:inline" onclick="google.sham(this);" aria-expanded="false" - aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA"> - <span class="_O0"> - </span> + <div class="ZINbbc xpd O9g5cc uUPGi"> + <div> + <div class="kCrYT"> + <a href="/url?q=http://this.should.be.the.link/"> + <div class="BNeawe"> + <b>This</b> is <b>the</b> title </div> - <div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1"> - <ul> - <li class="_Ykb"> - <a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent - .com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/"> - En cache - </a> - </li> - <li class="_Ykb"> - <a class="_Zkb" href="/search?safe=off&q=related:test.psy.com/"> - Pages similaires - </a> - </li> - </ul> + <div class="BNeawe"> + http://website + </div> + </a> + </div> + <div class="kCrYT"> + <div> + <div class="BNeawe"> + <div> + <div class="BNeawe"> + This should be the content. + </div> + </div> </div> </div> </div> - <span class="st"> - This should be the content. - </span> - <br> - <div class="osl"> - <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/"> - Test Personnalité - </a> - - <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/"> - Tests - Moi - </a> - - <a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple"> - Test Couple - </a> - - - <a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour"> - Test Amour + </div> + </p> + <div class="ZINbbc xpd O9g5cc uUPGi"> + <div> + <div class="kCrYT"> + <span> + <div class="BNeawe"> + Related searches + </div> + </span> + </div> + <div class="rVLSBd"> + <a> + <div> + <div class="BNeawe"> + suggestion title + </div> + </div> </a> </div> </div> - </div> - <div class="g"> - <h3 class="r"> - <a href="http://www.google.com/images?q=toto"> - <b>This</b> - </a> - </h3> - </div> - <div class="g"> - <h3 class="r"> - <a href="http://www.google.com/search?q=toto"> - <b>This</b> is - </a> - </h3> - </div> - <div class="g"> - <h3 class="r"> - <a href="€"> - <b>This</b> is <b>the</b> - </a> - </h3> - </div> - <div class="g"> - <h3 class="r"> - <a href="/url?q=url"> - <b>This</b> is <b>the</b> - </a> - </h3> - </div> - <p class="_Bmc" style="margin:3px 8px"> - <a href="/search?num=20&safe=off&q=t&revid=1754833769&sa=X&ei=-&ved="> - suggestion <b>title</b> - </a> </p> """ response = self.mock_response(html) @@ -198,29 +162,13 @@ class TestGoogleEngine(SearxTestCase): html = u""" <html> <body> - <table> - <tbody> - <tr> - <td> - <font> - <label> - <span id="ten">English</span> - </label> - </font> - </td> - <td> - <font> - <label> - <span id="tzh-CN">中文 (简体)</span> - </label> - <label> - <span id="tzh-TW">中文 (繁體)</span> - </label> - </font> - </td> - </tr> - </tbody> - </table> + <div id="langSec"> + <div> + <input name="lr" data-name="english" value="lang_en" /> + <input name="lr" data-name="中文 (简体)" value="lang_zh-CN" /> + <input name="lr" data-name="中文 (繁體)" value="lang_zh-TW" /> + </div> + </div> </body> </html> """ diff --git a/tests/unit/engines/test_google_news.py b/tests/unit/engines/test_google_news.py index a041a79b9..0a122ca6d 100644 --- a/tests/unit/engines/test_google_news.py +++ b/tests/unit/engines/test_google_news.py @@ -10,6 +10,7 @@ class TestGoogleNewsEngine(SearxTestCase): def test_request(self): google_news.supported_languages = ['en-US', 'fr-FR'] + google_news.language_aliases = {} query = 'test_query' dicto = defaultdict(dict) dicto['pageno'] = 1 @@ -20,6 +21,11 @@ class TestGoogleNewsEngine(SearxTestCase): self.assertIn(query, params['url']) self.assertIn('fr', params['url']) + dicto['language'] = 'all' + params = google_news.request(query, dicto) + self.assertIn('url', params) + self.assertNotIn('fr', params['url']) + def test_response(self): self.assertRaises(AttributeError, google_news.response, None) self.assertRaises(AttributeError, google_news.response, []) diff --git a/tests/unit/engines/test_google_videos.py b/tests/unit/engines/test_google_videos.py index a48e9a755..3b7edf373 100644 --- a/tests/unit/engines/test_google_videos.py +++ b/tests/unit/engines/test_google_videos.py @@ -30,16 +30,34 @@ class TestGoogleVideosEngine(SearxTestCase): <div> <div> <div class="g"> - <div> - <h3><a href="url_1">Title 1</h3> + <div class="r"> + <a href="url_1"><h3>Title 1</h3></a> + </div> + <div class="s"> + <div> + <a> + <g-img> + <img id="vidthumb1"> + </g-img> + </a> + </div> </div> <div> <span class="st">Content 1</span> </div> </div> <div class="g"> - <div> - <h3><a href="url_2">Title 2</h3> + <div class="r"> + <a href="url_2"><h3>Title 2</h3></a> + </div> + <div class="s"> + <div> + <a> + <g-img> + <img id="vidthumb2"> + </g-img> + </a> + </div> </div> <div> <span class="st">Content 2</span> @@ -47,6 +65,7 @@ class TestGoogleVideosEngine(SearxTestCase): </div> </div> </div> + <script>function _setImagesSrc(c,d,e){}</script> """ response = mock.Mock(text=html) results = google_videos.response(response) diff --git a/tests/unit/engines/test_mediawiki.py b/tests/unit/engines/test_mediawiki.py index 1b07dd8f4..b86372700 100644 --- a/tests/unit/engines/test_mediawiki.py +++ b/tests/unit/engines/test_mediawiki.py @@ -18,6 +18,10 @@ class TestMediawikiEngine(SearxTestCase): self.assertIn('wikipedia.org', params['url']) self.assertIn('fr', params['url']) + dicto['language'] = 'all' + params = mediawiki.request(query, dicto) + self.assertIn('en', params['url']) + mediawiki.base_url = "http://test.url/" mediawiki.search_url = mediawiki.base_url +\ 'w/api.php?action=query'\ diff --git a/tests/unit/engines/test_qwant.py b/tests/unit/engines/test_qwant.py index 86bfb22da..6611264f8 100644 --- a/tests/unit/engines/test_qwant.py +++ b/tests/unit/engines/test_qwant.py @@ -8,6 +8,7 @@ class TestQwantEngine(SearxTestCase): def test_request(self): qwant.supported_languages = ['en-US', 'fr-CA', 'fr-FR'] + qwant.language_aliases = {} query = 'test_query' dicto = defaultdict(dict) dicto['pageno'] = 0 @@ -20,11 +21,10 @@ class TestQwantEngine(SearxTestCase): self.assertIn('qwant.com', params['url']) self.assertIn('fr_fr', params['url']) - dicto['language'] = 'en-US' + dicto['language'] = 'all' qwant.categories = ['news'] params = qwant.request(query, dicto) self.assertFalse('fr' in params['url']) - self.assertIn('en_us', params['url']) self.assertIn('news', params['url']) dicto['language'] = 'fr' diff --git a/tests/unit/engines/test_seedpeer.py b/tests/unit/engines/test_seedpeer.py index 37b2de8e4..2057c1cb1 100644 --- a/tests/unit/engines/test_seedpeer.py +++ b/tests/unit/engines/test_seedpeer.py @@ -1,15 +1,11 @@ -import mock +# -*- coding: utf-8 -*- from collections import defaultdict +import mock from searx.engines import seedpeer from searx.testing import SearxTestCase -from datetime import datetime - -class TestSeedPeerEngine(SearxTestCase): - html = '' - with open('./tests/unit/engines/seedpeer_fixture.html') as fixture: - html += fixture.read() +class TestBtdiggEngine(SearxTestCase): def test_request(self): query = 'test_query' @@ -18,34 +14,53 @@ class TestSeedPeerEngine(SearxTestCase): params = seedpeer.request(query, dicto) self.assertIn('url', params) self.assertIn(query, params['url']) - self.assertIn('seedpeer.eu', params['url']) + self.assertIn('seedpeer', params['url']) - def test_response_raises_attr_error_on_empty_response(self): + def test_response(self): self.assertRaises(AttributeError, seedpeer.response, None) self.assertRaises(AttributeError, seedpeer.response, []) self.assertRaises(AttributeError, seedpeer.response, '') self.assertRaises(AttributeError, seedpeer.response, '[]') - def test_response_returns_empty_list(self): response = mock.Mock(text='<html></html>') self.assertEqual(seedpeer.response(response), []) - def test_response_returns_all_results(self): - response = mock.Mock(text=self.html) - results = seedpeer.response(response) - self.assertTrue(isinstance(results, list)) - self.assertEqual(len(results), 2) - - def test_response_returns_correct_results(self): - response = mock.Mock(text=self.html) + html = u""" + <html> + <head> + <script></script> + <script type="text/javascript" src="not_here.js"></script> + <script type="text/javascript"> + window.initialData= + {"data": {"list": [{"name": "Title", "seeds": "10", "peers": "20", "size": "1024", "hash": "abc123"}]}} + </script> + </head> + <body> + <table></table> + <table> + <thead><tr></tr></thead> + <tbody> + <tr> + <td><a href="link">Title</a></td> + <td>1 year</td> + <td>1 KB</td> + <td>10</td> + <td>20</td> + <td></td> + </tr> + </tbody> + </table> + </body> + </html> + """ + response = mock.Mock(text=html) results = seedpeer.response(response) - self.assertEqual( - results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG ' - ) - self.assertEqual( - results[0]['url'], - 'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html' - ) - self.assertEqual(results[0]['content'], '2.48 GB, 1 day') - self.assertEqual(results[0]['seed'], '861') - self.assertEqual(results[0]['leech'], '332') + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'https://seedpeer.me/link') + self.assertEqual(results[0]['seed'], 10) + self.assertEqual(results[0]['leech'], 20) + self.assertEqual(results[0]['filesize'], 1024) + self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123') + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123') diff --git a/tests/unit/engines/test_startpage.py b/tests/unit/engines/test_startpage.py index bdb3356b7..ac4454738 100644 --- a/tests/unit/engines/test_startpage.py +++ b/tests/unit/engines/test_startpage.py @@ -18,8 +18,9 @@ class TestStartpageEngine(SearxTestCase): self.assertIn('data', params) self.assertIn('query', params['data']) self.assertIn(query, params['data']['query']) - self.assertIn('with_language', params['data']) - self.assertIn('lang_fr', params['data']['with_language']) + + dicto['language'] = 'all' + params = startpage.request(query, dicto) def test_response(self): self.assertRaises(AttributeError, startpage.response, None) @@ -31,33 +32,32 @@ class TestStartpageEngine(SearxTestCase): self.assertEqual(startpage.response(response), []) html = """ - <div class='result' style=' *width : auto; *margin-right : 10%;'> - <h3> - <a href='http://this.should.be.the.link/' id='title_2' name='title_2' > - This should be the title +<div class="w-gl__result"> + <a + class="w-gl__result-title" + href="http://this.should.be.the.link/" + data-onw="1" + rel="noopener noreferrer" + target="_blank"> + + <h3>This should be the title</h3> </a> - <span id='title_stars_2' name='title_stars_2'> </span> - </h3> - <p class='desc clk'> - This should be the content. - </p> - <p> - <span class='url'>www.speed<b>test</b>.net/fr/ - </span> - - - <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata=" - class='proxy'> - Navigation avec Ixquick Proxy - </A> - - - <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid= - &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0= - &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'> - Mis en surbrillance - </A> - </p> - </div> - """ + <div class="w-gl__result-second-line-container"> + <div class="w-gl__result-url-container"> + <a + class="w-gl__result-url" + href="http://this.should.be.the.link/" + rel="noopener noreferrer" + target="_blank">https://www.cnbc.com/2019/10/12/dj-zedd-banned-in-china-for-liking-a-south-park-tweet.html</a> + </div> + <a + class="w-gl__anonymous-view-url" + href="https://eu-browse.startpage.com/do/proxy?ep=556b554d576b6f5054554546423167764b5445616455554d5342675441774659495246304848774f5267385453304941486b5949546c63704e33774f526b705544565647516d4a61554246304847674f4a556f6957415a4f436b455042426b6b4f7a64535a52784a56514a4f45307743446c567250445a4f4c52514e5677554e46776b4b545563704c7931554c5167465467644f42464d4f4255426f4d693152624634525741305845526c595746636b626d67494e42705743466c515252634f4267456e597a7346596b7856435134465345634f564249794b5752785643315863546769515773764a5163494c5877505246315865456f5141426b4f41774167596d6c5a4e30395758773442465251495677596c624770665a6b786344466b4151455663425249794d6a78525a55554157516f4342556766526b51314b57514e&ek=4q58686o5047786n6343527259445247576p6o38&ekdata=84abd523dc13cba5c65164d04d7d7263" + target="_blank">Anonymous View</a> + </div> + <p class="w-gl__description">This should be the content.</p> + </div> + """ # noqa response = mock.Mock(text=html.encode('utf-8')) results = startpage.response(response) self.assertEqual(type(results), list) @@ -65,72 +65,3 @@ class TestStartpageEngine(SearxTestCase): self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') - - html = """ - <div class='result' style=' *width : auto; *margin-right : 10%;'> - <h3> - <a href='http://www.google.com/aclk?sa=l&ai=C' id='title_2' name='title_2' > - This should be the title - </a> - <span id='title_stars_2' name='title_stars_2'> </span> - </h3> - <p class='desc clk'> - This should be the content. - </p> - <p> - <span class='url'>www.speed<b>test</b>.net/fr/ - </span> - - - <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata=" - class='proxy'> - Navigation avec Ixquick Proxy - </A> - - - <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid= - &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0= - &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'> - Mis en surbrillance - </A> - </p> - </div> - <div class='result' style=' *width : auto; *margin-right : 10%;'> - <h3> - <span id='title_stars_2' name='title_stars_2'> </span> - </h3> - <p class='desc clk'> - This should be the content. - </p> - <p> - <span class='url'>www.speed<b>test</b>.net/fr/ - </span> - </p> - </div> - <div class='result' style=' *width : auto; *margin-right : 10%;'> - <h3> - <a href='http://this.should.be.the.link/' id='title_2' name='title_2' > - This should be the title - </a> - <span id='title_stars_2' name='title_stars_2'> </span> - </h3> - <p> - <span class='url'>www.speed<b>test</b>.net/fr/ - </span> - - - <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata=" - class='proxy'> - Navigation avec Ixquick Proxy - </A> - - - <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid= - &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0= - &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'> - Mis en surbrillance - </A> - </p> - </div> - """ - response = mock.Mock(text=html.encode('utf-8')) - results = startpage.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['content'], '') diff --git a/tests/unit/engines/test_subtitleseeker.py b/tests/unit/engines/test_subtitleseeker.py deleted file mode 100644 index 96b253943..000000000 --- a/tests/unit/engines/test_subtitleseeker.py +++ /dev/null @@ -1,138 +0,0 @@ -from collections import defaultdict -import mock -from searx.engines import subtitleseeker -from searx.testing import SearxTestCase - - -class TestSubtitleseekerEngine(SearxTestCase): - - def test_request(self): - query = 'test_query' - dicto = defaultdict(dict) - dicto['pageno'] = 1 - dicto['language'] = 'fr-FR' - params = subtitleseeker.request(query, dicto) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('subtitleseeker.com' in params['url']) - - def test_response(self): - dicto = defaultdict(dict) - dicto['language'] = 'fr-FR' - response = mock.Mock(search_params=dicto) - - self.assertRaises(AttributeError, subtitleseeker.response, None) - self.assertRaises(AttributeError, subtitleseeker.response, []) - self.assertRaises(AttributeError, subtitleseeker.response, '') - self.assertRaises(AttributeError, subtitleseeker.response, '[]') - - response = mock.Mock(text='<html></html>', search_params=dicto) - self.assertEqual(subtitleseeker.response(response), []) - - html = """ - <div class="boxRows"> - <div class="boxRowsInner" style="width:600px;"> - <img src="http://static.subtitleseeker.com/images/movie.gif" - style="width:16px; height:16px;" class="icon"> - <a href="http://this.is.the.url/" - class="blue" title="Title subtitle" > - This is the Title - </a> - <br><br> - <span class="f10b grey-dark arial" style="padding:0px 0px 5px 20px"> - "Alternative Title" - </span> - </div> - <div class="boxRowsInner f12b red" style="width:70px;"> - 1998 - </div> - <div class="boxRowsInner grey-web f12" style="width:120px;"> - <img src="http://static.subtitleseeker.com/images/basket_put.png" - style="width:16px; height:16px;" class="icon"> - 1039 Subs - </div> - <div class="boxRowsInner grey-web f10" style="width:130px;"> - <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png" - style="width:16px; height:16px;" class="icon"> - 1 hours ago - </div> - <div class="clear"></div> - </div> - """ - response = mock.Mock(text=html, search_params=dicto) - results = subtitleseeker.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'This is the Title') - self.assertEqual(results[0]['url'], 'http://this.is.the.url/French/') - self.assertIn('1998', results[0]['content']) - self.assertIn('1039 Subs', results[0]['content']) - self.assertIn('Alternative Title', results[0]['content']) - - dicto['language'] = 'pt-BR' - results = subtitleseeker.response(response) - self.assertEqual(results[0]['url'], 'http://this.is.the.url/Brazilian/') - - html = """ - <div class="boxRows"> - <div class="boxRowsInner" style="width:600px;"> - <img src="http://static.subtitleseeker.com/images/movie.gif" - style="width:16px; height:16px;" class="icon"> - <a href="http://this.is.the.url/" - class="blue" title="Title subtitle" > - This is the Title - </a> - </div> - <div class="boxRowsInner f12b red" style="width:70px;"> - 1998 - </div> - <div class="boxRowsInner grey-web f12" style="width:120px;"> - <img src="http://static.subtitleseeker.com/images/basket_put.png" - style="width:16px; height:16px;" class="icon"> - 1039 Subs - </div> - <div class="boxRowsInner grey-web f10" style="width:130px;"> - <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png" - style="width:16px; height:16px;" class="icon"> - 1 hours ago - </div> - <div class="clear"></div> - </div> - """ - subtitleseeker.language = 'English' - response = mock.Mock(text=html, search_params=dicto) - results = subtitleseeker.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'This is the Title') - self.assertEqual(results[0]['url'], 'http://this.is.the.url/English/') - self.assertIn('1998', results[0]['content']) - self.assertIn('1039 Subs', results[0]['content']) - - html = """ - <div class="boxRowsInner" style="width:600px;"> - <img src="http://static.subtitleseeker.com/images/movie.gif" - style="width:16px; height:16px;" class="icon"> - <a href="http://this.is.the.url/" - class="blue" title="Title subtitle" > - This is the Title - </a> - </div> - <div class="boxRowsInner f12b red" style="width:70px;"> - 1998 - </div> - <div class="boxRowsInner grey-web f12" style="width:120px;"> - <img src="http://static.subtitleseeker.com/images/basket_put.png" - style="width:16px; height:16px;" class="icon"> - 1039 Subs - </div> - <div class="boxRowsInner grey-web f10" style="width:130px;"> - <img src="http://static.subtitleseeker.com/images/arrow_refresh_small.png" - style="width:16px; height:16px;" class="icon"> - 1 hours ago - </div> - """ - response = mock.Mock(text=html, search_params=dicto) - results = subtitleseeker.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) diff --git a/tests/unit/engines/test_swisscows.py b/tests/unit/engines/test_swisscows.py deleted file mode 100644 index 133f636de..000000000 --- a/tests/unit/engines/test_swisscows.py +++ /dev/null @@ -1,151 +0,0 @@ -from collections import defaultdict -import mock -from searx.engines import swisscows -from searx.testing import SearxTestCase - - -class TestSwisscowsEngine(SearxTestCase): - - def test_request(self): - swisscows.supported_languages = ['de-AT', 'de-DE'] - query = 'test_query' - dicto = defaultdict(dict) - dicto['pageno'] = 1 - dicto['language'] = 'de-DE' - params = swisscows.request(query, dicto) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('swisscows.ch' in params['url']) - self.assertTrue('uiLanguage=de' in params['url']) - self.assertTrue('region=de-DE' in params['url']) - - dicto['category'] = 'images' - params = swisscows.request(query, dicto) - self.assertIn('image', params['url']) - - def test_response(self): - self.assertRaises(AttributeError, swisscows.response, None) - self.assertRaises(AttributeError, swisscows.response, []) - self.assertRaises(AttributeError, swisscows.response, '') - self.assertRaises(AttributeError, swisscows.response, '[]') - - response = mock.Mock(text=b'<html></html>') - self.assertEqual(swisscows.response(response), []) - - response = mock.Mock(text=b'<html></html>') - self.assertEqual(swisscows.response(response), []) - - html = b""" - <script> - App.Dispatcher.dispatch("initialize", { - html5history: true, - initialData: {"Request": - {"Page":1, - "ItemsCount":1, - "Query":"This should ", - "NormalizedQuery":"This should ", - "Region":"de-AT", - "UILanguage":"de"}, - "Results":{"items":[ - {"Title":"\uE000This should\uE001 be the title", - "Description":"\uE000This should\uE001 be the content.", - "Url":"http://this.should.be.the.link/", - "DisplayUrl":"www.\uE000this.should.be.the\uE001.link", - "Id":"782ef287-e439-451c-b380-6ebc14ba033d"}, - {"Title":"Datei:This should1.svg", - "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png", - "SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should1.svg", - "DisplayUrl":"de.wikipedia.org/wiki/Datei:This should1.svg", - "Width":950, - "Height":534, - "FileSize":92100, - "ContentType":"image/jpeg", - "Thumbnail":{ - "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png", - "ContentType":"image/jpeg", - "Width":300, - "Height":168, - "FileSize":9134}, - "Id":"6a97a542-8f65-425f-b7f6-1178c3aba7be" - } - ],"TotalCount":55300, - "Query":"This should " - }, - "Images":[{"Title":"Datei:This should.svg", - "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png", - "SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should.svg", - "DisplayUrl":"de.wikipedia.org/wiki/Datei:This should.svg", - "Width":1280, - "Height":677, - "FileSize":50053, - "ContentType":"image/png", - "Thumbnail":{"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png", - "ContentType":"image/png", - "Width":300, - "Height":158, - "FileSize":8023}, - "Id":"ae230fd8-a06a-47d6-99d5-e74766d8143a"}]}, - environment: "production" - }).then(function (options) { - $('#Search_Form').on('submit', function (e) { - if (!Modernizr.history) return; - e.preventDefault(); - - var $form = $(this), - $query = $('#Query'), - query = $.trim($query.val()), - path = App.Router.makePath($form.attr('action'), null, $form.serializeObject()) - - if (query.length) { - options.html5history ? - ReactRouter.HistoryLocation.push(path) : - ReactRouter.RefreshLocation.push(path); - } - else $('#Query').trigger('blur'); - }); - - }); - </script> - """ - response = mock.Mock(text=html) - results = swisscows.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 3) - self.assertEqual(results[0]['title'], 'This should be the title') - self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') - self.assertEqual(results[0]['content'], 'This should be the content.') - self.assertEqual(results[1]['title'], 'Datei:This should1.svg') - self.assertEqual(results[1]['url'], 'http://de.wikipedia.org/wiki/Datei:This should1.svg') - self.assertEqual(results[1]['img_src'], 'http://ts2.mm.This/should1.png') - self.assertEqual(results[1]['template'], 'images.html') - self.assertEqual(results[2]['title'], 'Datei:This should.svg') - self.assertEqual(results[2]['url'], 'http://de.wikipedia.org/wiki/Datei:This should.svg') - self.assertEqual(results[2]['img_src'], 'http://ts2.mm.This/should.png') - self.assertEqual(results[2]['template'], 'images.html') - - def test_fetch_supported_languages(self): - html = """<html></html>""" - response = mock.Mock(text=html) - languages = swisscows._fetch_supported_languages(response) - self.assertEqual(type(languages), list) - self.assertEqual(len(languages), 0) - - html = """ - <html> - <div id="regions-popup"> - <div> - <ul> - <li><a data-search-language="browser"></a></li> - <li><a data-search-language="de-CH"></a></li> - <li><a data-search-language="fr-CH"></a></li> - </ul> - </div> - </div> - </html> - """ - response = mock.Mock(text=html) - languages = swisscows._fetch_supported_languages(response) - self.assertEqual(type(languages), list) - self.assertEqual(len(languages), 3) - self.assertIn('de-CH', languages) - self.assertIn('fr-CH', languages) diff --git a/tests/unit/engines/test_twitter.py b/tests/unit/engines/test_twitter.py index 10e4a0ba9..b444b48ee 100644 --- a/tests/unit/engines/test_twitter.py +++ b/tests/unit/engines/test_twitter.py @@ -20,6 +20,12 @@ class TestTwitterEngine(SearxTestCase): self.assertIn('lang', params['cookies']) self.assertIn('fr', params['cookies']['lang']) + dicto['language'] = 'all' + params = twitter.request(query, dicto) + self.assertIn('cookies', params) + self.assertIn('lang', params['cookies']) + self.assertIn('en', params['cookies']['lang']) + def test_response(self): self.assertRaises(AttributeError, twitter.response, None) self.assertRaises(AttributeError, twitter.response, []) diff --git a/tests/unit/engines/test_unsplash.py b/tests/unit/engines/test_unsplash.py new file mode 100644 index 000000000..4501de906 --- /dev/null +++ b/tests/unit/engines/test_unsplash.py @@ -0,0 +1,38 @@ +from collections import defaultdict +import mock +from searx.testing import SearxTestCase +from searx.engines import unsplash + + +class TestUnsplashEngine(SearxTestCase): + def test_request(self): + query = 'penguin' + _dict = defaultdict(dict) + _dict['pageno'] = 1 + params = unsplash.request(query, _dict) + + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + + def test_response(self): + resp = mock.Mock(text='{}') + result = unsplash.response(resp) + self.assertEqual([], result) + + resp.text = '{"results": []}' + result = unsplash.response(resp) + self.assertEqual([], result) + + # Sourced from https://unsplash.com/napi/search/photos?query=penguin&xp=&per_page=20&page=2 + with open('./tests/unit/engines/unsplash_fixture.json') as fixture: + resp.text = fixture.read() + + result = unsplash.response(resp) + self.assertEqual(len(result), 2) + self.assertEqual(result[0]['title'], 'low angle photography of swimming penguin') + self.assertEqual(result[0]['url'], 'https://unsplash.com/photos/FY8d721UO_4') + self.assertEqual(result[0]['thumbnail_src'], 'https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80\ +&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max') + self.assertEqual(result[0]['img_src'], 'https://images.unsplash.com/photo-1523557148507-1b77641c7e7c\ +?ixlib=rb-0.3.5') + self.assertEqual(result[0]['content'], '') diff --git a/tests/unit/engines/test_wikidata.py b/tests/unit/engines/test_wikidata.py index 62a409781..48be17bb4 100644 --- a/tests/unit/engines/test_wikidata.py +++ b/tests/unit/engines/test_wikidata.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from lxml.html import fromstring +from lxml import etree from collections import defaultdict import mock from searx.engines import wikidata @@ -11,11 +12,13 @@ class TestWikidataEngine(SearxTestCase): def test_request(self): query = 'test_query' dicto = defaultdict(dict) + dicto['language'] = 'all' params = wikidata.request(query, dicto) self.assertIn('url', params) self.assertIn(query, params['url']) self.assertIn('wikidata.org', params['url']) + dicto['language'] = 'es_ES' params = wikidata.request(query, dicto) self.assertIn(query, params['url']) @@ -27,12 +30,13 @@ class TestWikidataEngine(SearxTestCase): self.assertRaises(AttributeError, wikidata.response, '[]') wikidata.supported_languages = ['en', 'es'] - response = mock.Mock(text='<html></html>', search_params={"language": "en"}) + wikidata.language_aliases = {} + response = mock.Mock(content='<html></html>'.encode("utf-8"), search_params={"language": "en"}) self.assertEqual(wikidata.response(response), []) def test_getDetail(self): response = {} - results = wikidata.getDetail(response, "Q123", "en", "en-US") + results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser()) self.assertEqual(results, []) title_html = '<div><div class="wikibase-title-label">Test</div></div>' @@ -49,7 +53,7 @@ class TestWikidataEngine(SearxTestCase): """ response = {"parse": {"displaytitle": title_html, "text": html}} - results = wikidata.getDetail(response, "Q123", "en", "en-US") + results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser()) self.assertEqual(len(results), 1) self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test') @@ -89,7 +93,7 @@ class TestWikidataEngine(SearxTestCase): """ response = {"parse": {"displaytitle": title_html, "text": html}} - results = wikidata.getDetail(response, "Q123", "yua", "yua_MX") + results = wikidata.getDetail(response, "Q123", "yua", "yua_MX", etree.HTMLParser()) self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'Official website') self.assertEqual(results[0]['url'], 'https://officialsite.com') @@ -136,8 +140,8 @@ class TestWikidataEngine(SearxTestCase): </div> """ html_etree = fromstring(html) - - image_src = wikidata.add_image(html_etree) + id_cache = wikidata.get_id_cache(html_etree) + image_src = wikidata.add_image(id_cache) self.assertEqual(image_src, "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400") @@ -194,8 +198,9 @@ class TestWikidataEngine(SearxTestCase): </div> """ html_etree = fromstring(html) + id_cache = wikidata.get_id_cache(html_etree) - image_src = wikidata.add_image(html_etree) + image_src = wikidata.add_image(id_cache) self.assertEqual(image_src, "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400") @@ -229,11 +234,12 @@ class TestWikidataEngine(SearxTestCase): """ attributes = [] html_etree = fromstring(html) + id_cache = wikidata.get_id_cache(html_etree) - wikidata.add_attribute(attributes, html_etree, "Fail") + wikidata.add_attribute(attributes, id_cache, "Fail") self.assertEqual(attributes, []) - wikidata.add_attribute(attributes, html_etree, "P27") + wikidata.add_attribute(attributes, id_cache, "P27") self.assertEqual(len(attributes), 1) self.assertEqual(attributes[0]["label"], "Country of citizenship") self.assertEqual(attributes[0]["value"], "United Kingdom") @@ -268,7 +274,8 @@ class TestWikidataEngine(SearxTestCase): """ attributes = [] html_etree = fromstring(html) - wikidata.add_attribute(attributes, html_etree, "P569", date=True) + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_attribute(attributes, id_cache, "P569", date=True) self.assertEqual(len(attributes), 1) self.assertEqual(attributes[0]["label"], "Date of birth") self.assertEqual(attributes[0]["value"], "27 January 1832") @@ -316,14 +323,16 @@ class TestWikidataEngine(SearxTestCase): """ attributes = [] html_etree = fromstring(html) - wikidata.add_attribute(attributes, html_etree, "P6") + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_attribute(attributes, id_cache, "P6") self.assertEqual(len(attributes), 1) self.assertEqual(attributes[0]["label"], "Head of government") self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister") attributes = [] html_etree = fromstring(html) - wikidata.add_attribute(attributes, html_etree, "P6", trim=True) + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_attribute(attributes, id_cache, "P6", trim=True) self.assertEqual(len(attributes), 1) self.assertEqual(attributes[0]["value"], "Actual Prime Minister") @@ -354,12 +363,13 @@ class TestWikidataEngine(SearxTestCase): """ urls = [] html_etree = fromstring(html) - wikidata.add_url(urls, html_etree, 'P856') + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_url(urls, html_etree, id_cache, 'P856') self.assertEquals(len(urls), 1) self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls) urls = [] results = [] - wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results) + wikidata.add_url(urls, html_etree, id_cache, 'P856', 'custom label', results=results) self.assertEquals(len(urls), 1) self.assertEquals(len(results), 1) self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls) @@ -402,7 +412,8 @@ class TestWikidataEngine(SearxTestCase): """ urls = [] html_etree = fromstring(html) - wikidata.add_url(urls, html_etree, 'P856') + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_url(urls, html_etree, id_cache, 'P856') self.assertEquals(len(urls), 2) self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls) self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls) diff --git a/tests/unit/engines/test_wikipedia.py b/tests/unit/engines/test_wikipedia.py index 21d0225ed..316b12bc5 100644 --- a/tests/unit/engines/test_wikipedia.py +++ b/tests/unit/engines/test_wikipedia.py @@ -8,7 +8,8 @@ from searx.testing import SearxTestCase class TestWikipediaEngine(SearxTestCase): def test_request(self): - wikipedia.supported_languages = ['fr', 'en'] + wikipedia.supported_languages = ['fr', 'en', 'no'] + wikipedia.language_aliases = {'nb': 'no'} query = 'test_query' dicto = defaultdict(dict) @@ -25,10 +26,17 @@ class TestWikipediaEngine(SearxTestCase): self.assertIn('Test_Query', params['url']) self.assertNotIn('test_query', params['url']) - dicto['language'] = 'xx' + dicto['language'] = 'nb' + params = wikipedia.request(query, dicto) + self.assertIn('no.wikipedia.org', params['url']) + dicto['language'] = 'all' params = wikipedia.request(query, dicto) self.assertIn('en', params['url']) + dicto['language'] = 'xx' + params = wikipedia.request(query, dicto) + self.assertIn('en.wikipedia.org', params['url']) + def test_response(self): dicto = defaultdict(dict) dicto['language'] = 'fr' diff --git a/tests/unit/engines/test_wolframalpha_api.py b/tests/unit/engines/test_wolframalpha_api.py index 30d337645..0433b34aa 100644 --- a/tests/unit/engines/test_wolframalpha_api.py +++ b/tests/unit/engines/test_wolframalpha_api.py @@ -35,7 +35,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase): xml = '''<?xml version='1.0' encoding='UTF-8'?> <queryresult success='false' error='false' /> ''' - response = mock.Mock(text=xml.encode('utf-8')) + response = mock.Mock(content=xml.encode('utf-8')) self.assertEqual(wolframalpha_api.response(response), []) # test basic case @@ -83,7 +83,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase): </pod> </queryresult> """ - response = mock.Mock(text=xml, request=request) + response = mock.Mock(content=xml, request=request) results = wolframalpha_api.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 2) @@ -144,7 +144,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase): </pod> </queryresult> """ - response = mock.Mock(text=xml, request=request) + response = mock.Mock(content=xml, request=request) results = wolframalpha_api.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 2) diff --git a/tests/unit/engines/test_www1x.py b/tests/unit/engines/test_www1x.py index 9df8de6bf..40f5200fd 100644 --- a/tests/unit/engines/test_www1x.py +++ b/tests/unit/engines/test_www1x.py @@ -12,46 +12,3 @@ class TestWww1xEngine(SearxTestCase): self.assertTrue('url' in params) self.assertTrue(query in params['url']) self.assertTrue('1x.com' in params['url']) - - def test_response(self): - self.assertRaises(AttributeError, www1x.response, None) - self.assertRaises(AttributeError, www1x.response, []) - self.assertRaises(AttributeError, www1x.response, '') - self.assertRaises(AttributeError, www1x.response, '[]') - - response = mock.Mock(text='<html></html>') - self.assertEqual(www1x.response(response), []) - html = """ - <?xml version="1.0" encoding="UTF-8"?><!DOCTYPE characters - [ - <!ELEMENT characters (character*) > - <!ELEMENT character (#PCDATA ) > - - <!ENTITY iexcl "¡" > - <!ENTITY cent "¢" > - <!ENTITY pound "£" > - ] - ><root><searchresult><![CDATA[<table border="0" cellpadding="0" cellspacing="0" width="100%"> - <tr> - <td style="min-width: 220px;" valign="top"> - <div style="font-size: 30px; margin: 0px 0px 20px 0px;">Photos</div> - <div> - <a href="/photo/123456" class="dynamiclink"> -<img border="0" class="searchresult" src="/images/user/testimage-123456.jpg" style="width: 125px; height: 120px;"> - </a> - <a title="sjoerd lammers street photography" href="/member/sjoerdlammers" class="dynamiclink"> -<img border="0" class="searchresult" src="/images/profile/60c48b394c677d2fa4d9e7d263aabf44-square.jpg"> - </a> - </div> - </td> - </table> - ]]></searchresult></root> - """ - response = mock.Mock(text=html) - results = www1x.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['url'], 'https://1x.com/photo/123456') - self.assertEqual(results[0]['thumbnail_src'], 'https://1x.com/images/user/testimage-123456.jpg') - self.assertEqual(results[0]['content'], '') - self.assertEqual(results[0]['template'], 'images.html') diff --git a/tests/unit/engines/test_yacy.py b/tests/unit/engines/test_yacy.py index a60614d3d..f49532cf4 100644 --- a/tests/unit/engines/test_yacy.py +++ b/tests/unit/engines/test_yacy.py @@ -17,6 +17,11 @@ class TestYacyEngine(SearxTestCase): self.assertIn('localhost', params['url']) self.assertIn('fr', params['url']) + dicto['language'] = 'all' + params = yacy.request(query, dicto) + self.assertIn('url', params) + self.assertNotIn('lr=lang_', params['url']) + def test_response(self): self.assertRaises(AttributeError, yacy.response, None) self.assertRaises(AttributeError, yacy.response, []) diff --git a/tests/unit/engines/test_yahoo.py b/tests/unit/engines/test_yahoo.py index 5037bfc7d..e52c1109e 100644 --- a/tests/unit/engines/test_yahoo.py +++ b/tests/unit/engines/test_yahoo.py @@ -50,6 +50,13 @@ class TestYahooEngine(SearxTestCase): self.assertIn('zh_cht', params['url']) self.assertIn('zh_cht', params['cookies']['sB']) + dicto['language'] = 'all' + params = yahoo.request(query, dicto) + self.assertIn('cookies', params) + self.assertIn('sB', params['cookies']) + self.assertIn('en', params['cookies']['sB']) + self.assertIn('en', params['url']) + def test_no_url_in_request_year_time_range(self): dicto = defaultdict(dict) query = 'test_query' diff --git a/tests/unit/engines/test_yahoo_news.py b/tests/unit/engines/test_yahoo_news.py index c3297dacf..ae27df2a5 100644 --- a/tests/unit/engines/test_yahoo_news.py +++ b/tests/unit/engines/test_yahoo_news.py @@ -23,6 +23,13 @@ class TestYahooNewsEngine(SearxTestCase): self.assertIn('sB', params['cookies']) self.assertIn('fr', params['cookies']['sB']) + dicto['language'] = 'all' + params = yahoo_news.request(query, dicto) + self.assertIn('cookies', params) + self.assertIn('sB', params['cookies']) + self.assertIn('en', params['cookies']['sB']) + self.assertIn('en', params['url']) + def test_sanitize_url(self): url = "test.url" self.assertEqual(url, yahoo_news.sanitize_url(url)) diff --git a/tests/unit/engines/test_youtube_noapi.py b/tests/unit/engines/test_youtube_noapi.py index 41dcbb749..cbf7b9bcd 100644 --- a/tests/unit/engines/test_youtube_noapi.py +++ b/tests/unit/engines/test_youtube_noapi.py @@ -46,121 +46,71 @@ class TestYoutubeNoAPIEngine(SearxTestCase): self.assertEqual(youtube_noapi.response(response), []) html = """ - <ol id="item-section-063864" class="item-section"> - <li> - <div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile" - data-context-item-id="DIVZCPfAOeM" - data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB"> - <div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto"> - <a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link" - data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA"> - <div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg" - width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a> - <span class="thumb-menu dark-overflow-action-menu video-actions"> - </span> - </div> - <div class="yt-lockup-content"> - <h3 class="yt-lockup-title"> - <a href="/watch?v=DIVZCPfAOeM" - class="yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2 yt-uix-sessionlink spf-link" - data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA" - title="Top Speed Test Kawasaki Ninja H2 (Thailand) By. MEHAY SUPERBIKE" - aria-describedby="description-id-259079" rel="spf-prefetch" dir="ltr"> - Title - </a> - <span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span> - </h3> - <div class="yt-lockup-byline">de - <a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard" - data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA" - data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta"> - <ul class="yt-lockup-meta-info"> - <li>il y a 20 heures</li> - <li>8 424 vues</li> - </ul> - </div> - <div class="yt-lockup-description yt-ui-ellipsis yt-ui-ellipsis-2" dir="ltr"> - Description - </div> - <div class="yt-lockup-badges"> - <ul class="yt-badge-list "> - <li class="yt-badge-item" > - <span class="yt-badge">Nouveauté</span> - </li> - <li class="yt-badge-item" ><span class="yt-badge " >HD</span></li> - </ul> - </div> - <div class="yt-lockup-action-menu yt-uix-menu-container"> - <div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded" - data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu"> - </div> - </div> - </div> - </div> - </div> - </li> - </ol> + <div></div> + <script> + window["ytInitialData"] = { + "contents": { + "twoColumnSearchResultsRenderer": { + "primaryContents": { + "sectionListRenderer": { + "contents": [ + { + "itemSectionRenderer": { + "contents": [ + { + "videoRenderer": { + "videoId": "DIVZCPfAOeM", + "title": { + "simpleText": "Title" + }, + "descriptionSnippet": { + "runs": [ + { + "text": "Des" + }, + { + "text": "cription" + } + ] + } + } + }, + { + "videoRenderer": { + "videoId": "9C_HReR_McQ", + "title": { + "simpleText": "Title" + }, + "descriptionSnippet": { + "simpleText": "Description" + } + } + } + ] + } + } + ] + } + } + } + } + }; + </script> """ response = mock.Mock(text=html) results = youtube_noapi.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'Title') self.assertEqual(results[0]['url'], 'https://www.youtube.com/watch?v=DIVZCPfAOeM') self.assertEqual(results[0]['content'], 'Description') self.assertEqual(results[0]['thumbnail'], 'https://i.ytimg.com/vi/DIVZCPfAOeM/hqdefault.jpg') self.assertTrue('DIVZCPfAOeM' in results[0]['embedded']) - - html = """ - <ol id="item-section-063864" class="item-section"> - <li> - <div class="yt-lockup yt-lockup-tile yt-lockup-video vve-check clearfix yt-uix-tile" - data-context-item-id="DIVZCPfAOeM" - data-visibility-tracking="CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JECx_-GK5uqMpcIB"> - <div class="yt-lockup-dismissable"><div class="yt-lockup-thumbnail contains-addto"> - <a aria-hidden="true" href="/watch?v=DIVZCPfAOeM" class=" yt-uix-sessionlink pf-link" - data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JFIEdGVzdA"> - <div class="yt-thumb video-thumb"><img src="//i.ytimg.com/vi/DIVZCPfAOeM/mqdefault.jpg" - width="196" height="110"/></div><span class="video-time" aria-hidden="true">11:35</span></a> - <span class="thumb-menu dark-overflow-action-menu video-actions"> - </span> - </div> - <div class="yt-lockup-content"> - <h3 class="yt-lockup-title"> - <span class="accessible-description" id="description-id-259079"> - Durée : 11:35.</span> - </h3> - <div class="yt-lockup-byline">de - <a href="/user/mheejapan" class=" yt-uix-sessionlink spf-link g-hovercard" - data-sessionlink="itct=CBgQ3DAYACITCPGXnYau6sUCFZEIHAod-VQASCj0JA" data-ytid="UCzEesu54Hjs0uRKmpy66qeA" - data-name="">MEHAY SUPERBIKE</a></div><div class="yt-lockup-meta"> - <ul class="yt-lockup-meta-info"> - <li>il y a 20 heures</li> - <li>8 424 vues</li> - </ul> - </div> - <div class="yt-lockup-badges"> - <ul class="yt-badge-list "> - <li class="yt-badge-item" > - <span class="yt-badge">Nouveauté</span> - </li> - <li class="yt-badge-item" ><span class="yt-badge " >HD</span></li> - </ul> - </div> - <div class="yt-lockup-action-menu yt-uix-menu-container"> - <div class="yt-uix-menu yt-uix-videoactionmenu hide-until-delayloaded" - data-video-id="DIVZCPfAOeM" data-menu-content-id="yt-uix-videoactionmenu-menu"> - </div> - </div> - </div> - </div> - </div> - </li> - </ol> - """ - response = mock.Mock(text=html) - results = youtube_noapi.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(results[1]['title'], 'Title') + self.assertEqual(results[1]['url'], 'https://www.youtube.com/watch?v=9C_HReR_McQ') + self.assertEqual(results[1]['content'], 'Description') + self.assertEqual(results[1]['thumbnail'], 'https://i.ytimg.com/vi/9C_HReR_McQ/hqdefault.jpg') + self.assertTrue('9C_HReR_McQ' in results[1]['embedded']) html = """ <ol id="item-section-063864" class="item-section"> diff --git a/tests/unit/engines/unsplash_fixture.json b/tests/unit/engines/unsplash_fixture.json new file mode 100644 index 000000000..4c8db2a2c --- /dev/null +++ b/tests/unit/engines/unsplash_fixture.json @@ -0,0 +1,241 @@ +{ + "total": 2, + "total_pages": 1, + "results": [ + { + "id": "FY8d721UO_4", + "created_at": "2018-04-12T14:20:35-04:00", + "updated_at": "2018-08-28T20:58:33-04:00", + "width": 3891, + "height": 5829, + "color": "#152C33", + "description": "low angle photography of swimming penguin", + "urls": { + "raw": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&ixid=eyJhcHBfaWQiOjEyMDd9&s=095c5fc319c5a77c705f49ad63e0f195", + "full": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=85&fm=jpg&crop=entropy&cs=srgb&ixid=eyJhcHBfaWQiOjEyMDd9&s=74be977849c173d6929636d491a760c3", + "regular": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=1080&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=ad65df26970bd010085f0ca25434de33", + "small": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=400&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=5d2edfd073c31eb8ee7b305222bdc5a2", + "thumb": "https://images.unsplash.com/photo-1523557148507-1b77641c7e7c?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=a9b9e56e63efc6f4611a87ce7e9a48f8" + }, + "links": { + "self": "https://api.unsplash.com/photos/FY8d721UO_4", + "html": "https://unsplash.com/photos/FY8d721UO_4", + "download": "https://unsplash.com/photos/FY8d721UO_4/download", + "download_location": "https://api.unsplash.com/photos/FY8d721UO_4/download" + }, + "categories": [], + "sponsored": false, + "likes": 31, + "liked_by_user": false, + "current_user_collections": [], + "slug": null, + "user": { + "id": "N4gE4mrG8lE", + "updated_at": "2018-10-03T02:51:19-04:00", + "username": "gaspanik", + "name": "Masaaki Komori", + "first_name": "Masaaki", + "last_name": "Komori", + "twitter_username": "cipher", + "portfolio_url": "https://www.instagram.com/cipher/", + "bio": null, + "location": "Tokyo, JAPAN", + "links": { + "self": "https://api.unsplash.com/users/gaspanik", + "html": "https://unsplash.com/@gaspanik", + "photos": "https://api.unsplash.com/users/gaspanik/photos", + "likes": "https://api.unsplash.com/users/gaspanik/likes", + "portfolio": "https://api.unsplash.com/users/gaspanik/portfolio", + "following": "https://api.unsplash.com/users/gaspanik/following", + "followers": "https://api.unsplash.com/users/gaspanik/followers" + }, + "profile_image": { + "small": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=32&w=32&s=9fe12f6d177bd6fdbd56d233a80c01a3", + "medium": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=64&w=64&s=6ad7d156b62e438ae9dc794cba712fff", + "large": "https://images.unsplash.com/profile-fb-1502270358-e7c86c1011ce.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=128&w=128&s=13a08a2e72e7d11632410e92bd3a9406" + }, + "instagram_username": "cipher", + "total_collections": 0, + "total_likes": 406, + "total_photos": 196 + }, + "tags": [ + { + "title": "animal" + }, + { + "title": "water" + }, + { + "title": "swim" + }, + { + "title": "aquarium" + }, + { + "title": "wallpaper" + }, + { + "title": "blue" + }, + { + "title": "sealife" + }, + { + "title": "wildlife" + }, + { + "title": "bird" + }, + { + "title": "deep sea" + }, + { + "title": "fish" + }, + { + "title": "water life" + } + ], + "photo_tags": [ + { + "title": "animal" + }, + { + "title": "water" + }, + { + "title": "swim" + }, + { + "title": "aquarium" + }, + { + "title": "wallpaper" + } + ] + }, + { + "id": "ayKyc01xLWA", + "created_at": "2018-02-16T23:14:31-05:00", + "updated_at": "2018-08-28T20:48:27-04:00", + "width": 4928, + "height": 3264, + "color": "#161618", + "description": "black and white penguins on ice field", + "urls": { + "raw": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&ixid=eyJhcHBfaWQiOjEyMDd9&s=4e107a2bc49ab561ba6272eea2ec725d", + "full": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=85&fm=jpg&crop=entropy&cs=srgb&ixid=eyJhcHBfaWQiOjEyMDd9&s=f9b1e4d4572ab44efb2cf3d601d2b4d9", + "regular": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=1080&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=4430cedb63841f1fe055d5005316cc96", + "small": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=400&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=ee73c7af22ce445d408e240821ce07af", + "thumb": "https://images.unsplash.com/photo-1518840801558-9770b4a34eeb?ixlib=rb-0.3.5&q=80&fm=jpg&crop=entropy&cs=tinysrgb&w=200&fit=max&ixid=eyJhcHBfaWQiOjEyMDd9&s=934302390d383cad8c571905e3a80bac" + }, + "links": { + "self": "https://api.unsplash.com/photos/ayKyc01xLWA", + "html": "https://unsplash.com/photos/ayKyc01xLWA", + "download": "https://unsplash.com/photos/ayKyc01xLWA/download", + "download_location": "https://api.unsplash.com/photos/ayKyc01xLWA/download" + }, + "categories": [], + "sponsored": false, + "likes": 37, + "liked_by_user": false, + "current_user_collections": [], + "slug": null, + "user": { + "id": "tRb_KGw60Xk", + "updated_at": "2018-09-20T11:51:54-04:00", + "username": "ghost_cat", + "name": "Danielle Barnes", + "first_name": "Danielle", + "last_name": "Barnes", + "twitter_username": null, + "portfolio_url": null, + "bio": null, + "location": null, + "links": { + "self": "https://api.unsplash.com/users/ghost_cat", + "html": "https://unsplash.com/@ghost_cat", + "photos": "https://api.unsplash.com/users/ghost_cat/photos", + "likes": "https://api.unsplash.com/users/ghost_cat/likes", + "portfolio": "https://api.unsplash.com/users/ghost_cat/portfolio", + "following": "https://api.unsplash.com/users/ghost_cat/following", + "followers": "https://api.unsplash.com/users/ghost_cat/followers" + }, + "profile_image": { + "small": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=32&w=32&s=751bf6a557763648d52ffd7e60e79436", + "medium": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=64&w=64&s=e46cd1c8713035f045130e1b093b981e", + "large": "https://images.unsplash.com/profile-fb-1508491082-ae77f53e9ac3.jpg?ixlib=rb-0.3.5&q=80&fm=jpg&crop=faces&cs=tinysrgb&fit=crop&h=128&w=128&s=352eabcf107c3ce95fe51a18485f116b" + }, + "instagram_username": null, + "total_collections": 0, + "total_likes": 0, + "total_photos": 21 + }, + "tags": [ + { + "title": "ice" + }, + { + "title": "bird" + }, + { + "title": "ice field" + }, + { + "title": "iceberg" + }, + { + "title": "snow" + }, + { + "title": "frozen" + }, + { + "title": "animal" + }, + { + "title": "wildlife" + }, + { + "title": "wild" + }, + { + "title": "antarctica" + }, + { + "title": "sunshine" + }, + { + "title": "daylight" + }, + { + "title": "wilderness" + }, + { + "title": "south pole" + }, + { + "title": "flock" + } + ], + "photo_tags": [ + { + "title": "ice" + }, + { + "title": "bird" + }, + { + "title": "ice field" + }, + { + "title": "iceberg" + }, + { + "title": "snow" + } + ] + } + ] +}
\ No newline at end of file diff --git a/tests/unit/test_preferences.py b/tests/unit/test_preferences.py index c8f2a4419..61ac0e8e4 100644 --- a/tests/unit/test_preferences.py +++ b/tests/unit/test_preferences.py @@ -90,30 +90,25 @@ class TestSettings(SearxTestCase): # search language settings def test_lang_setting_valid_choice(self): - setting = SearchLanguageSetting('en', choices=['de', 'en']) + setting = SearchLanguageSetting('all', choices=['all', 'de', 'en']) setting.parse('de') self.assertEquals(setting.get_value(), 'de') def test_lang_setting_invalid_choice(self): - setting = SearchLanguageSetting('en', choices=['de', 'en']) + setting = SearchLanguageSetting('all', choices=['all', 'de', 'en']) setting.parse('xx') - self.assertEquals(setting.get_value(), 'en') + self.assertEquals(setting.get_value(), 'all') def test_lang_setting_old_cookie_choice(self): - setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES']) + setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES']) setting.parse('es_XA') self.assertEquals(setting.get_value(), 'es') def test_lang_setting_old_cookie_format(self): - setting = SearchLanguageSetting('en', choices=['en', 'es', 'es-ES']) + setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES']) setting.parse('es_ES') self.assertEquals(setting.get_value(), 'es-ES') - def test_lang_setting_old_default(self): - setting = SearchLanguageSetting('en', choices=['en', 'es', 'de']) - setting.parse('all') - self.assertEquals(setting.get_value(), 'en') - # plugins settings def test_plugins_setting_all_default_enabled(self): plugin1 = PluginStub('plugin1', True) @@ -127,3 +122,23 @@ class TestSettings(SearxTestCase): plugin3 = PluginStub('plugin3', True) setting = PluginsSetting('name', choices=[plugin1, plugin2, plugin3]) self.assertEquals(setting.get_enabled(), set(['plugin1', 'plugin3'])) + + +class TestPreferences(SearxTestCase): + + def test_encode(self): + from searx.preferences import Preferences + pref = Preferences(['oscar'], ['general'], {}, []) + url_params = 'eJx1VMmO2zAM_Zr6YrTocujJh6JF0QEKzKAz7VVgJNohLIseUU7ivy-VcWy5yyGOTVGP73GLKJNPYjiYgGeT4NB8BS9YOSY' \ + 'TUdifMDYM-vmGY1d5CN0EHTYOK88W_PXNkcDBozOjnzoK0vyi4bWnHs2RU4-zvHr_-RF9a-5Cy3GARByy7X7EkKMoBeMp9CuPQ-SzYMx' \ + '8Vr9P1qKI-XJ_p1fOkRJWNCgVM0a-zAttmBJbHkaPSZlNts-_jiuBFgUh2mPztkpHHLBhsRArDHvm356eHh5vATS0Mqagr0ZsZO_V8hT' \ + 'B9srt54_v6jewJugqL4Nn_hYSdhxnI-jRpi05GDQCStOT7UGVmJY8ZnltRKyF23SGiLWjqNcygKGkpyeGZIywJfD1gI5AjRTAmBM55Aw' \ + 'Q0Tn626lj7jzWo4e5hnEsIlprX6dTgdBRpyRBFKTDgBF8AasVyT4gvSTEoXRpXWRyG3CYQYld65I_V6lboILTMAlZY65_ejRDcHgp0Tv' \ + 'EPtGAsqTiBf3m76g7pP9B84mwjPvuUtASRDei1nDF2ix_JXW91UJkXrPh6RAhznVmKyQl7dwJdMJ6bz1QOmgzYlrEzHDMcEUuo44AgS1' \ + 'CvkjaOb2Q2AyY5oGDTs_OLXE_c2I5cg9hk3kEJZ0fu4SuktsIA2RhuJwP86AdripThCBeO9uVUejyPGmFSxPrqEYcuWi25zOEXV9tc1m' \ + '_KP1nafYtdfv6Q9hKfWmGm9A_3G635UwiVndLGdFCiLWkONk0xUxGLGGweGWTa2nZYZ0fS1YKlE3Uuw8fPl52E5U8HJYbC7sbjXUsrnT' \ + 'XHXRbELfO-1fGSqskiGnMK7B0dV3t8Lq08pbdtYpuVdoKWA2Yjuyah_vHp2rZWjo0zXL8Gw8DTj0=' + pref.parse_encoded_data(url_params) + self.assertEqual( + vars(pref.key_value_settings['categories']), + {'value': ['general'], 'choices': ['general', 'none']}) diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 49ccb608f..e4c0bdeed 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -62,3 +62,45 @@ class TestQuery(SearxTestCase): self.assertEquals(len(query.query_parts), 1) self.assertEquals(len(query.languages), 0) self.assertFalse(query.specific) + + def test_timeout_below100(self): + query_text = '<3 the query' + query = RawTextQuery(query_text, []) + query.parse_query() + + self.assertEquals(query.getFullQuery(), query_text) + self.assertEquals(len(query.query_parts), 3) + self.assertEquals(query.timeout_limit, 3) + self.assertFalse(query.specific) + + def test_timeout_above100(self): + query_text = '<350 the query' + query = RawTextQuery(query_text, []) + query.parse_query() + + self.assertEquals(query.getFullQuery(), query_text) + self.assertEquals(len(query.query_parts), 3) + self.assertEquals(query.timeout_limit, 0.35) + self.assertFalse(query.specific) + + def test_timeout_above1000(self): + query_text = '<3500 the query' + query = RawTextQuery(query_text, []) + query.parse_query() + + self.assertEquals(query.getFullQuery(), query_text) + self.assertEquals(len(query.query_parts), 3) + self.assertEquals(query.timeout_limit, 3.5) + self.assertFalse(query.specific) + + def test_timeout_invalid(self): + # invalid number: it is not bang but it is part of the query + query_text = '<xxx the query' + query = RawTextQuery(query_text, []) + query.parse_query() + + self.assertEquals(query.getFullQuery(), query_text) + self.assertEquals(len(query.query_parts), 1) + self.assertEquals(query.query_parts[0], query_text) + self.assertEquals(query.timeout_limit, None) + self.assertFalse(query.specific) diff --git a/tests/unit/test_search.py b/tests/unit/test_search.py index af5fffd8b..a39786d1a 100644 --- a/tests/unit/test_search.py +++ b/tests/unit/test_search.py @@ -2,9 +2,59 @@ from searx.testing import SearxTestCase +import searx.preferences +import searx.search +import searx.engines + -# TODO class SearchTestCase(SearxTestCase): - def test_(self): - pass + @classmethod + def setUpClass(cls): + searx.engines.initialize_engines([{ + 'name': 'general dummy', + 'engine': 'dummy', + 'categories': 'general', + 'shortcut': 'gd', + 'timeout': 3.0 + }]) + + def test_timeout_simple(self): + searx.search.max_request_timeout = None + search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}], + ['general'], 'en-US', 0, 1, None, None) + search = searx.search.Search(search_query) + search.search() + self.assertEquals(search.actual_timeout, 3.0) + + def test_timeout_query_above_default_nomax(self): + searx.search.max_request_timeout = None + search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}], + ['general'], 'en-US', 0, 1, None, 5.0) + search = searx.search.Search(search_query) + search.search() + self.assertEquals(search.actual_timeout, 3.0) + + def test_timeout_query_below_default_nomax(self): + searx.search.max_request_timeout = None + search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}], + ['general'], 'en-US', 0, 1, None, 1.0) + search = searx.search.Search(search_query) + search.search() + self.assertEquals(search.actual_timeout, 1.0) + + def test_timeout_query_below_max(self): + searx.search.max_request_timeout = 10.0 + search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}], + ['general'], 'en-US', 0, 1, None, 5.0) + search = searx.search.Search(search_query) + search.search() + self.assertEquals(search.actual_timeout, 5.0) + + def test_timeout_query_above_max(self): + searx.search.max_request_timeout = 10.0 + search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}], + ['general'], 'en-US', 0, 1, None, 15.0) + search = searx.search.Search(search_query) + search.search() + self.assertEquals(search.actual_timeout, 10.0) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 4854636c7..b09b9d414 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -90,6 +90,13 @@ class TestUtils(SearxTestCase): self.assertEqual(utils.match_language('iw-IL', ['he-IL']), 'he-IL') self.assertEqual(utils.match_language('he-IL', ['iw-IL'], aliases), 'iw-IL') + def test_ecma_unscape(self): + self.assertEqual(utils.ecma_unescape('text%20with%20space'), 'text with space') + self.assertEqual(utils.ecma_unescape('text using %xx: %F3'), + u'text using %xx: ó') + self.assertEqual(utils.ecma_unescape('text using %u: %u5409, %u4E16%u754c'), + u'text using %u: 吉, 世界') + class TestHTMLTextExtractor(SearxTestCase): @@ -128,3 +135,17 @@ class TestUnicodeWriter(SearxTestCase): rows = [1, 2, 3] self.unicode_writer.writerows(rows) self.assertEqual(self.unicode_writer.writerow.call_count, len(rows)) + + +class TestNewHmac(SearxTestCase): + + def test_bytes(self): + for secret_key in ['secret', b'secret', 1]: + if secret_key == 1: + with self.assertRaises(TypeError): + utils.new_hmac(secret_key, b'http://example.com') + continue + res = utils.new_hmac(secret_key, b'http://example.com') + self.assertEqual( + res, + '23e2baa2404012a5cc8e4a18b4aabf0dde4cb9b56f679ddc0fd6d7c24339d819') diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index e6739bfe5..72ace4850 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -15,7 +15,7 @@ class ViewsTestCase(SearxTestCase): self.app = webapp.app.test_client() # set some defaults - self.test_results = [ + test_results = [ { 'content': 'first test content', 'title': 'First Test', @@ -33,25 +33,39 @@ class ViewsTestCase(SearxTestCase): }, ] + timings = [ + { + 'engine': 'startpage', + 'total': 0.8, + 'load': 0.7 + }, + { + 'engine': 'youtube', + 'total': 0.9, + 'load': 0.6 + } + ] + def search_mock(search_self, *args): - search_self.result_container = Mock(get_ordered_results=lambda: self.test_results, + search_self.result_container = Mock(get_ordered_results=lambda: test_results, answers=set(), corrections=set(), suggestions=set(), infoboxes=[], unresponsive_engines=set(), - results=self.test_results, + results=test_results, results_number=lambda: 3, - results_length=lambda: len(self.test_results)) + results_length=lambda: len(test_results), + get_timings=lambda: timings) - Search.search = search_mock + self.setattr4test(Search, 'search', search_mock) def get_current_theme_name_mock(override=None): if override: return override return 'legacy' - webapp.get_current_theme_name = get_current_theme_name_mock + self.setattr4test(webapp, 'get_current_theme_name', get_current_theme_name_mock) self.maxDiff = None # to see full diffs @@ -77,6 +91,7 @@ class ViewsTestCase(SearxTestCase): result_dict = json.loads(result.data.decode('utf-8')) self.assertEqual('test', result_dict['query']) + self.assertEqual(len(result_dict['results']), 2) self.assertEqual(result_dict['results'][0]['content'], 'first test content') self.assertEqual(result_dict['results'][0]['url'], 'http://first.test.xyz') @@ -157,3 +172,9 @@ class ViewsTestCase(SearxTestCase): def test_favicon(self): result = self.app.get('/favicon.ico') self.assertEqual(result.status_code, 200) + + def test_config(self): + result = self.app.get('/config') + self.assertEqual(result.status_code, 200) + json_result = result.get_json() + self.assertTrue(json_result) |