From 2179079a9173b33b81e1084fc1e8e181c19ef8e9 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 2 Aug 2019 13:37:13 +0200 Subject: [fix] fix flickr_noapi decoding (#1655) Characters that were not ASCII were incorrectly decoded. Add an helper function: searx.utils.ecma_unescape (Python implementation of unescape Javascript function). --- tests/unit/test_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'tests/unit') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index fbaed2bd1..b09b9d414 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -90,6 +90,13 @@ class TestUtils(SearxTestCase): self.assertEqual(utils.match_language('iw-IL', ['he-IL']), 'he-IL') self.assertEqual(utils.match_language('he-IL', ['iw-IL'], aliases), 'iw-IL') + def test_ecma_unscape(self): + self.assertEqual(utils.ecma_unescape('text%20with%20space'), 'text with space') + self.assertEqual(utils.ecma_unescape('text using %xx: %F3'), + u'text using %xx: ó') + self.assertEqual(utils.ecma_unescape('text using %u: %u5409, %u4E16%u754c'), + u'text using %u: 吉, 世界') + class TestHTMLTextExtractor(SearxTestCase): -- cgit v1.2.3 From 72029d27ded8d93ab891c616d6bffbe8d3a67dd2 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 2 Aug 2019 13:50:51 +0200 Subject: [enh] Add timeout limit per request (#1640) The new url parameter "timeout_limit" set timeout limit defined in second. Example "timeout_limit=1.5" means the timeout limit is 1.5 seconds. In addition, the query can start with <[number] to set the timeout limit. For number between 0 and 99, the unit is the second : Example: "<30 searx" means the timeout limit is 3 seconds For number above 100, the unit is the millisecond: Example: "<850 searx" means the timeout is 850 milliseconds. In addition, there is a new optional setting: outgoing.max_request_timeout. If not set, the user timeout can't go above searx configuration (as before: the max timeout of selected engine for a query). If the value is set, the user can set a timeout between 0 and max_request_timeout using <[number] or timeout_limit query parameter. Related to #1077 Updated version of PR #1413 from @isj-privacore --- tests/unit/test_query.py | 42 +++++++++++++++++++++++++++++++++++ tests/unit/test_search.py | 56 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 95 insertions(+), 3 deletions(-) (limited to 'tests/unit') diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 49ccb608f..e4c0bdeed 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -62,3 +62,45 @@ class TestQuery(SearxTestCase): self.assertEquals(len(query.query_parts), 1) self.assertEquals(len(query.languages), 0) self.assertFalse(query.specific) + + def test_timeout_below100(self): + query_text = '<3 the query' + query = RawTextQuery(query_text, []) + query.parse_query() + + self.assertEquals(query.getFullQuery(), query_text) + self.assertEquals(len(query.query_parts), 3) + self.assertEquals(query.timeout_limit, 3) + self.assertFalse(query.specific) + + def test_timeout_above100(self): + query_text = '<350 the query' + query = RawTextQuery(query_text, []) + query.parse_query() + + self.assertEquals(query.getFullQuery(), query_text) + self.assertEquals(len(query.query_parts), 3) + self.assertEquals(query.timeout_limit, 0.35) + self.assertFalse(query.specific) + + def test_timeout_above1000(self): + query_text = '<3500 the query' + query = RawTextQuery(query_text, []) + query.parse_query() + + self.assertEquals(query.getFullQuery(), query_text) + self.assertEquals(len(query.query_parts), 3) + self.assertEquals(query.timeout_limit, 3.5) + self.assertFalse(query.specific) + + def test_timeout_invalid(self): + # invalid number: it is not bang but it is part of the query + query_text = ' Date: Sat, 3 Aug 2019 13:23:36 +0200 Subject: [fix] fix monkey patch in test_webapp.py (#1667) at the end of test_webapp.py, the monkey patch of searx.search.Search was not revert which lead to side effects on other tests close #1663 --- tests/unit/test_webapp.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'tests/unit') diff --git a/tests/unit/test_webapp.py b/tests/unit/test_webapp.py index dcf8b583c..72ace4850 100644 --- a/tests/unit/test_webapp.py +++ b/tests/unit/test_webapp.py @@ -15,7 +15,7 @@ class ViewsTestCase(SearxTestCase): self.app = webapp.app.test_client() # set some defaults - self.test_results = [ + test_results = [ { 'content': 'first test content', 'title': 'First Test', @@ -47,25 +47,25 @@ class ViewsTestCase(SearxTestCase): ] def search_mock(search_self, *args): - search_self.result_container = Mock(get_ordered_results=lambda: self.test_results, + search_self.result_container = Mock(get_ordered_results=lambda: test_results, answers=set(), corrections=set(), suggestions=set(), infoboxes=[], unresponsive_engines=set(), - results=self.test_results, + results=test_results, results_number=lambda: 3, - results_length=lambda: len(self.test_results), + results_length=lambda: len(test_results), get_timings=lambda: timings) - Search.search = search_mock + self.setattr4test(Search, 'search', search_mock) def get_current_theme_name_mock(override=None): if override: return override return 'legacy' - webapp.get_current_theme_name = get_current_theme_name_mock + self.setattr4test(webapp, 'get_current_theme_name', get_current_theme_name_mock) self.maxDiff = None # to see full diffs @@ -91,6 +91,7 @@ class ViewsTestCase(SearxTestCase): result_dict = json.loads(result.data.decode('utf-8')) self.assertEqual('test', result_dict['query']) + self.assertEqual(len(result_dict['results']), 2) self.assertEqual(result_dict['results'][0]['content'], 'first test content') self.assertEqual(result_dict['results'][0]['url'], 'http://first.test.xyz') -- cgit v1.2.3 From 9ff500181624e996febb60bf4f6378fc749f7f9a Mon Sep 17 00:00:00 2001 From: Dalf Date: Mon, 5 Aug 2019 15:43:01 +0200 Subject: [fix] arxiv engine --- tests/unit/engines/test_arxiv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/unit') diff --git a/tests/unit/engines/test_arxiv.py b/tests/unit/engines/test_arxiv.py index b32c0e605..83c4f8595 100644 --- a/tests/unit/engines/test_arxiv.py +++ b/tests/unit/engines/test_arxiv.py @@ -8,7 +8,7 @@ from searx.testing import SearxTestCase class TestBaseEngine(SearxTestCase): def test_request(self): - query = 'test_query' + query = 'test_query'.encode('utf-8') dicto = defaultdict(dict) dicto['pageno'] = 1 params = arxiv.request(query, dicto) -- cgit v1.2.3 From fcc9587ee9b1a4b31a511a8580f2d534e400e96b Mon Sep 17 00:00:00 2001 From: Dalf Date: Mon, 5 Aug 2019 15:44:02 +0200 Subject: [fix] fdroid engine --- tests/unit/engines/test_fdroid.py | 55 +++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 22 deletions(-) (limited to 'tests/unit') diff --git a/tests/unit/engines/test_fdroid.py b/tests/unit/engines/test_fdroid.py index d75f4f0b4..42a0a7148 100644 --- a/tests/unit/engines/test_fdroid.py +++ b/tests/unit/engines/test_fdroid.py @@ -13,29 +13,40 @@ class TestFdroidEngine(SearxTestCase): params = fdroid.request(query, dic) self.assertTrue('url' in params) self.assertTrue(query in params['url']) - self.assertTrue('f-droid.org' in params['url']) + self.assertTrue('search.f-droid.org' in params['url']) - def test_response(self): + def test_response_empty(self): resp = mock.Mock(text='') self.assertEqual(fdroid.response(resp), []) + def test_response_oneresult(self): html = """ - -
-
- -
-
-

Details...

-
-

- Sample title -
- Sample content -

-
-
+ + + + test + + + + + """ resp = mock.Mock(text=html) @@ -43,7 +54,7 @@ class TestFdroidEngine(SearxTestCase): self.assertEqual(type(results), list) self.assertEqual(len(results), 1) - self.assertEqual(results[0]['url'], 'https://google.com/qwerty') - self.assertEqual(results[0]['title'], 'Sample title') - self.assertEqual(results[0]['content'], 'Sample content') - self.assertEqual(results[0]['img_src'], 'http://example.com/image.png') + self.assertEqual(results[0]['url'], 'https://example.com/app.url') + self.assertEqual(results[0]['title'], 'App Example 1') + self.assertEqual(results[0]['content'], 'Description App Example 1 - GPL-3.0-only') + self.assertEqual(results[0]['img_src'], 'https://example.com/appexample.logo.png') -- cgit v1.2.3 From 88261e111ca2186f080c4048ab41b4c54cd5cf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Bourrel?= Date: Mon, 5 Aug 2019 16:15:40 +0200 Subject: Fix bing engine results count (#1387) This PR fixes the result count from bing which was throwing an (hidden) error and add a validation to avoid reading more results than avalaible. For example : If there is 100 results from some search and we try to get results from 120 to 130, Bing will send back the results from 0 to 10 and no error. If we compare results count with the first parameter of the request we can avoid this "invalid" results. --- tests/unit/engines/test_bing.py | 125 +++++++++++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 34 deletions(-) (limited to 'tests/unit') diff --git a/tests/unit/engines/test_bing.py b/tests/unit/engines/test_bing.py index 21191ff25..387034735 100644 --- a/tests/unit/engines/test_bing.py +++ b/tests/unit/engines/test_bing.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from collections import defaultdict import mock from searx.engines import bing @@ -10,7 +11,7 @@ class TestBingEngine(SearxTestCase): bing.supported_languages = ['en', 'fr', 'zh-CHS', 'zh-CHT', 'pt-PT', 'pt-BR'] query = u'test_query' dicto = defaultdict(dict) - dicto['pageno'] = 0 + dicto['pageno'] = 1 dicto['language'] = 'fr-FR' params = bing.request(query.encode('utf-8'), dicto) self.assertTrue('url' in params) @@ -23,70 +24,126 @@ class TestBingEngine(SearxTestCase): self.assertTrue('language' in params['url']) def test_response(self): + dicto = defaultdict(dict) + dicto['pageno'] = 1 + dicto['language'] = 'fr-FR' self.assertRaises(AttributeError, bing.response, None) self.assertRaises(AttributeError, bing.response, []) self.assertRaises(AttributeError, bing.response, '') self.assertRaises(AttributeError, bing.response, '[]') response = mock.Mock(text='') + response.search_params = dicto self.assertEqual(bing.response(response), []) response = mock.Mock(text='') + response.search_params = dicto self.assertEqual(bing.response(response), []) html = """ -
-
- -
this.meta.com - - - - -
-

This should be the content.

+
+
+ 23 900 000 résultats
+
    +
    +
    + +
    this.meta.com + + + + +
    +

    This should be the content.

    +
    +
    +
""" response = mock.Mock(text=html) + response.search_params = dicto results = bing.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') + self.assertEqual(results[-1]['number_of_results'], 23900000) html = """ -
  • -
    - -
    this.meta.com - - - - -
    -

    This should be the content.

    +
    +
    + 9-18 résultats sur 23 900 000
    -
  • +
      +
    1. +
      + +
      this.meta.com + + + + +
      +

      This should be the content.

      +
      +
    2. +
    +
    """ + dicto['pageno'] = 2 response = mock.Mock(text=html) + response.search_params = dicto results = bing.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) + self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') + self.assertEqual(results[-1]['number_of_results'], 23900000) + + html = """ +
    +
    + 23 900 000 résultats +
    +
      +
    1. +
      + +
      this.meta.com + + + + +
      +

      This should be the content.

      +
      +
    2. +
    +
    + """ + dicto['pageno'] = 33900000 + response = mock.Mock(text=html) + response.search_params = dicto + results = bing.response(response) + self.assertEqual(bing.response(response), []) def test_fetch_supported_languages(self): html = """""" -- cgit v1.2.3 From e74bdf84296a4173dd625e25b699196e87db8ac8 Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Mon, 14 Oct 2019 15:09:25 +0200 Subject: [fix] engine test --- tests/unit/engines/test_startpage.py | 123 +++++++---------------------------- 1 file changed, 25 insertions(+), 98 deletions(-) (limited to 'tests/unit') diff --git a/tests/unit/engines/test_startpage.py b/tests/unit/engines/test_startpage.py index a4704ce22..ac4454738 100644 --- a/tests/unit/engines/test_startpage.py +++ b/tests/unit/engines/test_startpage.py @@ -18,12 +18,9 @@ class TestStartpageEngine(SearxTestCase): self.assertIn('data', params) self.assertIn('query', params['data']) self.assertIn(query, params['data']['query']) - self.assertIn('with_language', params['data']) - self.assertIn('lang_fr', params['data']['with_language']) dicto['language'] = 'all' params = startpage.request(query, dicto) - self.assertNotIn('with_language', params['data']) def test_response(self): self.assertRaises(AttributeError, startpage.response, None) @@ -35,33 +32,32 @@ class TestStartpageEngine(SearxTestCase): self.assertEqual(startpage.response(response), []) html = """ -
  • -

    - - This should be the title +

    -

    - This should be the content. -

    -

    - www.speedtest.net/fr/ - - - - - Navigation avec Ixquick Proxy - - - - - Mis en surbrillance - -

    -
  • - """ + +

    This should be the content.

    +
    + """ # noqa response = mock.Mock(text=html.encode('utf-8')) results = startpage.response(response) self.assertEqual(type(results), list) @@ -69,72 +65,3 @@ class TestStartpageEngine(SearxTestCase): self.assertEqual(results[0]['title'], 'This should be the title') self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/') self.assertEqual(results[0]['content'], 'This should be the content.') - - html = """ -
  • -

    - - This should be the title - - -

    -

    - This should be the content. -

    -

    - www.speedtest.net/fr/ - - - - - Navigation avec Ixquick Proxy - - - - - Mis en surbrillance - -

    -
  • -
  • -

    - -

    -

    - This should be the content. -

    -

    - www.speedtest.net/fr/ - -

    -
  • -
  • -

    - - This should be the title - - -

    -

    - www.speedtest.net/fr/ - - - - - Navigation avec Ixquick Proxy - - - - - Mis en surbrillance - -

    -
  • - """ - response = mock.Mock(text=html.encode('utf-8')) - results = startpage.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['content'], '') -- cgit v1.2.3 From 3c425f09c14bbc297346271dac5b30af7a294b9e Mon Sep 17 00:00:00 2001 From: Adam Tauber Date: Wed, 16 Oct 2019 15:32:21 +0200 Subject: [fix] remove useless engine tests --- tests/unit/engines/test_deviantart.py | 71 ----------------------------- tests/unit/engines/test_digg.py | 85 ----------------------------------- tests/unit/engines/test_www1x.py | 43 ------------------ 3 files changed, 199 deletions(-) (limited to 'tests/unit') diff --git a/tests/unit/engines/test_deviantart.py b/tests/unit/engines/test_deviantart.py index bd2cf182f..a31151037 100644 --- a/tests/unit/engines/test_deviantart.py +++ b/tests/unit/engines/test_deviantart.py @@ -22,74 +22,3 @@ class TestDeviantartEngine(SearxTestCase): dicto['time_range'] = 'year' params = deviantart.request(query, dicto) self.assertEqual({}, params['url']) - - def test_response(self): - self.assertRaises(AttributeError, deviantart.response, None) - self.assertRaises(AttributeError, deviantart.response, []) - self.assertRaises(AttributeError, deviantart.response, '') - self.assertRaises(AttributeError, deviantart.response, '[]') - - response = mock.Mock(text='') - self.assertEqual(deviantart.response(response), []) - - response = mock.Mock(status_code=302) - self.assertEqual(deviantart.response(response), []) - - html = """ -
    - - - - - Title of image -
    - """ - response = mock.Mock(text=html) - results = deviantart.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Title of image') - self.assertEqual(results[0]['url'], 'https://url.of.image') - self.assertNotIn('content', results[0]) - self.assertEqual(results[0]['thumbnail_src'], 'https://url.of.thumbnail') - - html = """ - - - - - - Test - - - - - - - Title of image - - - - 5 years ago - - in Animation - - - - More Like This - - - """ - response = mock.Mock(text=html) - results = deviantart.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) diff --git a/tests/unit/engines/test_digg.py b/tests/unit/engines/test_digg.py index 6e7c9cc99..8bc4c67c2 100644 --- a/tests/unit/engines/test_digg.py +++ b/tests/unit/engines/test_digg.py @@ -14,88 +14,3 @@ class TestDiggEngine(SearxTestCase): self.assertIn('url', params) self.assertIn(query, params['url']) self.assertIn('digg.com', params['url']) - - def test_response(self): - self.assertRaises(AttributeError, digg.response, None) - self.assertRaises(AttributeError, digg.response, []) - self.assertRaises(AttributeError, digg.response, '') - self.assertRaises(AttributeError, digg.response, '[]') - - response = mock.Mock(text='{}') - self.assertEqual(digg.response(response), []) - - response = mock.Mock(text='{"data": []}') - self.assertEqual(digg.response(response), []) - - json = """ - { - "status": "ok", - "num": 10, - "next_position": 20, - "html": "" - } - """ - json = json.replace('\r\n', '').replace('\n', '').replace('\r', '') - response = mock.Mock(text=json) - results = digg.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Title of article') - self.assertEqual(results[0]['url'], 'http://url.of.link') - self.assertEqual(results[0]['thumbnail'], 'http://url.of.image.jpeg') - self.assertEqual(results[0]['content'], '') - - json = """ - { - "status": "error", - "num": 10, - "next_position": 20 - } - """ - response = mock.Mock(text=json) - results = digg.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) diff --git a/tests/unit/engines/test_www1x.py b/tests/unit/engines/test_www1x.py index 9df8de6bf..40f5200fd 100644 --- a/tests/unit/engines/test_www1x.py +++ b/tests/unit/engines/test_www1x.py @@ -12,46 +12,3 @@ class TestWww1xEngine(SearxTestCase): self.assertTrue('url' in params) self.assertTrue(query in params['url']) self.assertTrue('1x.com' in params['url']) - - def test_response(self): - self.assertRaises(AttributeError, www1x.response, None) - self.assertRaises(AttributeError, www1x.response, []) - self.assertRaises(AttributeError, www1x.response, '') - self.assertRaises(AttributeError, www1x.response, '[]') - - response = mock.Mock(text='') - self.assertEqual(www1x.response(response), []) - html = """ - - - - - - - ] - > - - -
    Photos
    - - - - ]]>
    - """ - response = mock.Mock(text=html) - results = www1x.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]['url'], 'https://1x.com/photo/123456') - self.assertEqual(results[0]['thumbnail_src'], 'https://1x.com/images/user/testimage-123456.jpg') - self.assertEqual(results[0]['content'], '') - self.assertEqual(results[0]['template'], 'images.html') -- cgit v1.2.3 From 9299355570e32c4d24d7274d716eca1a93119d13 Mon Sep 17 00:00:00 2001 From: Marc Abonce Seguin Date: Sun, 24 Nov 2019 20:21:37 -0700 Subject: add seedpeer again --- tests/unit/engines/test_seedpeer.py | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 tests/unit/engines/test_seedpeer.py (limited to 'tests/unit') diff --git a/tests/unit/engines/test_seedpeer.py b/tests/unit/engines/test_seedpeer.py new file mode 100644 index 000000000..2057c1cb1 --- /dev/null +++ b/tests/unit/engines/test_seedpeer.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from collections import defaultdict +import mock +from searx.engines import seedpeer +from searx.testing import SearxTestCase + + +class TestBtdiggEngine(SearxTestCase): + + def test_request(self): + query = 'test_query' + dicto = defaultdict(dict) + dicto['pageno'] = 1 + params = seedpeer.request(query, dicto) + self.assertIn('url', params) + self.assertIn(query, params['url']) + self.assertIn('seedpeer', params['url']) + + def test_response(self): + self.assertRaises(AttributeError, seedpeer.response, None) + self.assertRaises(AttributeError, seedpeer.response, []) + self.assertRaises(AttributeError, seedpeer.response, '') + self.assertRaises(AttributeError, seedpeer.response, '[]') + + response = mock.Mock(text='') + self.assertEqual(seedpeer.response(response), []) + + html = u""" + + + + + + + +
    + + + + + + + + + + + + +
    Title1 year1 KB1020
    + + + """ + response = mock.Mock(text=html) + results = seedpeer.response(response) + self.assertEqual(type(results), list) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['title'], 'Title') + self.assertEqual(results[0]['url'], 'https://seedpeer.me/link') + self.assertEqual(results[0]['seed'], 10) + self.assertEqual(results[0]['leech'], 20) + self.assertEqual(results[0]['filesize'], 1024) + self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123') + self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123') -- cgit v1.2.3 From ccaf6ca02c5bdc63f78e01a66429afaa5fb3cb68 Mon Sep 17 00:00:00 2001 From: Marc Abonce Seguin Date: Wed, 26 Jun 2019 00:45:20 -0500 Subject: [fix] update xpaths for new google results page --- tests/unit/engines/test_google.py | 117 ++++++++++++-------------------------- 1 file changed, 37 insertions(+), 80 deletions(-) (limited to 'tests/unit') diff --git a/tests/unit/engines/test_google.py b/tests/unit/engines/test_google.py index a73e9d2be..9d0edd439 100644 --- a/tests/unit/engines/test_google.py +++ b/tests/unit/engines/test_google.py @@ -58,93 +58,50 @@ class TestGoogleEngine(SearxTestCase): self.assertEqual(google.response(response), []) html = """ -
    -

    - - This is the title - -

    -
    -
    - - test.psychologies.com/ - -
    ‎ -