From 88261e111ca2186f080c4048ab41b4c54cd5cf87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9o=20Bourrel?= <leo.bourrel@epitech.eu>
Date: Mon, 5 Aug 2019 16:15:40 +0200
Subject: Fix bing engine results count (#1387)

This PR fixes the result count from bing which was throwing an (hidden) error and add a validation to avoid reading more results than avalaible.

For example :
If there is 100 results from some search and we try to get results from 120 to 130, Bing will send back the results from 0 to 10 and no error. If we compare results count with the first parameter of the request we can avoid this "invalid" results.
---
 searx/engines/bing.py | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

(limited to 'searx/engines')

diff --git a/searx/engines/bing.py b/searx/engines/bing.py
index ba22cc6b4..1e614867b 100644
--- a/searx/engines/bing.py
+++ b/searx/engines/bing.py
@@ -13,11 +13,15 @@
  @todo        publishedDate
 """
 
+import re
 from lxml import html
+from searx import logger, utils
 from searx.engines.xpath import extract_text
 from searx.url_utils import urlencode
 from searx.utils import match_language, gen_useragent
 
+logger = logger.getChild('bing engine')
+
 # engine dependent config
 categories = ['general']
 paging = True
@@ -30,9 +34,13 @@ base_url = 'https://www.bing.com/'
 search_string = 'search?{query}&first={offset}'
 
 
+def _get_offset_from_pageno(pageno):
+    return (pageno - 1) * 10 + 1
+
+
 # do search-request
 def request(query, params):
-    offset = (params['pageno'] - 1) * 10 + 1
+    offset = _get_offset_from_pageno(params.get('pageno', 0))
 
     if params['language'] == 'all':
         lang = 'EN'
@@ -53,15 +61,9 @@ def request(query, params):
 # get response from search-request
 def response(resp):
     results = []
+    result_len = 0
 
     dom = html.fromstring(resp.text)
-
-    try:
-        results.append({'number_of_results': int(dom.xpath('//span[@class="sb_count"]/text()')[0]
-                                                 .split()[0].replace(',', ''))})
-    except:
-        pass
-
     # parse results
     for result in dom.xpath('//div[@class="sa_cc"]'):
         link = result.xpath('.//h3/a')[0]
@@ -86,7 +88,24 @@ def response(resp):
                         'title': title,
                         'content': content})
 
-    # return results
+    try:
+        result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))
+        result_len_container = utils.to_string(result_len_container)
+        if "-" in result_len_container:
+            # Remove the part "from-to" for paginated request ...
+            result_len_container = result_len_container[result_len_container.find("-") * 2 + 2:]
+
+        result_len_container = re.sub('[^0-9]', '', result_len_container)
+        if len(result_len_container) > 0:
+            result_len = int(result_len_container)
+    except Exception as e:
+        logger.debug('result error :\n%s', e)
+        pass
+
+    if _get_offset_from_pageno(resp.search_params.get("pageno", 0)) > result_len:
+        return []
+
+    results.append({'number_of_results': result_len})
     return results
 
 
-- 
cgit v1.2.3