diff options
Diffstat (limited to 'searx/engines/duden.py')
| -rw-r--r-- | searx/engines/duden.py | 28 |
1 files changed, 16 insertions, 12 deletions
diff --git a/searx/engines/duden.py b/searx/engines/duden.py index 881ff9d9c..cf2f1a278 100644 --- a/searx/engines/duden.py +++ b/searx/engines/duden.py @@ -11,7 +11,8 @@ from lxml import html, etree import re from searx.engines.xpath import extract_text -from searx.url_utils import quote +from searx.utils import eval_xpath +from searx.url_utils import quote, urljoin from searx import logger categories = ['general'] @@ -20,7 +21,7 @@ language_support = False # search-url base_url = 'https://www.duden.de/' -search_url = base_url + 'suchen/dudenonline/{query}?page={offset}' +search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}' def request(query, params): @@ -35,7 +36,11 @@ def request(query, params): ''' offset = (params['pageno'] - 1) - params['url'] = search_url.format(offset=offset, query=quote(query)) + if offset == 0: + search_url_fmt = base_url + 'suchen/dudenonline/{query}' + params['url'] = search_url_fmt.format(query=quote(query)) + else: + params['url'] = search_url.format(offset=offset, query=quote(query)) return params @@ -48,9 +53,9 @@ def response(resp): dom = html.fromstring(resp.text) try: - number_of_results_string = re.sub('[^0-9]', '', dom.xpath( - '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0] - ) + number_of_results_string =\ + re.sub('[^0-9]', '', + eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]) results.append({'number_of_results': int(number_of_results_string)}) @@ -58,13 +63,12 @@ def response(resp): logger.debug("Couldn't read number of results.") pass - for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'): + for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'): try: - logger.debug("running for %s" % str(result)) - link = result.xpath('.//h2/a')[0] - url = link.attrib.get('href') - title = result.xpath('string(.//h2/a)') - content = extract_text(result.xpath('.//p')) + url = eval_xpath(result, './/h2/a')[0].get('href') + url = urljoin(base_url, url) + title = eval_xpath(result, 'string(.//h2/a)').strip() + content = extract_text(eval_xpath(result, './/p')) # append result results.append({'url': url, 'title': title, |