1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
# pylint: disable=missing-function-docstring
"""The XPath engine is a *generic* engine with which it is possible to configure
engines in the settings.
Here is a simple example of a XPath engine configured in the
:ref:`settings engine` section, further read :ref:`engines-dev`.
.. code:: yaml
- name : bitbucket
engine : xpath
paging : True
search_url : https://bitbucket.org/repo/all/{pageno}?name={query}
url_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]/@href
title_xpath : //article[@class="repo-summary"]//a[@class="repo-link"]
content_xpath : //article[@class="repo-summary"]/p
"""
from urllib.parse import urlencode
from lxml import html
from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list
from searx import logger
logger = logger.getChild('XPath engine')
search_url = None
"""
Search URL of the engine, replacements are:
``{query}``:
Search terms from user.
``{pageno}``:
Page number if engine supports pagging :py:obj:`paging`
``{lang}``:
ISO 639-1 language code (en, de, fr ..)
"""
lang_all='en'
'''Replacement ``{lang}`` in :py:obj:`search_url` if language ``all`` is
selected.
'''
soft_max_redirects = 0
'''Maximum redirects, soft limit. Record an error but don't stop the engine'''
results_xpath = ''
'''XPath selector for the list of result items'''
url_xpath = None
'''XPath selector of result's ``url``.'''
content_xpath = None
'''XPath selector of result's ``content``.'''
title_xpath = None
'''XPath selector of result's ``title``.'''
thumbnail_xpath = False
'''XPath selector of result's ``img_src``.'''
suggestion_xpath = ''
'''XPath selector of result's ``suggestion``.'''
cached_xpath = ''
cached_url = ''
paging = False
'''Engine supports paging [True or False].'''
page_size = 1
'''Number of results on each page. Only needed if the site requires not a page
number, but an offset.'''
first_page_num = 1
'''Number of the first page (usually 0 or 1).'''
def request(query, params):
'''Build request parameters (see :ref:`engine request`).
'''
lang = lang_all
if params['language'] != 'all':
lang = params['language'][:2]
fargs = {
'query': urlencode({'q': query})[2:],
'lang': lang,
'pageno': (params['pageno'] - 1) * page_size + first_page_num
}
params['url'] = search_url.format(**fargs)
params['soft_max_redirects'] = soft_max_redirects
logger.debug("query_url --> %s", params['url'])
return params
def response(resp):
'''Scrap *results* from the response (see :ref:`engine results`).
'''
results = []
dom = html.fromstring(resp.text)
is_onion = 'onions' in categories # pylint: disable=undefined-variable
if results_xpath:
for result in eval_xpath_list(dom, results_xpath):
url = extract_url(eval_xpath_list(result, url_xpath, min_len=1), search_url)
title = extract_text(eval_xpath_list(result, title_xpath, min_len=1))
content = extract_text(eval_xpath_list(result, content_xpath, min_len=1))
tmp_result = {'url': url, 'title': title, 'content': content}
# add thumbnail if available
if thumbnail_xpath:
thumbnail_xpath_result = eval_xpath_list(result, thumbnail_xpath)
if len(thumbnail_xpath_result) > 0:
tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)
# add alternative cached url if available
if cached_xpath:
tmp_result['cached_url'] = (
cached_url
+ extract_text(eval_xpath_list(result, cached_xpath, min_len=1))
)
if is_onion:
tmp_result['is_onion'] = True
results.append(tmp_result)
else:
if cached_xpath:
for url, title, content, cached in zip(
(extract_url(x, search_url) for
x in eval_xpath_list(dom, url_xpath)),
map(extract_text, eval_xpath_list(dom, title_xpath)),
map(extract_text, eval_xpath_list(dom, content_xpath)),
map(extract_text, eval_xpath_list(dom, cached_xpath))
):
results.append({
'url': url,
'title': title,
'content': content,
'cached_url': cached_url + cached, 'is_onion': is_onion
})
else:
for url, title, content in zip(
(extract_url(x, search_url) for
x in eval_xpath_list(dom, url_xpath)),
map(extract_text, eval_xpath_list(dom, title_xpath)),
map(extract_text, eval_xpath_list(dom, content_xpath))
):
results.append({
'url': url,
'title': title,
'content': content,
'is_onion': is_onion
})
if suggestion_xpath:
for suggestion in eval_xpath(dom, suggestion_xpath):
results.append({'suggestion': extract_text(suggestion)})
logger.debug("found %s results", len(results))
return results
|
