From d748b8419ad1ef875f34783bbbcf773ebc4cfb5e Mon Sep 17 00:00:00 2001 From: Kirill Isakov Date: Thu, 24 Mar 2016 01:57:27 +0600 Subject: Add Arch Linux Wiki search engine --- searx/engines/archlinux.py | 144 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 searx/engines/archlinux.py (limited to 'searx/engines/archlinux.py') diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py new file mode 100644 index 000000000..f12c4328a --- /dev/null +++ b/searx/engines/archlinux.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- + +""" + Arch Linux Wiki + + @website https://wiki.archlinux.org + @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it + @using-api no + @results HTML + @stable no (HTML can change) + @parse url, title, content +""" + +from urlparse import urljoin +from cgi import escape +from urllib import urlencode +from lxml import html +from searx.engines.xpath import extract_text + +# engine dependent config +categories = ['it'] +language_support = True +paging = True +base_url = 'https://wiki.archlinux.org' + +# xpath queries +xpath_results = '//ul[@class="mw-search-results"]/li' +xpath_link = './/div[@class="mw-search-result-heading"]/a' +xpath_content = './/div[@class="searchresult"]' + + +# cut 'en' from 'en_US', 'de' from 'de_CH', and so on +def locale_to_lang_code(locale): + if locale.find('_') >= 0: + locale = locale.split('_')[0] + return locale + +# wikis for some languages were moved off from the main site, we need to make +# requests to correct URLs to be able to get results in those languages +lang_urls = { + 'all': { + 'base': 'https://wiki.archlinux.org', + 'search': '/index.php?title=Special:Search&offset={offset}&{query}' + }, + 'de': { + 'base': 'https://wiki.archlinux.de', + 'search': '/index.php?title=Spezial:Suche&offset={offset}&{query}' + }, + 'fr': { + 'base': 'https://wiki.archlinux.fr', + 'search': '/index.php?title=Spécial:Recherche&offset={offset}&{query}' + }, + 'ja': { + 'base': 'https://wiki.archlinuxjp.org', + 'search': '/index.php?title=特別:検索&offset={offset}&{query}' + }, + 'ro': { + 'base': 'http://wiki.archlinux.ro', + 'search': '/index.php?title=Special:Căutare&offset={offset}&{query}' + }, + 'tr': { + 'base': 'http://archtr.org/wiki', + 'search': '/index.php?title=Özel:Ara&offset={offset}&{query}' + } +} + + +# get base & search URLs for selected language +def get_lang_urls(language): + if language in lang_urls: + return lang_urls[language] + return lang_urls['all'] + +# Language names to build search requests for +# those languages which are hosted on the main site. +main_langs = { + 'ar': 'العربية', + 'bg': 'Български', + 'cs': 'Česky', + 'da': 'Dansk', + 'el': 'Ελληνικά', + 'es': 'Español', + 'he': 'עברית', + 'hr': 'Hrvatski', + 'hu': 'Magyar', + 'it': 'Italiano', + 'ko': '한국어', + 'lt': 'Lietuviškai', + 'nl': 'Nederlands', + 'pl': 'Polski', + 'pt': 'Português', + 'ru': 'Русский', + 'sl': 'Slovenský', + 'th': 'ไทย', + 'uk': 'Українська', + 'zh': '简体中文' +} + + +# do search-request +def request(query, params): + # translate the locale (e.g. 'en_US') to language code ('en') + language = locale_to_lang_code(params['language']) + + # if our language is hosted on the main site, we need to add its name + # to the query in order to narrow the results to that language + if language in main_langs: + query += '(' + main_langs[language] + ')' + + # prepare the request parameters + query = urlencode({'search': query}) + offset = (params['pageno'] - 1) * 20 + + # get request URLs for our language of choice + urls = get_lang_urls(language) + search_url = urls['base'] + urls['search'] + + params['url'] = search_url.format(query=query, offset=offset) + + return params + + +# get response from search-request +def response(resp): + # get the base URL for the language in which request was made + language = locale_to_lang_code(resp.search_params['language']) + base_url = get_lang_urls(language)['base'] + + results = [] + + dom = html.fromstring(resp.text) + + # parse results + for result in dom.xpath(xpath_results): + link = result.xpath(xpath_link)[0] + href = urljoin(base_url, link.attrib.get('href')) + title = escape(extract_text(link)) + content = escape(extract_text(result.xpath(xpath_content))) + + results.append({'url': href, + 'title': title, + 'content': content}) + + return results -- cgit v1.2.3 From 8b7dc2acb9f670ba65e4b98eb310f04e4c212bd8 Mon Sep 17 00:00:00 2001 From: Kirill Isakov Date: Fri, 25 Mar 2016 00:38:48 +0600 Subject: Remove content field from ArchWiki results; reformat code in archlinux.py Content field in Arch Wiki search results is of no real use, more often than not it contains no usable information and includes too many markup tags which make the text unreadable. It is safe to remove it. --- searx/engines/archlinux.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'searx/engines/archlinux.py') diff --git a/searx/engines/archlinux.py b/searx/engines/archlinux.py index f12c4328a..84e0d0fba 100644 --- a/searx/engines/archlinux.py +++ b/searx/engines/archlinux.py @@ -3,12 +3,12 @@ """ Arch Linux Wiki - @website https://wiki.archlinux.org - @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it - @using-api no - @results HTML - @stable no (HTML can change) - @parse url, title, content + @website https://wiki.archlinux.org + @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it + @using-api no + @results HTML + @stable no (HTML can change) + @parse url, title """ from urlparse import urljoin @@ -26,7 +26,6 @@ base_url = 'https://wiki.archlinux.org' # xpath queries xpath_results = '//ul[@class="mw-search-results"]/li' xpath_link = './/div[@class="mw-search-result-heading"]/a' -xpath_content = './/div[@class="searchresult"]' # cut 'en' from 'en_US', 'de' from 'de_CH', and so on @@ -135,10 +134,8 @@ def response(resp): link = result.xpath(xpath_link)[0] href = urljoin(base_url, link.attrib.get('href')) title = escape(extract_text(link)) - content = escape(extract_text(result.xpath(xpath_content))) results.append({'url': href, - 'title': title, - 'content': content}) + 'title': title}) return results -- cgit v1.2.3