diff options
Diffstat (limited to 'searx/engines')
| -rw-r--r-- | searx/engines/__init__.py | 29 | ||||
| -rw-r--r-- | searx/engines/torznab.py | 145 |
2 files changed, 172 insertions, 2 deletions
diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py index e0b5796e4..44ea9a4bd 100644 --- a/searx/engines/__init__.py +++ b/searx/engines/__init__.py @@ -137,7 +137,7 @@ def update_engine_attributes(engine, engine_data): if isinstance(param_value, str): param_value = list(map(str.strip, param_value.split(','))) engine.categories = param_value - elif param_name != 'engine': + else: setattr(engine, param_name, param_value) # set default attributes @@ -147,11 +147,35 @@ def update_engine_attributes(engine, engine_data): def set_language_attributes(engine): - # pylint: disable=protected-access # assign supported languages from json file if engine.name in ENGINES_LANGUAGES: engine.supported_languages = ENGINES_LANGUAGES[engine.name] + elif engine.engine in ENGINES_LANGUAGES: + # The key of the dictionary ENGINES_LANGUAGES is the *engine name* + # configured in settings.xml. When multiple engines are configured in + # settings.yml to use the same origin engine (python module) these + # additional engines can use the languages from the origin engine. + # For this use the configured ``engine: ...`` from settings.yml + engine.supported_languages = ENGINES_LANGUAGES[engine.engine] + + if hasattr(engine, 'language'): + # For an engine, when there is `language: ...` in the YAML settings, the + # engine supports only one language, in this case + # engine.supported_languages should contains this value defined in + # settings.yml + if engine.language not in engine.supported_languages: + raise ValueError( + "settings.yml - engine: '%s' / language: '%s' not supported" % ( + engine.name, engine.language )) + + if isinstance(engine.supported_languages, dict): + engine.supported_languages = { + engine.language : engine.supported_languages[engine.language] + } + else: + engine.supported_languages = [engine.language] + # find custom aliases for non standard language codes for engine_lang in engine.supported_languages: iso_lang = match_language(engine_lang, BABEL_LANGS, fallback=None) @@ -172,6 +196,7 @@ def set_language_attributes(engine): 'Accept-Language': 'ja-JP,ja;q=0.8,en-US;q=0.5,en;q=0.3', # bing needs a non-English language } engine.fetch_supported_languages = ( + # pylint: disable=protected-access lambda: engine._fetch_supported_languages( get(engine.supported_languages_url, headers=headers)) ) diff --git a/searx/engines/torznab.py b/searx/engines/torznab.py new file mode 100644 index 000000000..960d1ee90 --- /dev/null +++ b/searx/engines/torznab.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Torznab WebAPI + +A engine that implements the `torznab WebAPI`_. + +.. _torznab WebAPI: https://torznab.github.io/spec-1.3-draft/torznab + +""" + +from datetime import datetime +from urllib.parse import quote +from lxml import etree + +from searx.exceptions import SearxEngineAPIException + +# about +about = { + "website": None, + "wikidata_id": None, + "official_api_documentation": "https://torznab.github.io/spec-1.3-draft", + "use_official_api": True, + "require_api_key": False, + "results": 'XML', +} + +categories = ['files'] +paging = False +time_range_support = False + +# defined in settings.yml +# example (Jackett): "http://localhost:9117/api/v2.0/indexers/all/results/torznab" +base_url = '' +api_key = '' +# https://newznab.readthedocs.io/en/latest/misc/api/#predefined-categories +torznab_categories = [] + +def init(engine_settings=None): # pylint: disable=unused-argument + if len(base_url) < 1: + raise ValueError('missing torznab base_url') + +def request(query, params): + + search_url = base_url + '?t=search&q={search_query}' + if len(api_key) > 0: + search_url += '&apikey={api_key}' + if len(torznab_categories) > 0: + search_url += '&cat={torznab_categories}' + + params['url'] = search_url.format( + search_query = quote(query), + api_key = api_key, + torznab_categories = ",".join([str(x) for x in torznab_categories]) + ) + + return params + +def response(resp): + results = [] + + search_results = etree.XML(resp.content) + + # handle errors + # https://newznab.readthedocs.io/en/latest/misc/api/#newznab-error-codes + if search_results.tag == "error": + raise SearxEngineAPIException(search_results.get("description")) + + for item in search_results[0].iterfind('item'): + result = {'template': 'torrent.html'} + + enclosure = item.find('enclosure') + + result["filesize"] = int(enclosure.get('length')) + + link = get_property(item, 'link') + guid = get_property(item, 'guid') + comments = get_property(item, 'comments') + + # define url + result["url"] = enclosure.get('url') + if comments is not None and comments.startswith('http'): + result["url"] = comments + elif guid is not None and guid.startswith('http'): + result["url"] = guid + + # define torrent file url + result["torrentfile"] = None + if enclosure.get('url').startswith("http"): + result["torrentfile"] = enclosure.get('url') + elif link is not None and link.startswith('http'): + result["torrentfile"] = link + + # define magnet link + result["magnetlink"] = get_torznab_attr(item, 'magneturl') + if result["magnetlink"] is None: + if enclosure.get('url').startswith("magnet"): + result["magnetlink"] = enclosure.get('url') + elif link is not None and link.startswith('magnet'): + result["magnetlink"] = link + + result["title"] = get_property(item, 'title') + result["files"] = get_property(item, 'files') + + result["publishedDate"] = None + try: + result["publishedDate"] = datetime.strptime( + get_property(item, 'pubDate'), '%a, %d %b %Y %H:%M:%S %z') + except (ValueError, TypeError) as e: + logger.debug("ignore exception (publishedDate): %s", e) + + result["seed"] = get_torznab_attr(item, 'seeders') + + # define leech + result["leech"] = get_torznab_attr(item, 'leechers') + if result["leech"] is None and result["seed"] is not None: + peers = get_torznab_attr(item, 'peers') + if peers is not None: + result["leech"] = int(peers) - int(result["seed"]) + + results.append(result) + + return results + + +def get_property(item, property_name): + property_element = item.find(property_name) + + if property_element is not None: + return property_element.text + + return None + + +def get_torznab_attr(item, attr_name): + element = item.find( + './/torznab:attr[@name="{attr_name}"]'.format(attr_name=attr_name), + { + 'torznab': 'http://torznab.com/schemas/2015/feed' + } + ) + + if element is not None: + return element.get("value") + + return None |