From b1912607ae9783d6ccf648bd7706a64eca5bedb9 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 12 Mar 2022 10:18:08 +0100 Subject: [mod] replace /help by /info pages and include pages in project docs This patch implements a bolierplate to share content from info-pages of the SearXNG instance (URL /info) with the project documentation (path /docs/user). The info pages are using Markdown (CommonMark), to include them in the project documentation (reST) the myst-parser [1] is used in the Sphinx-doc build chain. If base_url is known (defined in settings.yml) links to the instance are also inserted into the project documentation:: searxng_extra/docs_prebuild [1] https://www.sphinx-doc.org/en/master/usage/markdown.html Signed-off-by: Markus Heiser --- searx/infopage/__init__.py | 191 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 searx/infopage/__init__.py (limited to 'searx/infopage/__init__.py') diff --git a/searx/infopage/__init__.py b/searx/infopage/__init__.py new file mode 100644 index 000000000..d7736a934 --- /dev/null +++ b/searx/infopage/__init__.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +# pyright: basic +"""Render SearXNG instance documentation. + +Usage in a Flask app route: + +.. code:: python + + from searx import infopage + + _INFO_PAGES = infopage.InfoPageSet(infopage.MistletoePage) + + @app.route('/info/', methods=['GET']) + def info(pagename): + + locale = request.preferences.get_value('locale') + page = _INFO_PAGES.get_page(pagename, locale) + +""" + +__all__ = ['InfoPage', 'MistletoePage', 'InfoPageSet'] + +import os.path +import logging +from functools import cached_property +import typing + +import urllib.parse +import jinja2 +from flask.helpers import url_for +import mistletoe + +from .. import get_setting +from ..version import GIT_URL + +logger = logging.getLogger('doc') + + +class InfoPage: + """A page of the :py:obj:`online documentation `.""" + + def __init__(self, fname, base_url=None): + self.fname = fname + self.base_url = base_url + + @cached_property + def raw_content(self): + """Raw content of the page (without any jinja rendering)""" + with open(self.fname, 'r', encoding='utf-8') as f: + return f.read() + + @cached_property + def content(self): + """Content of the page (rendered in a Jinja conntext)""" + ctx = self.get_ctx() + template = jinja2.Environment().from_string(self.raw_content) + return template.render(**ctx) + + @cached_property + def title(self): + """Title of the content (without any markup)""" + t = "" + for l in self.raw_content.split('\n'): + if l.startswith('# '): + t = l.strip('# ') + return t + + def get_ctx(self): # pylint: disable=no-self-use + """Jinja context to render :py:obj:`InfoPage.content`""" + + def _md_link(name, url): + url = url_for(url) + if self.base_url: + url = self.base_url + url + return "[%s](%s)" % (name, url) + + def _md_search(query): + url = '%s?q=%s' % (url_for('search'), urllib.parse.quote(query)) + if self.base_url: + url = self.base_url + url + return '[%s](%s)' % (query, url) + + ctx = {} + ctx['GIT_URL'] = GIT_URL + ctx['get_setting'] = get_setting + ctx['link'] = _md_link + ctx['search'] = _md_search + + return ctx + + def render(self): + """Render / return content""" + return self.content + + +class MistletoePage(InfoPage): + """A HTML page of the :py:obj:`online documentation `.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @cached_property + def html(self): + """HTML representation of this page""" + return self.render() + + def render(self): + """Render Markdown (CommonMark_) to HTML by using mistletoe_. + + .. _CommonMark: https://commonmark.org/ + .. _mistletoe: https://github.com/miyuchina/mistletoe + + """ + return mistletoe.markdown(self.content) + + +_INFO_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'info')) + + +class InfoPageSet: # pylint: disable=too-few-public-methods + """Cached rendering of the online documentation a SearXNG instance has. + + :param page_class: render online documentation by :py:obj:`InfoPage` parser. + :type page_class: :py:obj:`InfoPage` + """ + + def __init__(self, page_class: typing.Type[InfoPage], base_url=None): + self.page_class = page_class + self.base_url = base_url + self.CACHE: typing.Dict[tuple, InfoPage] = {} + + # future: could be set from settings.xml + + self.folder: str = _INFO_FOLDER + """location of the Markdwon files""" + + self.i18n_origin: str = 'en' + """default language""" + + self.l10n: typing.List = [ + 'en', + ] + """list of supported languages (aka locales)""" + + self.toc: typing.List = [ + 'search-syntax', + 'about', + ] + """list of articles in the online documentation""" + + def get_page(self, pagename: str, locale: typing.Optional[str] = None): + """Return ``pagename`` instance of :py:obj:`InfoPage` + + :param pagename: name of the page, a value from :py:obj:`InfoPageSet.toc` + :type pagename: str + + :param locale: language of the page, e.g. ``en``, ``zh_Hans_CN`` + (default: :py:obj:`InfoPageSet.i18n_origin`) + :type locale: str + + """ + if pagename not in self.toc: + return None + if locale is not None and locale not in self.l10n: + return None + + locale = locale or self.i18n_origin + cache_key = (pagename, locale) + page = self.CACHE.get(cache_key) + + if page is not None: + return page + + # not yet instantiated + + fname = os.path.join(self.folder, locale, pagename) + '.md' + if not os.path.exists(fname): + logger.error('file %s does not exists', fname) + return None + + page = self.page_class(fname, self.base_url) + self.CACHE[cache_key] = page + return page + + def all_pages(self, locale: typing.Optional[str] = None): + """Iterate over all pages""" + locale = locale or self.i18n_origin + for pagename in self.toc: + page = self.get_page(pagename, locale) + yield pagename, page -- cgit v1.2.3 From 59100e852573cbc01b602b1637b0d4a02a863a9d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 12 Mar 2022 15:37:45 +0100 Subject: [fix] add module for backward compatibility cache_property has been added in py3.8 [1] To support cache_property in py3.7 the implementation from 3.8 has been copied to compat.py. This code can be cleanup with EOL of py3.7. [1] https://docs.python.org/3/library/functools.html#functools.cached_property Signed-off-by: Markus Heiser --- searx/infopage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'searx/infopage/__init__.py') diff --git a/searx/infopage/__init__.py b/searx/infopage/__init__.py index d7736a934..5c58193c1 100644 --- a/searx/infopage/__init__.py +++ b/searx/infopage/__init__.py @@ -23,7 +23,6 @@ __all__ = ['InfoPage', 'MistletoePage', 'InfoPageSet'] import os.path import logging -from functools import cached_property import typing import urllib.parse @@ -32,6 +31,7 @@ from flask.helpers import url_for import mistletoe from .. import get_setting +from ..compat import cached_property from ..version import GIT_URL logger = logging.getLogger('doc') -- cgit v1.2.3 From 1157462ff9fc08504297e259356ccbefb339965f Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sun, 13 Mar 2022 22:15:27 +0100 Subject: Various change on PR 930 --- searx/infopage/__init__.py | 92 ++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 52 deletions(-) (limited to 'searx/infopage/__init__.py') diff --git a/searx/infopage/__init__.py b/searx/infopage/__init__.py index 5c58193c1..9b181ad60 100644 --- a/searx/infopage/__init__.py +++ b/searx/infopage/__init__.py @@ -19,8 +19,9 @@ Usage in a Flask app route: """ -__all__ = ['InfoPage', 'MistletoePage', 'InfoPageSet'] +__all__ = ['InfoPage', 'InfoPageSet'] +import os import os.path import logging import typing @@ -33,16 +34,18 @@ import mistletoe from .. import get_setting from ..compat import cached_property from ..version import GIT_URL +from ..locales import LOCALE_NAMES -logger = logging.getLogger('doc') + +logger = logging.getLogger('searx.infopage') +_INFO_FOLDER = os.path.abspath(os.path.dirname(__file__)) class InfoPage: """A page of the :py:obj:`online documentation `.""" - def __init__(self, fname, base_url=None): + def __init__(self, fname): self.fname = fname - self.base_url = base_url @cached_property def raw_content(self): @@ -66,19 +69,25 @@ class InfoPage: t = l.strip('# ') return t + @cached_property + def html(self): + """Render Markdown (CommonMark_) to HTML by using mistletoe_. + + .. _CommonMark: https://commonmark.org/ + .. _mistletoe: https://github.com/miyuchina/mistletoe + + """ + return mistletoe.markdown(self.content) + def get_ctx(self): # pylint: disable=no-self-use """Jinja context to render :py:obj:`InfoPage.content`""" def _md_link(name, url): - url = url_for(url) - if self.base_url: - url = self.base_url + url + url = url_for(url, _external=True) return "[%s](%s)" % (name, url) def _md_search(query): - url = '%s?q=%s' % (url_for('search'), urllib.parse.quote(query)) - if self.base_url: - url = self.base_url + url + url = '%s?q=%s' % (url_for('search', _external=True), urllib.parse.quote(query)) return '[%s](%s)' % (query, url) ctx = {} @@ -89,33 +98,8 @@ class InfoPage: return ctx - def render(self): - """Render / return content""" - return self.content - - -class MistletoePage(InfoPage): - """A HTML page of the :py:obj:`online documentation `.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - @cached_property - def html(self): - """HTML representation of this page""" - return self.render() - - def render(self): - """Render Markdown (CommonMark_) to HTML by using mistletoe_. - - .. _CommonMark: https://commonmark.org/ - .. _mistletoe: https://github.com/miyuchina/mistletoe - - """ - return mistletoe.markdown(self.content) - - -_INFO_FOLDER = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'info')) + def __repr__(self): + return f'<{self.__class__.__name__} fname={self.fname!r}>' class InfoPageSet: # pylint: disable=too-few-public-methods @@ -123,24 +107,26 @@ class InfoPageSet: # pylint: disable=too-few-public-methods :param page_class: render online documentation by :py:obj:`InfoPage` parser. :type page_class: :py:obj:`InfoPage` + + :param info_folder: information directory + :type info_folder: str """ - def __init__(self, page_class: typing.Type[InfoPage], base_url=None): - self.page_class = page_class - self.base_url = base_url - self.CACHE: typing.Dict[tuple, InfoPage] = {} + def __init__( + self, page_class: typing.Optional[typing.Type[InfoPage]] = None, info_folder: typing.Optional[str] = None + ): + self.page_class = page_class or InfoPage + self.CACHE: typing.Dict[tuple, typing.Optional[InfoPage]] = {} # future: could be set from settings.xml - self.folder: str = _INFO_FOLDER + self.folder: str = info_folder or _INFO_FOLDER """location of the Markdwon files""" - self.i18n_origin: str = 'en' + self.locale_default: str = 'en' """default language""" - self.l10n: typing.List = [ - 'en', - ] + self.locales: typing.List = [locale for locale in os.listdir(_INFO_FOLDER) if locale in LOCALE_NAMES] """list of supported languages (aka locales)""" self.toc: typing.List = [ @@ -160,12 +146,13 @@ class InfoPageSet: # pylint: disable=too-few-public-methods :type locale: str """ + locale = locale or self.locale_default + if pagename not in self.toc: return None - if locale is not None and locale not in self.l10n: + if locale not in self.locales: return None - locale = locale or self.i18n_origin cache_key = (pagename, locale) page = self.CACHE.get(cache_key) @@ -176,16 +163,17 @@ class InfoPageSet: # pylint: disable=too-few-public-methods fname = os.path.join(self.folder, locale, pagename) + '.md' if not os.path.exists(fname): - logger.error('file %s does not exists', fname) + logger.info('file %s does not exists', fname) + self.CACHE[cache_key] = None return None - page = self.page_class(fname, self.base_url) + page = self.page_class(fname) self.CACHE[cache_key] = page return page def all_pages(self, locale: typing.Optional[str] = None): - """Iterate over all pages""" - locale = locale or self.i18n_origin + """Iterate over all pages of the TOC""" + locale = locale or self.locale_default for pagename in self.toc: page = self.get_page(pagename, locale) yield pagename, page -- cgit v1.2.3