diff options
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/brand.env | 5 | ||||
| -rw-r--r-- | utils/fetch_currencies.py | 8 | ||||
| -rwxr-xr-x | utils/fetch_firefox_version.py | 73 | ||||
| -rw-r--r-- | utils/fetch_languages.py | 275 | ||||
| -rw-r--r-- | utils/makefile.include | 128 | ||||
| -rw-r--r-- | utils/makefile.python | 263 | ||||
| -rw-r--r-- | utils/makefile.sphinx | 216 | ||||
| -rwxr-xr-x | utils/standalone_searx.py | 2 | ||||
| -rwxr-xr-x | utils/update-translations.sh | 6 |
9 files changed, 834 insertions, 142 deletions
diff --git a/utils/brand.env b/utils/brand.env new file mode 100644 index 000000000..7fe1a3911 --- /dev/null +++ b/utils/brand.env @@ -0,0 +1,5 @@ +export GIT_URL='https://github.com/asciimoo/searx' +export ISSUE_URL='https://github.com/asciimoo/searx/issues' +export SEARX_URL='https://searx.me' +export DOCS_URL='https://asciimoo.github.io/searx' +export PUBLIC_INSTANCES='https://searx.space' diff --git a/utils/fetch_currencies.py b/utils/fetch_currencies.py index 716b505ee..5605fb387 100644 --- a/utils/fetch_currencies.py +++ b/utils/fetch_currencies.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import print_function + import json import re import unicodedata @@ -38,13 +40,13 @@ def add_currency_name(name, iso4217): db_names = db['names'] if not isinstance(iso4217, basestring): - print "problem", name, iso4217 + print("problem", name, iso4217) return name = normalize_name(name) if name == '': - print "name empty", iso4217 + print("name empty", iso4217) return iso4217_set = db_names.get(name, None) @@ -127,7 +129,7 @@ def wdq_query(query): qlist = map(add_q, jsonresponse.get('items', {})) error = jsonresponse.get('status', {}).get('error', None) if error is not None and error != 'OK': - print "error for query '" + query + "' :" + error + print("error for query '" + query + "' :" + error) fetch_data_batch(qlist) diff --git a/utils/fetch_firefox_version.py b/utils/fetch_firefox_version.py new file mode 100755 index 000000000..722c48229 --- /dev/null +++ b/utils/fetch_firefox_version.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +# set path +from sys import path +from os.path import realpath, dirname, join +path.append(realpath(dirname(realpath(__file__)) + '/../')) + +# +import json +import requests +import re +from distutils.version import LooseVersion, StrictVersion +from lxml import html +from searx.url_utils import urlparse, urljoin +from searx import searx_dir + +URL = 'https://ftp.mozilla.org/pub/firefox/releases/' +RELEASE_PATH = '/pub/firefox/releases/' + +NORMAL_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?$') +# BETA_REGEX = re.compile('.*[0-9]b([0-9\-a-z]+)$') +# ESR_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?esr$') + +# +useragents = { + "versions": (), + "os": ('Windows NT 10.0; WOW64', + 'X11; Linux x86_64'), + "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}" +} + + +def fetch_firefox_versions(): + resp = requests.get(URL, timeout=2.0) + if resp.status_code != 200: + raise Exception("Error fetching firefox versions, HTTP code " + resp.status_code) + else: + dom = html.fromstring(resp.text) + versions = [] + + for link in dom.xpath('//a/@href'): + url = urlparse(urljoin(URL, link)) + path = url.path + if path.startswith(RELEASE_PATH): + version = path[len(RELEASE_PATH):-1] + if NORMAL_REGEX.match(version): + versions.append(LooseVersion(version)) + + list.sort(versions, reverse=True) + return versions + + +def fetch_firefox_last_versions(): + versions = fetch_firefox_versions() + + result = [] + major_last = versions[0].version[0] + major_list = (major_last, major_last - 1) + for version in versions: + major_current = version.version[0] + if major_current in major_list: + result.append(version.vstring) + + return result + + +def get_useragents_filename(): + return join(join(searx_dir, "data"), "useragents.json") + + +useragents["versions"] = fetch_firefox_last_versions() +with open(get_useragents_filename(), "w") as f: + json.dump(useragents, f, indent=4, ensure_ascii=False) diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py index 3241370c0..ddebb4111 100644 --- a/utils/fetch_languages.py +++ b/utils/fetch_languages.py @@ -2,170 +2,173 @@ # This script generates languages.py from intersecting each engine's supported languages. # -# The country names are obtained from http://api.geonames.org which requires registering as a user. -# # Output files (engines_languages.json and languages.py) # are written in current directory to avoid overwriting in case something goes wrong. -from requests import get -from urllib import urlencode -from lxml.html import fromstring -from json import loads, dumps +import json import io from sys import path +from babel import Locale, UnknownLocaleError +from babel.languages import get_global + path.append('../searx') # noqa from searx import settings from searx.engines import initialize_engines, engines -# Geonames API for country names. -geonames_user = '' # ADD USER NAME HERE -country_names_url = 'http://api.geonames.org/countryInfoJSON?{parameters}' - # Output files. engines_languages_file = 'engines_languages.json' languages_file = 'languages.py' -engines_languages = {} - - -# To filter out invalid codes and dialects. -def valid_code(lang_code): - # filter invalid codes - # sl-SL is technically not invalid, but still a mistake - invalid_codes = ['sl-SL', 'wt-WT', 'jw'] - invalid_countries = ['UK', 'XA', 'XL'] - if lang_code[:2] == 'xx'\ - or lang_code in invalid_codes\ - or lang_code[-2:] in invalid_countries\ - or is_dialect(lang_code): - return False - - return True +# Fetchs supported languages for each engine and writes json file with those. +def fetch_supported_languages(): -# Language codes with any additional tags other than language and country. -def is_dialect(lang_code): - lang_code = lang_code.split('-') - if len(lang_code) > 2 or len(lang_code[0]) > 3: - return True - if len(lang_code) == 2 and len(lang_code[1]) > 2: - return True - - return False + engines_languages = {} + names = list(engines) + names.sort() + for engine_name in names: + print("fetching languages of engine %s" % engine_name) -# Get country name in specified language. -def get_country_name(locale): - if geonames_user is '': - return '' + if hasattr(engines[engine_name], 'fetch_supported_languages'): + engines_languages[engine_name] = engines[engine_name].fetch_supported_languages() + if type(engines_languages[engine_name]) == list: + engines_languages[engine_name] = sorted(engines_languages[engine_name]) - locale = locale.split('-') - if len(locale) != 2: - return '' + # write json file + with open(engines_languages_file, 'w', encoding='utf-8') as f: + json.dump(engines_languages, f, indent=2, sort_keys=True) - url = country_names_url.format(parameters=urlencode({'lang': locale[0], - 'country': locale[1], - 'username': geonames_user})) - response = get(url) - json = loads(response.text) - content = json.get('geonames', None) - if content is None or len(content) != 1: - print "No country name found for " + locale[0] + "-" + locale[1] - return '' + return engines_languages - return content[0].get('countryName', '') +# Get babel Locale object from lang_code if possible. +def get_locale(lang_code): + try: + locale = Locale.parse(lang_code, sep='-') + return locale + except (UnknownLocaleError, ValueError): + return None -# Fetchs supported languages for each engine and writes json file with those. -def fetch_supported_languages(): - initialize_engines(settings['engines']) - for engine_name in engines: - if hasattr(engines[engine_name], 'fetch_supported_languages'): - try: - engines_languages[engine_name] = engines[engine_name].fetch_supported_languages() - except Exception as e: - print e - # write json file - with io.open(engines_languages_file, "w", encoding="utf-8") as f: - f.write(unicode(dumps(engines_languages, ensure_ascii=False, encoding="utf-8"))) +# Append engine_name to list of engines that support locale. +def add_engine_counter(lang_code, engine_name, languages): + if lang_code in languages: + if 'counter' not in languages[lang_code]: + languages[lang_code]['counter'] = [engine_name] + elif engine_name not in languages[lang_code]['counter']: + languages[lang_code]['counter'].append(engine_name) # Join all language lists. -# Iterate all languages supported by each engine. -def join_language_lists(): - global languages - # include wikipedia first for more accurate language names - languages = {code: lang for code, lang - in engines_languages['wikipedia'].iteritems() - if valid_code(code)} - +# TODO: Add language names from engine's language list if name not known by babel. +def join_language_lists(engines_languages): + language_list = {} for engine_name in engines_languages: - for locale in engines_languages[engine_name]: - if valid_code(locale): - # if language is not on list or if it has no name yet - if locale not in languages or not languages[locale].get('name'): - if isinstance(engines_languages[engine_name], dict): - languages[locale] = engines_languages[engine_name][locale] - else: - languages[locale] = {} - - # add to counter of engines that support given language - lang = locale.split('-')[0] - if lang in languages: - if 'counter' not in languages[lang]: - languages[lang]['counter'] = [engine_name] - elif engine_name not in languages[lang]['counter']: - languages[lang]['counter'].append(engine_name) - - # filter list to include only languages supported by most engines - min_supported_engines = int(0.70 * len(engines_languages)) - languages = {code: lang for code, lang - in languages.iteritems() - if len(lang.get('counter', [])) >= min_supported_engines or - len(languages.get(code.split('-')[0], {}).get('counter', [])) >= min_supported_engines} - - # get locales that have no name or country yet - for locale in languages.keys(): - # try to get language names - if not languages[locale].get('name'): - name = languages.get(locale.split('-')[0], {}).get('name', None) - if name: - languages[locale]['name'] = name - else: - # filter out locales with no name - del languages[locale] - continue - - # try to get language name in english - if not languages[locale].get('english_name'): - languages[locale]['english_name'] = languages.get(locale.split('-')[0], {}).get('english_name', '') - - # try to get country name - if locale.find('-') > 0 and not languages[locale].get('country'): - languages[locale]['country'] = get_country_name(locale) or '' - - -# Remove countryless language if language is featured in only one country. -def filter_single_country_languages(): - prev_lang = None - prev_code = None - for code in sorted(languages): - lang = code.split('-')[0] - if lang == prev_lang: + for lang_code in engines_languages[engine_name]: + + # apply custom fixes if necessary + if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values(): + lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items() + if lang_code == alias) + + locale = get_locale(lang_code) + + # ensure that lang_code uses standard language and country codes + if locale and locale.territory: + lang_code = locale.language + '-' + locale.territory + + # add locale if it's not in list + if lang_code not in language_list: + if locale: + language_list[lang_code] = {'name': locale.get_language_name().title(), + 'english_name': locale.english_name, + 'country': locale.get_territory_name() or ''} + + # also add language without country + if locale.language not in language_list: + language_list[locale.language] = {'name': locale.get_language_name().title(), + 'english_name': locale.english_name} + else: + language_list[lang_code] = {} + + # count engine for both language_country combination and language alone + add_engine_counter(lang_code, engine_name, language_list) + add_engine_counter(lang_code.split('-')[0], engine_name, language_list) + + return language_list + + +# Filter language list so it only includes the most supported languages and countries. +def filter_language_list(all_languages): + min_supported_engines = 10 + main_engines = [engine_name for engine_name in engines.keys() + if 'general' in engines[engine_name].categories and + engines[engine_name].supported_languages and + not engines[engine_name].disabled] + + # filter list to include only languages supported by most engines or all default general engines + filtered_languages = {code: lang for code, lang + in all_languages.items() + if (len(lang.get('counter', [])) >= min_supported_engines or + all(main_engine in lang.get('counter', []) + for main_engine in main_engines))} + + return filtered_languages + + +# Add country codes to languages without one and filter out language codes. +def assign_country_codes(filtered_languages, all_languages): + sorted_languages = sorted(all_languages, + key=lambda lang: len(all_languages[lang].get('counter', [])), + reverse=True) + previous_lang = None + previous_code = None + countries = 0 + for current_code in sorted(filtered_languages): + current_lang = current_code.split('-')[0] + + # count country codes per language + if current_lang == previous_lang: countries += 1 + else: - if prev_lang is not None and countries == 1: - del languages[prev_lang] - languages[prev_code]['country'] = '' + if previous_lang is not None: + # if language has no single country code + if countries == 0: + # try to get country code with most supported engines + for l in sorted_languages: + l_parts = l.split('-') + if len(l_parts) == 2 and l_parts[0] == previous_lang: + filtered_languages[l] = all_languages[l] + filtered_languages[l]['country'] = '' + countries = 1 + break + + if countries == 0: + # get most likely country code from babel + subtags = get_global('likely_subtags').get(previous_lang) + if subtags: + subtag_parts = subtags.split('_') + new_code = subtag_parts[0] + '-' + subtag_parts[-1] + filtered_languages[new_code] = all_languages[previous_lang] + countries = 1 + + if countries == 1: + # remove countryless version of language if there's only one country + del filtered_languages[previous_lang] + if previous_code in filtered_languages: + filtered_languages[previous_code]['country'] = '' + countries = 0 - prev_lang = lang - prev_code = code + previous_lang = current_lang + + previous_code = current_code # Write languages.py. -def write_languages_file(): - new_file = open(languages_file, 'w') +def write_languages_file(languages): + new_file = open(languages_file, 'wb') file_content = '# -*- coding: utf-8 -*-\n'\ + '# list of language codes\n'\ + '# this file is generated automatically by utils/update_search_languages.py\n'\ @@ -183,7 +186,9 @@ def write_languages_file(): if __name__ == "__main__": - fetch_supported_languages() - join_language_lists() - filter_single_country_languages() - write_languages_file() + initialize_engines(settings['engines']) + engines_languages = fetch_supported_languages() + all_languages = join_language_lists(engines_languages) + filtered_languages = filter_language_list(all_languages) + assign_country_codes(filtered_languages, all_languages) + write_languages_file(filtered_languages) diff --git a/utils/makefile.include b/utils/makefile.include new file mode 100644 index 000000000..716889c02 --- /dev/null +++ b/utils/makefile.include @@ -0,0 +1,128 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +make-help: + @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' + @echo ' make V=2 [targets] 2 => give reason for rebuild of target' + +quiet_cmd_common_clean = CLEAN $@ + cmd_common_clean = \ + rm -rf tests/build ;\ + find . -name '*.orig' -exec rm -f {} + ;\ + find . -name '*.rej' -exec rm -f {} + ;\ + find . -name '*~' -exec rm -f {} + ;\ + find . -name '*.bak' -exec rm -f {} + ;\ + +FMT = cat +ifeq ($(shell which fmt >/dev/null 2>&1; echo $$?), 0) +FMT = fmt +endif + +# MS-Windows +# +# For a minimal *make-environment*, I'am using the gnu-tools from: +# +# - GNU MCU Eclipse Windows Build Tools, which brings 'make', 'rm' etc. +# https://github.com/gnu-mcu-eclipse/windows-build-tools/releases +# +# - git for Windows, which brings 'find', 'grep' etc. +# https://git-scm.com/download/win + + +# normpath +# +# System-dependent normalization of the path name +# +# usage: $(call normpath,/path/to/file) + +normpath = $1 +ifeq ($(OS),Windows_NT) + normpath = $(subst /,\,$1) +endif + + +# stolen from linux/Makefile +# + +ifeq ("$(origin V)", "command line") + KBUILD_VERBOSE = $(V) +endif +ifndef KBUILD_VERBOSE + KBUILD_VERBOSE = 0 +endif + +ifeq ($(KBUILD_VERBOSE),1) + quiet = + Q = +else + quiet=quiet_ + Q = @ +endif + +# stolen from linux/scripts/Kbuild.include +# + +# Convenient variables +comma := , +quote := " +#" this comment is only for emacs highlighting +squote := ' +#' this comment is only for emacs highlighting +empty := +space := $(empty) $(empty) +space_escape := _-_SPACE_-_ + +# Find any prerequisites that is newer than target or that does not exist. +# PHONY targets skipped in both cases. +any-prereq = $(filter-out $(PHONY),$?) $(filter-out $(PHONY) $(wildcard $^),$^) +# +### +# why - tell why a a target got build +# enabled by make V=2 +# Output (listed in the order they are checked): +# (1) - due to target is PHONY +# (2) - due to target missing +# (3) - due to: file1.h file2.h +# (4) - due to command line change +# (5) - due to missing .cmd file +# (6) - due to target not in $(targets) +# (1) PHONY targets are always build +# (2) No target, so we better build it +# (3) Prerequisite is newer than target +# (4) The command line stored in the file named dir/.target.cmd +# differed from actual command line. This happens when compiler +# options changes +# (5) No dir/.target.cmd file (used to store command line) +# (6) No dir/.target.cmd file and target not listed in $(targets) +# This is a good hint that there is a bug in the kbuild file +ifeq ($(KBUILD_VERBOSE),2) +why = \ + $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ + $(if $(wildcard $@), \ + $(if $(strip $(any-prereq)),- due to: $(any-prereq), \ + $(if $(arg-check), \ + $(if $(cmd_$@),- due to command line change, \ + $(if $(filter $@, $(targets)), \ + - due to missing .cmd file, \ + - due to $(notdir $@) not in $$(targets) \ + ) \ + ) \ + ) \ + ), \ + - due to target missing \ + ) \ + ) + +echo-why = $(call escsq, $(strip $(why))) +endif +# +### +# Escape single quote for use in echo statements +escsq = $(subst $(squote),'\$(squote)',$1) +# +# echo command. +# Short version is used, if $(quiet) equals `quiet_', otherwise full one. +echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(echo-why)';) +# +# printing commands +cmd = @$(echo-cmd) $(cmd_$(1)) + diff --git a/utils/makefile.python b/utils/makefile.python new file mode 100644 index 000000000..590bbdb46 --- /dev/null +++ b/utils/makefile.python @@ -0,0 +1,263 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +# list of python packages (folders) or modules (files) of this build +PYOBJECTS ?= + +SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python +export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH +export PY_ENV PYDIST PYBUILD + +# folder where the python distribution takes place +PYDIST ?= ./py_dist +# folder where the python intermediate build files take place +PYBUILD ?= ./py_build +# python version to use +PY ?=3 +# $(PYTHON) points to the python interpreter from the OS! The python from the +# OS is needed e.g. to create a virtualenv. For tasks inside the virtualenv the +# interpeter from '$(PY_ENV_BIN)/python' is used. +PYTHON ?= python$(PY) +PIP ?= pip$(PY) +PIP_INST ?= --user + +# https://www.python.org/dev/peps/pep-0508/#extras +#PY_SETUP_EXTRAS ?= \[develop,test\] +PY_SETUP_EXTRAS ?= + +PYDEBUG ?= --pdb +PYLINT_RC ?= .pylintrc + +TEST_FOLDER ?= ./tests +TEST ?= . + +VTENV_OPTS = "--no-site-packages" +PY_ENV = ./local/py$(PY) +PY_ENV_BIN = $(PY_ENV)/bin +PY_ENV_ACT = . $(PY_ENV_BIN)/activate + +ifeq ($(OS),Windows_NT) + PYTHON = python + PY_ENV_BIN = $(PY_ENV)/Scripts + PY_ENV_ACT = $(PY_ENV_BIN)/activate +endif + +ifeq ($(PYTHON),python) + VIRTUALENV = virtualenv +else + VIRTUALENV = virtualenv --python=$(PYTHON) +endif + +ifeq ($(KBUILD_VERBOSE),1) + PIP_VERBOSE = + VIRTUALENV_VERBOSE = +else + PIP_VERBOSE = "-q" + VIRTUALENV_VERBOSE = "-q" +endif + +python-help:: + @echo 'makefile.python:' + @echo ' pyenv | pyenv[un]install' + @echo ' build $(PY_ENV) & [un]install python objects' + @echo ' targts using pyenv $(PY_ENV):' + @echo ' pylint - run pylint *linting*' + @echo ' pytest - run *tox* test on python objects' + @echo ' pydebug - run tests within a PDB debug session' + @echo ' pybuild - build python packages ($(PYDIST) $(PYBUILD))' + @echo ' pyclean - clean intermediate python objects' + @echo ' targets using system users environment:' + @echo ' py[un]install - [un]install python objects in editable mode' + @echo ' upload-pypi - upload $(PYDIST)/* files to PyPi' + @echo 'options:' + @echo ' make PY=2 [targets] => to eval targets with python 2 ($(PY))' + @echo ' make PIP_INST= => to set/unset pip install options ($(PIP_INST))' + @echo ' make TEST=. => choose test from $(TEST_FOLDER) (default "." runs all)' + @echo ' make DEBUG= => target "debug": do not invoke PDB on errors' + @echo ' make PY_SETUP_EXTRAS => also install extras_require from setup.py \[develop,test\]' + @echo ' when using target "pydebug", set breakpoints within py-source by adding::' + @echo ' DEBUG()' + +# ------------------------------------------------------------------------------ +# OS requirements +# ------------------------------------------------------------------------------ + +PHONY += msg-python-exe python-exe +msg-python-exe: + @echo "\n $(PYTHON) is required\n\n\ + Make sure you have $(PYTHON) installed, grab it from\n\ + https://www.python.org or install it from your package\n\ + manager. On debian based OS these requirements are\n\ + installed by::\n\n\ + sudo -H apt-get install $(PYTHON)\n" | $(FMT) + +ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1) +python-exe: msg-python-exe + $(error The '$(PYTHON)' command was not found) +else +python-exe: + @: +endif + +# ------------------------------------------------------------------------------ +# commands +# ------------------------------------------------------------------------------ + +# $2 path to folder with setup.py, this uses pip from the OS +quiet_cmd_pyinstall = INSTALL $2 + cmd_pyinstall = $(PIP) $(PIP_VERBOSE) install $(PIP_INST) -e $2$(PY_SETUP_EXTRAS) + +# $2 path to folder with setup.py, this uses pip from pyenv (not OS!) +quiet_cmd_pyenvinstall = PYENV install $2 + cmd_pyenvinstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) + +# Uninstall the package. Since pip does not uninstall the no longer needed +# depencies (something like autoremove) the depencies remain. + +# $2 package name to uninstall, this uses pip from the OS. +quiet_cmd_pyuninstall = UNINSTALL $2 + cmd_pyuninstall = $(PIP) $(PIP_VERBOSE) uninstall --yes $2 + +# $2 path to folder with setup.py, this uses pip from pyenv (not OS!) +quiet_cmd_pyenvuninstall = PYENV uninstall $2 + cmd_pyenvuninstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes $2 + +# $2 path to folder where virtualenv take place +quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate + cmd_virtualenv = \ + if [ ! -d "./$(PY_ENV)" ];then \ + $(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2; \ + else \ + echo "PYENV using virtualenv from $2"; \ + fi + +# $2 path to lint +quiet_cmd_pylint = LINT $@ + cmd_pylint = $(PY_ENV_BIN)/python -m pylint --rcfile $(PYLINT_RC) $2 + +quiet_cmd_pytest = TEST $@ + cmd_pytest = $(PY_ENV_BIN)/python -m tox -vv + +# setuptools, pip, easy_install its a mess full of cracks, a documentation hell +# and broken by design ... all sucks, I really, really hate all this ... aaargh! +# +# About python packaging see `Python Packaging Authority`_. Most of the names +# here are mapped to ``setup(<name1>=..., <name2>=...)`` arguments in +# ``setup.py``. See `Packaging and distributing projects`_ about ``setup(...)`` +# arguments. If this is all new for you, start with `PyPI Quick and Dirty`_. +# +# Further read: +# +# - pythonwheels_ +# - setuptools_ +# - packaging_ +# - sdist_ +# - installing_ +# +# .. _`Python Packaging Authority`: https://www.pypa.io +# .. _`Packaging and distributing projects`: https://packaging.python.org/guides/distributing-packages-using-setuptools/ +# .. _`PyPI Quick and Dirty`: https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/ +# .. _pythonwheels: https://pythonwheels.com/ +# .. _setuptools: https://setuptools.readthedocs.io/en/latest/setuptools.html +# .. _packaging: https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-and-distributing-projects +# .. _sdist: https://packaging.python.org/guides/distributing-packages-using-setuptools/#source-distributions +# .. _bdist_wheel: https://packaging.python.org/guides/distributing-packages-using-setuptools/#pure-python-wheels +# .. _installing: https://packaging.python.org/tutorials/installing-packages/ +# +quiet_cmd_pybuild = BUILD $@ + cmd_pybuild = $(PY_ENV_BIN)/python setup.py \ + sdist -d $(PYDIST) \ + bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST) + +quiet_cmd_pyclean = CLEAN $@ +# remove 'build' folder since bdist_wheel does not care the --bdist-dir + cmd_pyclean = \ + rm -rf $(PYDIST) $(PYBUILD) $(PY_ENV) ./.tox *.egg-info ;\ + find . -name '*.pyc' -exec rm -f {} + ;\ + find . -name '*.pyo' -exec rm -f {} + ;\ + find . -name __pycache__ -exec rm -rf {} + + +# ------------------------------------------------------------------------------ +# targets +# ------------------------------------------------------------------------------ + +# for installation use the pip from the OS! +PHONY += pyinstall +pyinstall: pip-exe + $(call cmd,pyinstall,.) + +PHONY += pyuninstall +pyuninstall: pip-exe + $(call cmd,pyuninstall,$(PYOBJECTS)) + +# for installation use the pip from PY_ENV (not the OS)! +PHONY += pyenvinstall +pyenvinstall: $(PY_ENV) + $(call cmd,pyenvinstall,.) + +PHONY += pyenvuninstall +pyenvuninstall: $(PY_ENV) + $(call cmd,pyenvuninstall,$(PYOBJECTS)) + +PHONY += pyclean +pyclean: + $(call cmd,pyclean) + +# to build *local* environment, python from the OS is needed! +pyenv: $(PY_ENV) +$(PY_ENV): python-exe + $(call cmd,virtualenv,$(PY_ENV)) + $(Q)$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel pip setuptools + $(Q)$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt + +PHONY += pylint-exe +pylint-exe: $(PY_ENV) + @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install pylint + +PHONY += pylint +pylint: pylint-exe + $(call cmd,pylint,$(PYOBJECTS)) + +PHONY += pybuild +pybuild: $(PY_ENV) + $(call cmd,pybuild) + +PHONY += pytest +pytest: $(PY_ENV) + $(call cmd,pytest) + +PHONY += pydebug +# set breakpoint with: +# DEBUG() +# e.g. to run tests in debug mode in emacs use: +# 'M-x pdb' ... 'make pydebug' +pydebug: $(PY_ENV) + DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST) + +# install / uninstall python objects into virtualenv (PYENV) +pyenv-install: $(PY_ENV) + @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e . + @echo "ACTIVATE $(call normpath,$(PY_ENV_ACT)) " + +pyenv-uninstall: $(PY_ENV) + @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes . + +# runs python interpreter from ./local/py<N>/bin/python +pyenv-python: pyenv-install + $(PY_ENV_BIN)/python -i + +# With 'dependency_links=' setuptools supports dependencies on packages hosted +# on other reposetories then PyPi, see "Packages Not On PyPI" [1]. The big +# drawback is, due to security reasons (I don't know where the security gate on +# PyPi is), this feature is not supported by pip [2]. Thats why an upload to +# PyPi is required and since uploads via setuptools is not recommended, we have +# to imstall / use twine ... its really a mess. +# +# [1] http://python-packaging.readthedocs.io/en/latest/dependencies.html#packages-not-on-pypi +# [2] https://github.com/pypa/pip/pull/1519 + +# https://github.com/pypa/twine +PHONY += upload-pypi +upload-pypi: pyclean pyenvinstall pybuild + @$(PY_ENV_BIN)/twine upload $(PYDIST)/* + +.PHONY: $(PHONY) diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx new file mode 100644 index 000000000..2c1922fc9 --- /dev/null +++ b/utils/makefile.sphinx @@ -0,0 +1,216 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build +SPHINX_CONF ?= conf.py + +DOCS_FOLDER ?= docs +DOCS_BUILD ?= build/docs +DOCS_DIST ?= dist/docs +GH_PAGES ?= gh-pages + +BOOKS_FOLDER ?= docs +BOOKS_DIST ?= dist/books + +ifeq ($(KBUILD_VERBOSE),1) + SPHINX_VERBOSE = "-v" +else + SPHINX_VERBOSE = +endif + +## SPHINXVERS variable +## =================== +## +## .. _requirement-specifiers: https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers +## +## Sphinx version to use, when building documentation. Set this when calling +## build target. The default value is empty (install latest), to select a +## specific version use a requirement-specifiers_. E.g. to build your target +## 'doc' with a select sphinx-doc_ version 1.7.9:: +## +## make SPHINXVERS='==1.7.9' docs +## +## To build with latest 1.7:: +## +## make SPHINXVERS='>=1.7,<1.8' docs +## +SPHINXVERS ?= + +docs-help: + @echo 'makefile.sphinx:' + @echo ' docs-clean - clean intermediate doc objects' + @echo ' $(GH_PAGES) - create & upload github pages' + @echo ' sphinx-pdf - run sphinx latex & pdf targets' + echo '' + @echo ' books/{name}.html : build only the HTML of document {name}' + @echo ' valid values for books/{name}.html are:' + @echo ' $(BOOKS_HTML)' | $(FMT) + @echo ' books/{name}.pdf : build only the PDF of document {name}' + @echo ' valid values for books/{name}.pdf are:' + @echo ' $(BOOKS_PDF) ' | $(FMT) + +# ------------------------------------------------------------------------------ +# requirements +# ------------------------------------------------------------------------------ + +sphinx-doc: $(PY_ENV) + @echo "PYENV installing Sphinx$(SPHINXVERS)" + $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' + +sphinx-live: $(PY_ENV) + @echo "PYENV installing Sphinx$(SPHINXVERS)" + $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild + + +PHONY += msg-texlive texlive + +ifeq ($(shell which xelatex >/dev/null 2>&1; echo $$?), 1) +texlive: msg-TeXLive + $(error The 'xelatex' command was not found) +else +texlive: + @: +endif + +msg-texlive: + $(Q)echo "\n\ +The TeX/PDF output and the *math* extension require TexLive and latexmk:\n\n\ + Make sure you have a updated TeXLive with XeTeX engine installed, grab it\n\ + it from https://www.tug.org/texlive or install it from your package manager.\n\n\ + Install latexmk from your package manager or visit https://ctan.org/pkg/latexmk\n\n\ + Sphinx-doc produce (Xe)LaTeX files which might use additional TeX-packages\n\ + and fonts. To process these LaTeX files, a TexLive installation with the\n\ + additional packages is required. On debian based OS these requirements\n\ + are installed by::\n\n\ + sudo -H apt-get install\n\ + latexmk\n\ + texlive-base texlive-xetex texlive-latex-recommended\n\ + texlive-extra-utils dvipng ttf-dejavu\n" + +# ------------------------------------------------------------------------------ +# commands +# ------------------------------------------------------------------------------ + +# $2 sphinx builder e.g. "html" +# $3 path where configuration file (conf.py) is located +# $4 sourcedir +# $5 dest subfolder e.g. "man" for man pages at $(DOCS_DIST)/man + +quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(DOCS_DIST)/$5) + cmd_sphinx = SPHINX_CONF=$(abspath $4/$(SPHINX_CONF))\ + $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ + -b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5 + +quiet_cmd_sphinx_autobuild = SPHINX $@ --> file://$(abspath $(DOCS_DIST)/$5) + cmd_sphinx_autobuild = PATH="$(PY_ENV_BIN):$(PATH)" $(PY_ENV_BIN)/sphinx-autobuild $(SPHINX_VERBOSE) --poll -B --host 0.0.0.0 --port 8080 $(SPHINXOPTS)\ + -b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5 + +quiet_cmd_sphinx_clean = CLEAN $@ + cmd_sphinx_clean = rm -rf $(DOCS_BUILD) $(DOCS_DIST) $(GH_PAGES)/* $(GH_PAGES)/.buildinfo + +# ------------------------------------------------------------------------------ +# targets +# ------------------------------------------------------------------------------ + +# build PDF of whole documentation in: $(DOCS_DIST)/pdf + +PHONY += sphinx-pdf +sphinx-pdf: sphinx-latex + $(Q)cd $(DOCS_BUILD)/latex/; make all-pdf + $(Q)mkdir -p $(DOCS_DIST)/pdf + $(Q)cp $(DOCS_BUILD)/latex/*.pdf $(DOCS_DIST)/pdf + @echo "SPHINX *.pdf --> file://$(abspath $(DOCS_DIST)/pdf)" + +PHONY += sphinx-latex +sphinx-latex: texlive sphinx-doc + $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ + -b latex \ + -c $(DOCS_FOLDER) \ + -d $(DOCS_BUILD)/.doctrees \ + $(DOCS_FOLDER) \ + $(DOCS_BUILD)/latex + +# Sphinx projects, we call them *books* (what is more common). Books are +# folders under $(BOOKS_FOLDER) containing a conf.py file. The HTML output goes +# to folder $(BOOKS_DIST)/<name> while PDF is placed (BOOKS_DIST)/<name>/pdf + +BOOKS=$(patsubst $(BOOKS_FOLDER)/%/conf.py,books/%,$(wildcard $(BOOKS_FOLDER)/*/conf.py)) + +# fine grained targets +BOOKS_HTML = $(patsubst %,%.html,$(BOOKS)) +BOOKS_CLEAN = $(patsubst %,%.clean,$(BOOKS)) +BOOKS_LATEX = $(patsubst %,%.latex,$(BOOKS)) +BOOKS_PDF = $(patsubst %,%.pdf,$(BOOKS)) +BOOKS_LIVE = $(patsubst %,%.live,$(BOOKS)) + +$(BOOKS_DIST): + mkdir -p $(BOOKS_DIST) + +PHONY += $(BOOKS_HTML) +$(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST) + SPHINX_CONF=$(patsubst books/%.html,%,$@)/conf.py \ + $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ + -b html \ + -c $(DOCS_FOLDER) \ + -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \ + $(patsubst books/%.html,%,$@) \ + $(BOOKS_DIST)/$(patsubst books/%.html,%,$@) + @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))" + +PHONY += $(BOOKS_HTML) +$(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST) + PATH="$(PY_ENV_BIN):$(PATH)" \ + SPHINX_CONF=$(patsubst books/%.live,%,$@)/conf.py \ + $(PY_ENV_BIN)/sphinx-autobuild --poll -B --host 0.0.0.0 --port 8080 $(SPHINX_VERBOSE) $(SPHINXOPTS)\ + -b html \ + -c $(DOCS_FOLDER) \ + -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \ + $(patsubst books/%.live,%,$@) \ + $(BOOKS_DIST)/$(patsubst books/%.live,%,$@) + +$(BOOKS_PDF): %.pdf : %.latex + $(Q)cd $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@); make all-pdf + $(Q)mkdir -p $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf + $(Q)cp -v $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@)/*.pdf $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf + @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@))/pdf" + +PHONY += $(BOOKS_LATEX) +$(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST) + SPHINX_CONF=$(patsubst books/%.latex,%,$@)/conf.py \ + $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ + -b latex \ + -c $(DOCS_FOLDER) \ + -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \ + $(patsubst books/%.latex,%,$@) \ + $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@) + @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))" + +$(BOOKS_CLEAN): + $(Q)rm -rf $(BOOKS_DIST)/$(patsubst books/%.clean,%,$@) \ + $(DOCS_BUILD)/books/$(patsubst books/%.clean,%,$@) \ + $(DOCS_BUILD)/latex/$(patsubst books/%.clean,%,$@) + +# github pages + +PHONY += $(GH_PAGES) +$(GH_PAGES):: + $(MAKE) docs + [ -d "gh-pages/.git" ] || git clone $(GIT_URL) gh-pages + -cd $(GH_PAGES); git checkout gh-pages >/dev/null + -cd $(GH_PAGES); git pull + -cd $(GH_PAGES); ls -A | grep -v '.git$$' | xargs rm -rf + cp -r $(DOCS_DIST)/* $(GH_PAGES)/ + touch $(GH_PAGES)/.nojekyll + echo "<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>" > $(GH_PAGES)/404.html + cd $(GH_PAGES);\ + git add --all . ;\ + git commit -m "gh-pages: updated" ;\ + git push origin gh-pages + + +PHONY += docs-clean +docs-clean: $(BOOKS_CLEAN) + $(call cmd,sphinx_clean) + +.PHONY: $(PHONY) diff --git a/utils/standalone_searx.py b/utils/standalone_searx.py index 223163639..7bc1d32ed 100755 --- a/utils/standalone_searx.py +++ b/utils/standalone_searx.py @@ -64,7 +64,7 @@ form = { preferences = searx.preferences.Preferences(['oscar'], searx.engines.categories.keys(), searx.engines.engines, []) preferences.key_value_settings['safesearch'].parse(args.safesearch) -search_query = searx.search.get_search_query_from_webapp(preferences, form) +search_query, raw_text_query = searx.search.get_search_query_from_webapp(preferences, form) search = searx.search.Search(search_query) result_container = search.search() diff --git a/utils/update-translations.sh b/utils/update-translations.sh index 00e7fb1e0..240387ae7 100755 --- a/utils/update-translations.sh +++ b/utils/update-translations.sh @@ -7,9 +7,9 @@ SEARX_DIR='searx' -pybabel extract -F babel.cfg -o messages.pot $SEARX_DIR -for f in `ls $SEARX_DIR'/translations/'`; do - pybabel update -N -i messages.pot -d $SEARX_DIR'/translations/' -l $f +pybabel extract -F babel.cfg -o messages.pot "$SEARX_DIR" +for f in `ls "$SEARX_DIR"'/translations/'`; do + pybabel update -N -i messages.pot -d "$SEARX_DIR"'/translations/' -l "$f" done echo '[!] update done, edit .po files if required and run pybabel compile -d searx/translations/' |