diff options
Diffstat (limited to 'manage.sh')
| -rwxr-xr-x | manage.sh | 275 |
1 files changed, 275 insertions, 0 deletions
diff --git a/manage.sh b/manage.sh new file mode 100755 index 000000000..a450c6dfd --- /dev/null +++ b/manage.sh @@ -0,0 +1,275 @@ +#!/bin/sh + +BASE_DIR="$(dirname -- "`readlink -f -- "$0"`")" + +cd -- "$BASE_DIR" +set -e + +# subshell +PYTHONPATH="$BASE_DIR" +SEARX_DIR="$BASE_DIR/searx" +ACTION="$1" + + +# +# Python +# + +update_packages() { + pip install --upgrade pip + pip install --upgrade setuptools + pip install -r "$BASE_DIR/requirements.txt" +} + +update_dev_packages() { + update_packages + pip install -r "$BASE_DIR/requirements-dev.txt" +} + +install_geckodriver() { + echo '[!] Checking geckodriver' + # TODO : check the current geckodriver version + set -e + geckodriver -V > /dev/null 2>&1 || NOTFOUND=1 + set +e + if [ -z "$NOTFOUND" ]; then + return + fi + GECKODRIVER_VERSION="v0.24.0" + PLATFORM="`python -c "import six; import platform; six.print_(platform.system().lower(), platform.architecture()[0])"`" + case "$PLATFORM" in + "linux 32bit" | "linux2 32bit") ARCH="linux32";; + "linux 64bit" | "linux2 64bit") ARCH="linux64";; + "windows 32 bit") ARCH="win32";; + "windows 64 bit") ARCH="win64";; + "mac 64bit") ARCH="macos";; + esac + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; + + if [ -z "$1" ]; then + if [ -z "$VIRTUAL_ENV" ]; then + printf "geckodriver can't be installed because VIRTUAL_ENV is not set, you should download it from\n %s" "$GECKODRIVER_URL" + exit + else + GECKODRIVER_DIR="$VIRTUAL_ENV/bin" + fi + else + GECKODRIVER_DIR="$1" + mkdir -p -- "$GECKODRIVER_DIR" + fi + + printf "Installing %s/geckodriver from\n %s" "$GECKODRIVER_DIR" "$GECKODRIVER_URL" + + FILE="`mktemp`" + wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "$GECKODRIVER_DIR" -f "$FILE" geckodriver + rm -- "$FILE" + chmod 777 -- "$GECKODRIVER_DIR/geckodriver" +} + +locales() { + pybabel compile -d "$SEARX_DIR/translations" +} + +update_useragents() { + echo '[!] Updating user agent versions' + python utils/fetch_firefox_version.py +} + +pep8_check() { + echo '[!] Running pep8 check' + # ignored rules: + # E402 module level import not at top of file + # W503 line break before binary operator + pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" "$SEARX_DIR" "$BASE_DIR/tests" +} + +unit_tests() { + echo '[!] Running unit tests' + python -m nose2 -s "$BASE_DIR/tests/unit" +} + +py_test_coverage() { + echo '[!] Running python test coverage' + PYTHONPATH="`pwd`" python -m nose2 -C --log-capture --with-coverage --coverage "$SEARX_DIR" -s "$BASE_DIR/tests/unit" \ + && coverage report \ + && coverage html +} + +robot_tests() { + echo '[!] Running robot tests' + PYTHONPATH="`pwd`" python "$SEARX_DIR/testing.py" robot +} + +tests() { + set -e + pep8_check + unit_tests + install_geckodriver + robot_tests + set +e +} + + +# +# Web +# + +npm_path_setup() { + which npm || (printf 'Error: npm is not found\n'; exit 1) + export PATH="$(npm bin)":$PATH +} + +npm_packages() { + npm_path_setup + + echo '[!] install NPM packages' + cd -- "$BASE_DIR" + npm install less@2.7 less-plugin-clean-css grunt-cli + + echo '[!] install NPM packages for oscar theme' + cd -- "$BASE_DIR/searx/static/themes/oscar" + npm install + + echo '[!] install NPM packages for simple theme' + cd -- "$BASE_DIR/searx/static/themes/simple" + npm install +} + +build_style() { + npm_path_setup + + lessc --clean-css="--s1 --advanced --compatibility=ie9" "$BASE_DIR/searx/static/$1" "$BASE_DIR/searx/static/$2" +} + +styles() { + npm_path_setup + + echo '[!] Building legacy style' + build_style themes/legacy/less/style.less themes/legacy/css/style.css + build_style themes/legacy/less/style-rtl.less themes/legacy/css/style-rtl.css + echo '[!] Building courgette style' + build_style themes/courgette/less/style.less themes/courgette/css/style.css + build_style themes/courgette/less/style-rtl.less themes/courgette/css/style-rtl.css + echo '[!] Building pix-art style' + build_style themes/pix-art/less/style.less themes/pix-art/css/style.css + echo '[!] Building bootstrap style' + build_style less/bootstrap/bootstrap.less css/bootstrap.min.css +} + +grunt_build() { + npm_path_setup + + echo '[!] Grunt build : oscar theme' + grunt --gruntfile "$SEARX_DIR/static/themes/oscar/gruntfile.js" + echo '[!] Grunt build : simple theme' + grunt --gruntfile "$SEARX_DIR/static/themes/simple/gruntfile.js" +} + +docker_build() { + # Check if it is a git repository + if [ ! -d .git ]; then + echo "This is not Git repository" + exit 1 + fi + + if [ ! -x "$(which git)" ]; then + echo "git is not installed" + exit 1 + fi + + if [ ! git remote get-url origin 2> /dev/null ]; then + echo "there is no remote origin" + exit 1 + fi + + # This is a git repository + + # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e) + # awk to remove the "v" and the "g" + SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); $3=substr($3, 2); print}') + + # add the suffix "-dirty" if the repository has uncommited change + git update-index -q --refresh + if [ ! -z "$(git diff-index --name-only HEAD --)" ]; then + SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty" + fi + + # Get the last git commit id, will be added to the Searx version (see Dockerfile) + VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4) + echo "Last commit : $VERSION_GITCOMMIT" + + # Check consistency between the git tag and the searx/version.py file + # /!\ HACK : parse Python file with bash /!\ + # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py ) + # SEARX_PYTHON_VERSION=$(python -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)") + SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -) + if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then + echo "Inconsistency between the last git tag and the searx/version.py file" + echo "git tag: $SEARX_GIT_VERSION" + echo "searx/version.py: $SEARX_PYTHON_VERSION" + exit 1 + fi + + # define the docker image name + # /!\ HACK to get the user name /!\ + GITHUB_USER=$(git remote get-url origin | sed 's/.*github\.com\/\([^\/]*\).*/\1/') + SEARX_IMAGE_NAME="${GITHUB_USER:-searx}/searx" + + # build Docker image + echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" + sudo docker build \ + --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \ + --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ + --build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \ + --build-arg LABEL_VCS_URL=$(git remote get-url origin) \ + --build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \ + --build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \ + -t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} . + + if [ "$1" = "push" ]; then + sudo docker push ${SEARX_IMAGE_NAME}:latest + sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} + fi +} + +# +# Help +# + +help() { + [ -z "$1" ] || printf 'Error: %s\n' "$1" + echo "Searx manage.sh help + +Commands +======== + help - This text + + Build requirements + ------------------ + update_packages - Check & update production dependency changes + update_dev_packages - Check & update development and production dependency changes + install_geckodriver - Download & install geckodriver if not already installed (required for robot_tests) + npm_packages - Download & install npm dependencies + update_useragents - Update useragents.json with the most recent versions of Firefox + + Build + ----- + locales - Compile locales + styles - Build less files + grunt_build - Build files for themes + docker_build - Build Docker image + + Tests + ----- + unit_tests - Run unit tests + pep8_check - Pep8 validation + robot_tests - Run selenium tests + tests - Run all python tests (pep8, unit, robot_tests) + py_test_coverage - Unit test coverage +" +} + +[ "$(command -V "$ACTION" | grep ' function$')" = "" ] \ + && help "action not found" \ + || "$ACTION" "$2" |