diff options
| -rw-r--r-- | .github/workflows/container.yml | 183 | ||||
| -rw-r--r-- | .github/workflows/integration.yml | 46 | ||||
| -rw-r--r-- | Makefile | 10 | ||||
| -rw-r--r-- | container/Dockerfile | 100 | ||||
| -rwxr-xr-x | container/docker-entrypoint.sh (renamed from dockerfiles/docker-entrypoint.sh) | 4 | ||||
| -rw-r--r-- | container/legacy/Dockerfile (renamed from Dockerfile) | 13 | ||||
| -rw-r--r-- | container/uwsgi.ini (renamed from dockerfiles/uwsgi.ini) | 0 | ||||
| -rw-r--r-- | docs/admin/installation-docker.rst | 11 | ||||
| -rwxr-xr-x | manage | 91 | ||||
| -rw-r--r-- | searx/version.py | 6 | ||||
| -rw-r--r-- | utils/lib_sxng_container.sh | 319 |
11 files changed, 628 insertions, 155 deletions
diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 000000000..d232a0737 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,183 @@ +--- +name: Container + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +permissions: + contents: read + # Organization GHCR + packages: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + build: + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + name: Build (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + outputs: + version_string: ${{ steps.build.outputs.version_string }} + version_tag: ${{ steps.build.outputs.version_tag }} + docker_tag: ${{ steps.build.outputs.docker_tag }} + git_url: ${{ steps.build.outputs.git_url }} + git_branch: ${{ steps.build.outputs.git_branch }} + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup cache container mounts + uses: actions/cache@v4 + with: + # yamllint disable-line rule:line-length + key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./container/Dockerfile ./container/legacy/Dockerfile') }}" + restore-keys: "container-mounts-${{ matrix.arch }}-" + path: | + /var/tmp/buildah-cache/ + /var/tmp/buildah-cache-*/ + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Build + id: build + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make podman.build + + test: + name: Test (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + needs: build + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Test + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + GIT_URL: "${{ needs.build.outputs.git_url }}" + run: make container.test + + release: + if: github.repository_owner == 'searxng' && github.ref_name == 'master' + name: Release + runs-on: ubuntu-24.04-arm + needs: + - build + - test + + steps: + - if: env.DOCKERHUB_USERNAME != '' + name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: "docker.io" + username: "${{ env.DOCKERHUB_USERNAME }}" + password: "${{ secrets.DOCKERHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Release + env: + GIT_URL: "${{ needs.build.outputs.git_url }}" + DOCKER_TAG: "${{ needs.build.outputs.docker_tag }}" + run: make container.push diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index abdaf0c18..b40ae26ab 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -94,49 +94,3 @@ jobs: - name: Build run: make themes.all - - dockers: - name: Docker - if: github.ref == 'refs/heads/master' - needs: - - test - - theme - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - runs-on: ubuntu-24.04 - steps: - - name: Checkout - if: env.DOCKERHUB_USERNAME != null - uses: actions/checkout@v4 - with: - # make sure "make docker.push" can get the git history - fetch-depth: '0' - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: Set up QEMU - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - if: env.DOCKERHUB_USERNAME != null - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - if: env.DOCKERHUB_USERNAME != null - run: make -e GIT_URL=$(git remote get-url origin) docker.buildx @@ -54,7 +54,7 @@ ci.test: test.yamllint test.black test.types.ci test.pylint test.unit test.robo test: test.yamllint test.black test.types.dev test.pylint test.unit test.robot test.rst test.shell test.shell: $(Q)shellcheck -x -s dash \ - dockerfiles/docker-entrypoint.sh + container/docker-entrypoint.sh $(Q)shellcheck -x -s bash \ utils/brand.sh \ $(MTOOLS) \ @@ -77,7 +77,9 @@ test.shell: MANAGE += weblate.translations.commit weblate.push.translations MANAGE += data.all data.traits data.useragents data.locales data.currencies MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean -MANAGE += docker.build docker.push docker.buildx +MANAGE += podman.build +MANAGE += docker.build docker.buildx +MANAGE += container.build container.test container.push MANAGE += gecko.driver MANAGE += node.env node.env.dev node.clean MANAGE += py.build py.clean @@ -95,8 +97,8 @@ $(MANAGE): # short hands of selected targets -PHONY += docs docker themes +PHONY += docs container themes docs: docs.html -docker: docker.build +container: container.build themes: themes.all diff --git a/container/Dockerfile b/container/Dockerfile new file mode 100644 index 000000000..b0530dfec --- /dev/null +++ b/container/Dockerfile @@ -0,0 +1,100 @@ +FROM docker.io/library/python:3.13-slim AS builder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + brotli \ + # uwsgi + libpcre3-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/local/searxng/ + +COPY ./requirements.txt ./requirements.txt + +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ + && . ./venv/bin/activate \ + && pip install -r requirements.txt \ + && pip install "uwsgi~=2.0" + +COPY ./searx/ ./searx/ + +ARG TIMESTAMP_SETTINGS=0 +ARG TIMESTAMP_UWSGI=0 + +RUN python -m compileall -q searx \ + && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \ + && touch -c --date=@$TIMESTAMP_UWSGI ./container/uwsgi.ini \ + && find /usr/local/searxng/searx/static \ + \( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \ + -type f -exec gzip -9 -k {} + -exec brotli --best {} + + +ARG SEARXNG_UID=977 +ARG SEARXNG_GID=977 + +RUN grep -m1 root /etc/group > /tmp/.searxng.group \ + && grep -m1 root /etc/passwd > /tmp/.searxng.passwd \ + && echo "searxng:x:$SEARXNG_GID:" >> /tmp/.searxng.group \ + && echo "searxng:x:$SEARXNG_UID:$SEARXNG_GID:searxng:/usr/local/searxng:/bin/bash" >> /tmp/.searxng.passwd + +FROM docker.io/library/python:3.13-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + # healthcheck + wget \ + # uwsgi + libpcre3 \ + libxml2 \ + mailcap \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=root:root --from=builder /tmp/.searxng.passwd /etc/passwd +COPY --chown=root:root --from=builder /tmp/.searxng.group /etc/group + +ARG LABEL_DATE="0001-01-01T00:00:00Z" +ARG GIT_URL="unspecified" +ARG SEARXNG_GIT_VERSION="unspecified" +ARG LABEL_VCS_REF="unspecified" +ARG LABEL_VCS_URL="unspecified" + +WORKDIR /usr/local/searxng/ + +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ +COPY --chown=searxng:searxng ./container/ ./container/ + +LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ + org.opencontainers.image.created=$LABEL_DATE \ + org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \ + org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \ + org.opencontainers.image.licenses="AGPL-3.0-or-later" \ + org.opencontainers.image.revision=$LABEL_VCS_REF \ + org.opencontainers.image.source=$LABEL_VCS_URL \ + org.opencontainers.image.title="searxng" \ + org.opencontainers.image.url=$LABEL_VCS_URL \ + org.opencontainers.image.version=$SEARXNG_GIT_VERSION + +ENV CONFIG_PATH=/etc/searxng \ + DATA_PATH=/var/cache/searxng + +ENV SEARXNG_VERSION=$SEARXNG_GIT_VERSION \ + INSTANCE_NAME=searxng \ + AUTOCOMPLETE="" \ + BASE_URL="" \ + BIND_ADDRESS=[::]:8080 \ + MORTY_KEY="" \ + MORTY_URL="" \ + SEARXNG_SETTINGS_PATH=$CONFIG_PATH/settings.yml \ + UWSGI_SETTINGS_PATH=$CONFIG_PATH/uwsgi.ini \ + UWSGI_WORKERS=%k \ + UWSGI_THREADS=4 + +VOLUME $CONFIG_PATH +VOLUME $DATA_PATH + +EXPOSE 8080 + +HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1 + +ENTRYPOINT ["/usr/local/searxng/container/docker-entrypoint.sh"] diff --git a/dockerfiles/docker-entrypoint.sh b/container/docker-entrypoint.sh index 3668fb589..72d020dcf 100755 --- a/dockerfiles/docker-entrypoint.sh +++ b/container/docker-entrypoint.sh @@ -140,14 +140,14 @@ if [ "$SEARX_CONF" -eq "1" ]; then cat << EOF > /etc/searx/deprecated_volume_read_me.txt This Docker image uses the volume /etc/searxng Update your configuration: -* remove uwsgi.ini (or very carefully update your existing uwsgi.ini using https://github.com/searxng/searxng/blob/master/dockerfiles/uwsgi.ini ) +* remove uwsgi.ini (or very carefully update your existing uwsgi.ini using https://github.com/searxng/searxng/blob/master/container/uwsgi.ini ) * mount /etc/searxng instead of /etc/searx EOF fi # end of searx compatibility # make sure there are uwsgi settings -update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searxng/dockerfiles/uwsgi.ini" "patch_uwsgi_settings" +update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searxng/container/uwsgi.ini" "patch_uwsgi_settings" # make sure there are searxng settings update_conf "${FORCE_CONF_UPDATE}" "${SEARXNG_SETTINGS_PATH}" "/usr/local/searxng/searx/settings.yml" "patch_searxng_settings" diff --git a/Dockerfile b/container/legacy/Dockerfile index 9aeb28214..5436ea5da 100644 --- a/Dockerfile +++ b/container/legacy/Dockerfile @@ -1,3 +1,5 @@ +# For armv7 architecture + FROM docker.io/library/python:3.13-slim AS builder RUN apt-get update \ @@ -16,8 +18,7 @@ WORKDIR /usr/local/searxng/ COPY ./requirements.txt ./requirements.txt -# Readd on #4707 "--mount=type=cache,id=pip,target=/root/.cache/pip" -RUN python -m venv ./venv \ +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ && . ./venv/bin/activate \ && pip install -r requirements.txt \ && pip install "uwsgi~=2.0" @@ -29,7 +30,7 @@ ARG TIMESTAMP_UWSGI=0 RUN python -m compileall -q searx \ && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \ - && touch -c --date=@$TIMESTAMP_UWSGI ./dockerfiles/uwsgi.ini \ + && touch -c --date=@$TIMESTAMP_UWSGI ./container/uwsgi.ini \ && find /usr/local/searxng/searx/static \ \( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \ -type f -exec gzip -9 -k {} + -exec brotli --best {} + @@ -69,7 +70,7 @@ WORKDIR /usr/local/searxng/ COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ -COPY --chown=searxng:searxng ./dockerfiles/ ./dockerfiles/ +COPY --chown=searxng:searxng ./container/ ./container/ LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ org.opencontainers.image.created=$LABEL_DATE \ @@ -90,8 +91,6 @@ ENV SEARXNG_VERSION=$SEARXNG_GIT_VERSION \ AUTOCOMPLETE="" \ BASE_URL="" \ BIND_ADDRESS=[::]:8080 \ - MORTY_KEY="" \ - MORTY_URL="" \ SEARXNG_SETTINGS_PATH=$CONFIG_PATH/settings.yml \ UWSGI_SETTINGS_PATH=$CONFIG_PATH/uwsgi.ini \ UWSGI_WORKERS=%k \ @@ -104,4 +103,4 @@ EXPOSE 8080 HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1 -ENTRYPOINT ["/usr/local/searxng/dockerfiles/docker-entrypoint.sh"] +ENTRYPOINT ["/usr/local/searxng/container/docker-entrypoint.sh"] diff --git a/dockerfiles/uwsgi.ini b/container/uwsgi.ini index 3bfd49e72..3bfd49e72 100644 --- a/dockerfiles/uwsgi.ini +++ b/container/uwsgi.ini diff --git a/docs/admin/installation-docker.rst b/docs/admin/installation-docker.rst index 09471891b..06b3fe465 100644 --- a/docs/admin/installation-docker.rst +++ b/docs/admin/installation-docker.rst @@ -145,13 +145,6 @@ shell inside container - `How to make bash scripts work in dash <http://mywiki.wooledge.org/Bashism>`_ - `Checking for Bashisms <https://dev.to/bowmanjd/writing-bash-scripts-that-are-not-only-bash-checking-for-bashisms-and-testing-with-dash-1bli>`_ -Like in many other distributions, Alpine's `/bin/sh -<https://wiki.ubuntu.com/DashAsBinSh>`__ is :man:`dash`. Dash is meant to be -`POSIX-compliant <https://pubs.opengroup.org/onlinepubs/9699919799>`__. -Compared to debian, in the Alpine image :man:`bash` is not installed. The -:origin:`dockerfiles/docker-entrypoint.sh` script is checked *against dash* -(``make tests.shell``). - To open a shell inside the container: .. code:: sh @@ -188,10 +181,10 @@ Command line <https://docs.docker.com/engine/reference/run/#foreground>`__. In the :origin:`Dockerfile` the ENTRYPOINT_ is defined as -:origin:`dockerfiles/docker-entrypoint.sh` +:origin:`container/docker-entrypoint.sh` .. code:: sh docker run --rm -it searxng/searxng -h -.. program-output:: ../dockerfiles/docker-entrypoint.sh -h +.. program-output:: ../container/docker-entrypoint.sh -h @@ -11,6 +11,9 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh" +# shellcheck source=utils/lib_sxng_container.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_container.sh" + # shellcheck source=utils/lib_sxng_data.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh" @@ -77,9 +80,6 @@ docs.: gh-pages : deploy on gh-pages branch prebuild : build reST include files (./${DOCS_BUILD}/includes) clean : clean documentation build -docker.: - build : build docker image - push : build and push docker image gecko.driver: download & install geckodriver if not already installed (required for robot_tests) @@ -101,6 +101,7 @@ EOF go.help node.help weblate.help + container.help data.help test.help themes.help @@ -136,90 +137,6 @@ webapp.run() { SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp } -docker.push() { - docker.build push -} - -docker.buildx() { - docker.build buildx -} - -# shellcheck disable=SC2119 -docker.build() { - pyenv.install - - local SEARXNG_GIT_VERSION - local VERSION_GITCOMMIT - local GITHUB_USER - local SEARXNG_IMAGE_NAME - local BUILD - - build_msg DOCKER build - # run installation in a subprocess and activate pyenv - - # See https://www.shellcheck.net/wiki/SC1001 and others .. - # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 - ( set -e - pyenv.activate - - # Check if it is a git repository - if [ ! -d .git ]; then - die 1 "This is not Git repository" - fi - if [ ! -x "$(which git)" ]; then - die 1 "git is not installed" - fi - - if ! git remote get-url origin 2> /dev/null; then - die 1 "there is no remote origin" - fi - - # This is a git repository - git update-index -q --refresh - python -m searx.version freeze - eval "$(python -m searx.version)" - - # Get the last git commit id - VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2) - build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" - - # define the docker image name - GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/') - SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}" - - BUILD="build" - if [ "$1" = "buildx" ]; then - # buildx includes the push option - CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache" - BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max" - shift - fi - build_msg DOCKER "Build command: ${BUILD}" - - # build Docker image - build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" - # shellcheck disable=SC2086 - docker $BUILD \ - --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ - --build-arg GIT_URL="${GIT_URL}" \ - --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \ - --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ - --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ - --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ - --build-arg LABEL_VCS_URL="${GIT_URL}" \ - --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ - --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ - -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" . - - if [ "$1" = "push" ]; then - docker push "${SEARXNG_IMAGE_NAME}:latest" - docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" - fi - ) - dump_return $? -} - # shellcheck disable=SC2119 gecko.driver() { pyenv.install diff --git a/searx/version.py b/searx/version.py index d2013808b..565cc7e7a 100644 --- a/searx/version.py +++ b/searx/version.py @@ -41,6 +41,12 @@ def subprocess_run(args, **kwargs): def get_git_url_and_branch(): + # handle GHA directly + if "GITHUB_REPOSITORY" in os.environ and "GITHUB_REF_NAME" in os.environ: + git_url = f"https://github.com/{os.environ['GITHUB_REPOSITORY']}" + git_branch = os.environ["GITHUB_REF_NAME"] + return git_url, git_branch + try: ref = subprocess_run("git rev-parse --abbrev-ref @{upstream}") except subprocess.CalledProcessError: diff --git a/utils/lib_sxng_container.sh b/utils/lib_sxng_container.sh new file mode 100644 index 000000000..b3f84594f --- /dev/null +++ b/utils/lib_sxng_container.sh @@ -0,0 +1,319 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later + +container.help() { + cat <<EOF +container.: + build : build container image +EOF +} + +CONTAINER_IMAGE_ORGANIZATION=${GITHUB_REPOSITORY_OWNER:-"searxng"} +CONTAINER_IMAGE_NAME="searxng" + +container.build() { + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local container_engine + local dockerfile + local arch + local variant + local platform + + # Check if git is installed + if ! command -v git &>/dev/null; then + die 1 "Git is not installed" + fi + + # Check if podman or docker is installed + if [ "$1" = "docker" ]; then + if command -v docker &>/dev/null; then + container_engine="docker" + else + die 1 "Docker is not installed" + fi + elif [ "$1" = "podman" ]; then + if command -v podman &>/dev/null; then + container_engine="podman" + else + die 1 "Podman is not installed" + fi + else + # If no explicit engine is passed, prioritize podman over docker + if command -v podman &>/dev/null; then + container_engine="podman" + elif command -v docker &>/dev/null; then + container_engine="docker" + else + die 1 "Podman/Docker is not installed" + fi + fi + info_msg "Selected engine: $container_engine" + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + dockerfile="Dockerfile" + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + dockerfile="Dockerfile" + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + dockerfile="legacy/Dockerfile" + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; $parch" + exit 1 + ;; + esac + info_msg "Selected platform: $platform" + + pyenv.install + + ( + set -e + pyenv.activate + + # Check if it is a git repository + if [ ! -d .git ]; then + die 1 "This is not Git repository" + fi + + if ! git remote get-url origin &>/dev/null; then + die 1 "There is no remote origin" + fi + + # This is a git repository + git update-index -q --refresh + python -m searx.version freeze + eval "$(python -m searx.version)" + + info_msg "Set \$VERSION_STRING: $VERSION_STRING" + info_msg "Set \$VERSION_TAG: $VERSION_TAG" + info_msg "Set \$DOCKER_TAG: $DOCKER_TAG" + info_msg "Set \$GIT_URL: $GIT_URL" + info_msg "Set \$GIT_BRANCH: $GIT_BRANCH" + + if [ "$container_engine" = "podman" ]; then + params_build_builder="build --format=docker --platform=$platform --target=builder --layers --identity-label=false" + params_build="build --format=docker --platform=$platform --layers --squash-all --omit-history --identity-label=false" + else + params_build_builder="build --platform=$platform --target=builder" + params_build="build --platform=$platform --squash" + fi + + if [ "$GITHUB_ACTIONS" = "true" ]; then + params_build_builder+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache" + params_build+=" --cache-from=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache --cache-to=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache" + + # Tags + params_build+=" --tag=ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + else + # Tags + params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:latest" + params_build+=" --tag=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$DOCKER_TAG" + fi + + # shellcheck disable=SC2086 + "$container_engine" $params_build_builder \ + --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ + --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./container/uwsgi.ini)" \ + --tag="localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder" \ + --file="./container/$dockerfile" \ + . + build_msg CONTAINER "Image \"builder\" built" + + # shellcheck disable=SC2086 + "$container_engine" $params_build \ + --build-arg="GIT_URL=$GIT_URL" \ + --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ + --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ + --build-arg="LABEL_VCS_URL=$GIT_URL" \ + --file="./container/$dockerfile" \ + . + build_msg CONTAINER "Image built" + + if [ "$GITHUB_ACTIONS" = "true" ]; then + "$container_engine" push "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + + # Output to GHA + { + echo "version_string=$VERSION_STRING" + echo "version_tag=$VERSION_TAG" + echo "docker_tag=$DOCKER_TAG" + echo "git_url=$GIT_URL" + echo "git_branch=$GIT_BRANCH" + } >>"$GITHUB_OUTPUT" + fi + ) + dump_return $? +} + +container.test() { + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local arch + local variant + local platform + + if [ "$GITHUB_ACTIONS" != "true" ]; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; $parch" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + ( + set -e + + podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" + + name="$CONTAINER_IMAGE_NAME-$(date +%N)" + + podman create --name="$name" --rm --timeout=60 --network="host" \ + "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-$arch$variant" >/dev/null + + podman start "$name" >/dev/null + podman logs -f "$name" & + pid_logs=$! + + # Wait until container is ready + sleep 5 + + curl -vf --max-time 5 "http://localhost:8080/healthz" + + kill $pid_logs &>/dev/null || true + podman stop "$name" >/dev/null + ) + dump_return $? +} + +container.push() { + # Architectures on manifest + local release_archs=("amd64" "arm64" "armv7") + + local archs=() + local variants=() + local platforms=() + + if [ "$GITHUB_ACTIONS" != "true" ]; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + for arch in "${release_archs[@]}"; do + case $arch in + "X64" | "x86_64" | "amd64") + archs+=("amd64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARM64" | "aarch64" | "arm64") + archs+=("arm64") + variants+=("") + platforms+=("linux/${archs[-1]}") + ;; + "ARMV7" | "armv7" | "armhf" | "arm") + archs+=("arm") + variants+=("v7") + platforms+=("linux/${archs[-1]}/${variants[-1]}") + ;; + *) + err_msg "Unsupported architecture; $arch" + exit 1 + ;; + esac + done + + ( + set -e + + # Pull archs + for i in "${!archs[@]}"; do + podman pull "ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" + done + + # Manifest tags + release_tags=("latest") + release_tags+=("$DOCKER_TAG") + + # Create manifests + for tag in "${release_tags[@]}"; do + if ! podman manifest exists "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag"; then + podman manifest create "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + fi + + # Add archs to manifest + for i in "${!archs[@]}"; do + podman manifest add \ + "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ + "containers-storage:ghcr.io/$CONTAINER_IMAGE_ORGANIZATION/cache:$CONTAINER_IMAGE_NAME-${archs[$i]}${variants[$i]}" + done + done + + podman image list + + # Push manifests + for tag in "${release_tags[@]}"; do + build_msg CONTAINER "Pushing manifest with tag: $tag" + + podman manifest push \ + "localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" \ + "docker://docker.io/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:$tag" + done + ) + dump_return $? +} + +# Alias +podman.build() { + container.build podman +} + +# Alias +docker.build() { + container.build docker +} + +# Alias +docker.buildx() { + container.build docker +} |