From 3de7a6da2dba72057a75d54b4b9fd864dee230e9 Mon Sep 17 00:00:00 2001 From: Ivan Gabaldon Date: Thu, 7 Aug 2025 10:46:26 +0200 Subject: [enh] container: tidy builds (#5086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building the container currently does not work properly. When rebuilding several times with `make container`, `version_frozen.py` is recreated, which wouldn't be an issue if the file’s timestamp was constant. Now, when creating `version_frozen.py`, it will have the same timestamp as the commit when it was created. (`version_frozen.py` is moved to a dedicated layer). Reusing "builder" cache when building "dist" could be slow (CD reports 2 seconds, but locally I've seen it take up to 10 seconds), so the Dockerfile is now split and we save a couple steps by importing the "builder" image directly. The last changes made it possible to remove the layer cache in "builder", since the overhead is now greater than building the layers from scratch. Until now, all "dist" layers were squashed into a single layer, which in most cases is a good idea (except for storage/delivery pricing/overhead), but in our case, since we manage the entire pipeline, we can ignore this and share layers between builds. This means (for example) that if we change files unrelated to the container in several consecutive commits (documentation changes), we don't have to push the entire image to registry, but only the different layers (`version_frozen.py` in this example). The same applies when pulling, as only the layers that have changed compared to the local layers will be downloaded (that's the theory, we'll see if this works as expected or if we need to tweak something else). --- container/Dockerfile | 61 -------------------------------------------- container/builder.dockerfile | 24 +++++++++++++++++ container/dist.dockerfile | 44 ++++++++++++++++++++++++++++++++ container/entrypoint.sh | 2 +- container/template/.empty | 0 5 files changed, 69 insertions(+), 62 deletions(-) delete mode 100644 container/Dockerfile create mode 100644 container/builder.dockerfile create mode 100644 container/dist.dockerfile delete mode 100644 container/template/.empty (limited to 'container') diff --git a/container/Dockerfile b/container/Dockerfile deleted file mode 100644 index 1010e2523..000000000 --- a/container/Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -FROM ghcr.io/searxng/base:searxng-builder AS builder - -COPY ./requirements*.txt ./ - -RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ - && . ./venv/bin/activate \ - && pip install -r requirements.txt \ - && pip install -r requirements-server.txt - -COPY ./searx/ ./searx/ - -ARG TIMESTAMP_SETTINGS="0" - -RUN python -m compileall -q searx \ - && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \ - && find ./searx/static \ - \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" -o -name "*.ttf" -o -name "*.eot" \) \ - -type f -exec gzip -9 -k {} + -exec brotli --best {} + - -FROM ghcr.io/searxng/base:searxng AS dist - -ARG LABEL_DATE="0001-01-01T00:00:00Z" -ARG GIT_URL="unspecified" -ARG SEARXNG_GIT_VERSION="unspecified" -ARG LABEL_VCS_REF="unspecified" -ARG LABEL_VCS_URL="unspecified" - -COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ -COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ -COPY --chown=searxng:searxng ./container/template/ ./.template/ -COPY --chown=searxng:searxng ./container/entrypoint.sh ./entrypoint.sh - -LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ - org.opencontainers.image.created="$LABEL_DATE" \ - org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \ - org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \ - org.opencontainers.image.licenses="AGPL-3.0-or-later" \ - org.opencontainers.image.revision="$LABEL_VCS_REF" \ - org.opencontainers.image.source="$LABEL_VCS_URL" \ - org.opencontainers.image.title="searxng" \ - org.opencontainers.image.url="$LABEL_VCS_URL" \ - org.opencontainers.image.version="$SEARXNG_GIT_VERSION" - -ENV SEARXNG_VERSION="$SEARXNG_GIT_VERSION" \ - SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \ - GRANIAN_PROCESS_NAME="searxng" \ - GRANIAN_INTERFACE="wsgi" \ - GRANIAN_HOST="::" \ - GRANIAN_PORT="8080" \ - GRANIAN_WEBSOCKETS="false" \ - GRANIAN_LOOP="uvloop" \ - GRANIAN_BLOCKING_THREADS="4" \ - GRANIAN_WORKERS_KILL_TIMEOUT="30s" \ - GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m" - -VOLUME $CONFIG_PATH -VOLUME $DATA_PATH - -EXPOSE 8080 - -ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"] diff --git a/container/builder.dockerfile b/container/builder.dockerfile new file mode 100644 index 000000000..dc2279dd9 --- /dev/null +++ b/container/builder.dockerfile @@ -0,0 +1,24 @@ +FROM ghcr.io/searxng/base:searxng-builder AS builder + +COPY ./requirements*.txt ./ + +RUN --mount=type=cache,id=pip,target=/root/.cache/pip set -eux; \ + python -m venv ./.venv/; \ + . ./.venv/bin/activate; \ + pip install -r ./requirements.txt -r ./requirements-server.txt + +COPY ./searx/ ./searx/ + +ARG TIMESTAMP_SETTINGS="0" + +RUN set -eux; \ + python -m compileall -q ./searx/; \ + touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml; \ + find ./searx/static/ -type f \ + \( -name "*.html" -o -name "*.css" -o -name "*.js" -o -name "*.svg" \) \ + -exec gzip -9 -k {} + \ + -exec brotli -9 -k {} + \ + -exec gzip --test {}.gz + \ + -exec brotli --test {}.br +; \ + # Move always changing files to /usr/local/searxng/ + mv ./searx/version_frozen.py ./ diff --git a/container/dist.dockerfile b/container/dist.dockerfile new file mode 100644 index 000000000..326f62b0d --- /dev/null +++ b/container/dist.dockerfile @@ -0,0 +1,44 @@ +FROM ghcr.io/searxng/base:searxng AS dist + +ARG CONTAINER_IMAGE_ORGANIZATION="searxng" +ARG CONTAINER_IMAGE_NAME="searxng" + +COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/.venv/ ./.venv/ +COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/searx/ ./searx/ +COPY --chown=searxng:searxng ./container/ ./ +COPY --chown=searxng:searxng --from=localhost/$CONTAINER_IMAGE_ORGANIZATION/$CONTAINER_IMAGE_NAME:builder /usr/local/searxng/version_frozen.py ./searx/ + +ARG CREATED="0001-01-01T00:00:00Z" +ARG VERSION="unknown" +ARG VCS_URL="unknown" +ARG VCS_REVISION="unknown" + +LABEL org.opencontainers.image.created="$CREATED" \ + org.opencontainers.image.description="SearXNG is a metasearch engine. Users are neither tracked nor profiled." \ + org.opencontainers.image.documentation="https://docs.searxng.org/admin/installation-docker" \ + org.opencontainers.image.licenses="AGPL-3.0-or-later" \ + org.opencontainers.image.revision="$VCS_REVISION" \ + org.opencontainers.image.source="$VCS_URL" \ + org.opencontainers.image.title="SearXNG" \ + org.opencontainers.image.url="https://searxng.org" \ + org.opencontainers.image.version="$VERSION" + +ENV SEARXNG_VERSION="$VERSION" \ + SEARXNG_SETTINGS_PATH="$CONFIG_PATH/settings.yml" \ + GRANIAN_PROCESS_NAME="searxng" \ + GRANIAN_INTERFACE="wsgi" \ + GRANIAN_HOST="::" \ + GRANIAN_PORT="8080" \ + GRANIAN_WEBSOCKETS="false" \ + GRANIAN_LOOP="uvloop" \ + GRANIAN_BLOCKING_THREADS="4" \ + GRANIAN_WORKERS_KILL_TIMEOUT="30s" \ + GRANIAN_BLOCKING_THREADS_IDLE_TIMEOUT="5m" + +# "*_PATH" ENVs are defined in base images +VOLUME $CONFIG_PATH +VOLUME $DATA_PATH + +EXPOSE 8080 + +ENTRYPOINT ["/usr/local/searxng/entrypoint.sh"] diff --git a/container/entrypoint.sh b/container/entrypoint.sh index 7a09cce13..2e45bca21 100755 --- a/container/entrypoint.sh +++ b/container/entrypoint.sh @@ -127,4 +127,4 @@ volume_handler "$DATA_PATH" # Check for files config_handler "$SEARXNG_SETTINGS_PATH" "/usr/local/searxng/searx/settings.yml" -exec /usr/local/searxng/venv/bin/granian searx.webapp:app +exec /usr/local/searxng/.venv/bin/granian searx.webapp:app diff --git a/container/template/.empty b/container/template/.empty deleted file mode 100644 index e69de29bb..000000000 -- cgit v1.2.3