diff --git a/scripts/images/backend-python/Dockerfile b/scripts/images/backend-python/Dockerfile index cf24083cd..7819f92cc 100644 --- a/scripts/images/backend-python/Dockerfile +++ b/scripts/images/backend-python/Dockerfile @@ -26,7 +26,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ POETRY_VERSION=2.2.1 \ POETRY_NO_INTERACTION=1 \ POETRY_VIRTUALENVS_CREATE=false \ - POETRY_CACHE_DIR=/tmp/poetry_cache + POETRY_CACHE_DIR=/tmp/poetry_cache \ + TIKTOKEN_CACHE_DIR=/opt/tiktoken-cache ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk @@ -49,6 +50,10 @@ COPY runtime/datamate-python/pyproject.toml runtime/datamate-python/poetry.lock* RUN --mount=type=cache,target=$POETRY_CACHE_DIR \ poetry install --no-root --only main +# Preload the tokenizer vocabulary so knowledge-base operations work offline. +RUN mkdir -p "$TIKTOKEN_CACHE_DIR" \ + && python -c "import tiktoken; [tiktoken.get_encoding(name) for name in ('cl100k_base', 'o200k_base')]" + # Download NLTK data RUN python -c "import nltk; nltk.download(['punkt_tab','averaged_perceptron_tagger_eng'], download_dir='/usr/local/nltk_data')" ENV NLTK_DATA=/usr/local/nltk_data