From f2707eb2688a6ac04619daa447a186c04633074c Mon Sep 17 00:00:00 2001 From: cosmincc <22967286+cosmincc@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:09:40 +0200 Subject: [PATCH] llama-cpp: support every Linux prebuilt backend var_backend now covers all Linux builds llama.cpp publishes, each with the runtime its release workflow builds against: - cuda: CUDA 12 or 13 build chosen from the host driver (<580 -> 12), plus the matching cudart tarball deployed into /opt/llama-cpp - rocm: ubuntu-rocm--x64 with the same ROCm release of AMD's TheRock wheels (rocm[libraries]) in a uv venv, added to LD_LIBRARY_PATH; updates follow upstream when the build moves to a newer ROCm - sycl-fp16 / sycl-fp32: oneAPI runtime libraries from Intel's apt repo, registered with ld.so; Level Zero loader from oneapi-src/level-zero and libze-intel-gpu1 from intel/compute-runtime - openvino: libtbb12, OpenCL ICD loader and Level Zero; GGML_OPENVINO_DEVICE is set to GPU when a render node is present auto stays the default (Vulkan with a GPU, otherwise CPU). The rocm->vulkan remap and the "no Linux ROCm/CUDA build" comments are gone, since upstream now ships both. Release lookups use the b tag prefix, the only tags carrying binaries, and updates keep the recorded backend. The install-time variables are now exported from the CT script so app_vars actually reach the container. Co-Authored-By: Claude Opus 5.5 (1M context) --- ct/llama-cpp.sh | 60 +++++++++++-- install/llama-cpp-install.sh | 168 +++++++++++++++++++++++++++++------ json/llama-cpp.json | 11 ++- 3 files changed, 200 insertions(+), 39 deletions(-) diff --git a/ct/llama-cpp.sh b/ct/llama-cpp.sh index 37ca9c27f5..6a1c70a54b 100644 --- a/ct/llama-cpp.sh +++ b/ct/llama-cpp.sh @@ -19,6 +19,13 @@ var_gpu="${var_gpu:-yes}" var_unprivileged="${var_unprivileged:-1}" var_testurl="${var_testurl:-https://github.com/community-scripts/DevScripts/issues/2124}" +export var_backend="${var_backend:-}" +export var_model_repo="${var_model_repo:-}" +export var_ctx_size="${var_ctx_size:-}" +export var_offload_layers="${var_offload_layers:-}" +export var_api_key="${var_api_key:-}" +export var_port="${var_port:-}" + header_info "$APP" variables color @@ -34,26 +41,61 @@ function update_script() { exit fi - if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp"; then + # Only the bNNNN tags carry the prebuilt binaries; other tags have no assets. + if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp" "" "" "b"; then msg_info "Stopping Service" systemctl stop llama-cpp msg_ok "Stopped Service" # Keep the build variant the install picked; the CPU tarball would otherwise - # overwrite a Vulkan install and quietly end GPU offload. - # - # rocm reads as vulkan: upstream publishes no Linux ROCm build, so anything - # recorded as rocm predates that being noticed and would otherwise fail - # every update on an asset that does not exist. + # overwrite a GPU build and quietly end offload. LLAMA_BACKEND="$(cat /opt/llama-cpp_data/.backend 2>/dev/null || echo cpu)" + LLAMA_ARCH="$(arch_resolve x64 arm64)" + # Installs from before the ROCm build existed recorded rocm while actually + # running the Vulkan build. + [[ "$LLAMA_BACKEND" == "rocm" && ! -d /opt/llama-cpp_data/rocm ]] && LLAMA_BACKEND="vulkan" + + LLAMA_CUDART_ASSET="" case "$LLAMA_BACKEND" in - vulkan | rocm) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-$(arch_resolve x64 arm64).tar.gz" ;; - *) LLAMA_ASSET="llama-*-bin-ubuntu-$(arch_resolve x64 arm64).tar.gz" ;; + vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;; + cuda-12 | cuda-13) + LLAMA_ASSET="llama-*-bin-ubuntu-cuda-${LLAMA_BACKEND#cuda-}.*-${LLAMA_ARCH}.tar.gz" + LLAMA_CUDART_ASSET="cudart-${LLAMA_ASSET}" + ;; + rocm) LLAMA_ASSET="llama-*-bin-ubuntu-rocm-*-x64.tar.gz" ;; + sycl-fp16 | sycl-fp32) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_BACKEND}-x64.tar.gz" ;; + openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;; + *) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;; esac - CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET" + CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "${CHECK_UPDATE_RELEASE}" "/opt/llama-cpp" "$LLAMA_ASSET" "b" + # The wipe above also removed the CUDA runtime, so it always goes back in. + if [[ -n "$LLAMA_CUDART_ASSET" ]]; then + rm -f ~/.llama-cpp-cudart + fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "${CHECK_UPDATE_RELEASE}" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET" "b" + fi chmod +x /opt/llama-cpp/llama-* + case "$LLAMA_BACKEND" in + sycl-* | openvino) + fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*+u24.04_amd64.deb" || true + if [[ -d /dev/dri ]]; then + fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true + fi + ;; + rocm) + # The ROCm build links against the ROCm release it was built with, so + # follow it when upstream moves to a newer one. + ROCM_VERSION="$(jq -r 'if type == "array" then .[0] else . end | .assets[].name' "$TOOLS_GH_REL_JSON" 2>/dev/null | grep -oP 'bin-ubuntu-rocm-\K[0-9.]+(?=-x64)' | head -1)" + if [[ -n "$ROCM_VERSION" && "$ROCM_VERSION" != "$(cat /opt/llama-cpp_data/rocm/.version 2>/dev/null)" ]]; then + msg_info "Updating ROCm Runtime to ${ROCM_VERSION}" + $STD uv pip install --python /opt/llama-cpp_data/rocm --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries]==${ROCM_VERSION}.*" + echo "$ROCM_VERSION" >/opt/llama-cpp_data/rocm/.version + msg_ok "Updated ROCm Runtime to ${ROCM_VERSION}" + fi + ;; + esac + msg_info "Starting Service" systemctl start llama-cpp msg_ok "Started Service" diff --git a/install/llama-cpp-install.sh b/install/llama-cpp-install.sh index 9aff3556e1..8f7e3a258c 100644 --- a/install/llama-cpp-install.sh +++ b/install/llama-cpp-install.sh @@ -13,23 +13,33 @@ setting_up_container network_check update_os -var_backend="${var_backend:-auto}" var_model_repo="${var_model_repo:-ggml-org/gemma-3-1b-it-GGUF}" var_port="${var_port:-8080}" var_ctx_size="${var_ctx_size:-4096}" var_offload_layers="${var_offload_layers:-}" var_api_key="${var_api_key:-}" -# Which prebuilt upstream ships decides what the container can offload to, and -# for Linux that list is short: ubuntu-x64, ubuntu-arm64, ubuntu-vulkan-x64, -# ubuntu-vulkan-arm64, plus the Intel sycl and openvino builds. There is no -# Linux CUDA build and no Linux ROCm build either -- the only ROCm asset -# upstream publishes is win-rocm-*-x64.zip. Vulkan is therefore the GPU path for -# every vendor here, AMD included, and it reaches an AMD card through RADV. -# -# Asking for a ubuntu-rocm asset failed the install outright with "No asset -# matching 'llama--bin-ubuntu-rocm--x64.tar.gz' found" on any host where the -# engine had set up /dev/kfd. +# The Linux prebuilts upstream publishes (see .github/workflows/release.yml): +# cpu ubuntu-{x64,arm64} +# vulkan ubuntu-vulkan-{x64,arm64} any GPU vendor (AMD through RADV) +# cuda ubuntu-cuda-{12.x,13.x}-x64, ubuntu-cuda-13.x-arm64, plus a cudart tarball +# rocm ubuntu-rocm--x64 needs the matching ROCm (TheRock) runtime +# sycl ubuntu-sycl-{fp16,fp32}-x64 Intel GPUs, needs oneAPI + Level Zero +# openvino ubuntu-openvino--x64 Intel CPU/GPU, OpenVINO runtime is bundled +if [[ -z "${var_backend:-}" ]]; then + LLAMA_CHOICE=$(prompt_select "Select the llama.cpp build:" 1 60 \ + "auto - Vulkan when a GPU is passed through, otherwise CPU" \ + "cpu - CPU only" \ + "vulkan - Vulkan (Intel, AMD and NVIDIA GPUs)" \ + "cuda - NVIDIA CUDA" \ + "rocm - AMD ROCm" \ + "sycl-fp16 - Intel GPU via oneAPI SYCL (FP16)" \ + "sycl-fp32 - Intel GPU via oneAPI SYCL (FP32)" \ + "openvino - Intel CPU/GPU via OpenVINO") + var_backend="${LLAMA_CHOICE%% *}" +fi +var_backend="${var_backend:-auto}" + if [[ "$var_backend" == "auto" ]]; then if [[ -e /dev/kfd || -e /dev/nvidia0 || -d /dev/dri ]]; then var_backend="vulkan" @@ -37,37 +47,116 @@ if [[ "$var_backend" == "auto" ]]; then var_backend="cpu" fi fi - -# rocm was a documented value, so keep accepting it rather than failing anyone -# who pinned it -- but say where they are actually being sent. -if [[ "$var_backend" == "rocm" ]]; then - msg_warn "llama.cpp publishes no Linux ROCm build - using the Vulkan build, which drives an AMD card through RADV" - var_backend="vulkan" -fi +[[ "$var_backend" == "sycl" ]] && var_backend="sycl-fp16" LLAMA_ARCH="$(arch_resolve x64 arm64)" - case "$var_backend" in -vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;; +rocm | sycl-fp16 | sycl-fp32 | openvino) + if [[ "$LLAMA_ARCH" != "x64" ]]; then + msg_error "llama.cpp publishes the ${var_backend} build for x64 only" + exit 1 + fi + ;; +cpu | vulkan | cuda) ;; *) - var_backend="cpu" - LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" + msg_error "Unknown var_backend '${var_backend}' (expected auto, cpu, vulkan, cuda, rocm, sycl-fp16, sycl-fp32 or openvino)" + exit 1 ;; esac msg_info "Installing Dependencies" LLAMA_DEPS=(libgomp1) -[[ "$var_backend" == "vulkan" ]] && LLAMA_DEPS+=(libvulkan1 mesa-vulkan-drivers vulkan-tools) +case "$var_backend" in +vulkan) LLAMA_DEPS+=(libvulkan1 mesa-vulkan-drivers vulkan-tools) ;; +openvino) LLAMA_DEPS+=(libtbb12 ocl-icd-libopencl1) ;; +sycl-*) LLAMA_DEPS+=(ocl-icd-libopencl1) ;; +esac $STD apt install -y "${LLAMA_DEPS[@]}" msg_ok "Installed Dependencies" setup_hwaccel -fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET" +# SYCL and OpenVINO reach an Intel GPU through Level Zero. release.yml builds +# SYCL against the loader from oneapi-src/level-zero, so use the same package. +if [[ "$var_backend" == sycl-* || "$var_backend" == "openvino" ]]; then + fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*+u24.04_amd64.deb" + if [[ -d /dev/dri ]]; then + fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true + fi +fi + +# The SYCL tarball ships only llama.cpp itself; the DPC++/oneMKL/oneDNN/TBB +# libraries it links against come from oneAPI. release.yml builds with the full +# toolkit, but running needs only the runtime libraries. +if [[ "$var_backend" == sycl-* ]]; then + setup_deb822_repo \ + "oneapi" \ + "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB" \ + "https://apt.repos.intel.com/oneapi" \ + "all" \ + "main" + msg_info "Installing Intel oneAPI Runtime" + $STD apt install -y intel-oneapi-runtime-libs + # The runtime packages unpack under /opt/intel/oneapi without telling the + # dynamic linker, so register every directory that holds one of them. + find /opt/intel/oneapi -type f \( -name 'libsycl.so*' -o -name 'libur_loader.so*' -o -name 'libmkl_core.so*' \ + -o -name 'libdnnl.so*' -o -name 'libtbb.so*' -o -name 'libsvml.so' -o -name 'libiomp5.so' -o -name 'libumf.so*' \) \ + -printf '%h\n' 2>/dev/null | sort -u >/etc/ld.so.conf.d/oneapi-llama-cpp.conf + ldconfig + msg_ok "Installed Intel oneAPI Runtime" +fi + +LLAMA_CUDART_ASSET="" +case "$var_backend" in +vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;; +cuda) + # CUDA 13 needs a 580+ host driver; anything older gets the CUDA 12 build, + # which upstream publishes for x64 only. + NVIDIA_DRIVER_MAJOR="$(grep -oP '\d{3,}(?=\.\d+)' /proc/driver/nvidia/version 2>/dev/null | head -1)" + if [[ "$LLAMA_ARCH" == "x64" && -n "$NVIDIA_DRIVER_MAJOR" && "$NVIDIA_DRIVER_MAJOR" -lt 580 ]]; then + var_backend="cuda-12" + else + var_backend="cuda-13" + fi + [[ -z "$NVIDIA_DRIVER_MAJOR" ]] && msg_warn "No NVIDIA host driver visible - installing the CUDA 13 build, which needs driver 580 or newer" + LLAMA_ASSET="llama-*-bin-ubuntu-cuda-${var_backend#cuda-}.*-${LLAMA_ARCH}.tar.gz" + LLAMA_CUDART_ASSET="cudart-${LLAMA_ASSET}" + ;; +rocm) LLAMA_ASSET="llama-*-bin-ubuntu-rocm-*-x64.tar.gz" ;; +sycl-fp16 | sycl-fp32) LLAMA_ASSET="llama-*-bin-ubuntu-${var_backend}-x64.tar.gz" ;; +openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;; +*) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;; +esac + +# Only the bNNNN tags carry the prebuilt binaries; other tags have no assets. +fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET" "b" +LLAMA_RELEASE="$(cat ~/.llama-cpp)" +if [[ -n "$LLAMA_CUDART_ASSET" ]]; then + fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET" "b" +fi + +mkdir -p /opt/llama-cpp_data/models +LLAMA_LIBPATH="/opt/llama-cpp" + +# The ROCm build carries no ROCm libraries. release.yml builds it against AMD's +# TheRock wheels, so install the same ROCm release of those as the runtime. +if [[ "$var_backend" == "rocm" ]]; then + ROCM_VERSION="$(jq -r --arg t "$LLAMA_RELEASE" 'if type == "array" then .[] else . end | select(.tag_name == $t) | .assets[].name' "$TOOLS_GH_REL_JSON" 2>/dev/null | grep -oP 'bin-ubuntu-rocm-\K[0-9.]+(?=-x64)' | head -1)" + if [[ -z "$ROCM_VERSION" ]]; then + msg_error "Could not read the ROCm release the llama.cpp ${LLAMA_RELEASE} build targets" + exit 1 + fi + PYTHON_VERSION="3.12" setup_uv + msg_info "Installing ROCm ${ROCM_VERSION} Runtime" + $STD uv venv --python 3.12 /opt/llama-cpp_data/rocm + $STD uv pip install --python /opt/llama-cpp_data/rocm --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries]==${ROCM_VERSION}.*" + echo "$ROCM_VERSION" >/opt/llama-cpp_data/rocm/.version + LLAMA_LIBPATH="${LLAMA_LIBPATH}:$(/opt/llama-cpp_data/rocm/bin/rocm-sdk path --root)/lib" + msg_ok "Installed ROCm ${ROCM_VERSION} Runtime" +fi msg_info "Configuring llama.cpp" chmod +x /opt/llama-cpp/llama-* -mkdir -p /opt/llama-cpp_data/models echo "$var_backend" >/opt/llama-cpp_data/.backend if [[ -z "$var_offload_layers" ]]; then [[ "$var_backend" == "cpu" ]] && var_offload_layers=0 || var_offload_layers=999 @@ -85,6 +174,27 @@ LLAMA_CACHE=/opt/llama-cpp_data/models LLAMA_ARG_CTX_SIZE=${var_ctx_size} LLAMA_ARG_N_GPU_LAYERS=${var_offload_layers} EOF +case "$var_backend" in +sycl-*) + cat <>/opt/llama-cpp.env + +# SYCL: cache JIT-compiled kernels across restarts; pick GPUs with the selector +SYCL_CACHE_PERSISTENT=1 +SYCL_CACHE_DIR=/opt/llama-cpp_data/sycl-cache +#ONEAPI_DEVICE_SELECTOR=level_zero:0 +EOF + ;; +openvino) + OV_DEVICE="CPU" + compgen -G "/dev/dri/renderD*" >/dev/null && OV_DEVICE="GPU" + cat <>/opt/llama-cpp.env + +# OpenVINO device: CPU, GPU (GPU.0, GPU.1, ...) or NPU +GGML_OPENVINO_DEVICE=${OV_DEVICE} +GGML_OPENVINO_CACHE_DIR=/opt/llama-cpp_data/openvino-cache +EOF + ;; +esac # An empty LLAMA_ARG_API_KEY makes llama-server demand an empty bearer token. if [[ -n "$var_api_key" ]]; then echo "LLAMA_ARG_API_KEY=${var_api_key}" >>/opt/llama-cpp.env @@ -95,10 +205,14 @@ chmod 600 /opt/llama-cpp.env msg_ok "Configured llama.cpp (${var_backend} build)" if [[ "$var_backend" != "cpu" ]]; then - if LD_LIBRARY_PATH=/opt/llama-cpp ldd /opt/llama-cpp/llama-server 2>/dev/null | grep -q "not found"; then + if LD_LIBRARY_PATH="$LLAMA_LIBPATH" ldd /opt/llama-cpp/llama-server /opt/llama-cpp/libggml-*.so 2>/dev/null | grep -q "not found"; then msg_warn "The ${var_backend} build is missing shared libraries in this container - it will fall back to the CPU" elif [[ "$var_backend" == "vulkan" ]] && ! vulkaninfo --summary >/dev/null 2>&1; then msg_warn "Vulkan build installed but no Vulkan device is visible - check GPU passthrough and the driver libraries in the container" + elif [[ "$var_backend" == cuda-* ]] && ! ldconfig -p | grep -q 'libcuda[.]so'; then + msg_warn "CUDA build installed but the NVIDIA driver library (libcuda) is missing - check GPU passthrough" + elif [[ "$var_backend" == sycl-* || "$var_backend" == "rocm" ]] && [[ ! -d /dev/dri && ! -e /dev/kfd ]]; then + msg_warn "${var_backend} build installed but no GPU is passed through - it will fall back to the CPU" fi fi @@ -114,7 +228,7 @@ Type=simple User=root WorkingDirectory=/opt/llama-cpp EnvironmentFile=/opt/llama-cpp.env -Environment=LD_LIBRARY_PATH=/opt/llama-cpp +Environment=LD_LIBRARY_PATH=${LLAMA_LIBPATH} ExecStart=/opt/llama-cpp/llama-server Restart=on-failure RestartSec=10 diff --git a/json/llama-cpp.json b/json/llama-cpp.json index d856ac36d8..f7b8a26ffd 100644 --- a/json/llama-cpp.json +++ b/json/llama-cpp.json @@ -43,10 +43,15 @@ "options": [ "auto", "cpu", - "vulkan" + "vulkan", + "cuda", + "rocm", + "sycl-fp16", + "sycl-fp32", + "openvino" ], "default": "auto", - "help": "auto takes vulkan for any passed-through GPU (AMD via RADV, NVIDIA and Intel too), cpu otherwise. Upstream ships no Linux ROCm or CUDA build." + "help": "auto takes vulkan for any passed-through GPU, cpu otherwise. cuda picks the CUDA 12 or 13 build from the host driver; rocm, sycl-fp16/fp32 and openvino are x64 only." }, { "name": "var_model_repo", @@ -100,7 +105,7 @@ "type": "warning" }, { - "text": "The build variant is chosen at install time: cpu or vulkan, or auto to follow the passed-through GPU. Upstream publishes no Linux CUDA or ROCm build, so every GPU is served by the vulkan build - AMD through RADV, NVIDIA through the driver's Vulkan ICD, which has to be present in the container. The choice is remembered in /opt/llama-cpp_data/.backend so updates keep it.", + "text": "The build variant is chosen at install time: cpu, vulkan (any GPU vendor), cuda (NVIDIA; CUDA 13 with a 580+ host driver, CUDA 12 otherwise), rocm (AMD, with the matching ROCm runtime from AMD's TheRock wheels), sycl-fp16/sycl-fp32 (Intel GPU, with the oneAPI runtime and Level Zero) or openvino (Intel CPU/GPU), or auto to take vulkan when a GPU is passed through. The choice is remembered in /opt/llama-cpp_data/.backend so updates keep it.", "type": "info" }, {