llama-cpp: support every Linux prebuilt backend (CUDA, ROCm, SYCL, OpenVINO)

ct/llama-cpp.sh

@@ -19,6 +19,13 @@ var_gpu="${var_gpu:-yes}"
 var_unprivileged="${var_unprivileged:-1}"
 var_testurl="${var_testurl:-https://github.com/community-scripts/DevScripts/issues/2124}"
 
+export var_backend="${var_backend:-}"
+export var_model_repo="${var_model_repo:-}"
+export var_ctx_size="${var_ctx_size:-}"
+export var_offload_layers="${var_offload_layers:-}"
+export var_api_key="${var_api_key:-}"
+export var_port="${var_port:-}"
+
 header_info "$APP"
 variables
 color
@@ -34,26 +41,61 @@ function update_script() {
     exit
   fi
 
-  if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp"; then
+  # Only the bNNNN tags carry the prebuilt binaries; other tags have no assets.
+  if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp" "" "" "b"; then
     msg_info "Stopping Service"
     systemctl stop llama-cpp
     msg_ok "Stopped Service"
 
     # Keep the build variant the install picked; the CPU tarball would otherwise
-    # overwrite a Vulkan install and quietly end GPU offload.
-    #
-    # rocm reads as vulkan: upstream publishes no Linux ROCm build, so anything
-    # recorded as rocm predates that being noticed and would otherwise fail
-    # every update on an asset that does not exist.
+    # overwrite a GPU build and quietly end offload.
     LLAMA_BACKEND="$(cat /opt/llama-cpp_data/.backend 2>/dev/null || echo cpu)"
+    LLAMA_ARCH="$(arch_resolve x64 arm64)"
+    # Installs from before the ROCm build existed recorded rocm while actually
+    # running the Vulkan build.
+    [[ "$LLAMA_BACKEND" == "rocm" && ! -d /opt/llama-cpp_data/rocm ]] && LLAMA_BACKEND="vulkan"
+
+    LLAMA_CUDART_ASSET=""
     case "$LLAMA_BACKEND" in
-    vulkan | rocm) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-$(arch_resolve x64 arm64).tar.gz" ;;
-    *) LLAMA_ASSET="llama-*-bin-ubuntu-$(arch_resolve x64 arm64).tar.gz" ;;
+    vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
+    cuda-12 | cuda-13)
+      LLAMA_ASSET="llama-*-bin-ubuntu-cuda-${LLAMA_BACKEND#cuda-}.*-${LLAMA_ARCH}.tar.gz"
+      LLAMA_CUDART_ASSET="cudart-${LLAMA_ASSET}"
+      ;;
+    rocm) LLAMA_ASSET="llama-*-bin-ubuntu-rocm-*-x64.tar.gz" ;;
+    sycl-fp16 | sycl-fp32) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_BACKEND}-x64.tar.gz" ;;
+    openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;;
+    *) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;;
     esac
 
-    CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET"
+    CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "${CHECK_UPDATE_RELEASE}" "/opt/llama-cpp" "$LLAMA_ASSET" "b"
+    # The wipe above also removed the CUDA runtime, so it always goes back in.
+    if [[ -n "$LLAMA_CUDART_ASSET" ]]; then
+      rm -f ~/.llama-cpp-cudart
+      fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "${CHECK_UPDATE_RELEASE}" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET" "b"
+    fi
     chmod +x /opt/llama-cpp/llama-*
 
+    case "$LLAMA_BACKEND" in
+    sycl-* | openvino)
+      fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*+u24.04_amd64.deb" || true
+      if [[ -d /dev/dri ]]; then
+        fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true
+      fi
+      ;;
+    rocm)
+      # The ROCm build links against the ROCm release it was built with, so
+      # follow it when upstream moves to a newer one.
+      ROCM_VERSION="$(jq -r 'if type == "array" then .[0] else . end | .assets[].name' "$TOOLS_GH_REL_JSON" 2>/dev/null | grep -oP 'bin-ubuntu-rocm-\K[0-9.]+(?=-x64)' | head -1)"
+      if [[ -n "$ROCM_VERSION" && "$ROCM_VERSION" != "$(cat /opt/llama-cpp_data/rocm/.version 2>/dev/null)" ]]; then
+        msg_info "Updating ROCm Runtime to ${ROCM_VERSION}"
+        $STD uv pip install --python /opt/llama-cpp_data/rocm --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries]==${ROCM_VERSION}.*"
+        echo "$ROCM_VERSION" >/opt/llama-cpp_data/rocm/.version
+        msg_ok "Updated ROCm Runtime to ${ROCM_VERSION}"
+      fi
+      ;;
+    esac
+
     msg_info "Starting Service"
     systemctl start llama-cpp
     msg_ok "Started Service"

install/llama-cpp-install.sh

@@ -13,61 +13,150 @@ setting_up_container
 network_check
 update_os
 
-var_backend="${var_backend:-auto}"
 var_model_repo="${var_model_repo:-ggml-org/gemma-3-1b-it-GGUF}"
 var_port="${var_port:-8080}"
 var_ctx_size="${var_ctx_size:-4096}"
 var_offload_layers="${var_offload_layers:-}"
 var_api_key="${var_api_key:-}"
 
-# Which prebuilt upstream ships decides what the container can offload to, and
-# for Linux that list is short: ubuntu-x64, ubuntu-arm64, ubuntu-vulkan-x64,
-# ubuntu-vulkan-arm64, plus the Intel sycl and openvino builds. There is no
-# Linux CUDA build and no Linux ROCm build either -- the only ROCm asset
-# upstream publishes is win-rocm-*-x64.zip. Vulkan is therefore the GPU path for
-# every vendor here, AMD included, and it reaches an AMD card through RADV.
-#
-# Asking for a ubuntu-rocm asset failed the install outright with "No asset
-# matching 'llama--bin-ubuntu-rocm--x64.tar.gz' found" on any host where the
-# engine had set up /dev/kfd.
+# The Linux prebuilts upstream publishes (see .github/workflows/release.yml):
+#   cpu       ubuntu-{x64,arm64}
+#   vulkan    ubuntu-vulkan-{x64,arm64}       any GPU vendor (AMD through RADV)
+#   cuda      ubuntu-cuda-{12.x,13.x}-x64, ubuntu-cuda-13.x-arm64, plus a cudart tarball
+#   rocm      ubuntu-rocm-<ver>-x64           needs the matching ROCm (TheRock) runtime
+#   sycl      ubuntu-sycl-{fp16,fp32}-x64     Intel GPUs, needs oneAPI + Level Zero
+#   openvino  ubuntu-openvino-<ver>-x64       Intel CPU/GPU, OpenVINO runtime is bundled
+if [[ -z "${var_backend:-}" ]]; then
+  LLAMA_CHOICE=$(prompt_select "Select the llama.cpp build:" 1 60 \
+    "auto      - Vulkan when a GPU is passed through, otherwise CPU" \
+    "cpu       - CPU only" \
+    "vulkan    - Vulkan (Intel, AMD and NVIDIA GPUs)" \
+    "cuda      - NVIDIA CUDA" \
+    "rocm      - AMD ROCm" \
+    "sycl-fp16 - Intel GPU via oneAPI SYCL (FP16)" \
+    "sycl-fp32 - Intel GPU via oneAPI SYCL (FP32)" \
+    "openvino  - Intel CPU/GPU via OpenVINO")
+  var_backend="${LLAMA_CHOICE%% *}"
+fi
+var_backend="${var_backend:-auto}"
+
 if [[ "$var_backend" == "auto" ]]; then
   if [[ -e /dev/kfd || -e /dev/nvidia0 || -d /dev/dri ]]; then
     var_backend="vulkan"
   else
     var_backend="cpu"
   fi
 fi
-
-# rocm was a documented value, so keep accepting it rather than failing anyone
-# who pinned it -- but say where they are actually being sent.
-if [[ "$var_backend" == "rocm" ]]; then
-  msg_warn "llama.cpp publishes no Linux ROCm build - using the Vulkan build, which drives an AMD card through RADV"
-  var_backend="vulkan"
-fi
+[[ "$var_backend" == "sycl" ]] && var_backend="sycl-fp16"
 
 LLAMA_ARCH="$(arch_resolve x64 arm64)"
-
 case "$var_backend" in
-vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
+rocm | sycl-fp16 | sycl-fp32 | openvino)
+  if [[ "$LLAMA_ARCH" != "x64" ]]; then
+    msg_error "llama.cpp publishes the ${var_backend} build for x64 only"
+    exit 1
+  fi
+  ;;
+cpu | vulkan | cuda) ;;
 *)
-  var_backend="cpu"
-  LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz"
+  msg_error "Unknown var_backend '${var_backend}' (expected auto, cpu, vulkan, cuda, rocm, sycl-fp16, sycl-fp32 or openvino)"
+  exit 1
   ;;
 esac
 
 msg_info "Installing Dependencies"
 LLAMA_DEPS=(libgomp1)
-[[ "$var_backend" == "vulkan" ]] && LLAMA_DEPS+=(libvulkan1 mesa-vulkan-drivers vulkan-tools)
+case "$var_backend" in
+vulkan) LLAMA_DEPS+=(libvulkan1 mesa-vulkan-drivers vulkan-tools) ;;
+openvino) LLAMA_DEPS+=(libtbb12 ocl-icd-libopencl1) ;;
+sycl-*) LLAMA_DEPS+=(ocl-icd-libopencl1) ;;
+esac
 $STD apt install -y "${LLAMA_DEPS[@]}"
 msg_ok "Installed Dependencies"
 
 setup_hwaccel
 
-fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET"
+# SYCL and OpenVINO reach an Intel GPU through Level Zero. release.yml builds
+# SYCL against the loader from oneapi-src/level-zero, so use the same package.
+if [[ "$var_backend" == sycl-* || "$var_backend" == "openvino" ]]; then
+  fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*+u24.04_amd64.deb"
+  if [[ -d /dev/dri ]]; then
+    fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true
+  fi
+fi
+
+# The SYCL tarball ships only llama.cpp itself; the DPC++/oneMKL/oneDNN/TBB
+# libraries it links against come from oneAPI. release.yml builds with the full
+# toolkit, but running needs only the runtime libraries.
+if [[ "$var_backend" == sycl-* ]]; then
+  setup_deb822_repo \
+    "oneapi" \
+    "https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB" \
+    "https://apt.repos.intel.com/oneapi" \
+    "all" \
+    "main"
+  msg_info "Installing Intel oneAPI Runtime"
+  $STD apt install -y intel-oneapi-runtime-libs
+  # The runtime packages unpack under /opt/intel/oneapi without telling the
+  # dynamic linker, so register every directory that holds one of them.
+  find /opt/intel/oneapi -type f \( -name 'libsycl.so*' -o -name 'libur_loader.so*' -o -name 'libmkl_core.so*' \
+    -o -name 'libdnnl.so*' -o -name 'libtbb.so*' -o -name 'libsvml.so' -o -name 'libiomp5.so' -o -name 'libumf.so*' \) \
+    -printf '%h\n' 2>/dev/null | sort -u >/etc/ld.so.conf.d/oneapi-llama-cpp.conf
+  ldconfig
+  msg_ok "Installed Intel oneAPI Runtime"
+fi
+
+LLAMA_CUDART_ASSET=""
+case "$var_backend" in
+vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
+cuda)
+  # CUDA 13 needs a 580+ host driver; anything older gets the CUDA 12 build,
+  # which upstream publishes for x64 only.
+  NVIDIA_DRIVER_MAJOR="$(grep -oP '\d{3,}(?=\.\d+)' /proc/driver/nvidia/version 2>/dev/null | head -1)"
+  if [[ "$LLAMA_ARCH" == "x64" && -n "$NVIDIA_DRIVER_MAJOR" && "$NVIDIA_DRIVER_MAJOR" -lt 580 ]]; then
+    var_backend="cuda-12"
+  else
+    var_backend="cuda-13"
+  fi
+  [[ -z "$NVIDIA_DRIVER_MAJOR" ]] && msg_warn "No NVIDIA host driver visible - installing the CUDA 13 build, which needs driver 580 or newer"
+  LLAMA_ASSET="llama-*-bin-ubuntu-cuda-${var_backend#cuda-}.*-${LLAMA_ARCH}.tar.gz"
+  LLAMA_CUDART_ASSET="cudart-${LLAMA_ASSET}"
+  ;;
+rocm) LLAMA_ASSET="llama-*-bin-ubuntu-rocm-*-x64.tar.gz" ;;
+sycl-fp16 | sycl-fp32) LLAMA_ASSET="llama-*-bin-ubuntu-${var_backend}-x64.tar.gz" ;;
+openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;;
+*) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;;
+esac
+
+# Only the bNNNN tags carry the prebuilt binaries; other tags have no assets.
+fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET" "b"
+LLAMA_RELEASE="$(cat ~/.llama-cpp)"
+if [[ -n "$LLAMA_CUDART_ASSET" ]]; then
+  fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET" "b"
+fi
+
+mkdir -p /opt/llama-cpp_data/models
+LLAMA_LIBPATH="/opt/llama-cpp"
+
+# The ROCm build carries no ROCm libraries. release.yml builds it against AMD's
+# TheRock wheels, so install the same ROCm release of those as the runtime.
+if [[ "$var_backend" == "rocm" ]]; then
+  ROCM_VERSION="$(jq -r --arg t "$LLAMA_RELEASE" 'if type == "array" then .[] else . end | select(.tag_name == $t) | .assets[].name' "$TOOLS_GH_REL_JSON" 2>/dev/null | grep -oP 'bin-ubuntu-rocm-\K[0-9.]+(?=-x64)' | head -1)"
+  if [[ -z "$ROCM_VERSION" ]]; then
+    msg_error "Could not read the ROCm release the llama.cpp ${LLAMA_RELEASE} build targets"
+    exit 1
+  fi
+  PYTHON_VERSION="3.12" setup_uv
+  msg_info "Installing ROCm ${ROCM_VERSION} Runtime"
+  $STD uv venv --python 3.12 /opt/llama-cpp_data/rocm
+  $STD uv pip install --python /opt/llama-cpp_data/rocm --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries]==${ROCM_VERSION}.*"
+  echo "$ROCM_VERSION" >/opt/llama-cpp_data/rocm/.version
+  LLAMA_LIBPATH="${LLAMA_LIBPATH}:$(/opt/llama-cpp_data/rocm/bin/rocm-sdk path --root)/lib"
+  msg_ok "Installed ROCm ${ROCM_VERSION} Runtime"
+fi
 
 msg_info "Configuring llama.cpp"
 chmod +x /opt/llama-cpp/llama-*
-mkdir -p /opt/llama-cpp_data/models
 echo "$var_backend" >/opt/llama-cpp_data/.backend
 if [[ -z "$var_offload_layers" ]]; then
   [[ "$var_backend" == "cpu" ]] && var_offload_layers=0 || var_offload_layers=999
@@ -85,6 +174,27 @@ LLAMA_CACHE=/opt/llama-cpp_data/models
 LLAMA_ARG_CTX_SIZE=${var_ctx_size}
 LLAMA_ARG_N_GPU_LAYERS=${var_offload_layers}
 EOF
+case "$var_backend" in
+sycl-*)
+  cat <<EOF >>/opt/llama-cpp.env
+
+# SYCL: cache JIT-compiled kernels across restarts; pick GPUs with the selector
+SYCL_CACHE_PERSISTENT=1
+SYCL_CACHE_DIR=/opt/llama-cpp_data/sycl-cache
+#ONEAPI_DEVICE_SELECTOR=level_zero:0
+EOF
+  ;;
+openvino)
+  OV_DEVICE="CPU"
+  compgen -G "/dev/dri/renderD*" >/dev/null && OV_DEVICE="GPU"
+  cat <<EOF >>/opt/llama-cpp.env
+
+# OpenVINO device: CPU, GPU (GPU.0, GPU.1, ...) or NPU
+GGML_OPENVINO_DEVICE=${OV_DEVICE}
+GGML_OPENVINO_CACHE_DIR=/opt/llama-cpp_data/openvino-cache
+EOF
+  ;;
+esac
 # An empty LLAMA_ARG_API_KEY makes llama-server demand an empty bearer token.
 if [[ -n "$var_api_key" ]]; then
   echo "LLAMA_ARG_API_KEY=${var_api_key}" >>/opt/llama-cpp.env
@@ -95,10 +205,14 @@ chmod 600 /opt/llama-cpp.env
 msg_ok "Configured llama.cpp (${var_backend} build)"
 
 if [[ "$var_backend" != "cpu" ]]; then
-  if LD_LIBRARY_PATH=/opt/llama-cpp ldd /opt/llama-cpp/llama-server 2>/dev/null | grep -q "not found"; then
+  if LD_LIBRARY_PATH="$LLAMA_LIBPATH" ldd /opt/llama-cpp/llama-server /opt/llama-cpp/libggml-*.so 2>/dev/null | grep -q "not found"; then
     msg_warn "The ${var_backend} build is missing shared libraries in this container - it will fall back to the CPU"
   elif [[ "$var_backend" == "vulkan" ]] && ! vulkaninfo --summary >/dev/null 2>&1; then
     msg_warn "Vulkan build installed but no Vulkan device is visible - check GPU passthrough and the driver libraries in the container"
+  elif [[ "$var_backend" == cuda-* ]] && ! ldconfig -p | grep -q 'libcuda[.]so'; then
+    msg_warn "CUDA build installed but the NVIDIA driver library (libcuda) is missing - check GPU passthrough"
+  elif [[ "$var_backend" == sycl-* || "$var_backend" == "rocm" ]] && [[ ! -d /dev/dri && ! -e /dev/kfd ]]; then
+    msg_warn "${var_backend} build installed but no GPU is passed through - it will fall back to the CPU"
   fi
 fi
 
@@ -114,7 +228,7 @@ Type=simple
 User=root
 WorkingDirectory=/opt/llama-cpp
 EnvironmentFile=/opt/llama-cpp.env
-Environment=LD_LIBRARY_PATH=/opt/llama-cpp
+Environment=LD_LIBRARY_PATH=${LLAMA_LIBPATH}
 ExecStart=/opt/llama-cpp/llama-server
 Restart=on-failure
 RestartSec=10

json/llama-cpp.json

@@ -43,10 +43,15 @@
       "options": [
         "auto",
         "cpu",
-        "vulkan"
+        "vulkan",
+        "cuda",
+        "rocm",
+        "sycl-fp16",
+        "sycl-fp32",
+        "openvino"
       ],
       "default": "auto",
-      "help": "auto takes vulkan for any passed-through GPU (AMD via RADV, NVIDIA and Intel too), cpu otherwise. Upstream ships no Linux ROCm or CUDA build."
+      "help": "auto takes vulkan for any passed-through GPU, cpu otherwise. cuda picks the CUDA 12 or 13 build from the host driver; rocm, sycl-fp16/fp32 and openvino are x64 only."
     },
     {
       "name": "var_model_repo",
@@ -100,7 +105,7 @@
       "type": "warning"
     },
     {
-      "text": "The build variant is chosen at install time: cpu or vulkan, or auto to follow the passed-through GPU. Upstream publishes no Linux CUDA or ROCm build, so every GPU is served by the vulkan build - AMD through RADV, NVIDIA through the driver's Vulkan ICD, which has to be present in the container. The choice is remembered in /opt/llama-cpp_data/.backend so updates keep it.",
+      "text": "The build variant is chosen at install time: cpu, vulkan (any GPU vendor), cuda (NVIDIA; CUDA 13 with a 580+ host driver, CUDA 12 otherwise), rocm (AMD, with the matching ROCm runtime from AMD's TheRock wheels), sycl-fp16/sycl-fp32 (Intel GPU, with the oneAPI runtime and Level Zero) or openvino (Intel CPU/GPU), or auto to take vulkan when a GPU is passed through. The choice is remembered in /opt/llama-cpp_data/.backend so updates keep it.",
       "type": "info"
     },
     {