abetlen · abetlen · Jun 8, 2026 · Jun 8, 2026
diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
@@ -1,6 +1,12 @@
 name: Build Wheels (CUDA)
 
-on: workflow_dispatch
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: Release tag to upload wheel assets to
+        required: false
+        type: string
 
 permissions:
   contents: write
@@ -131,7 +137,9 @@ jobs:
           if ($IsWindows) {
             python -m pip install build wheel ninja
           } else {
-            python -m pip install build wheel
+            sudo apt-get update
+            sudo apt-get install -y patchelf
+            python -m pip install auditwheel build wheel
           }
 
       - name: Build Wheel
@@ -169,6 +177,12 @@ jobs:
             $env:CPLUS_INCLUDE_PATH = "$cudaRoot/include$pathSeparator$env:CPLUS_INCLUDE_PATH"
             $env:LIBRARY_PATH = "$cudaRoot/lib$pathSeparator$env:CONDA_PREFIX/lib$pathSeparator$env:LIBRARY_PATH"
             $env:LD_LIBRARY_PATH = "$cudaRoot/lib$pathSeparator$env:CONDA_PREFIX/lib$pathSeparator$env:LD_LIBRARY_PATH"
+            $cudaLibraryPaths = @(
+              (Join-Path $cudaRoot 'lib'),
+              (Join-Path $cudaRoot 'lib64'),
+              (Join-Path $env:CONDA_PREFIX 'lib')
+            ) | Where-Object { Test-Path $_ }
+            Write-Output "CUDA_LIBRARY_PATHS=$($cudaLibraryPaths -join ':')" >> $env:GITHUB_ENV
           } elseif ($IsWindows) {
             $ninjaPath = ((Get-Command ninja -ErrorAction Stop).Source).Replace('\', '/')
             $env:CMAKE_GENERATOR = 'Ninja'
@@ -218,15 +232,44 @@ jobs:
           # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit.
           $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=$cudaArchs -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS"
           $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off'
+          if ($IsLinux) {
+            $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_OPENMP=OFF'
+          }
           python -m build --wheel
           # Publish tags that reflect the actual installed toolkit version.
           Write-Output "CUDA_VERSION=$cudaTagVersion" >> $env:GITHUB_ENV
 
+      - name: Repair Linux wheel
+        if: runner.os == 'Linux'
+        shell: bash
+        run: |
+          set -euxo pipefail
+          mkdir -p wheelhouse
+          export LD_LIBRARY_PATH="$PWD/llama_cpp/lib:${CUDA_LIBRARY_PATHS}:${LD_LIBRARY_PATH:-}"
+          auditwheel_bin="${CONDA}/envs/llamacpp/bin/auditwheel"
+          "${auditwheel_bin}" repair \
+            --exclude libcuda.so \
+            --exclude libcuda.so.1 \
+            --exclude libcudart.so.11.0 \
+            --exclude libcudart.so.12 \
+            --exclude libcudart.so.13 \
+            --exclude libcublas.so.11 \
+            --exclude libcublas.so.12 \
+            --exclude libcublas.so.13 \
+            --exclude libcublasLt.so.11 \
+            --exclude libcublasLt.so.12 \
+            --exclude libcublasLt.so.13 \
+            -w wheelhouse \
+            dist/*.whl
+          rm dist/*.whl
+          cp wheelhouse/*.whl dist/
+          "${auditwheel_bin}" show dist/*.whl
+
       - uses: softprops/action-gh-release@v3
-        if: startsWith(github.ref, 'refs/tags/')
+        if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.release_tag != '')
         with:
           files: dist/*
-          # Set tag_name to <tag>-cu<cuda_version>
-          tag_name: ${{ github.ref_name }}-cu${{ env.CUDA_VERSION }}
+          # Set tag_name to <tag>-cu<cuda_version>.
+          tag_name: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }}-cu${{ env.CUDA_VERSION }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/build-wheels-rocm.yaml b/.github/workflows/build-wheels-rocm.yaml
@@ -1,6 +1,20 @@
 name: Build Wheels (ROCm)
 
-on: workflow_dispatch
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: Release tag to upload wheel assets to
+        required: false
+        type: string
+      amdgpu_targets:
+        description: AMDGPU targets to compile into the Linux ROCm wheel
+        required: false
+        default: gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1200;gfx1201
+      windows_amdgpu_targets:
+        description: AMDGPU targets to compile into the Windows HIP Radeon wheel
+        required: false
+        default: gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032
 
 permissions:
   contents: write
@@ -24,7 +38,7 @@ jobs:
       - name: Install system dependencies
         run: |
           apt-get update
-          apt-get install -y --no-install-recommends git cmake lsb-release ninja-build
+          apt-get install -y --no-install-recommends git cmake lsb-release ninja-build patchelf
 
       - uses: actions/checkout@v6
         with:
@@ -37,9 +51,12 @@ jobs:
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install build wheel
+          python -m pip install auditwheel build wheel
 
       - name: Build ROCm wheel
+        env:
+          MATRIX_AMDGPU_TARGETS: ${{ matrix.amdgpu_targets }}
+          INPUT_AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }}
         run: |
           export ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
           export HIP_PATH="${HIP_PATH:-$ROCM_PATH}"
@@ -56,11 +73,38 @@ jobs:
           rocm_tag="$(hipconfig --version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1\2/')"
           echo "ROCM_VERSION=$rocm_tag" >> "$GITHUB_ENV"
 
-          amdgpu_targets="${{ matrix.amdgpu_targets }}"
-          export CMAKE_ARGS="-DGGML_HIP=on -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off -DAMDGPU_TARGETS=$amdgpu_targets -DCMAKE_HIP_ARCHITECTURES=$amdgpu_targets"
+          amdgpu_targets="${INPUT_AMDGPU_TARGETS:-$MATRIX_AMDGPU_TARGETS}"
+          export CMAKE_ARGS="-DGGML_HIP=on -DGGML_NATIVE=off -DGGML_OPENMP=OFF -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off -DAMDGPU_TARGETS=$amdgpu_targets -DCMAKE_HIP_ARCHITECTURES=$amdgpu_targets"
           python -m build --wheel
+
+      - name: Repair Linux wheel
+        run: |
+          export ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
+          export LD_LIBRARY_PATH="$PWD/llama_cpp/lib:$ROCM_PATH/lib:$ROCM_PATH/lib64:${LD_LIBRARY_PATH:-}"
           mkdir -p wheelhouse
-          cp dist/*.whl wheelhouse/
+          python -m auditwheel repair \
+            --exclude libamdhip64.so \
+            --exclude libamdhip64.so.6 \
+            --exclude libamdhip64.so.7 \
+            --exclude libhiprtc.so \
+            --exclude libhiprtc.so.6 \
+            --exclude libhiprtc.so.7 \
+            --exclude libhipblas.so \
+            --exclude libhipblas.so.2 \
+            --exclude libhipblas.so.3 \
+            --exclude libhipblaslt.so \
+            --exclude libhipblaslt.so.0 \
+            --exclude libhipblaslt.so.1 \
+            --exclude librocblas.so \
+            --exclude librocblas.so.4 \
+            --exclude librocblas.so.5 \
+            --exclude libhsa-runtime64.so.1 \
+            --exclude libhsakmt.so.1 \
+            -w wheelhouse \
+            dist/*.whl
+          rm dist/*.whl
+          cp wheelhouse/*.whl dist/
+          python -m auditwheel show dist/*.whl
 
       - uses: actions/upload-artifact@v7
         with:
@@ -139,11 +183,14 @@ jobs:
           & $clangPath.FullName --version
 
       - name: Build HIP wheel
+        env:
+          MATRIX_AMDGPU_TARGETS: ${{ matrix.amdgpu_targets }}
+          INPUT_AMDGPU_TARGETS: ${{ inputs.windows_amdgpu_targets }}
         run: |
           $ErrorActionPreference = "Stop"
           $hipPath = Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Split-Path | Split-Path
           $rocwmmaInclude = (Join-Path $PWD 'opt\rocm-7.2.1\include').Replace('\', '/')
-          $amdgpuTargets = "${{ matrix.amdgpu_targets }}"
+          $amdgpuTargets = if ($env:INPUT_AMDGPU_TARGETS) { $env:INPUT_AMDGPU_TARGETS } else { $env:MATRIX_AMDGPU_TARGETS }
 
           $env:HIP_PATH = $hipPath
           $env:ROCM_PATH = $hipPath
@@ -198,7 +245,7 @@ jobs:
   release_rocm:
     name: Release ROCm
     needs: [build_wheels]
-    if: startsWith(github.ref, 'refs/tags/')
+    if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.release_tag != '')
     runs-on: ubuntu-latest
 
     steps:
@@ -211,14 +258,14 @@ jobs:
         with:
           files: dist/*
           # Set release name to <tag>-rocm<rocm_version>.
-          tag_name: ${{ github.ref_name }}-rocm72
+          tag_name: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }}-rocm72
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   release_hip:
     name: Release HIP
     needs: [build_wheels_windows_hip]
-    if: startsWith(github.ref, 'refs/tags/')
+    if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.release_tag != '')
     runs-on: ubuntu-latest
 
     steps:
@@ -231,6 +278,6 @@ jobs:
         with:
           files: dist/*
           # Set release name to <tag>-hip-radeon.
-          tag_name: ${{ github.ref_name }}-hip-radeon
+          tag_name: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }}-hip-radeon
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/build-wheels-vulkan.yaml b/.github/workflows/build-wheels-vulkan.yaml
@@ -1,6 +1,12 @@
 name: Build Wheels (Vulkan)
 
-on: workflow_dispatch
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: Release tag to upload wheel assets to
+        required: false
+        type: string
 
 permissions:
   contents: write
@@ -40,23 +46,6 @@ jobs:
           python-version: ${{ matrix.pyver }}
           cache: "pip"
 
-      - name: Install Vulkan SDK
-        if: runner.os == 'Linux'
-        run: |
-          curl -fL \
-            "https://sdk.lunarg.com/sdk/download/${VULKAN_SDK_VERSION}/linux/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.xz" \
-            -o vulkan-sdk.tar.xz
-          echo "${VULKAN_SDK_LINUX_SHA256}  vulkan-sdk.tar.xz" | sha256sum -c -
-          mkdir -p "$RUNNER_TEMP/vulkan-sdk"
-          tar -xf vulkan-sdk.tar.xz -C "$RUNNER_TEMP/vulkan-sdk"
-          source "$RUNNER_TEMP/vulkan-sdk/${VULKAN_SDK_VERSION}/setup-env.sh"
-          {
-            echo "VULKAN_SDK=$VULKAN_SDK"
-            echo "LD_LIBRARY_PATH=$VULKAN_SDK/lib:${LD_LIBRARY_PATH:-}"
-          } >> "$GITHUB_ENV"
-          echo "$VULKAN_SDK/bin" >> "$GITHUB_PATH"
-          "$VULKAN_SDK/bin/glslc" --version
-
       - name: Install Vulkan SDK
         if: runner.os == 'Windows'
         shell: pwsh
@@ -71,6 +60,7 @@ jobs:
           & "$vulkanSdk\Bin\glslc.exe" --version
 
       - name: Install build dependencies
+        if: runner.os == 'Windows'
         run: |
           python -m pip install --upgrade pip
           python -m pip install build wheel
@@ -81,11 +71,28 @@ jobs:
 
       - name: Build Vulkan wheel
         if: runner.os == 'Linux'
-        run: |
-          export CMAKE_ARGS="-DGGML_NATIVE=off -DGGML_METAL=OFF -DGGML_VULKAN=on"
-          python -m build --wheel
-          mkdir -p wheelhouse
-          cp dist/*.whl wheelhouse/
+        uses: pypa/cibuildwheel@v3.4.1
+        env:
+          CIBW_BUILD: "cp38-manylinux_*"
+          CIBW_ARCHS: "auto64"
+          CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014"
+          CIBW_BEFORE_ALL_LINUX: >
+            yum install -y xz &&
+            curl -L https://micro.mamba.pm/api/micromamba/linux-64/latest -o /tmp/micromamba.tar.bz2 &&
+            mkdir -p /tmp/micromamba &&
+            tar -xjf /tmp/micromamba.tar.bz2 -C /tmp/micromamba bin/micromamba &&
+            /tmp/micromamba/bin/micromamba create -y -p /opt/vulkan -c conda-forge shaderc libvulkan-loader spirv-headers &&
+            /opt/vulkan/bin/glslc --version &&
+            curl -fL "https://sdk.lunarg.com/sdk/download/${{ env.VULKAN_SDK_VERSION }}/linux/vulkansdk-linux-x86_64-${{ env.VULKAN_SDK_VERSION }}.tar.xz" -o /tmp/vulkan-sdk.tar.xz &&
+            echo "${{ env.VULKAN_SDK_LINUX_SHA256 }}  /tmp/vulkan-sdk.tar.xz" | sha256sum -c - &&
+            mkdir -p /opt/vulkan-sdk &&
+            tar -xf /tmp/vulkan-sdk.tar.xz -C /opt/vulkan-sdk
+          CIBW_ENVIRONMENT_LINUX: >
+            CMAKE_ARGS="-DGGML_NATIVE=off -DGGML_METAL=OFF -DGGML_OPENMP=OFF -DGGML_VULKAN=on -DCMAKE_PREFIX_PATH=/opt/vulkan -DVulkan_INCLUDE_DIR=/opt/vulkan-sdk/${{ env.VULKAN_SDK_VERSION }}/x86_64/include -DVulkan_LIBRARY=/opt/vulkan/lib/libvulkan.so -DVulkan_GLSLC_EXECUTABLE=/opt/vulkan/bin/glslc"
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: "LD_LIBRARY_PATH=/project/llama_cpp/lib:/opt/vulkan/lib auditwheel repair --exclude libvulkan.so.1 -w {dest_dir} {wheel}"
+        with:
+          package-dir: .
+          output-dir: wheelhouse
 
       - name: Build Vulkan wheel
         if: runner.os == 'Windows'
@@ -105,7 +112,7 @@ jobs:
   release:
     name: Release
     needs: [build_wheels]
-    if: startsWith(github.ref, 'refs/tags/')
+    if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.release_tag != '')
     runs-on: ubuntu-latest
 
     steps:
@@ -118,6 +125,6 @@ jobs:
         with:
           files: dist/*
           # Set release name to <tag>-vulkan.
-          tag_name: ${{ github.ref_name }}-vulkan
+          tag_name: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }}-vulkan
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- fix(ci): Repair Linux accelerator wheels for manylinux publishing
+
 ## [0.3.28]
 
 - feat(example): align server MTP support with llama.cpp by @abetlen in #2283