diff --git a/README.md b/README.md
index ed3ce7f..97446f3 100644
--- a/README.md
+++ b/README.md
@@ -61,8 +61,8 @@ The following table lists the supported accelerated backends and their correspon
| CUDA Version
(Variant) | vLLM | SGLang | VoxBox |
|------------------------------|------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------|----------|
-| 13.0 | **`0.27.1`**, `0.25.1`, `0.24.0`, `0.22.1`,
`0.21.0`, `0.20.2`,
`0.19.1`, `0.18.1` | `0.5.18`, `0.5.15.post1`,
`0.5.14`, `0.5.12.post1` | |
-| 12.9 | **`0.27.1`**, `0.25.1`, `0.24.0`, `0.22.1`,
`0.21.0`, `0.20.2`,
`0.19.1`, `0.18.1`,
`0.17.1`, `0.16.0`,
`0.15.1`, `0.14.1`,
`0.13.0`, `0.12.0`,
`0.11.2` | `0.5.18`, `0.5.15.post1`, `0.5.14`, `0.5.12.post1`,
`0.5.9`, `0.5.8.post1`,
`0.5.7`, `0.5.6.post2` | |
+| 13.0 | **`0.28.0`**, `0.27.1`, `0.25.1`, `0.24.0`, `0.22.1`,
`0.21.0`, `0.20.2`,
`0.19.1`, `0.18.1` | `0.5.18`, `0.5.15.post1`,
`0.5.14`, `0.5.12.post1` | |
+| 12.9 | **`0.28.0`**, `0.27.1`, `0.25.1`, `0.24.0`, `0.22.1`,
`0.21.0`, `0.20.2`,
`0.19.1`, `0.18.1`,
`0.17.1`, `0.16.0`,
`0.15.1`, `0.14.1`,
`0.13.0`, `0.12.0`,
`0.11.2` | `0.5.18`, `0.5.15.post1`, `0.5.14`, `0.5.12.post1`,
`0.5.9`, `0.5.8.post1`,
`0.5.7`, `0.5.6.post2` | |
| 12.8 | `0.17.1`, `0.16.0`,
`0.15.1`, `0.14.1`,
`0.13.0`, `0.12.0`,
`0.11.2`, `0.10.2` | `0.5.9`, `0.5.8.post1`,
`0.5.7`, `0.5.6.post2`,
`0.5.5.post3` | `0.0.21` |
| 12.6 | `0.15.1`, `0.14.1`,
`0.13.0`, `0.12.0`,
`0.11.2`, `0.10.2` | | `0.0.21` |
@@ -114,7 +114,7 @@ The following table lists the supported accelerated backends and their correspon
| ROCm Version
(Variant) | vLLM | SGLang |
|------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------|
-| 7.2 | **`0.27.1`**, `0.25.1`, `0.24.0`, `0.22.1`,
`0.21.0`, `0.20.2`,
`0.19.1` | `0.5.18`, `0.5.15.post1`,
`0.5.14`, `0.5.12.post1` |
+| 7.2 | **`0.28.0`**, `0.27.1`, `0.25.1`, `0.24.0`, `0.22.1`,
`0.21.0`, `0.20.2`,
`0.19.1` | `0.5.18`, `0.5.15.post1`,
`0.5.14`, `0.5.12.post1` |
| 7.1 | `0.17.1` | |
| 7.0 | `0.18.1`,
`0.16.0`, `0.15.1`,
`0.14.1`, `0.13.0`,
`0.12.0`, `0.11.2` | `0.5.9`, `0.5.8.post1`,
`0.5.7`, `0.5.6.post2` |
| 6.4 | `0.16.0`,
`0.15.1`, `0.14.1`,
`0.13.0`, `0.12.0`,
`0.11.2`, `0.10.2` | `0.5.8.post1`, `0.5.7`,
`0.5.6.post2`, `0.5.5.post3` |
diff --git a/pack/cuda/Dockerfile.vllm b/pack/cuda/Dockerfile.vllm
index fa431fd..d0bc829 100644
--- a/pack/cuda/Dockerfile.vllm
+++ b/pack/cuda/Dockerfile.vllm
@@ -2,8 +2,8 @@ ARG PYTHON_VERSION=3.12
ARG CMAKE_MAX_JOBS
ARG CUDA_VERSION=13.0.1
ARG CUDA_ARCHS
-ARG VLLM_BASE_IMAGE=vllm/vllm-openai:v0.27.1-ubuntu2404
-ARG VLLM_VERSION=0.27.1
+ARG VLLM_BASE_IMAGE=vllm/vllm-openai:v0.28.0-ubuntu2404
+ARG VLLM_VERSION=0.28.0
ARG VLLM_TORCH_VERSION=2.13.0
ARG VLLM_TORCH_CUDA_VERSION=${CUDA_VERSION}
## Built from source, replacing the base's own lmcache: that one is a PyPI wheel compiled
@@ -11,7 +11,7 @@ ARG VLLM_TORCH_CUDA_VERSION=${CUDA_VERSION}
## on a mismatch lmcache silently degrades to its torch baseline (vllm-project/vllm#53424).
## Keep in sync with the ROCm and SGLang pins —— the MP wire protocol has no version handshake.
ARG VLLM_LMCACHE_VERSION=0.5.4
-ARG VLLM_OMNI_COMMIT=d77a35a3
+ARG VLLM_OMNI_COMMIT=0d582c2da0
FROM ${VLLM_BASE_IMAGE} AS vllm-build
SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
diff --git a/pack/cuda/patches/vllm/001_wrong_dp_ray.patch b/pack/cuda/patches/vllm/001_wrong_dp_ray.patch
index f49b54b..7d19307 100644
--- a/pack/cuda/patches/vllm/001_wrong_dp_ray.patch
+++ b/pack/cuda/patches/vllm/001_wrong_dp_ray.patch
@@ -1,33 +1,28 @@
diff --git a/vllm/utils/network_utils.py b/vllm/utils/network_utils.py
--- a/vllm/utils/network_utils.py
+++ b/vllm/utils/network_utils.py
-@@ -146,6 +146,8 @@
- def get_open_zmq_inproc_path() -> str:
- return f"inproc://{uuid4()}"
-
+@@ -171,6 +171,9 @@
+ return range(dp_master_port, dp_master_port + 10)
+
+
+_next_port: int | None = None
++
+
def get_open_port() -> int:
"""
Get an open port for the vLLM process to listen on.
-@@ -156,13 +158,17 @@
+@@ -180,10 +183,13 @@
Right now we reserve 10 ports for the data parallel master
process. Currently it uses 2 ports.
"""
+ global _next_port
- if "VLLM_DP_MASTER_PORT" in os.environ:
- dp_master_port = envs.VLLM_DP_MASTER_PORT
- reserved_port_range = range(dp_master_port, dp_master_port + 10)
- while True:
-- candidate_port = _get_open_port()
-+ candidate_port = _get_open_port(_next_port)
-+ _next_port = candidate_port + 1
- if candidate_port not in reserved_port_range:
- return candidate_port
-- return _get_open_port()
-+ port = _get_open_port(_next_port)
+ reserved_port_range = _get_reserved_port_range()
+- port = _get_open_port()
++ port = _get_open_port(start_port=_next_port)
+ _next_port = port + 1
-+ return port
-
-
- def get_open_ports_list(count: int = 5) -> list[int]:
+ if port in reserved_port_range:
+ port = _get_open_port(start_port=reserved_port_range.stop, max_attempts=1000)
++ _next_port = port + 1
+ return port
+
+
diff --git a/pack/cuda/patches/vllm/002_shm_broadcast_port_race.patch b/pack/cuda/patches/vllm/002_shm_broadcast_port_race.patch
deleted file mode 100644
index e3ea0f8..0000000
--- a/pack/cuda/patches/vllm/002_shm_broadcast_port_race.patch
+++ /dev/null
@@ -1,31 +0,0 @@
-diff --git a/vllm/distributed/device_communicators/shm_broadcast.py b/vllm/distributed/device_communicators/shm_broadcast.py
---- a/vllm/distributed/device_communicators/shm_broadcast.py
-+++ b/vllm/distributed/device_communicators/shm_broadcast.py
-@@ -34,10 +34,9 @@
- from vllm.distributed.utils import StatelessProcessGroup, sched_yield
- from vllm.logger import init_logger
- from vllm.platforms import current_platform
- from vllm.utils.network_utils import (
- get_ip,
-- get_open_port,
- get_open_zmq_inproc_path,
- get_open_zmq_ipc_path,
- is_valid_ipv6_address,
- )
-@@ -525,13 +524,13 @@
- connect_ip = get_ip()
- self.remote_socket = context.socket(XPUB)
- self.remote_socket.setsockopt(XPUB_VERBOSE, True)
-- remote_subscribe_port = get_open_port()
- if is_valid_ipv6_address(connect_ip):
- self.remote_socket.setsockopt(IPV6, 1)
- remote_addr_ipv6 = True
- connect_ip = f"[{connect_ip}]"
-- socket_addr = f"tcp://{connect_ip}:{remote_subscribe_port}"
-- self.remote_socket.bind(socket_addr)
-+ self.remote_socket.bind(f"tcp://{connect_ip}:0")
-+ last_endpoint = self.remote_socket.getsockopt(zmq.LAST_ENDPOINT)
-+ remote_subscribe_port = last_endpoint.decode().rsplit(":", 1)[1]
- remote_subscribe_addr = f"tcp://{connect_ip}:{remote_subscribe_port}"
- else:
- remote_subscribe_addr = None
diff --git a/pack/matrix.yaml b/pack/matrix.yaml
index 68df19d..f85578a 100644
--- a/pack/matrix.yaml
+++ b/pack/matrix.yaml
@@ -81,8 +81,8 @@ rules:
- "linux/amd64"
args:
- "ROCM_VERSION=7.2.3"
- - "VLLM_VERSION=0.27.1"
- - "VLLM_BASE_IMAGE=vllm/vllm-openai-rocm:v0.27.1"
+ - "VLLM_VERSION=0.28.0"
+ - "VLLM_BASE_IMAGE=vllm/vllm-openai-rocm:v0.28.0"
## AMD ROCm 7.2.1 - SGLang 0.5.18 (rocm720-mi30x)
##
- backend: "rocm"
@@ -106,8 +106,8 @@ rules:
- "vllm"
args:
- "CUDA_VERSION=13.0.1"
- - "VLLM_VERSION=0.27.1"
- - "VLLM_BASE_IMAGE=vllm/vllm-openai:v0.27.1-ubuntu2404"
+ - "VLLM_VERSION=0.28.0"
+ - "VLLM_BASE_IMAGE=vllm/vllm-openai:v0.28.0-ubuntu2404"
## NVIDIA CUDA 13.0.1 - SGLang 0.5.18
##
- backend: "cuda"
@@ -124,8 +124,8 @@ rules:
- "vllm"
args:
- "CUDA_VERSION=12.9.1"
- - "VLLM_VERSION=0.27.1"
- - "VLLM_BASE_IMAGE=vllm/vllm-openai:v0.27.1-cu129-ubuntu2404"
+ - "VLLM_VERSION=0.28.0"
+ - "VLLM_BASE_IMAGE=vllm/vllm-openai:v0.28.0-cu129-ubuntu2404"
## NVIDIA CUDA 12.9.1 - SGLang 0.5.18
##
- backend: "cuda"
diff --git a/pack/rocm/Dockerfile.vllm b/pack/rocm/Dockerfile.vllm
index 38b8b0c..fdd72b2 100644
--- a/pack/rocm/Dockerfile.vllm
+++ b/pack/rocm/Dockerfile.vllm
@@ -2,15 +2,15 @@ ARG PYTHON_VERSION=3.12
ARG CMAKE_MAX_JOBS
ARG ROCM_VERSION=7.2.3
ARG ROCM_ARCHS
-ARG VLLM_BASE_IMAGE=vllm/vllm-openai-rocm:v0.27.1
-ARG VLLM_VERSION=0.27.1
-ARG VLLM_TORCH_VERSION=2.13.0
+ARG VLLM_BASE_IMAGE=vllm/vllm-openai-rocm:v0.28.0
+ARG VLLM_VERSION=0.28.0
+ARG VLLM_TORCH_VERSION=2.12.0
ARG VLLM_TORCH_ROCM_VERSION=${ROCM_VERSION}
## The ROCm base ships no lmcache of its own, so it is always built from source here.
## Keep in sync with the CUDA and SGLang pins —— see pack/cuda/Dockerfile.vllm.
ARG VLLM_LMCACHE_VERSION=0.5.4
ARG VLLM_MOONCAKE_VERSION=0.3.11.post1
-ARG VLLM_OMNI_COMMIT=d77a35a3
+ARG VLLM_OMNI_COMMIT=0d582c2da0
FROM ${VLLM_BASE_IMAGE} AS vllm-build
SHELL ["/bin/bash", "-eo", "pipefail", "-c"]
diff --git a/pack/rocm/patches/vllm/001_wrong_dp_ray.patch b/pack/rocm/patches/vllm/001_wrong_dp_ray.patch
index f49b54b..7d19307 100644
--- a/pack/rocm/patches/vllm/001_wrong_dp_ray.patch
+++ b/pack/rocm/patches/vllm/001_wrong_dp_ray.patch
@@ -1,33 +1,28 @@
diff --git a/vllm/utils/network_utils.py b/vllm/utils/network_utils.py
--- a/vllm/utils/network_utils.py
+++ b/vllm/utils/network_utils.py
-@@ -146,6 +146,8 @@
- def get_open_zmq_inproc_path() -> str:
- return f"inproc://{uuid4()}"
-
+@@ -171,6 +171,9 @@
+ return range(dp_master_port, dp_master_port + 10)
+
+
+_next_port: int | None = None
++
+
def get_open_port() -> int:
"""
Get an open port for the vLLM process to listen on.
-@@ -156,13 +158,17 @@
+@@ -180,10 +183,13 @@
Right now we reserve 10 ports for the data parallel master
process. Currently it uses 2 ports.
"""
+ global _next_port
- if "VLLM_DP_MASTER_PORT" in os.environ:
- dp_master_port = envs.VLLM_DP_MASTER_PORT
- reserved_port_range = range(dp_master_port, dp_master_port + 10)
- while True:
-- candidate_port = _get_open_port()
-+ candidate_port = _get_open_port(_next_port)
-+ _next_port = candidate_port + 1
- if candidate_port not in reserved_port_range:
- return candidate_port
-- return _get_open_port()
-+ port = _get_open_port(_next_port)
+ reserved_port_range = _get_reserved_port_range()
+- port = _get_open_port()
++ port = _get_open_port(start_port=_next_port)
+ _next_port = port + 1
-+ return port
-
-
- def get_open_ports_list(count: int = 5) -> list[int]:
+ if port in reserved_port_range:
+ port = _get_open_port(start_port=reserved_port_range.stop, max_attempts=1000)
++ _next_port = port + 1
+ return port
+
+
diff --git a/pack/rocm/patches/vllm/002_shm_broadcast_port_race.patch b/pack/rocm/patches/vllm/002_shm_broadcast_port_race.patch
deleted file mode 100644
index e3ea0f8..0000000
--- a/pack/rocm/patches/vllm/002_shm_broadcast_port_race.patch
+++ /dev/null
@@ -1,31 +0,0 @@
-diff --git a/vllm/distributed/device_communicators/shm_broadcast.py b/vllm/distributed/device_communicators/shm_broadcast.py
---- a/vllm/distributed/device_communicators/shm_broadcast.py
-+++ b/vllm/distributed/device_communicators/shm_broadcast.py
-@@ -34,10 +34,9 @@
- from vllm.distributed.utils import StatelessProcessGroup, sched_yield
- from vllm.logger import init_logger
- from vllm.platforms import current_platform
- from vllm.utils.network_utils import (
- get_ip,
-- get_open_port,
- get_open_zmq_inproc_path,
- get_open_zmq_ipc_path,
- is_valid_ipv6_address,
- )
-@@ -525,13 +524,13 @@
- connect_ip = get_ip()
- self.remote_socket = context.socket(XPUB)
- self.remote_socket.setsockopt(XPUB_VERBOSE, True)
-- remote_subscribe_port = get_open_port()
- if is_valid_ipv6_address(connect_ip):
- self.remote_socket.setsockopt(IPV6, 1)
- remote_addr_ipv6 = True
- connect_ip = f"[{connect_ip}]"
-- socket_addr = f"tcp://{connect_ip}:{remote_subscribe_port}"
-- self.remote_socket.bind(socket_addr)
-+ self.remote_socket.bind(f"tcp://{connect_ip}:0")
-+ last_endpoint = self.remote_socket.getsockopt(zmq.LAST_ENDPOINT)
-+ remote_subscribe_port = last_endpoint.decode().rsplit(":", 1)[1]
- remote_subscribe_addr = f"tcp://{connect_ip}:{remote_subscribe_port}"
- else:
- remote_subscribe_addr = None