Restore model probe for service routers (#4322)

Since #4270, a router replica of a service with `model` and no
`probes` was probed with `GET /health` instead of the chat completions
request. The latter deadlocked: the router only answers it once
workers are registered, and the worker sync skipped routers that were
not ready. #4313 dropped the readiness check from the sync, so this
deadlock is gone.

`/health` passes before the router has any workers. In a rolling
deployment, the replacement router was considered ready, and the old
one was scaled down, before the replacement could serve requests. Now
that #4320 syncs workers with every running router, the replacement
passes the chat completions probe once it has workers.

Workers behind a router still get no default probe, as they may not
serve HTTP at all (gRPC workers).

With a Dynamo router, a router rolling deployment now waits for the
replacement indefinitely instead of scaling down the old router, as
workers stay attached to the old router's etcd/NATS. Such deployments
were broken before as well and are to be fixed separately.

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Author: un-def

src/dstack/_internal/core/models/configurations.py

@@ -72,7 +72,6 @@
 DEFAULT_PROBE_READY_AFTER = 1
 DEFAULT_PROBE_METHOD = "get"
 DEFAULT_PROBE_UNTIL_READY = False
-ROUTER_HEALTH_PROBE_URL = "/health"
 MAX_PROBE_URL_LEN = 2048
 DEFAULT_REPLICA_GROUP_NAME = "0"
 OPENAI_MODEL_PROBE_TIMEOUT = 30

src/dstack/_internal/server/services/jobs/configurators/base.py

@@ -27,7 +27,6 @@
     DEFAULT_REPLICA_GROUP_NAME,
     LEGACY_REPO_DIR,
     OPENAI_MODEL_PROBE_TIMEOUT,
-    ROUTER_HEALTH_PROBE_URL,
     HTTPHeaderSpec,
     NodeGroup,
     PortMapping,
@@ -513,17 +512,17 @@ def _probes(self) -> list[ProbeSpec]:
         model = conf.model
         if not isinstance(model, OpenAIChatModel):
             return []
-        if all(group.router is None for group in conf.replica_groups):
-            # No router: every replica serves the model itself, so a chat completions
-            # request is a genuine end-to-end readiness check.
-            return [_openai_model_probe_spec(model.name, model.prefix)]
-        group = self._replica_group()
-        if group is not None and group.router is not None:
-            # Probe the router's own liveness endpoint, which does not depend on any
-            # worker. Workers get no default probe: they may not serve HTTP at all
-            # (gRPC workers), and they don't receive traffic directly.
-            return [_router_health_probe_spec()]
-        return []
+        if any(group.router is not None for group in conf.replica_groups):
+            group = self._replica_group()
+            if group is None or group.router is None:
+                # Workers get no default probe: they may not serve HTTP at all (gRPC
+                # workers), and they don't receive traffic directly.
+                return []
+            # The router answers chat completions only once it has workers, which it gets
+            # regardless of its own readiness. For SGLang routers, dstack registers workers
+            # via `ServiceRouterWorkerSyncWorker`, which doesn't check router readiness.
+            # Dynamo workers register themselves with the router via etcd/NATS.
+        return [_openai_model_probe_spec(model.name, model.prefix)]
 
 
 def interpolate_job_volumes(
@@ -598,19 +597,6 @@ def _openai_model_probe_spec(model_name: str, prefix: str) -> ProbeSpec:
     )
 
 
-def _router_health_probe_spec() -> ProbeSpec:
-    # Both supported routers (SGLang/SMG and Dynamo) serve `/health` independently of
-    # whether any worker is registered.
-    return ProbeSpec(
-        type="http",
-        method=DEFAULT_PROBE_METHOD,
-        url=ROUTER_HEALTH_PROBE_URL,
-        timeout=DEFAULT_PROBE_TIMEOUT,
-        interval=DEFAULT_PROBE_INTERVAL,
-        ready_after=DEFAULT_PROBE_READY_AFTER,
-    )
-
-
 def _join_shell_commands(commands: List[str]) -> str:
     for i, cmd in enumerate(commands):
         cmd = cmd.strip()

src/tests/_internal/server/services/jobs/configurators/test_service.py

@@ -6,7 +6,6 @@
 from dstack._internal import settings
 from dstack._internal.core.models.configurations import (
     OPENAI_MODEL_PROBE_TIMEOUT,
-    ROUTER_HEALTH_PROBE_URL,
     ProbeConfig,
     PythonVersion,
     ReplicaGroup,
@@ -116,9 +115,9 @@ def _router_worker_configuration() -> ServiceConfiguration:
             ],
         )
 
-    async def test_router_group_gets_health_probe(self):
-        """The router must not be probed with chat completions: it only answers those once
-        dstack has registered workers, and registration requires the router to be ready."""
+    async def test_router_group_gets_model_probe(self):
+        """The router is probed with chat completions like a replica serving the model
+        itself: worker registration doesn't wait for the router to be ready."""
         run_spec = get_run_spec(
             run_name="run", repo_id="id", configuration=self._router_worker_configuration()
         )
@@ -128,9 +127,9 @@ async def test_router_group_gets_health_probe(self):
 
         probes = job_specs[0].probes
         assert len(probes) == 1
-        assert probes[0].url == ROUTER_HEALTH_PROBE_URL
-        assert probes[0].method == "get"
-        assert probes[0].body is None
+        assert probes[0].method == "post"
+        assert probes[0].url == "/v1/chat/completions"
+        assert "meta-llama/Meta-Llama-3.1-8B-Instruct" in (probes[0].body or "")
 
     async def test_worker_group_gets_no_derived_probe(self):
         """Workers behind a router may speak gRPC, so no probe can be derived from `model`."""