Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions livekit-agents/livekit/agents/ipc/job_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ async def aclose(self) -> None: ...

async def launch_job(self, info: RunningJobInfo) -> None: ...

async def prewarm_connection(self, url: str) -> None:
"""Ask the idle executor to keep a connection to the LiveKit server at `url` ready"""
...

def logging_extra(self) -> dict[str, Any]: ...


Expand Down
7 changes: 7 additions & 0 deletions livekit-agents/livekit/agents/ipc/job_proc_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,13 @@ async def _do_inference_task(self, inf_req: proto.InferenceRequest) -> None:
proto.InferenceResponse(request_id=inf_req.request_id, error=str(e)),
)

async def prewarm_connection(self, url: str) -> None:
"""keep a connection to the LiveKit server ready while the process is idle"""
if self._running_job is not None or not self._initialize_fut.done():
return

await channel.asend_message(self._pch, proto.PrewarmConnectionRequest(url=url))

async def launch_job(self, info: RunningJobInfo) -> None:
"""start/assign a job to the process"""
if self._running_job is not None:
Expand Down
71 changes: 71 additions & 0 deletions livekit-agents/livekit/agents/ipc/job_proc_lazy_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
InferenceResponse,
InitializeRequest,
JoinTiming,
PrewarmConnectionRequest,
ShutdownRequest,
ShutdownRequestAck,
ShuttingDown,
Expand All @@ -54,6 +55,13 @@
if TYPE_CHECKING:
from ..voice.remote_session import SessionHost

# An idle process keeps one connection to the LiveKit server open for its next job,
# replacing it as soon as the server closes it, or once it is older than the max age.
# Proxies close a connection that sends no request within their read timeout (60s on
# LiveKit Cloud), so max age + check interval must stay below that.
PREWARM_CONNECTION_MAX_AGE = 45.0
PREWARM_CONNECTION_CHECK_INTERVAL = 10.0

# Defensive timeout for AgentSession.aclose() during job shutdown. Hardcoded for now
# as a guardrail against close paths that hang indefinitely. If aclose() does not
# return in this window, the rest of the shutdown sequence (on_session_end,
Expand Down Expand Up @@ -219,6 +227,12 @@ def __init__(
self._job_task: asyncio.Task[None] | None = None
self._join_timing_task: asyncio.Task[None] | None = None
self._entrypoint_started_mono = 0.0
self._prewarm_task: asyncio.Task[None] | None = None
self._prewarm_url = ""
self._prewarm_wake = asyncio.Event()
self._prewarmed: rtc.PrewarmedConnection | None = None
self._prewarmed_url = ""
self._prewarmed_at = 0.0

# used to warn users if both connect and shutdown are not called inside the job_entry
self._ctx_connect_called = False
Expand Down Expand Up @@ -260,6 +274,13 @@ async def _read_ipc_task() -> None:
continue

self._start_job(msg)
if isinstance(msg, PrewarmConnectionRequest) and not self.has_running_job:
self._prewarm_url = msg.url
self._prewarm_wake.set()
if self._prewarm_task is None:
self._prewarm_task = asyncio.create_task(
self._prewarm_connection(), name="prewarm_connection"
)
if isinstance(msg, ShutdownRequest):
await self._client.send(ShutdownRequestAck())

Expand Down Expand Up @@ -302,8 +323,52 @@ def _on_read_done(t: asyncio.Task[None]) -> None:
# ensure cleanup on cancellation (e.g. parent channel closes)
if self._job_task is not None:
await aio.cancel_and_wait(self._job_task)
if self._prewarm_task is not None:
await aio.cancel_and_wait(self._prewarm_task)
await aio.cancel_and_wait(read_task)

async def _prewarm_connection(self) -> None:
"""Keep a connection to the server ready for the next job's room connection"""
logged_failure = False
while not self.has_running_job:
conn = self._prewarmed
if (
conn is None
or self._prewarmed_url != self._prewarm_url
or time.monotonic() - self._prewarmed_at > PREWARM_CONNECTION_MAX_AGE
or not conn.alive
):
url = self._prewarm_url
try:
new_conn = await rtc.PrewarmedConnection.create(url)
logged_failure = False
except Exception as e:
if not logged_failure:
logger.debug("failed to prewarm the connection to the server", exc_info=e)
logged_failure = True
else:
if self.has_running_job:
# the job started while this was opening, without it
new_conn.close()
break
if conn is not None:
conn.close()
self._prewarmed = new_conn
self._prewarmed_url = url
self._prewarmed_at = time.monotonic()

self._prewarm_wake.clear()
with contextlib.suppress(asyncio.TimeoutError):
await asyncio.wait_for(
self._prewarm_wake.wait(), timeout=PREWARM_CONNECTION_CHECK_INTERVAL
)

def _take_prewarmed_connection(self) -> rtc.PrewarmedConnection | None:
conn, self._prewarmed = self._prewarmed, None
# wake the loop so it exits now that a job is running
self._prewarm_wake.set()
return conn

def _start_job(self, msg: StartJobRequest) -> None:
if msg.running_job.fake_job:
from .mock_room import create_mock_room
Expand Down Expand Up @@ -350,6 +415,12 @@ def _on_ctx_shutdown(reason: str) -> None:
on_shutdown=_on_ctx_shutdown,
inference_executor=self._inf_client,
)
prewarmed = self._take_prewarmed_connection()
if prewarmed is not None:
if msg.running_job.fake_job:
prewarmed.close()
else:
self._job_ctx._prewarmed_connection = prewarmed
# Reachable from SessionHost (via get_job_context) when a simulation finalizes.
self._job_ctx._simulation_end_fnc = self._simulation_end_fnc

Expand Down
7 changes: 7 additions & 0 deletions livekit-agents/livekit/agents/ipc/job_thread_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,13 @@ async def _do_inference_task(self, inf_req: proto.InferenceRequest) -> None:
proto.InferenceResponse(request_id=inf_req.request_id, error=str(e)),
)

async def prewarm_connection(self, url: str) -> None:
"""keep a connection to the LiveKit server ready while the executor is idle"""
if self._running_job is not None or not self._initialize_fut.done():
return

await channel.asend_message(self._pch, proto.PrewarmConnectionRequest(url=url))

async def launch_job(self, info: RunningJobInfo) -> None:
"""start/assign a job to the executor"""
if self._running_job is not None:
Expand Down
32 changes: 31 additions & 1 deletion livekit-agents/livekit/agents/ipc/proc_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,14 @@ def __init__(
self._default_num_idle_processes = num_idle_processes
self._http_proxy = http_proxy
self._target_idle_processes = num_idle_processes
self._prewarm_url: str | None = None

self._init_sem = asyncio.Semaphore(MAX_CONCURRENT_INITIALIZATIONS)
self._warmed_proc_queue = asyncio.Queue[JobExecutor]()
self._executors: list[JobExecutor] = []
self._spawn_tasks: set[asyncio.Task[None]] = set()
self._close_tasks: set[asyncio.Task[None]] = set()
self._prewarm_tasks: set[asyncio.Task[None]] = set()
self._monitor_tasks: set[asyncio.Task[None]] = set()
self._started = False
self._closed = False
Expand Down Expand Up @@ -201,6 +203,32 @@ async def launch_job(self, info: RunningJobInfo) -> None:
extra={"job_id": info.job.id, "attempt": attempt + 1},
)

def set_prewarm_url(self, url: str | None) -> None:
"""Set the LiveKit server URL that idle processes keep a connection open to.

Jobs connect to the URL of their assignment, which can differ from the URL the
worker registered with (e.g. a region-specific URL on LiveKit Cloud), so this is
updated whenever an assignment reveals a new one.
"""
if url == self._prewarm_url:
return

self._prewarm_url = url
if not url:
return

for proc in self._executors:
if proc.running_job is None:
task = asyncio.create_task(self._send_prewarm(proc, url))
self._prewarm_tasks.add(task)
task.add_done_callback(self._prewarm_tasks.discard)

async def _send_prewarm(self, proc: JobExecutor, url: str) -> None:
try:
await proc.prewarm_connection(url)
except Exception:
logger.debug("failed to send prewarm request", extra=proc.logging_extra())

def set_target_idle_processes(self, num_idle_processes: int) -> None:
self._target_idle_processes = num_idle_processes

Expand Down Expand Up @@ -263,6 +291,8 @@ def _on_join_timing(timing: agent.JobJoinTiming) -> None:
await proc.start()
self.emit("process_started", proc)
await proc.initialize()
if self._prewarm_url:
await self._send_prewarm(proc, self._prewarm_url)
self.emit("process_ready", proc)
self._warmed_proc_queue.put_nowait(proc)
if self._warmed_proc_queue.qsize() >= self._default_num_idle_processes:
Expand Down Expand Up @@ -321,7 +351,7 @@ async def _main_task(self) -> None:

await asyncio.sleep(0.1)
except asyncio.CancelledError:
await aio.cancel_and_wait(*self._spawn_tasks)
await aio.cancel_and_wait(*self._spawn_tasks, *self._prewarm_tasks)
await asyncio.gather(*[proc.aclose() for proc in self._executors])
await asyncio.gather(*self._close_tasks)
await asyncio.gather(*self._monitor_tasks)
16 changes: 16 additions & 0 deletions livekit-agents/livekit/agents/ipc/proto.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,21 @@ def read(self, b: io.BytesIO) -> None:
self.timing.ParseFromString(channel.read_bytes(b))


@dataclass
class PrewarmConnectionRequest:
"""sent by the main process to an idle subprocess so it keeps a transport connection
to the LiveKit server open; the next job's room connection then skips TCP/TLS setup"""

MSG_ID: ClassVar[int] = 13
url: str = ""

def write(self, b: io.BytesIO) -> None:
channel.write_string(b, self.url)

def read(self, b: io.BytesIO) -> None:
self.url = channel.read_string(b)


IPC_MESSAGES = {
InitializeRequest.MSG_ID: InitializeRequest,
InitializeResponse.MSG_ID: InitializeResponse,
Expand All @@ -276,4 +291,5 @@ def read(self, b: io.BytesIO) -> None:
ShutdownRequestAck.MSG_ID: ShutdownRequestAck,
ShuttingDown.MSG_ID: ShuttingDown,
JoinTiming.MSG_ID: JoinTiming,
PrewarmConnectionRequest.MSG_ID: PrewarmConnectionRequest,
}
12 changes: 10 additions & 2 deletions livekit-agents/livekit/agents/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,8 @@ def __init__(
# monotonic, connect() returned: the SDK returns once the primary transport connected
self._connected_mono = 0.0
self._lock = asyncio.Lock()
# opened by the idle process before this job arrived; used by the first connect()
self._prewarmed_connection: rtc.PrewarmedConnection | None = None
self._tagger = Tagger()
self._recording_initialized = False
self._redaction_enabled = info.job.enable_redaction
Expand Down Expand Up @@ -716,11 +718,13 @@ async def connect(
self._connect_called_mono = self._connect_called_mono or time.monotonic()

encryption = encryption or e2ee
prewarmed, self._prewarmed_connection = self._prewarmed_connection, None
room_options = rtc.RoomOptions(
encryption=encryption,
auto_subscribe=auto_subscribe == AutoSubscribe.SUBSCRIBE_ALL,
rtc_config=rtc_config,
single_peer_connection=single_peer_connection,
prewarmed_connection=prewarmed,
)

with session_context.session_span(
Expand All @@ -733,8 +737,12 @@ async def connect(
},
job_ctx=self,
) as connect_span:
await self._room.connect(self._info.url, self._info.token, options=room_options)
self._connected_mono = time.monotonic()
try:
await self._room.connect(self._info.url, self._info.token, options=room_options)
self._connected_mono = time.monotonic()
finally:
if prewarmed is not None:
prewarmed.close()
connect_span.set_attributes(
{
trace_types.ATTR_PARTICIPANT_ID: self._room.local_participant.sid,
Expand Down
Loading
Loading