Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions livekit-agents/livekit/agents/_token_mode.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
"""Token mode: the process holds no API secret, only a worker token minted for it.

Token mode is on when ``LIVEKIT_TOKEN`` is set and neither ``LIVEKIT_API_KEY`` nor
``LIVEKIT_API_SECRET`` is. The worker uses the token to register and exports it to
``LIVEKIT_TOKEN`` for the jobs it runs. Code in a job reads it only through
:func:`token_mode_token`, so a refreshed token can later be put in this one place.
"""

from __future__ import annotations

import os


def token_mode_token() -> str | None:
"""The worker token when this process runs in token mode, else None."""
if os.environ.get("LIVEKIT_API_KEY") or os.environ.get("LIVEKIT_API_SECRET"):
return None

return os.environ.get("LIVEKIT_TOKEN") or None


def require_api_secret(feature: str) -> None:
"""Raises when ``feature``, which signs tokens with the API secret, runs in token mode."""
if token_mode_token() is not None:
raise RuntimeError(
f"{feature} signs tokens with LIVEKIT_API_KEY and LIVEKIT_API_SECRET, and is not "
"supported in token mode (LIVEKIT_TOKEN without an API key and secret)"
)
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from livekit import api, rtc

from ... import llm, stt, tts, utils, vad
from ..._token_mode import require_api_secret
from ...job import DEFAULT_PARTICIPANT_KINDS, get_job_context
from ...llm.chat_context import Instructions
from ...llm.tool_context import ToolError, ToolFlag, function_tool
Expand Down Expand Up @@ -301,6 +302,7 @@ async def _dial_human_agent(self) -> AgentSession:
job_ctx = get_job_context()
ws_url = job_ctx._info.url

require_api_secret("WarmTransferTask")
# create a new room for the human agent
human_agent_room_name = self._caller_room.name + "-human-agent"
room = rtc.Room()
Expand Down
43 changes: 27 additions & 16 deletions livekit-agents/livekit/agents/cli/watcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from livekit.protocol.agent_pb import agent_dev

from .. import utils
from ..job import RunningJobInfo
from ..log import logger
from ..worker import AgentServer
from . import proto
Expand Down Expand Up @@ -37,6 +38,7 @@ def __init__(
self._worker = worker
self._cli_addr = cli_addr
self._main_task: asyncio.Task | None = None
self._reload_task: asyncio.Task | None = None

def start(self) -> None:
self._main_task = self._loop.create_task(self._run())
Expand All @@ -47,34 +49,26 @@ async def _run(self) -> None:
reader, writer = await asyncio.open_connection(host, int(port_str))

try:
# On startup: tell the CLI about this agent server (agent name + URL) so
# it can surface things like a Cloud console link, then sync running jobs.
# On startup: tell the CLI about this agent server (agent name + URL, and that
# it supports token mode) so it can surface things like a Cloud console link,
# then sync running jobs.
info = agent_dev.AgentDevMessage(
server_info=agent_dev.ServerInfo(
agent_name=self._worker._agent_name,
url=self._worker._ws_url,
supports_token_mode=True,
)
)
await _send_proto(writer, info.SerializeToString())

# send GetRunningJobsRequest to Go, recv response, reload jobs
# send GetRunningJobsRequest to Go; the response reloads the jobs
req = agent_dev.AgentDevMessage(
get_running_jobs_request=agent_dev.GetRunningAgentJobsRequest()
)
await _send_proto(writer, req.SerializeToString())

data = await _recv_proto(reader)
resp = agent_dev.AgentDevMessage()
resp.ParseFromString(data)

if resp.HasField("get_running_jobs_response"):
jobs_resp = resp.get_running_jobs_response
if jobs_resp.jobs:
running_jobs = [proto.running_job_from_proto(j) for j in jobs_resp.jobs]
logger.info(f"reloading {len(running_jobs)} job(s)")
await self._worker._reload_jobs(running_jobs)

# Listen for GetRunningJobsRequest from Go (capture before restart)
# Handle messages from Go: the jobs to reload, GetRunningJobsRequest (capture
# before restart), and Credentials (a refreshed worker token in token mode)
while True:
try:
data = await _recv_proto(reader)
Expand All @@ -84,7 +78,15 @@ async def _run(self) -> None:
msg = agent_dev.AgentDevMessage()
msg.ParseFromString(data)

if msg.HasField("get_running_jobs_request"):
if msg.HasField("get_running_jobs_response"):
jobs_resp = msg.get_running_jobs_response
if jobs_resp.jobs:
running_jobs = [proto.running_job_from_proto(j) for j in jobs_resp.jobs]
logger.info(f"reloading {len(running_jobs)} job(s)")
# not awaited here: in token mode the reload waits for registration,
# which can need a token that comes later on this channel (Credentials)
self._reload_task = self._loop.create_task(self._reload_jobs(running_jobs))
elif msg.HasField("get_running_jobs_request"):
jobs = self._worker.active_jobs
job_protos = [proto.running_job_to_proto(j) for j in jobs]
resp = agent_dev.AgentDevMessage(
Expand All @@ -93,17 +95,26 @@ async def _run(self) -> None:
)
)
await _send_proto(writer, resp.SerializeToString())
elif msg.HasField("credentials"):
await self._worker._update_token(msg.credentials.token)
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
Resolved
except (asyncio.IncompleteReadError, ConnectionError, OSError):
pass
finally:
writer.close()
with contextlib.suppress(Exception):
await writer.wait_closed()

@utils.log_exceptions(logger=logger)
async def _reload_jobs(self, jobs: list[RunningJobInfo]) -> None:
await self._worker._reload_jobs(jobs)

async def aclose(self) -> None:
if not self._main_task:
return

self._main_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await self._main_task

if self._reload_task:
await utils.aio.cancel_and_wait(self._reload_task)
12 changes: 9 additions & 3 deletions livekit-agents/livekit/agents/inference/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from livekit import api

from .._token_mode import token_mode_token
from ..utils.misc import shortuuid
from ..version import __version__

Expand Down Expand Up @@ -82,20 +83,21 @@ def resolve_credentials(
resolved_api_key = api_key or os.getenv(
"LIVEKIT_INFERENCE_API_KEY", os.getenv("LIVEKIT_API_KEY", "")
)
if not resolved_api_key:
if not resolved_api_key and token_mode_token() is None:
raise ValueError(
"api_key is required, either as argument or set LIVEKIT_API_KEY environmental variable"
)

resolved_api_secret = api_secret or os.getenv(
"LIVEKIT_INFERENCE_API_SECRET", os.getenv("LIVEKIT_API_SECRET", "")
)
if not resolved_api_secret:
if not resolved_api_secret and token_mode_token() is None:
raise ValueError(
"api_secret is required, either as argument or set LIVEKIT_API_SECRET environmental variable"
)

return resolved_api_key, resolved_api_secret
# both are empty in token mode, where create_access_token returns the worker token
return resolved_api_key or "", resolved_api_secret or ""


def get_inference_headers(*, inference_class: str | None = None) -> dict[str, str]:
Expand Down Expand Up @@ -159,6 +161,10 @@ def create_inference_request_id(


def create_access_token(api_key: str | None, api_secret: str | None, ttl: float = 600) -> str:
if (not api_key or not api_secret) and (token := token_mode_token()) is not None:
# token mode: no secret to sign with, the worker token is sent as is
return token

grant = api.access_token.InferenceGrants(perform=True)
return (
api.AccessToken(api_key, api_secret)
Expand Down
5 changes: 3 additions & 2 deletions livekit-agents/livekit/agents/inference/avatar.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
APITimeoutError,
create_api_error_from_http,
)
from .._token_mode import token_mode_token
from ..job import get_job_context
from ..log import logger
from ..types import (
Expand Down Expand Up @@ -185,7 +186,7 @@ def __init__(
if is_given(api_key)
else os.getenv("LIVEKIT_INFERENCE_API_KEY", os.getenv("LIVEKIT_API_KEY", ""))
)
if not self._api_key:
if not self._api_key and token_mode_token() is None:
raise ValueError(
"api_key is required, either as argument or set LIVEKIT_API_KEY environment variable"
)
Expand All @@ -194,7 +195,7 @@ def __init__(
if is_given(api_secret)
else os.getenv("LIVEKIT_INFERENCE_API_SECRET", os.getenv("LIVEKIT_API_SECRET", ""))
)
if not self._api_secret:
if not self._api_secret and token_mode_token() is None:
raise ValueError(
"api_secret is required, either as argument or set LIVEKIT_API_SECRET environment variable"
)
Expand Down
5 changes: 3 additions & 2 deletions livekit-agents/livekit/agents/inference/eot/detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import aiohttp

from ... import utils
from ..._token_mode import token_mode_token
from ...language import LanguageCode
from ...log import logger
from ...types import (
Expand Down Expand Up @@ -84,9 +85,9 @@ def __init__(
missing: list[str] = []
if not lk_base_url:
missing.append("LIVEKIT_INFERENCE_URL")
if not lk_api_key:
if not lk_api_key and token_mode_token() is None:
missing.append("LIVEKIT_API_KEY")
if not lk_api_secret:
if not lk_api_secret and token_mode_token() is None:
missing.append("LIVEKIT_API_SECRET")
if missing:
if auto:
Expand Down
5 changes: 3 additions & 2 deletions livekit-agents/livekit/agents/inference/interruption.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

from .. import utils
from .._exceptions import APIConnectionError, APIError, APIStatusError
from .._token_mode import token_mode_token
from ..log import logger
from ..metrics.base import InterruptionMetrics, Metadata
from ..telemetry import trace_types
Expand Down Expand Up @@ -314,7 +315,7 @@ def __init__(
if api_key
else os.getenv("LIVEKIT_INFERENCE_API_KEY", os.getenv("LIVEKIT_API_KEY", ""))
)
if not lk_api_key:
if not lk_api_key and token_mode_token() is None:
raise ValueError(
"api_key is required, either as argument or set LIVEKIT_API_KEY environmental variable"
)
Expand All @@ -324,7 +325,7 @@ def __init__(
if api_secret
else os.getenv("LIVEKIT_INFERENCE_API_SECRET", os.getenv("LIVEKIT_API_SECRET", ""))
)
if not lk_api_secret:
if not lk_api_secret and token_mode_token() is None:
raise ValueError(
"api_secret is required, either as argument or set LIVEKIT_API_SECRET environmental variable"
)
Expand Down
5 changes: 3 additions & 2 deletions livekit-agents/livekit/agents/inference/stt.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
APITimeoutError,
create_api_error_from_http,
)
from .._token_mode import token_mode_token
from ..language import LanguageCode
from ..log import logger
from ..types import (
Expand Down Expand Up @@ -796,7 +797,7 @@ def __init__(
if is_given(api_key)
else os.getenv("LIVEKIT_INFERENCE_API_KEY", os.getenv("LIVEKIT_API_KEY", ""))
)
if not lk_api_key:
if not lk_api_key and token_mode_token() is None:
raise ValueError(
"api_key is required, either as argument or set LIVEKIT_API_KEY environmental variable"
)
Expand All @@ -806,7 +807,7 @@ def __init__(
if is_given(api_secret)
else os.getenv("LIVEKIT_INFERENCE_API_SECRET", os.getenv("LIVEKIT_API_SECRET", ""))
)
if not lk_api_secret:
if not lk_api_secret and token_mode_token() is None:
raise ValueError(
"api_secret is required, either as argument or set LIVEKIT_API_SECRET environmental variable"
)
Expand Down
5 changes: 3 additions & 2 deletions livekit-agents/livekit/agents/inference/tts.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
APITimeoutError,
create_api_error_from_http,
)
from .._token_mode import token_mode_token
from ..language import LanguageCode
from ..log import logger
from ..tts._provider_format import drop_bracket_cues
Expand Down Expand Up @@ -473,7 +474,7 @@ def __init__(
if is_given(api_key)
else os.getenv("LIVEKIT_INFERENCE_API_KEY", os.getenv("LIVEKIT_API_KEY", ""))
)
if not lk_api_key:
if not lk_api_key and token_mode_token() is None:
raise ValueError(
"api_key is required, either as argument or set LIVEKIT_API_KEY environmental variable"
)
Expand All @@ -483,7 +484,7 @@ def __init__(
if is_given(api_secret)
else os.getenv("LIVEKIT_INFERENCE_API_SECRET", os.getenv("LIVEKIT_API_SECRET", ""))
)
if not lk_api_secret:
if not lk_api_secret and token_mode_token() is None:
raise ValueError(
"api_secret is required, either as argument or set LIVEKIT_API_SECRET environmental variable"
)
Expand Down
32 changes: 31 additions & 1 deletion livekit-agents/livekit/agents/ipc/job_proc_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from livekit.protocol import agent

from .._token_mode import token_mode_token
from ..job import JobContext, JobProcess, RunningJobInfo
from ..log import logger
from ..telemetry import metrics
Expand Down Expand Up @@ -42,6 +43,8 @@ def __init__(
mp_ctx: BaseContext,
loop: asyncio.AbstractEventLoop,
on_join_timing: Callable[[agent.JobJoinTiming], None] | None = None,
create_job_token: Callable[[str, str], Awaitable[agent.CreateJobTokenResponse]]
| None = None,
) -> None:
super().__init__(
initialize_timeout=initialize_timeout,
Expand All @@ -67,6 +70,8 @@ def __init__(
self._inference_executor = inference_executor
self._on_join_timing = on_join_timing
self._inference_tasks: set[asyncio.Task[None]] = set()
self._create_job_token = create_job_token
self._job_token_tasks: set[asyncio.Task[None]] = set()
self._id = shortuuid("PCEXEC_")

@property
Expand Down Expand Up @@ -131,8 +136,12 @@ async def _main_task(self, ipc_ch: aio.ChanReceiver[channel.Message]) -> None:
task.add_done_callback(self._inference_tasks.discard)
if isinstance(msg, proto.JoinTiming) and self._on_join_timing:
self._on_join_timing(msg.timing)
if isinstance(msg, proto.CreateJobTokenRequest):
token_task = asyncio.create_task(self._do_create_job_token(msg))
self._job_token_tasks.add(token_task)
token_task.add_done_callback(self._job_token_tasks.discard)
finally:
await aio.cancel_and_wait(*self._inference_tasks)
await aio.cancel_and_wait(*self._inference_tasks, *self._job_token_tasks)

@log_exceptions(logger=logger)
async def _supervise_task(self) -> None:
Expand Down Expand Up @@ -166,6 +175,19 @@ async def _do_inference_task(self, inf_req: proto.InferenceRequest) -> None:
proto.InferenceResponse(request_id=inf_req.request_id, error=str(e)),
)

async def _do_create_job_token(self, req: proto.CreateJobTokenRequest) -> None:
resp = proto.CreateJobTokenResponse(request_id=req.request_id)
try:
if self._create_job_token is None or self._running_job is None:
raise RuntimeError("cannot create a job token: no running job")

res = await self._create_job_token(self._running_job.job.id, req.claims)
resp.token, resp.expires_at = res.token, res.expires_at
except Exception as e:
resp.error = str(e)

await channel.asend_message(self._pch, resp)

async def launch_job(self, info: RunningJobInfo) -> None:
"""start/assign a job to the process"""
if self._running_job is not None:
Expand All @@ -181,13 +203,21 @@ async def launch_job(self, info: RunningJobInfo) -> None:
start_req = proto.StartJobRequest()
start_req.running_job = info
try:
if (token := token_mode_token()) is not None:
# the process may hold an older worker token in its environment, from when
# it started
await channel.asend_message(self._pch, proto.UpdateWorkerToken(token=token))
await channel.asend_message(self._pch, start_req)
except Exception:
self._running_job = None
self._job_status = None
metrics.job_ended()
raise

async def update_worker_token(self, token: str) -> None:
"""sends a refreshed worker token (token mode) to the process running a job"""
await channel.asend_message(self._pch, proto.UpdateWorkerToken(token=token))

def logging_extra(self) -> dict[str, Any]:
extra = super().logging_extra()

Expand Down
Loading
Loading