diff --git a/src/livepeer_gateway/errors.py b/src/livepeer_gateway/errors.py index bddd26e..64dca2f 100644 --- a/src/livepeer_gateway/errors.py +++ b/src/livepeer_gateway/errors.py @@ -1,3 +1,5 @@ +"""Exception types raised by the Livepeer gateway SDK.""" + from __future__ import annotations from dataclasses import dataclass @@ -8,7 +10,13 @@ class LivepeerGatewayError(RuntimeError): class LivepeerHTTPError(LivepeerGatewayError): - """Raised when an HTTP endpoint returns a non-success status.""" + """A non-success HTTP response from an endpoint. + + Attributes: + status_code: HTTP status code returned by the endpoint. + url: Endpoint that produced the error. + body: Raw response body, when available. + """ def __init__(self, status_code: int, url: str, body: str = "", message: str | None = None) -> None: self.status_code = int(status_code) @@ -19,28 +27,53 @@ def __init__(self, status_code: int, url: str, body: str = "", message: str | No @dataclass class OrchestratorRejection: - """Records a single orchestrator that was tried and rejected.""" + """A single orchestrator that was tried during selection and rejected. + + Attributes: + url: Endpoint URL of the orchestrator that was attempted. + reason: Human-readable explanation of why it was rejected. + """ url: str reason: str @dataclass class RunnerRejection: - """Records a single runner that was tried and rejected.""" + """A single runner that was tried during selection and rejected. + + Attributes: + url: Endpoint URL of the runner that was attempted. + reason: Human-readable explanation of why it was rejected. + """ url: str reason: str class NoOrchestratorAvailableError(LivepeerGatewayError): - """Raised when no orchestrator could be selected.""" + """No orchestrator is available; every candidate was rejected during selection. + + Attributes: + rejections: Each orchestrator that was tried and why it was rejected. + """ def __init__(self, message: str, rejections: list[OrchestratorRejection] | None = None) -> None: super().__init__(message) self.rejections: list[OrchestratorRejection] = rejections or [] + def __str__(self) -> str: + message = super().__str__() + if not self.rejections: + return message + reasons = "; ".join(f"{r.url}: {r.reason}" for r in self.rejections) + return f"{message}: {reasons}" + class NoRunnerAvailableError(LivepeerGatewayError): - """Raised when no runner could be selected.""" + """No runner is available; every candidate was rejected during selection. + + Attributes: + rejections: Each runner that was tried and why it was rejected. + """ def __init__(self, message: str, rejections: list[RunnerRejection] | None = None) -> None: super().__init__(message) @@ -55,7 +88,11 @@ def __str__(self) -> str: class SignerRefreshRequired(LivepeerGatewayError): - """Raised when the remote signer returns HTTP 480 and a refresh is required.""" + """The remote signer requires a credential refresh. + + Attributes: + orchestrator_url: Orchestrator whose signer requested the refresh, if known. + """ def __init__( self, @@ -68,8 +105,8 @@ def __init__( class SkipPaymentCycle(LivepeerGatewayError): - """Raised when the signer returns HTTP 482 to skip a payment cycle.""" + """A signer HTTP 482 response requesting that a payment cycle be skipped.""" class PaymentError(LivepeerGatewayError): - """Raised when a PaymentSession operation fails.""" + """A failed PaymentSession operation.""" diff --git a/src/livepeer_gateway/live_runner.py b/src/livepeer_gateway/live_runner.py index b965a3e..9a4bfad 100644 --- a/src/livepeer_gateway/live_runner.py +++ b/src/livepeer_gateway/live_runner.py @@ -1,3 +1,5 @@ +"""Live runner discovery, registration, sessions, trickle channels, and calls.""" + from __future__ import annotations import asyncio @@ -36,33 +38,72 @@ class LiveRunnerTrickleChannelRequest(TypedDict): + """A trickle channel to create. + + Attributes: + name: Caller-chosen channel name, unique within the session. + mime_type: MIME type of the media carried on the channel. + """ + name: str mime_type: str class LiveRunnerTrickleChannel(TypedDict): + """A trickle channel created on a live runner session. + + Attributes: + name: The channel name requested by the caller. + channel_name: The orchestrator-assigned channel identifier. + url: Public/external trickle URL. + internal_url: Private-network runner-to-orchestrator URL. When the runner + and orchestrator share a network (e.g. Docker), this bypasses public + TLS/routing; present only when the orchestrator is configured to + return one. + mime_type: MIME type of the media carried on the channel. + """ + name: str channel_name: str - # Public/external trickle URL. url: str - # Optional private-network URL for runner-to-orchestrator traffic. When the - # runner and orchestrator share a network, such as Docker, this can bypass - # public TLS/routing, but it is only present when the orchestrator is - # configured to return one. internal_url: NotRequired[str] mime_type: str class LiveRunnerSessionHeaders(Protocol): - def get(self, key: str, default: str = "") -> str: ... + """Mapping-like session headers exposing a ``get(key, default)`` accessor.""" + + def get(self, key: str, default: str = "") -> str: + """Return the value for ``key``, or ``default`` if absent.""" + ... class LiveRunnerSessionRequest(Protocol): + """An incoming request carrying the orchestrator's live-runner session headers. + + Any object exposing a ``headers`` mapping (e.g. a web framework request) + satisfies this protocol. + + Attributes: + headers: Session headers set by the orchestrator proxy, such as + ``Livepeer-Session-Id``, ``Livepeer-Session-Token``, + ``Livepeer-Runner-Route``, and ``Livepeer-Session-Control``. + """ + headers: LiveRunnerSessionHeaders @dataclass(frozen=True) class LiveRunnerSessionEvent: + """A session reservation lifecycle event delivered over the O2R channel. + + Attributes: + session_id: The session the event concerns. + event: Whether the session was ``"reserved"`` or ``"released"``. + timestamp: Orchestrator-supplied event timestamp, if any. + raw: The raw event message as received. + """ + session_id: str event: Literal["reserved", "released"] timestamp: Optional[str] @@ -74,7 +115,16 @@ class LiveRunnerSessionEvent: @dataclass(frozen=True) class LiveRunnerInstance: - """A normalized live runner discovered from an orchestrator entry.""" + """A normalized live runner discovered from an orchestrator entry. + + Attributes: + url: Runner endpoint used to call the app. + app: App the runner serves. + runner_id: Orchestrator-assigned runner identifier. + mode: Runner mode, e.g. ``"persistent"`` or ``"single-shot"``. + orchestrator_url: URL of the orchestrator advertising the runner. + raw: The raw runner entry as received from discovery. + """ url: str app: str @@ -86,6 +136,17 @@ class LiveRunnerInstance: @dataclass(frozen=True) class LiveRunnerSession: + """A reserved session on a live runner. + + Attributes: + session_id: Identifier of the reserved session. + app_url: URL where the app serves this session. Clients send their session + requests here (e.g. ``{app_url}/echo``). + runner_url: Base URL of the runner hosting the session, used to manage the + session lifecycle such as stopping it. + runner: The runner instance backing the session, when known. + """ + session_id: str app_url: str runner_url: str @@ -94,12 +155,33 @@ class LiveRunnerSession: @dataclass(frozen=True) class LiveRunnerProxy: + """A public proxy exposing a runner-served URL to the public internet. + + The orchestrator maps a public URL onto a target URL the runner serves, forwarding + inbound traffic to it. + + Attributes: + proxy_id: Identifier of the created proxy. + url: Public URL that forwards to the runner-served target. + """ + proxy_id: str url: str @dataclass(frozen=True) class LiveRunnerCallResult: + """The result of a call to a live runner. + + Attributes: + data: The runner's JSON response body. + runner_url: URL of the runner that produced the response. + runner: The runner instance that was called, when known. + session_id: Session identifier associated with the call, if any. + payment_session: Payment session used to settle a paid call. ``None`` for + unpaid calls. Excluded from ``repr`` and equality. + """ + data: dict[str, Any] runner_url: str runner: Optional[LiveRunnerInstance] = None @@ -113,11 +195,20 @@ class LiveRunnerCallResult: @dataclass(frozen=True) class LiveRunnerGPU: + """GPU details advertised when registering a runner. + + Attributes: + id: GPU identifier (e.g. device index or UUID). + name: Human-readable GPU model name. + vram_mb: Video memory in megabytes. + """ + id: str = "" name: str = "" vram_mb: int = 0 def to_json(self) -> dict[str, Any]: + """Return the JSON payload, omitting empty/zero fields.""" data: dict[str, Any] = {} if self.id: data["id"] = self.id @@ -130,11 +221,21 @@ def to_json(self) -> dict[str, Any]: @dataclass(frozen=True) class LiveRunnerPriceInfo: + """Pricing advertised for a runner's work. + + Attributes: + price: Amount charged per ``unit``, in ``currency`` (a number or numeric + string). + currency: Currency of ``price`` (e.g. ``"usd"``). + unit: Billing unit that ``price`` applies to (e.g. ``"hour"``). + """ + price: int | float | str currency: str = "usd" unit: str = "hour" def to_json(self) -> dict[str, Any]: + """Return the pricing as a JSON payload.""" return { "price": self.price, "currency": str(self.currency or "usd").strip().lower(), @@ -143,6 +244,25 @@ def to_json(self) -> dict[str, Any]: class LiveRunnerRegistration: + """A live runner's registration with an orchestrator. + + Keeps the runner registered by sending periodic heartbeats and tracks the + sessions the orchestrator has reserved on it. Prefer ``register_runner`` + to construct and start one, and use it as an async context manager to + unregister on exit. Also exposes session-scoped helpers + (``create_trickle_channels``, ``remove_trickle_channels``, ``create_proxy``) + pre-bound to this registration's orchestrator and runner id. + + Attributes: + orchestrator_url: Orchestrator the runner is registered with. May be updated to + the orchestrator returned by the initial heartbeat. + runner_id: Orchestrator-assigned runner id, populated after the first heartbeat. + heartbeat_interval_s: How often the runner sends heartbeats, in seconds. + heartbeat_ttl_s: How long the orchestrator keeps the runner registered without + a heartbeat, in seconds, if advertised. + o2r_channel: Orchestrator-to-runner control channel, when established. + """ + def __init__( self, *, @@ -164,6 +284,37 @@ def __init__( on_session_reserve: Optional[LiveRunnerSessionCallback] = None, on_session_release: Optional[LiveRunnerSessionCallback] = None, ) -> None: + """Construct a registration without starting it. + + Prefer ``register_runner``, which assembles ``price_info``, + optionally auto-detects the GPU, and starts heartbeating. When + constructing directly, call ``start`` (or use the instance as an + async context manager) before the runner is registered. + + Args: + orchestrator_url: Orchestrator to register with. + secret: Bootstrap secret authorizing the initial registration. + runner_url: URL at which this runner serves the app. + app: App this runner serves. + price_info: Pricing advertised for the runner's work. + runner_id: Preferred runner id. The orchestrator may assign one otherwise. + mode: Runner mode, ``"persistent"`` or ``"single-shot"``. + label: Optional human-readable label for the runner. + version: Optional runner/app version string. + status: Initial advertised status, e.g. ``"ready"``. + capacity: Number of concurrent sessions the runner can serve. + gpu: GPU details to advertise, if any. + timeout: Per-request timeout, in seconds. + heartbeat_interval_s: Override the heartbeat interval. Defaults to the + orchestrator-advertised value. + unregister_on_close: Unregister the runner when the registration closes. + on_session_reserve: Callback invoked when a session is reserved. + on_session_release: Callback invoked when a session is released. + + Raises: + LivepeerGatewayError: If ``orchestrator_url`` is invalid. + ValueError: If ``mode`` is not ``"persistent"`` or ``"single-shot"``. + """ self.orchestrator_url = _normalize_http_base(orchestrator_url) self.runner_id = runner_id self.heartbeat_interval_s = heartbeat_interval_s or _DEFAULT_HEARTBEAT_INTERVAL_S @@ -193,16 +344,30 @@ def __init__( self._o2r_task: Optional[asyncio.Task[None]] = None async def start(self) -> "LiveRunnerRegistration": + """Send the initial heartbeat and start the background heartbeat loop. + + Returns: + This registration, for chaining. + + Raises: + LivepeerGatewayError: If the initial registration heartbeat fails. + """ await self._send_heartbeat() self._task = asyncio.create_task(self._heartbeat_loop()) return self @property def active_session_ids(self) -> tuple[str, ...]: + """Immutable snapshot of currently reserved session ids, in reservation order.""" # Return an immutable snapshot; internal storage stays list-backed to preserve reservation order. return tuple(self._active_session_ids) async def close(self) -> None: + """Stop heartbeating and, unless disabled, unregister the runner. + + Idempotent and safe to call from ``__aexit__``. Shutdown errors are logged + rather than raised. + """ self._closed = True o2r_reader = self._o2r_reader self._o2r_reader = None @@ -258,6 +423,22 @@ async def create_trickle_channels( This is intended for apps running behind the orchestrator's live-runner proxy, not end-user clients. Apps should normally pass the incoming request so the orchestrator-provided session headers are used. + + Args: + session: The session to target: a ``session_id`` string, or a + ``LiveRunnerSessionRequest`` from which the session id and token are + read. + channels: Channels to create, each a dict with ``name`` and ``mime_type``. + session_token: Session auth token. Overrides the value carried by + ``session``. Required when ``session`` is a plain id string. + + Returns: + The created channels as returned by the runner. + + Raises: + LivepeerGatewayError: If credentials are missing or the response omits the + created channels. + TypeError: If a channel entry is malformed. """ return await create_trickle_channels( session, @@ -280,6 +461,21 @@ async def remove_trickle_channels( This is intended for apps running behind the orchestrator's live-runner proxy, not end-user clients. Apps should normally pass the incoming request so the orchestrator-provided session headers are used. + + Args: + session: The session to target: a ``session_id`` string, or a + ``LiveRunnerSessionRequest`` from which the session id and token are + read. + channels: Names of the channels to remove. + session_token: Session auth token. Overrides the value carried by + ``session``. Required when ``session`` is a plain id string. + + Returns: + Names of the channels that were removed. + + Raises: + LivepeerGatewayError: If credentials are missing, the response is not a + JSON object, or it omits the deleted channel list. """ return await remove_trickle_channels( session, @@ -298,7 +494,30 @@ async def create_proxy( session_token: str = "", timeout: Optional[float] = None, ) -> LiveRunnerProxy: - """Create a public proxy URL for a live runner app target.""" + """Create a public proxy URL for a live runner app target. + + This is intended for apps running behind the orchestrator's live-runner + proxy, not end-user clients. Apps should normally pass the incoming + request so the orchestrator-provided session headers are used. + + Args: + session: The session to target: a ``session_id`` string, or a + ``LiveRunnerSessionRequest`` from which the session id and token are + read. + target_url: The runner-served URL to expose through the proxy. + session_token: Session auth token. Overrides the value carried by + ``session``. Required when ``session`` is a plain id string. + timeout: Per-request timeout, in seconds. Defaults to the registration's + timeout. + + Returns: + The created proxy, carrying its ``proxy_id`` and public ``url``. + + Raises: + LivepeerGatewayError: If credentials or ``target_url`` are missing, + the response is not a JSON object, or it omits ``proxy_id`` or + ``url``. + """ return await create_proxy( session, target_url, @@ -491,9 +710,44 @@ async def register_runner( unregister_on_close: bool = True, on_session_reserve: Optional[LiveRunnerSessionCallback] = None, on_session_release: Optional[LiveRunnerSessionCallback] = None, -) -> LiveRunnerRegistration: +) -> "LiveRunnerRegistration": + """Register a live runner with an orchestrator and start heartbeating. + + Args: + orchestrator_url: Orchestrator to register with. + secret: Bootstrap secret authorizing the initial registration. + runner_url: URL at which this runner serves the app. + app: App this runner serves. + price: Amount charged per ``unit``, in ``currency`` (a number or numeric + string). + currency: Currency of ``price`` (e.g. ``"usd"``). + unit: Billing unit that ``price`` applies to (e.g. ``"hour"``). + runner_id: Preferred runner id. The orchestrator may assign one otherwise. + mode: Runner mode, ``"persistent"`` or ``"single-shot"``. + label: Optional human-readable label for the runner. + version: Optional runner/app version string. + status: Initial advertised status, e.g. ``"ready"``. + capacity: Number of concurrent sessions the runner can serve. + gpu: GPU details to advertise. Auto-detected when omitted and + ``auto_detect_gpu`` is set. + auto_detect_gpu: Detect local GPU details when ``gpu`` is not provided. + timeout: Per-request timeout, in seconds. + heartbeat_interval_s: Override the heartbeat interval. Defaults to the + orchestrator-advertised value. + unregister_on_close: Unregister the runner when the registration closes. + on_session_reserve: Callback invoked when a session is reserved. + on_session_release: Callback invoked when a session is released. + + Returns: + A started ``LiveRunnerRegistration`` whose heartbeat loop is running. + + Raises: + LivepeerGatewayError: If ``orchestrator_url`` is invalid or the initial + registration heartbeat fails. + ValueError: If ``mode`` is not ``"persistent"`` or ``"single-shot"``. + """ if gpu is None and auto_detect_gpu: - gpu = detect_process_gpu() + gpu = _detect_process_gpu() registration = LiveRunnerRegistration( orchestrator_url=orchestrator_url, @@ -526,7 +780,30 @@ async def create_trickle_channels( session_token: str = "", timeout: float = 5.0, ) -> list[LiveRunnerTrickleChannel]: - """Create trickle channels for a live runner app session.""" + """Create trickle channels for a live runner app session. + + Args: + session: The session to target: a ``session_id`` string, or a + ``LiveRunnerSessionRequest`` from which the session id, token, runner route, + and control URL are read. + channels: Channels to create, each a dict with ``name`` and ``mime_type``. + orchestrator_url: Orchestrator base URL, e.g. ``https://orch.example`` (the + request path is appended, so pass no path). Falls back to the session's + control URL when empty. + runner_id: Runner route for the request. Overrides the value carried by + ``session``. Required when ``session`` is a plain id string. + session_token: Session auth token. Overrides the value carried by + ``session``. Required when ``session`` is a plain id string. + timeout: Per-request timeout, in seconds. + + Returns: + The created channels as returned by the runner. + + Raises: + LivepeerGatewayError: If credentials are missing or the response omits the + created channels. + TypeError: If a channel entry is malformed. + """ runner, session_id, token, control_url = _resolve_session_credentials( session, runner_id=runner_id, @@ -556,7 +833,29 @@ async def remove_trickle_channels( session_token: str = "", timeout: float = 5.0, ) -> list[str]: - """Remove trickle channels for a live runner app session.""" + """Remove trickle channels for a live runner app session. + + Args: + session: The session to target: a ``session_id`` string, or a + ``LiveRunnerSessionRequest`` from which the session id, token, runner route, + and control URL are read. + channels: Names of the channels to remove. + orchestrator_url: Orchestrator base URL, e.g. ``https://orch.example`` (the + request path is appended, so pass no path). Falls back to the session's + control URL when empty. + runner_id: Runner route for the request. Overrides the value carried by + ``session``. Required when ``session`` is a plain id string. + session_token: Session auth token. Overrides the value carried by ``session``. + Required when ``session`` is a plain id string. + timeout: Per-request timeout, in seconds. + + Returns: + Names of the channels that were removed. + + Raises: + LivepeerGatewayError: If credentials are missing, the response is not a JSON + object, or it omits the deleted channel list. + """ runner, session_id, token, control_url = _resolve_session_credentials( session, runner_id=runner_id, @@ -588,7 +887,29 @@ async def create_proxy( session_token: str = "", timeout: float = 5.0, ) -> LiveRunnerProxy: - """Create a public proxy URL for a target served by a live runner app session.""" + """Create a public proxy URL for a target served by a live runner app session. + + Args: + session: The session to target: a ``session_id`` string, or a + ``LiveRunnerSessionRequest`` from which the session id, token, runner route, + and control URL are read. + target_url: The runner-served URL to expose through the proxy. + orchestrator_url: Orchestrator base URL, e.g. ``https://orch.example`` (the + request path is appended, so pass no path). Falls back to the session's + control URL when empty. + runner_id: Runner route for the request. Overrides the value carried by + ``session``. Required when ``session`` is a plain id string. + session_token: Session auth token. Overrides the value carried by + ``session``. Required when ``session`` is a plain id string. + timeout: Per-request timeout, in seconds. + + Returns: + The created proxy, carrying its ``proxy_id`` and public ``url``. + + Raises: + LivepeerGatewayError: If credentials or ``target_url`` are missing, the response + is not a JSON object, or it omits ``proxy_id`` or ``url``. + """ runner, session_id, token, control_url = _resolve_session_credentials( session, runner_id=runner_id, @@ -625,6 +946,32 @@ async def call_runner( timeout: float = 5.0, max_payment_challenge_retries: int = 3, ) -> LiveRunnerCallResult: + """Send a request to a live runner, handling payment challenges transparently. + + If ``signer_url`` is set, a 402 payment challenge is answered and the request + retried. Paid calls without a signer fail. + + Args: + runner_url: Runner endpoint to call. Optional if ``runner`` is given. + runner: Discovered runner instance to call. Its URL is used when ``runner_url`` + is empty. + payload: JSON request body sent to the runner. Defaults to an empty body. + method: HTTP method for the request. + signer_url: Signer service used to answer 402 payment challenges. Required for + paid runners. + signer_headers: Extra HTTP headers sent to the signer service. + timeout: Per-request timeout, in seconds. + max_payment_challenge_retries: Maximum number of payment challenges to answer + before giving up. + + Returns: + A ``LiveRunnerCallResult`` wrapping the runner's JSON response. + + Raises: + LivepeerGatewayError: If neither target is given, a paid call is made without + ``signer_url``, the response is not a JSON object, or the payment-challenge + retries are exhausted. + """ runner_url = runner_url.strip() or (runner.url.strip() if runner is not None else "") if not runner_url: raise LivepeerGatewayError("Live runner call requires runner_url") @@ -787,6 +1134,18 @@ async def stop_runner_session( *, timeout: float = 5.0, ) -> None: + """Stop a live runner session, releasing its reservation. + + Args: + session: The session to stop: a ``LiveRunnerSession`` (stopped via its runner + URL) or a ``LiveRunnerSessionRequest`` carrying the orchestrator's + session-control headers. + timeout: Per-request timeout, in seconds. + + Raises: + LivepeerGatewayError: If the required runner URL, session id, or session control + URL is missing, or the stop request fails. + """ request_headers: dict[str, str] = {} if isinstance(session, LiveRunnerSession): runner_url = session.runner_url.strip() @@ -813,7 +1172,7 @@ async def stop_runner_session( ) -def detect_process_gpu() -> Optional[LiveRunnerGPU]: +def _detect_process_gpu() -> Optional[LiveRunnerGPU]: for detector in (_detect_gpu_pynvml, _detect_gpu_torch, _detect_gpu_nvidia_smi): try: gpu = detector() diff --git a/src/livepeer_gateway/media_decode.py b/src/livepeer_gateway/media_decode.py index 4413f01..e562e5b 100644 --- a/src/livepeer_gateway/media_decode.py +++ b/src/livepeer_gateway/media_decode.py @@ -1,3 +1,5 @@ +"""Decode trickle MPEG-TS media into frames and demuxed packets.""" + from __future__ import annotations import queue @@ -12,6 +14,22 @@ @dataclass(frozen=True) class DecodedMediaFrame: + """A single decoded media frame with its source timing metadata. + + Base type for ``VideoDecodedMediaFrame`` and ``AudioDecodedMediaFrame``. + + Attributes: + kind: Stream kind this frame came from, e.g. ``"video"`` or ``"audio"``. + stream_index: Index of the source stream within the container. + frame: The underlying decoded PyAV frame. + pts: Presentation timestamp in ``time_base`` units, or ``None`` if unset. + time_base: Stream time base (seconds per ``pts`` unit), or ``None``. + pts_time: Presentation timestamp in seconds (``pts * time_base``). Media time + relative to the stream start, not wall-clock time. + demuxed_at: Wall-clock time (``time.time()``) when the packet was demuxed. + decoded_at: Wall-clock time (``time.time()``) when the frame was decoded. + """ + kind: str stream_index: int frame: Union[av.VideoFrame, av.AudioFrame] @@ -25,6 +43,16 @@ class DecodedMediaFrame: @dataclass(frozen=True) class VideoDecodedMediaFrame(DecodedMediaFrame): + """A decoded video frame. + + Extends ``DecodedMediaFrame`` with video-specific attributes. + + Attributes: + width: Frame width in pixels. + height: Frame height in pixels. + pix_fmt: Pixel format name (e.g. ``"yuv420p"``), or ``None`` if unknown. + """ + width: int height: int pix_fmt: Optional[str] @@ -32,6 +60,17 @@ class VideoDecodedMediaFrame(DecodedMediaFrame): @dataclass(frozen=True) class AudioDecodedMediaFrame(DecodedMediaFrame): + """A decoded audio frame. + + Extends ``DecodedMediaFrame`` with audio-specific attributes. + + Attributes: + sample_rate: Samples per second, or ``None`` if unknown. + layout: Channel layout name (e.g. ``"stereo"``), or ``None`` if unknown. + format: Sample format name (e.g. ``"fltp"``), or ``None`` if unknown. + samples: Number of samples per channel in this frame, or ``None``. + """ + sample_rate: Optional[int] layout: Optional[str] format: Optional[str] @@ -40,6 +79,26 @@ class AudioDecodedMediaFrame(DecodedMediaFrame): @dataclass(frozen=True) class DemuxedMediaPacket: + """A single demuxed media packet with its source timing metadata. + + Yielded by ``MediaOutput.packets()`` (and passed to ``on_packet``) before + decoding. + + Attributes: + kind: Stream kind this packet came from, e.g. ``"video"`` or ``"audio"``. + stream_index: Index of the source stream within the container. + packet: The underlying demuxed PyAV packet. + pts: Presentation timestamp in ``time_base`` units, or ``None`` if unset. + dts: Decode timestamp in ``time_base`` units, or ``None`` if unset. + time_base: Stream time base (seconds per timestamp unit), or ``None``. + pts_time: Presentation timestamp in seconds (``pts * time_base``), or ``None``. + Media time relative to the stream start, not wall-clock time. + dts_time: Decode timestamp in seconds (``dts * time_base``), or ``None``. + is_keyframe: Whether the packet is a keyframe. + size: Packet size in bytes. + demuxed_at: Wall-clock time (``time.time()``) when the packet was demuxed. + """ + kind: str stream_index: int packet: av.Packet @@ -59,6 +118,12 @@ class DemuxedMediaPacket: @dataclass(frozen=True) class DecoderQueueStats: + """Best-effort snapshot of the decoder's cross-thread queue metrics. + + Fields are lock-free single-writer totals read across threads; treat them as + telemetry, not exact queue state. + """ + # These queue metrics are intentionally best-effort snapshots. They are # derived from single-writer totals that are updated from different threads # without per-operation locks, and rely on CPython's implementation details @@ -415,6 +480,8 @@ def _run(self) -> None: class MpegTsDecoder(_MpegTsOutputWorker): + """Background worker that decodes an MPEG-TS byte stream into media frames.""" + def __init__(self) -> None: super().__init__(thread_name="MpegTsDecoder") @@ -455,6 +522,8 @@ def _run(self) -> None: class MpegTsPacketDemuxer(_MpegTsOutputWorker): + """Background worker that demuxes an MPEG-TS byte stream into packets.""" + def __init__(self) -> None: super().__init__(thread_name="MpegTsPacketDemuxer") @@ -485,10 +554,12 @@ def _run(self) -> None: def is_decoder_end(item: object) -> bool: + """Return ``True`` if ``item`` is the decoder's end-of-stream sentinel.""" return item is _END def decoder_error(item: object) -> Optional[BaseException]: + """Return the exception carried by a decoder error ``item``, or ``None``.""" if isinstance(item, _DecoderError): return item.error return None diff --git a/src/livepeer_gateway/media_output.py b/src/livepeer_gateway/media_output.py index 1f820a8..42c9e04 100644 --- a/src/livepeer_gateway/media_output.py +++ b/src/livepeer_gateway/media_output.py @@ -1,5 +1,4 @@ -""" -Helpers for consuming trickle media outputs as segments, bytes, or frames. +"""Helpers for consuming trickle media outputs as segments, bytes, or frames. """ from __future__ import annotations @@ -41,15 +40,39 @@ class LagPolicy(Enum): - """ - Policy for handling consumers that fall behind the segment window. - """ + """Policy for handling consumers that fall behind the segment window.""" FAIL = "fail" LATEST = "latest" EARLIEST = "earliest" @dataclass(frozen=True) class MediaOutputStats: + """Snapshot of a MediaOutput's consumption, demux, and decode statistics. + + Attributes: + elapsed_s: Seconds since the output started consuming. + segments_consumed: Number of segments read to completion. + bytes_read: Total bytes read across all segments. + chunks_read: Number of byte chunks yielded. + content_type_errors: Segments rejected for an unaccepted Content-Type. + segment_read_errors: Read errors reported by the underlying segments. + segment_max_bytes_exceeded: Segments that exceeded the max-bytes bound. + consumer_lag_skip_latest: Times a lagging consumer skipped to the latest + segment. + consumer_lag_retry_earliest: Times a lagging consumer retried from the earliest + segment. + consumer_lag_fail: Times a lagging consumer failed under LagPolicy.FAIL. + video_packets_demuxed: Video packets emitted by the demuxer. + audio_packets_demuxed: Audio packets emitted by the demuxer. + other_packets_demuxed: Non-audio/video packets emitted by the demuxer. + video_frames_decoded: Video frames emitted by the decoder. + audio_frames_decoded: Audio frames emitted by the decoder. + packet_errors: Errors raised while demuxing. + decode_errors: Errors raised while decoding. + decoder: Decode-pipeline queue metrics, or None when unavailable. + subscriber: Underlying trickle subscriber stats, or None when not subscribed. + """ + elapsed_s: float segments_consumed: int bytes_read: int @@ -100,8 +123,7 @@ def __str__(self) -> str: class MediaOutput: - """ - Access a trickle media output + """Access a trickle media output. Exposes: - per-segment iteration (SegmentReader objects) @@ -109,28 +131,9 @@ class MediaOutput: - demuxed MPEG-TS packets - individual audio and video frames - Segments are sourced from a single shared subscriber so that multiple - iterators can consume the same output concurrently without duplicate - network requests. - - Attributes: - subscribe_url: Trickle subscribe URL for this output. - start_seq: Initial server sequence when subscribing. - max_retries: Max retries for segment fetches. - max_segment_bytes: Safety bound for a single segment size. - connection_close: Whether to close connections after each segment. - chunk_size: Byte chunk size yielded by bytes()/packets()/frames(). - max_segments: Max number of segments retained in memory. - on_lag: Behavior when a consumer falls behind the segment window. - - LagPolicy.FAIL: raise LivepeerGatewayError. - - LagPolicy.LATEST: skip to the newest available segment. - - LagPolicy.EARLIEST: retry from the oldest available segment. - _sub: Shared trickle subscriber. - _segments: In-memory window of SegmentReader objects. - _lock: Coroutine-level lock for segment fetching/eviction. - _eos: End-of-stream indicator. - _next_local_seq: Local sequence counter for fetched segments. - _base_seq: Local sequence of _segments[0]. + Segments are sourced from a single shared subscriber so that multiple iterators can + consume the same output concurrently without duplicate network requests. Constructor + parameters are documented on ``__init__``. """ def __init__( @@ -149,6 +152,33 @@ def __init__( on_frame: Optional[MediaFrameCallback] = None, on_packet: Optional[MediaPacketCallback] = None, ) -> None: + """Configure a trickle media output. + + Args: + subscribe_url: Trickle subscribe URL for this output. + start_seq: Initial server sequence to subscribe from. + max_retries: Maximum retries per segment fetch. + max_segment_bytes: Safety bound on a single segment's size, unbounded when + None. + connection_close: Close the connection after each segment. + chunk_size: Byte chunk size yielded by ``bytes()``/``packets()``/ + ``frames()``. + max_segments: Maximum segments retained in memory. Must be >= 1. + on_lag: Behavior when a consumer falls behind the retained segment window + (LagPolicy.FAIL raises, LATEST skips to newest, EARLIEST retries from + oldest). + accepted_content_types: Content types accepted for incoming segments. A + segment with any other type raises. + on_bytes: Optional callback invoked with each byte chunk, started + automatically when an event loop is running. + on_frame: Optional callback invoked with each decoded audio/video frame, + started automatically when an event loop is running. + on_packet: Optional callback invoked with each demuxed packet, started + automatically when an event loop is running. + + Raises: + ValueError: If ``max_segments`` < 1 or ``accepted_content_types`` is empty. + """ if max_segments < 1: raise ValueError("max_segments must be >= 1") self.subscribe_url = subscribe_url @@ -199,8 +229,7 @@ def __init__( self.start_callbacks() def start_callbacks(self) -> list[asyncio.Task[None]]: - """ - Start configured frame/packet callback consumers. + """Start configured frame/packet callback consumers. This is idempotent. If called without a running event loop, no tasks are started and callers may retry later from async code. @@ -244,6 +273,7 @@ def start_callbacks(self) -> list[asyncio.Task[None]]: return started def callback_tasks(self) -> tuple[asyncio.Task[None], ...]: + """Return the running callback consumer tasks as a tuple.""" tasks = [] if self._bytes_callback_task is not None: tasks.append(self._bytes_callback_task) @@ -254,8 +284,7 @@ def callback_tasks(self) -> tuple[asyncio.Task[None], ...]: return tuple(tasks) async def wait_callbacks(self, timeout: Optional[float] = None) -> tuple[object, ...]: - """ - Wait for configured callback consumers to finish. + """Wait for configured callback consumers to finish. Raises the first callback error, matching close(). """ @@ -307,8 +336,7 @@ def _record_callback_task_result(self, task: asyncio.Task[None]) -> None: def segments( self, ) -> AsyncIterator[SegmentReader]: - """ - Read the trickle media channel and yield SegmentReader objects. + """Read the trickle media channel and yield SegmentReader objects. Segments are shared across iterators. """ @@ -326,9 +354,7 @@ async def _iter() -> AsyncIterator[SegmentReader]: def bytes( self, ) -> AsyncIterator[bytes]: - """ - Read the trickle media channel and yield a continuous byte stream. - """ + """Read the trickle media channel and yield a continuous byte stream.""" async def _iter() -> AsyncIterator[bytes]: async for chunk in self._iter_bytes(): @@ -339,9 +365,7 @@ async def _iter() -> AsyncIterator[bytes]: def packets( self, ) -> AsyncIterator[DemuxedMediaPacket]: - """ - Read the trickle media channel, demux MPEG-TS, and yield packets. - """ + """Read the trickle media channel, demux MPEG-TS, and yield packets.""" async def _iter() -> AsyncIterator[DemuxedMediaPacket]: demuxer = MpegTsPacketDemuxer() @@ -393,9 +417,7 @@ async def _feed() -> None: def frames( self, ) -> AsyncIterator[AudioDecodedMediaFrame | VideoDecodedMediaFrame]: - """ - Read the trickle media channel, decode MPEG-TS, and yield raw frames. - """ + """Read the trickle media channel, decode MPEG-TS, and yield raw frames.""" async def _iter() -> AsyncIterator[AudioDecodedMediaFrame | VideoDecodedMediaFrame]: decoder = MpegTsDecoder() @@ -479,9 +501,7 @@ async def _next_segment( self, seq: int, ) -> Optional[SegmentReader]: - """ - Return the segment at seq, lazily advancing the subscriber if needed. - """ + """Return the segment at seq, lazily advancing the subscriber if needed.""" # Safe lock-free read: asyncio only context-switches on awaits, and this # block has no awaits. That means _segments/_base_seq cannot change # until we return or enter the locked slow path below. @@ -547,6 +567,15 @@ async def _next_segment( return None async def close(self, *, wait_callbacks: bool = True, timeout: Optional[float] = 10.0) -> None: + """Close the output, its subscriber, and any callback consumers. + + Args: + wait_callbacks: Wait for callback consumers to drain before cancelling them. + timeout: Seconds to wait for callbacks. None waits indefinitely. + + Raises: + The first error raised by a callback consumer, if any. + """ callback_tasks = self.callback_tasks() if callback_tasks: if wait_callbacks and (timeout is None or timeout > 0): @@ -574,6 +603,7 @@ async def __aexit__(self, exc_type, exc_value, traceback) -> None: await self.close() def get_stats(self) -> MediaOutputStats: + """Return a snapshot of consumption, demux, and decode statistics.""" decoder_stats = ( self._processor.get_stats() if self._processor is not None else self._last_decoder_stats ) diff --git a/src/livepeer_gateway/media_publish.py b/src/livepeer_gateway/media_publish.py index bf11725..1e6b1a0 100644 --- a/src/livepeer_gateway/media_publish.py +++ b/src/livepeer_gateway/media_publish.py @@ -1,3 +1,5 @@ +"""Publish media as segmented MPEG-TS over trickle.""" + from __future__ import annotations import asyncio @@ -55,8 +57,7 @@ def _normalize_fps(fps: Optional[float]) -> int: @dataclass(frozen=True) class VideoOutputConfig: - """ - Output settings for one video track. + """Output settings for one video track. Stream creation is intentionally config-first where possible, but video stream initialization still waits for the first frame because width/height @@ -80,14 +81,13 @@ class VideoOutputConfig: @dataclass(frozen=True) class AudioOutputConfig: - """ - Output settings for one audio track. + """Output settings for one audio track. Audio stream creation is first-frame driven, like video. - `sample_rate` and `layout` are optional: + ``sample_rate`` and ``layout`` are optional: - when set, they are enforced as output targets - - when unset (`None`), values are derived from the first audio frame + - when unset (``None``), values are derived from the first audio frame - if unset and first-frame metadata is missing, internal defaults apply """ @@ -105,12 +105,22 @@ class AudioOutputConfig: @dataclass(frozen=True) class MediaPublishConfig: - """ - Top-level media publish configuration. + """Top-level configuration for a ``MediaPublish`` stream. - `tracks` defines the full set of output tracks for the muxed stream. Each + ``tracks`` defines the full set of output tracks for the muxed stream. Each entry becomes its own runtime track handle, queue, and encoder state inside one shared output container. + + Attributes: + mime_type: MIME type advertised for the published trickle stream. + tracks: Output track configs. Each becomes an independent runtime track. + track_wait_timeout_s: Max seconds to wait for a track's first frame once + the first frame has arrived on any track. Tracks still missing a + first frame at the deadline are dropped. + min_segment_wallclock_s: Best-effort lower bound on the wall-clock + lifetime of each trickle segment. + segment_post_idle_timeout_s: Seconds of PyAV idleness before the current + trickle transport segment is rotated. """ mime_type: str = "video/mp2t" @@ -164,6 +174,23 @@ def __str__(self) -> str: @dataclass(frozen=True) class MediaPublishStats: + """Snapshot of ``MediaPublish`` throughput and segment statistics. + + Attributes: + elapsed_s: Seconds since the publisher started. + segments_started: Trickle segments opened. + segments_completed: Trickle segments closed and marked complete. + segments_failed: Trickle segments that failed or were dropped mid-stream. + bytes_streamed_to_trickle: Payload bytes written to trickle segments. + bytes_drained: Payload bytes discarded while draining a failed or rotated + segment. + segment_writer_put_timeouts: Segment-writer enqueue timeouts observed. + terminal_failures: Unrecoverable publisher failures. + encoder_errors: Errors raised by the encoder thread. + publisher: Statistics from the underlying trickle publisher. + track_queue_stats: Per-track queue statistics. + """ + elapsed_s: float segments_started: int segments_completed: int @@ -202,11 +229,17 @@ def _new_track_stats() -> dict[str, int]: class MediaPublishTrack: - """ - Runtime handle for one configured output track. + """Runtime handle for one configured output track. + + Owns its queue and encoder state. Obtain instances from + ``MediaPublish.get_tracks()`` rather than constructing directly. Call + ``write_frame()`` to enqueue frames for encoding and ``resize()`` to adjust + queue capacity at runtime. - Owns its queue and encoder state. Call `write_frame()` to enqueue frames - for encoding. Use `resize()` to adjust queue capacity at runtime. + Attributes: + kind: Track media kind, ``"video"`` or ``"audio"``. + config: Output config this track was created from. + index: Zero-based position among tracks of the same kind. """ def __init__( @@ -244,6 +277,7 @@ def __init__( self._audio_resampler: Any = None async def write_frame(self, frame: av.VideoFrame | av.AudioFrame) -> None: + """Enqueue a frame on this track for encoding.""" await self._owner._write_frame_to_track(self, frame) def resize(self, queue_size: int) -> None: @@ -255,13 +289,12 @@ def __repr__(self) -> str: class MediaPublish: - """ - Publish muxed media as segmented MPEG-TS over trickle. + """Publish muxed media as segmented MPEG-TS over trickle. - One `MediaPublish` owns a single output container and one runtime track + One ``MediaPublish`` owns a single output container and one runtime track state per configured output track. Both audio and video tracks are first-frame driven. Once the first frame is observed for any track, a - startup timeout begins; tracks that do not deliver their first frame before + startup timeout begins. Tracks that do not deliver their first frame before the deadline are dropped before container initialization. """ @@ -271,6 +304,17 @@ def __init__( *, config: MediaPublishConfig = MediaPublishConfig(), ) -> None: + """Configure a publisher for a trickle channel. + + Args: + publish_url: Trickle channel URL the muxed stream is published to. + config: Output configuration, including the set of tracks to mux. + + Raises: + ValueError: If ``config`` has no tracks, or any of its timeouts + (``track_wait_timeout_s``, ``min_segment_wallclock_s``, + ``segment_post_idle_timeout_s``) is out of range. + """ self.publish_url = publish_url self._channel_name = publish_url.rstrip("/").rsplit("/", 1)[-1] if not config.tracks: @@ -374,9 +418,21 @@ def __init__( @property def tracks(self) -> tuple[MediaPublishTrack, ...]: + """Immutable snapshot of all runtime tracks.""" return tuple(self._tracks) def get_tracks(self, kind: Optional[str] = None) -> list[MediaPublishTrack]: + """Return runtime tracks, optionally filtered by kind. + + Args: + kind: ``"video"`` or ``"audio"`` to filter. ``None`` returns all tracks. + + Returns: + The matching runtime tracks. + + Raises: + ValueError: If ``kind`` is not ``None``, ``"video"``, or ``"audio"``. + """ if kind is None: return list(self._tracks) normalized = kind.strip().lower() @@ -400,6 +456,15 @@ def resize_track_queue(self, track: MediaPublishTrack, queue_size: int) -> None: track._queue.resize(queue_size) async def write_frame(self, frame: av.VideoFrame | av.AudioFrame) -> None: + """Enqueue a frame, routing it to the single track of its media kind. + + Use ``get_tracks()`` and the per-track ``write_frame()`` instead when + more than one track of the frame's kind is configured. + + Raises: + TypeError: If the frame type is unsupported, no track of its kind is + enabled, or multiple tracks of its kind make routing ambiguous. + """ track = self._resolve_track_for_frame(frame) await self._write_frame_to_track(track, frame) @@ -463,6 +528,11 @@ async def _suppress_close_step(self, step_name: str, awaitable: Awaitable[Any]) _LOG.warning("MediaPublish close suppressed %s failure", step_name, exc_info=True) async def close(self) -> None: + """Stop encoding and release the encoder thread, segments, and publisher. + + Best-effort and idempotent: individual shutdown steps are suppressed and + logged so one failure does not block the rest of cleanup. + """ if self._closed: return self._closed = True @@ -1051,6 +1121,7 @@ async def _drain() -> None: pass def get_stats(self) -> MediaPublishStats: + """Return a snapshot of publish throughput and per-track statistics.""" publisher_stats = self._publisher.get_stats() per_track = tuple( TrackQueueStats( diff --git a/src/livepeer_gateway/selection.py b/src/livepeer_gateway/selection.py index 3a1c6a1..d5f6c24 100644 --- a/src/livepeer_gateway/selection.py +++ b/src/livepeer_gateway/selection.py @@ -1,3 +1,5 @@ +"""Orchestrator and live-runner selection cursors and entry points.""" + from __future__ import annotations import logging @@ -27,8 +29,7 @@ class SelectionCursor: - """ - Stateful selector that advances through orchestrators in batches. + """Stateful selector that advances through orchestrators in batches. For each batch, all ``get_orch_info`` calls run in parallel and successful responses are cached in arrival order. Consumers pop cached successes one at @@ -54,6 +55,14 @@ def __init__( self.rejections: list[OrchestratorRejection] = [] def next(self) -> Tuple[str, lp_rpc_pb2.OrchestratorInfo]: + """Return the next orchestrator that responded, probing batches as needed. + + Returns: + The ``(url, OrchestratorInfo)`` pair of the next successful orchestrator. + + Raises: + NoOrchestratorAvailableError: If every orchestrator was rejected. + """ while True: if self._pending_successes: selected = self._pending_successes.pop(0) @@ -127,6 +136,28 @@ def orchestrator_selector( capabilities: Optional[lp_rpc_pb2.Capabilities] = None, use_tofu: bool = True, ) -> SelectionCursor: + """Discover orchestrators and return a cursor that tries them in batches. + + Calling the returned cursor's ``.next()`` returns the next orchestrator that + responds to ``get_orch_info``; candidates are probed in parallel batches. + + Args: + orchestrators: Explicit orchestrator addresses, as a sequence or a + comma-delimited string. If provided, these are used instead of + discovering via ``discovery_url``/``signer_url``. + signer_url: Signer service used to authenticate discovery. + signer_headers: Extra HTTP headers sent to the signer service. + discovery_url: Discovery endpoint queried when ``orchestrators`` is omitted. + discovery_headers: Extra HTTP headers sent to the discovery endpoint. + capabilities: Capabilities used to filter and query orchestrators. + use_tofu: Trust the orchestrator's TLS certificate on first use. + + Returns: + A cursor whose ``.next()`` yields the next responsive orchestrator. + + Raises: + NoOrchestratorAvailableError: If discovery returns no orchestrators. + """ orch_list = discover_orchestrators( orchestrators, signer_url=signer_url, @@ -150,8 +181,7 @@ def orchestrator_selector( class RunnerSelectionCursor: - """ - Stateful selector that advances through live runners sequentially. + """Stateful selector that advances through live runners sequentially. Runner attempts are intentionally not parallelized: selecting a persistent runner reserves capacity, and selecting a single-shot runner may perform @@ -179,9 +209,18 @@ def __init__( @property def candidates(self) -> tuple[LiveRunnerInstance, ...]: + """The runners this cursor will attempt, in order.""" return tuple(self._candidates) async def next(self) -> LiveRunnerCallResult: + """Call the next runner and return its result, skipping ones that fail. + + Returns: + The ``LiveRunnerCallResult`` from the first runner that succeeds. + + Raises: + NoRunnerAvailableError: If every remaining runner is rejected. + """ while self._next_index < len(self._candidates): runner = self._candidates[self._next_index] self._next_index += 1 @@ -233,6 +272,35 @@ async def runner_selector( gpu: Optional[FilterValue] = None, timeout: float = 5.0, ) -> RunnerSelectionCursor: + """Discover live runners and return a cursor that attempts them one at a time. + + Awaiting the returned cursor's ``.next()`` calls the next runner (see + ``call_runner``) with ``body``/``method`` and returns its ``LiveRunnerCallResult``. + Runners are tried sequentially, not in parallel, because selecting one may reserve + capacity or perform the actual operation. + + Args: + body: Request payload forwarded to each runner attempt. Defaults to an empty + body. + method: HTTP method used for each runner attempt. + orchestrators: Explicit orchestrator addresses, as a sequence or a + comma-delimited string. If provided, runners are discovered from these + instead of via ``discovery_url``/``signer_url``. + signer_url: Signer service used to authenticate discovery and to answer payment + challenges on subsequent runner calls. + signer_headers: Extra HTTP headers sent to the signer service. + discovery_url: Discovery endpoint queried when ``orchestrators`` is omitted. + discovery_headers: Extra HTTP headers sent to the discovery endpoint. + app: Restrict candidates to runners serving this app (string or sequence). + gpu: Restrict candidates to runners on this GPU (string or sequence). + timeout: Per-request timeout, in seconds, for each runner attempt. + + Returns: + A cursor whose ``.next()`` yields the next successful runner's result. + + Raises: + NoRunnerAvailableError: If discovery returns no candidate runners. + """ if orchestrators is not None: entries = await discover_orchestrator_runners( orchestrators, @@ -276,6 +344,31 @@ async def reserve_session( gpu: Optional[FilterValue] = None, timeout: float = 5.0, ) -> LiveRunnerSession: + """Select a runner and reserve a session on it in one step. + + Convenience wrapper over ``runner_selector`` that calls the first working + runner and returns its session details. + + Args: + signer_url: Signer service used to authenticate discovery and to answer + payment challenges on the runner call. + signer_headers: Extra HTTP headers sent to the signer service. + discovery_url: Discovery endpoint queried when discovering runners. + discovery_headers: Extra HTTP headers sent to the discovery endpoint. + orchestrators: Explicit orchestrator addresses, as a sequence or a + comma-delimited string. If provided, runners are discovered from + these instead of via ``discovery_url``/``signer_url``. + app: Restrict candidates to runners serving this app (string or sequence). + gpu: Restrict candidates to runners on this GPU (string or sequence). + timeout: Per-request timeout, in seconds, for the runner call. + + Returns: + A ``LiveRunnerSession`` carrying the reserved ``session_id`` and ``app_url``. + + Raises: + LivepeerGatewayError: If the runner response omits ``session_id`` or ``app_url``. + NoRunnerAvailableError: If no runner succeeds. + """ cursor = await runner_selector( orchestrators=orchestrators, signer_url=signer_url,