diff --git a/sdk/ai/azure-ai-agents/CHANGELOG.md b/sdk/ai/azure-ai-agents/CHANGELOG.md index e6560c78eed1..2d52918fa011 100644 --- a/sdk/ai/azure-ai-agents/CHANGELOG.md +++ b/sdk/ai/azure-ai-agents/CHANGELOG.md @@ -7,17 +7,33 @@ - Added `VersionSelector.setVersionSelectionRule` as a convenience for configuring a single version selection rule. - Added public `StreamingResponseUtils` in the `com.azure.ai.agents.util` package for converting OpenAI streaming responses to Azure SDK `IterableStream` and Reactor `Flux` types. -- Added `BetaVoiceAgentsTelephonyClient` and `BetaVoiceAgentsTelephonyAsyncClient`, built through - `AgentsClientBuilder.beta()`, for managing voice-agent outbound call jobs and telephony campaigns. -- Added `BetaVoiceAgentsConversationsClient` and `BetaVoiceAgentsConversationsAsyncClient`, built through - `AgentsClientBuilder.beta()`, for managing persisted voice-agent conversations and their responses, items, and - audio content. +- Added raw JSON WebSocket sends, complete unknown-event payloads, UTF-8 binary JSON reception, configurable receive + limits and overflow policies, and opt-in recovery from malformed events. +- Added custom WebSocket close codes and reasons, and per-event synchronous receive timeouts. +- Added realtime handshake options for session IDs, structured inputs, API versions, credential scopes, preview features, extra headers and query parameters, and same-host secure connection URL overrides. + +- Added preview `BetaVoiceAgentsTelephonyClient` and `BetaVoiceAgentsTelephonyAsyncClient` for outbound call jobs and campaign management, including recipient import, validation, publishing, pausing, resuming, and cancellation. +- Added preview `BetaVoiceAgentsConversationsClient` and `BetaVoiceAgentsConversationsAsyncClient` for managing + persisted voice-agent conversations and their responses, items, and audio content. +- Added preview `BetaVoiceAgentWebSocketClient`, `BetaVoiceAgentWebSocketAsyncClient`, + `BetaVoiceAgentWebSocketSessionClient`, and `BetaVoiceAgentWebSocketSessionAsyncClient` with typed realtime events, + text and PCM16 audio input, response cancellation, function-call output, persisted-conversation options, and + authenticated `wss://` transport. +- Added synchronous and asynchronous live text conversation samples, an asynchronous Java Sound microphone/speaker sample with barge-in, and a live client-executed function-tool sample. ### Breaking Changes +- Voice-agent WebSocket connections now require secure endpoints, including localhost. Synchronous sessions now + enforce a 32 MiB default message limit. +- Replaced `generateAgent` and `generateAgentWithResponse` on `AgentsClient` and `AgentsAsyncClient` with + `createAgentFromPrompt` and `createAgentFromPromptWithResponse` on `BetaAgentsClient` and `BetaAgentsAsyncClient`. +- Moved `getId()` and `getConversationId()` from `VoiceResponseBase` to `VoiceResponse`. + ### Bugs Fixed -- Fixed polling for optimization jobs and telephony operations that return the `cancelled` status spelling. +- Reject insecure voice-agent WebSocket URLs before token acquisition to prevent sending credentials over plaintext. +- Made synchronous voice-agent receive-buffer overflow signaling atomic across concurrent callbacks. +- Fixed polling for telephony operations that return the `cancelled` status spelling. ### Other Changes diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index 3fbb328e5558..5cd2bd34ce0f 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -68,6 +68,9 @@ The Agents client library has the following sub-clients which group the differen - `ResponsesClient` / `ResponsesAsyncClient`: Create responses that require Azure-specific request fields, such as an explicit `AgentReference` or structured inputs. For standard OpenAI Responses API calls through a configured agent endpoint, use an agent-scoped OpenAI client. See the [OpenAI Responses API documentation][openai_responses_api_docs] for more information. - `BetaMemoryStoresClient` / `BetaMemoryStoresAsyncClient` **(preview)**: Manage memory stores and individual memory items for agents. - `ToolboxesClient` / `ToolboxesAsyncClient`: Manage toolboxes and toolbox versions. +- `BetaVoiceAgentWebSocketClient` / `BetaVoiceAgentWebSocketAsyncClient` **(preview)**: Open typed realtime WebSocket sessions with voice agents. +- `BetaVoiceAgentsTelephonyClient` / `BetaVoiceAgentsTelephonyAsyncClient` **(preview)**: Manage voice-agent outbound calls and telephony campaigns. +- `BetaVoiceAgentsConversationsClient` / `BetaVoiceAgentsConversationsAsyncClient` **(preview)**: Read persisted voice-agent conversations, transcripts, and audio metadata. Conversation operations are accessed through the [OpenAI Official Java SDK][openai_java_sdk]'s `ConversationService`. See the [OpenAI's Conversation API documentation][openai_conversations_api_docs] for more information. @@ -114,6 +117,26 @@ ResponseService responseService = responsesClient.getResponseService(); ConversationService conversationService = openAIClient.conversations(); ``` +### Realtime connection options + +Use `VoiceAgentWebSocketConnectionOptions` with the synchronous or asynchronous beta voice-agent client's +`openWebSocketSession` method to set session IDs, agent version overrides, structured inputs, API versions, credential +scopes, preview features, and extra handshake headers or query parameters. + +```java +VoiceAgentWebSocketConnectionOptions options = new VoiceAgentWebSocketConnectionOptions() + .setAgentSessionId("session-id") + .setAgentVersionOverride("2") + .setStructuredInputs("{\"language\":\"en\"}") + .setExtraHeaders(Collections.singletonMap("User-Agent", "my-application/1.0")); +``` + +Extra query parameters and non-protected headers override defaults. Authentication and WebSocket protocol +headers remain transport-controlled. An explicitly empty `Foundry-Features` value is preserved. +`setConnectionUrl` accepts a full `wss://` URI on the project endpoint's host and port, with no user information +or fragment. Existing query parameters are preserved unless overridden. URL validation happens before token +acquisition; cross-host overrides are rejected to prevent credentials from being sent to another host. + ### Agent version drafts Draft agent versions are preview candidates that are not promoted to the agent's latest released version. Create one with @@ -188,9 +211,18 @@ Build clients whose names start with `Beta` from `AgentsClientBuilder.beta()`. T |---|---| | `BetaAgentsClient` | `WorkflowAgents=V1Preview,ExternalAgents=V1Preview,DraftAgents=V1Preview,AgentsOptimization=V2Preview` | | `BetaMemoryStoresClient` | `MemoryStores=V1Preview` | +| `BetaVoiceAgentWebSocketClient` | `VoiceAgents=V1Preview` | +| `BetaVoiceAgentsTelephonyClient` | `VoiceAgents=V1Preview` | +| `BetaVoiceAgentsConversationsClient` | `VoiceAgents=V1Preview` | The async `Beta*AsyncClient` counterparts follow the same behavior. +### Realtime voice-agent sessions + +Use `BetaVoiceAgentWebSocketClient` or `BetaVoiceAgentWebSocketAsyncClient` to open a typed, bidirectional session with an existing voice agent. The client acquires a token for `https://ai.azure.com/.default`, negotiates the `realtime` WebSocket subprotocol, and sends the required `VoiceAgents=V1Preview` feature header automatically. + +The session API supports text and PCM16 audio input, typed streaming server events, response cancellation, client-executed function tools, and persisted conversations. See [Realtime voice-agent WebSocket examples](#realtime-voice-agent-websocket-examples-preview) for a walkthrough and complete samples. + ### Agent optimization The preview `BetaAgentsClient` and `BetaAgentsAsyncClient` can create and monitor agent optimization jobs. These jobs @@ -928,6 +960,141 @@ See the full sample in [CreateResponseWithStructuredInput.java](https://github.c --- +### Voice agent samples (preview) + +The [voice-agent samples](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice) cover agent management and persisted conversations. + +| Scenario | Samples | +|---|---| +| Lifecycle | [VoiceAgentBasicSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java) and [VoiceAgentBasicAsyncSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java) create, retrieve, update, list, enable, disable, and delete voice agents. | +| Versions and drafts | [VoiceAgentVersionsSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java) creates and lists released and draft versions. | +| Guided generation | [VoiceAgentGenerateSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java) generates a voice-agent definition. | +| Audio and tools | [VoiceAgentWithToolsSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java) configures PCM audio, transcription, voice activity detection, function tools, and system tools. | +| Persisted conversations | [VoiceAgentReadConversationSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java) reads responses and transcripts, while [VoiceAgentReadConversationAudioSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java) downloads call and item audio. | + +Authenticate with `DefaultAzureCredential`. Every voice sample requires `FOUNDRY_PROJECT_ENDPOINT`. Samples that create explicit definitions optionally use `FOUNDRY_VOICE_MODEL`, `FOUNDRY_VOICE_MODEL_TYPE`, and `FOUNDRY_VOICE_AGENT_NAME`. The persisted-conversation samples require `FOUNDRY_VOICE_AGENT_NAME` and `FOUNDRY_VOICE_CONVERSATION_ID`. + +### Realtime voice-agent WebSocket examples (preview) + +Realtime WebSocket sessions provide bidirectional text and audio communication with a voice agent. Create the voice agent before opening a session; the lifecycle samples above demonstrate how to create one. + +#### Create a realtime WebSocket client + +Build a synchronous or asynchronous preview client from the same `AgentsClientBuilder`. Beta clients automatically send the required preview feature header. + +```java +AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint); + +BetaVoiceAgentWebSocketClient realtimeClient + = builder.beta().buildBetaVoiceAgentWebSocketClient(); +BetaVoiceAgentWebSocketAsyncClient realtimeAsyncClient + = builder.beta().buildBetaVoiceAgentWebSocketAsyncClient(); +``` + +#### Send a synchronous text turn + +Connections require an `https://` or `wss://` project endpoint. Insecure endpoints and untrusted connection URL overrides +are rejected before acquiring a token. This also applies to localhost; use certificate-verified TLS for local servers. + +Unknown server event types are returned as `RawRealtimeServerEvent`; `getRawEvent()` preserves the complete JSON object. +Use `sendEvent(BinaryData)` to send raw JSON objects, including event types or fields not modeled by this SDK. Both +clients accept UTF-8 JSON in text or binary WebSocket messages. + +Configure `VoiceAgentWebSocketConnectionOptions` before connecting and do not modify it while the session is active: + +- `setReceiveBufferCapacity` sets a bounded event queue (default 256, range 1-65536). +- `setOverflowStrategy` defaults to `ERROR`, which closes an overflowing connection. `DROP_OLDEST` and `DROP_LATEST` + explicitly permit data loss and should only be used when the application can tolerate missing events. +- `setMaxMessageSize` limits accepted message bytes (default 32 MiB). Oversized messages terminate the connection. + The sync transport checks size after receiving a complete message; this does not bound the transport's allocation. +- Malformed JSON or invalid UTF-8 terminates reception by default. Set `setMalformedEventHandler` to report and skip + malformed events while continuing reception. This callback must not block; throwing from it terminates the session. +```java com.azure.ai.agents.realtime_forward_compatibility +VoiceAgentWebSocketConnectionOptions options + = new VoiceAgentWebSocketConnectionOptions() + .setReceiveBufferCapacity(512) + .setMaxMessageSize(8 * 1024 * 1024) + .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.ERROR); +try (BetaVoiceAgentWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName, options)) { + session.sendEvent(BinaryData.fromString( + "{\"type\":\"response.create\",\"event_id\":\"response-1\"}")); + for (RealtimeServerEvent event : session.receiveEvents()) { + if (event instanceof RawRealtimeServerEvent) { + BinaryData payload + = ((RawRealtimeServerEvent) event).getRawEvent(); + System.out.println("Received an unrecognized event with " + payload.getLength() + " bytes."); + } + } +} +``` + +Connect to the voice agent, add the user's text to the conversation, and request a response. Consume the typed server events until the response finishes. A session supports only one consumer of `receiveEvents()`. + +For bounded synchronous waits, use `receiveEvents(Duration)` with a positive per-event timeout. A timeout raises +`IllegalStateException` with a `TimeoutException` cause, leaves the session open, and allows the same iterator to retry. +Use `close(code, reason)` or asynchronous `closeAsync(code, reason)` to send a custom close frame. Close reasons must +fit in 123 UTF-8 bytes and close codes must be valid WebSocket codes. The first asynchronous close request wins. + +```java +try (BetaVoiceAgentWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName)) { + session.sendText("Hello! Tell me about the services you provide."); + session.createResponse(); + + for (RealtimeServerEvent event : session.receiveEvents()) { + if (event instanceof RealtimeResponseTextDeltaEvent) { + System.out.print(((RealtimeResponseTextDeltaEvent) event).getDelta()); + } else if (event instanceof RealtimeErrorEvent) { + RealtimeErrorEvent error = (RealtimeErrorEvent) event; + System.out.println("Session error: " + error.getError().message()); + } else if (event instanceof RealtimeResponseDoneEvent) { + break; + } + } +} +``` + +Use `sendText` and `createResponse` again for subsequent turns while the session remains open. Call `cancelResponse` to interrupt an active response. + +#### Send an asynchronous text turn + +The asynchronous client returns a `Mono` when connecting and a `Flux` when receiving events. `Mono.usingWhen` closes the session on completion, error, or cancellation. + +```java +Mono.usingWhen( + realtimeAsyncClient.openWebSocketSession(agentName), + session -> session.sendText("Hello! Tell me about the services you provide.") + .then(session.createResponse()) + .thenMany(session.receiveEvents()) + .doOnNext(event -> { + if (event instanceof RealtimeResponseTextDeltaEvent) { + System.out.print(((RealtimeResponseTextDeltaEvent) event).getDelta()); + } + }) + .takeUntil(event -> event instanceof RealtimeResponseDoneEvent) + .then(), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, + (session, error) -> session.closeAsync(), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync) + .block(); +``` + +#### Stream audio and handle function tools + +Use `appendInputAudio` to send PCM16 chunks, `commitInputAudio` to commit buffered audio when server-side voice activity detection is not configured, and `clearInputAudio` to discard pending input. Audio output arrives through `RealtimeResponseAudioDeltaEvent` events. When a `RealtimeResponseFunctionCallArgumentsDoneEvent` event requests a client-side tool, execute the function and call `sendFunctionCallOutput` with its call ID and serialized result. + +| Scenario | Complete sample | +|---|---| +| Synchronous live text | [VoiceAgentLiveTextConversationSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java) | +| Asynchronous live text | [VoiceAgentLiveTextConversationAsyncSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java) | +| Asynchronous live audio | [VoiceAgentLiveAudioConversationAsyncSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java) | +| Live function tool | [VoiceAgentLiveFunctionToolSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java) | + +All realtime examples require `FOUNDRY_PROJECT_ENDPOINT` and optionally use `FOUNDRY_VOICE_AGENT_NAME`. The function-tool example also optionally uses `FOUNDRY_VOICE_MODEL` and `FOUNDRY_VOICE_MODEL_TYPE`. The asynchronous text and audio examples delete their generated agents by default; set `FOUNDRY_KEEP_VOICE_AGENT=true` to retain them. + +The live audio example requires a Java Sound-compatible microphone and speaker. It streams signed, little-endian, mono PCM16 audio at 24 kHz. These examples use WebSocket transport. Although the generated protocol models include WebRTC signaling events, the Java client does not provide a WebRTC peer connection or media implementation. + ### Service API versions The client library targets the latest service API version by default. diff --git a/sdk/ai/azure-ai-agents/pom.xml b/sdk/ai/azure-ai-agents/pom.xml index 0e49aae3f313..ba9357dcb283 100644 --- a/sdk/ai/azure-ai-agents/pom.xml +++ b/sdk/ai/azure-ai-agents/pom.xml @@ -47,6 +47,8 @@ 0.0 0.0 + --add-modules java.desktop + --add-reads com.azure.ai.agents=java.desktop --add-exports com.azure.core/com.azure.core.implementation.util=ALL-UNNAMED --add-opens com.azure.core/com.azure.core.implementation.util=ALL-UNNAMED @@ -68,6 +70,52 @@ azure-core-http-netty 1.16.7 + + io.projectreactor.netty + reactor-netty-http + 1.2.18 + + + io.netty + netty-codec-http + 4.1.137.Final + + + io.netty + netty-transport + 4.1.137.Final + + + io.netty + netty-common + 4.1.137.Final + + + io.netty + netty-codec + 4.1.137.Final + + + io.netty + netty-buffer + 4.1.137.Final + + + com.squareup.okhttp3 + okhttp + 4.12.0 + + + com.squareup.okio + okio + + + + + com.squareup.okio + okio-jvm + 3.18.1 + @@ -102,6 +150,14 @@ com.openai:openai-java:[4.45.0] + io.projectreactor.netty:reactor-netty-http:[1.2.18] + io.netty:netty-codec-http:[4.1.137.Final] + io.netty:netty-transport:[4.1.137.Final] + io.netty:netty-common:[4.1.137.Final] + io.netty:netty-codec:[4.1.137.Final] + io.netty:netty-buffer:[4.1.137.Final] + com.squareup.okhttp3:okhttp:[4.12.0] + com.squareup.okio:okio-jvm:[3.18.1] diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java index 824df8d8f9be..8e7bede71955 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/AgentsClientBuilder.java @@ -9,6 +9,7 @@ import com.azure.ai.agents.implementation.http.HttpClientHelper; import com.azure.ai.agents.implementation.models.AgentDefinitionOptInKeys; import com.azure.ai.agents.implementation.models.FoundryFeaturesOptInKeys; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; import com.azure.ai.agents.implementation.utils.Beta; import com.azure.core.annotation.Generated; import com.azure.core.annotation.ServiceClientBuilder; @@ -22,6 +23,7 @@ import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpPipelineBuilder; import com.azure.core.http.HttpPipelinePosition; +import com.azure.core.http.ProxyOptions; import com.azure.core.http.policy.AddDatePolicy; import com.azure.core.http.policy.AddHeadersFromContextPolicy; import com.azure.core.http.policy.AddHeadersPolicy; @@ -37,6 +39,7 @@ import com.azure.core.util.ClientOptions; import com.azure.core.util.Configuration; import com.azure.core.util.CoreUtils; +import com.azure.core.util.UserAgentUtil; import com.azure.core.util.builder.ClientBuilderUtil; import com.azure.core.util.logging.ClientLogger; import com.azure.core.util.serializer.JacksonAdapter; @@ -47,6 +50,7 @@ import com.openai.client.okhttp.OpenAIOkHttpClient; import com.openai.client.okhttp.OpenAIOkHttpClientAsync; import com.openai.credential.BearerTokenCredential; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -563,10 +567,12 @@ public BetaAgentsClientBuilder beta() { serviceClients = { BetaAgentsClient.class, BetaMemoryStoresClient.class, + BetaVoiceAgentWebSocketClient.class, BetaVoiceAgentsTelephonyClient.class, BetaVoiceAgentsConversationsClient.class, BetaAgentsAsyncClient.class, BetaMemoryStoresAsyncClient.class, + BetaVoiceAgentWebSocketAsyncClient.class, BetaVoiceAgentsTelephonyAsyncClient.class, BetaVoiceAgentsConversationsAsyncClient.class }) public final class BetaAgentsClientBuilder { @@ -670,6 +676,26 @@ public BetaMemoryStoresClient buildBetaMemoryStoresClient() { return new BetaMemoryStoresClient(buildInnerClient(MEMORY_STORES_PREVIEW_FEATURES).getBetaMemoryStores()); } + /** + * Builds an asynchronous client for realtime voice-agent WebSocket sessions. + * + * @return an asynchronous voice-agent WebSocket client. + */ + @Beta + public BetaVoiceAgentWebSocketAsyncClient buildBetaVoiceAgentWebSocketAsyncClient() { + return new BetaVoiceAgentWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); + } + + /** + * Builds a synchronous client for realtime voice-agent WebSocket sessions. + * + * @return a synchronous voice-agent WebSocket client. + */ + @Beta + public BetaVoiceAgentWebSocketClient buildBetaVoiceAgentWebSocketClient() { + return new BetaVoiceAgentWebSocketClient(createVoiceAgentWebSocketConfiguration()); + } + /** * Builds a synchronous beta client for preview voice-agent telephony operations. *

@@ -794,4 +820,43 @@ private BetaVoiceAgentsConversationsClient buildBetaVoiceAgentsConversationsClie private BetaVoiceAgentsTelephonyClient buildBetaVoiceAgentsTelephonyClient() { return new BetaVoiceAgentsTelephonyClient(buildInnerClient().getBetaVoiceAgentsTelephonies()); } + + private VoiceAgentWebSocketClientConfiguration createVoiceAgentWebSocketConfiguration() { + validateClient(); + Objects.requireNonNull(tokenCredential, + "'credential' must be configured to build a voice-agent WebSocket client."); + Configuration buildConfiguration + = configuration == null ? Configuration.getGlobalConfiguration() : configuration; + ClientOptions localClientOptions = clientOptions == null ? new ClientOptions() : clientOptions; + HttpLogOptions localLogOptions = httpLogOptions == null ? new HttpLogOptions() : httpLogOptions; + String clientName = PROPERTIES.getOrDefault(SDK_NAME, "azure-ai-agents"); + String clientVersion = PROPERTIES.getOrDefault(SDK_VERSION, "unknown"); + String applicationId = CoreUtils.getApplicationId(localClientOptions, localLogOptions); + String userAgent + = UserAgentUtil.toUserAgentString(applicationId, clientName, clientVersion, buildConfiguration); + HttpHeaders headers = CoreUtils.createHttpHeadersFromClientOptions(localClientOptions); + ProxyOptions proxyOptions = ProxyOptions.fromConfiguration(buildConfiguration); + AgentsServiceVersion localServiceVersion + = serviceVersion == null ? AgentsServiceVersion.getLatest() : serviceVersion; + return new VoiceAgentWebSocketClientConfiguration(URI.create(endpoint), tokenCredential, + localServiceVersion.getVersion(), userAgent, headers, proxyOptions); + } + + /** + * Builds an asynchronous client for realtime voice-agent WebSocket sessions. + * + * @return an asynchronous voice-agent WebSocket client. + */ + private BetaVoiceAgentWebSocketAsyncClient buildBetaVoiceAgentWebSocketAsyncClient() { + return new BetaVoiceAgentWebSocketAsyncClient(createVoiceAgentWebSocketConfiguration()); + } + + /** + * Builds a synchronous client for realtime voice-agent WebSocket sessions. + * + * @return a synchronous voice-agent WebSocket client. + */ + private BetaVoiceAgentWebSocketClient buildBetaVoiceAgentWebSocketClient() { + return new BetaVoiceAgentWebSocketClient(createVoiceAgentWebSocketConfiguration()); + } } diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketAsyncClient.java new file mode 100644 index 000000000000..4b5ed49dece9 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketAsyncClient.java @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.core.annotation.ServiceClient; +import reactor.core.publisher.Mono; + +import java.util.Objects; + +/** + * An asynchronous client for opening realtime WebSocket sessions with Foundry voice agents. + */ +@ServiceClient(builder = AgentsClientBuilder.class, isAsync = true) +@Beta(warningText = "This class is in preview and may change in future releases.") +public final class BetaVoiceAgentWebSocketAsyncClient { + private final VoiceAgentWebSocketClientConfiguration configuration; + + BetaVoiceAgentWebSocketAsyncClient(VoiceAgentWebSocketClientConfiguration configuration) { + this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); + } + + /** + * Opens a realtime WebSocket session using the voice agent's persisted configuration. + * + * @param agentName the voice agent name. + * @return a connected session. + */ + public Mono openWebSocketSession(String agentName) { + return openWebSocketSession(agentName, new VoiceAgentWebSocketConnectionOptions()); + } + + /** + * Opens a realtime WebSocket session. + * + * @param agentName the voice agent name. + * @param options connection options. + * @return a connected session. + */ + public Mono openWebSocketSession(String agentName, + VoiceAgentWebSocketConnectionOptions options) { + Objects.requireNonNull(agentName, "'agentName' cannot be null."); + Objects.requireNonNull(options, "'options' cannot be null."); + VoiceAgentWebSocketConnectionOptions optionsSnapshot = new VoiceAgentWebSocketConnectionOptions(options); + return Mono.defer(() -> { + BetaVoiceAgentWebSocketSessionAsyncClient session + = new BetaVoiceAgentWebSocketSessionAsyncClient(configuration, agentName, optionsSnapshot); + return session.connect().thenReturn(session); + }); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketClient.java new file mode 100644 index 000000000000..232c956f8431 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketClient.java @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.core.annotation.ServiceClient; +import com.azure.core.util.logging.ClientLogger; + +import java.util.Objects; + +/** + * A synchronous client for opening realtime WebSocket sessions with Foundry voice agents. + */ +@ServiceClient(builder = AgentsClientBuilder.class) +@Beta(warningText = "This class is in preview and may change in future releases.") +public final class BetaVoiceAgentWebSocketClient { + private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentWebSocketClient.class); + private final VoiceAgentWebSocketClientConfiguration configuration; + + BetaVoiceAgentWebSocketClient(VoiceAgentWebSocketClientConfiguration configuration) { + this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); + } + + /** + * Opens a realtime WebSocket session using the voice agent's persisted configuration. + * + * @param agentName the voice agent name. + * @return a connected session. + */ + public BetaVoiceAgentWebSocketSessionClient openWebSocketSession(String agentName) { + return openWebSocketSession(agentName, new VoiceAgentWebSocketConnectionOptions()); + } + + /** + * Opens a realtime WebSocket session. + * + * @param agentName the voice agent name. + * @param options connection options. + * @throws IllegalArgumentException if {@code agentName} is empty. + * @return a connected session. + */ + public BetaVoiceAgentWebSocketSessionClient openWebSocketSession(String agentName, + VoiceAgentWebSocketConnectionOptions options) { + Objects.requireNonNull(agentName, "'agentName' cannot be null."); + if (agentName.isEmpty()) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("'agentName' cannot be empty.")); + } + Objects.requireNonNull(options, "'options' cannot be null."); + return BetaVoiceAgentWebSocketSessionClient.connect(configuration, agentName, + new VoiceAgentWebSocketConnectionOptions(options)); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java new file mode 100644 index 000000000000..3dfe0a837b20 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionAsyncClient.java @@ -0,0 +1,628 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHandshakeHandler; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHttpResponse; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.ai.agents.models.RealtimeClientEvent; +import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; +import com.azure.ai.agents.models.RealtimeInputAudioBufferAppendEvent; +import com.azure.ai.agents.models.RealtimeInputAudioBufferClearEvent; +import com.azure.ai.agents.models.RealtimeInputAudioBufferCommitEvent; +import com.azure.ai.agents.models.RealtimeResponseCancelEvent; +import com.azure.ai.agents.models.RealtimeResponseCreateEvent; +import com.azure.ai.agents.models.RealtimeConversationItem; +import com.azure.ai.agents.models.RealtimeConversationItemFunctionCallOutput; +import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.VoiceAgentResponseCreateOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.core.credential.AccessToken; +import com.azure.core.credential.TokenRequestContext; +import com.azure.core.exception.ClientAuthenticationException; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.exception.ResourceModifiedException; +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.core.http.HttpHeader; +import com.azure.core.http.ProxyOptions; +import com.azure.core.util.AsyncCloseable; +import com.azure.core.util.BinaryData; +import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; +import com.azure.core.util.logging.ClientLogger; +import io.netty.channel.Channel; +import io.netty.channel.ChannelOption; +import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame; +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; +import io.netty.handler.codec.http.websocketx.WebSocketClientHandshakeException; +import io.netty.handler.codec.http.websocketx.WebSocketFrame; +import java.io.IOException; +import java.net.URI; +import java.time.Duration; +import java.util.Base64; +import java.util.Collections; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CancellationException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import reactor.core.Disposable; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.netty.Connection; +import reactor.netty.http.client.HttpClient; +import reactor.netty.http.client.WebsocketClientSpec; +import reactor.netty.http.websocket.WebsocketInbound; +import reactor.netty.http.websocket.WebsocketOutbound; +import reactor.netty.transport.ProxyProvider; + +/** + * An asynchronous bidirectional realtime session connected to a Foundry voice agent. + * + *

Instances are created by {@link BetaVoiceAgentWebSocketAsyncClient#openWebSocketSession(String)}. A session supports one + * subscriber to {@link #receiveEvents()}. Close the session when it is no longer needed.

+ */ +@Beta(warningText = "This class is in preview and may change in future releases.") +public final class BetaVoiceAgentWebSocketSessionAsyncClient implements AsyncCloseable, AutoCloseable { + private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentWebSocketSessionAsyncClient.class); + private static final int MAX_OUTSTANDING_SENDS = 256; + + private final VoiceAgentWebSocketClientConfiguration configuration; + private final VoiceAgentWebSocketConnectionOptions options; + private final HttpClient httpClient; + private final URI websocketUri; + private final AtomicReference state = new AtomicReference<>(State.NEW); + private final AtomicReference channel = new AtomicReference<>(); + private final AtomicReference outbound = new AtomicReference<>(); + private final AtomicReference connectionOperation = new AtomicReference<>(); + private final AtomicReference lifecycle = new AtomicReference<>(); + private final AtomicBoolean receiveClaimed = new AtomicBoolean(); + private final Semaphore sendPermits = new Semaphore(MAX_OUTSTANDING_SENDS); + private final Queue eventQueue; + private final Sinks.Many events; + private final Sinks.One ready = Sinks.one(); + private final Sinks.One closeSignal = Sinks.one(); + private final AtomicReference> closeOperation = new AtomicReference<>(); + + private volatile Integer closeCode; + private volatile String closeReason; + + BetaVoiceAgentWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, + VoiceAgentWebSocketConnectionOptions options) { + this(configuration, agentName, options, HttpClient.create()); + } + + BetaVoiceAgentWebSocketSessionAsyncClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, + VoiceAgentWebSocketConnectionOptions options, HttpClient httpClient) { + this.configuration = Objects.requireNonNull(configuration, "'configuration' cannot be null."); + Objects.requireNonNull(agentName, "'agentName' cannot be null."); + if (agentName.isEmpty()) { + throw new IllegalArgumentException("'agentName' cannot be empty."); + } + this.options = options == null ? new VoiceAgentWebSocketConnectionOptions() : options; + this.eventQueue = new ArrayBlockingQueue<>(this.options.getReceiveBufferCapacity()); + this.events = Sinks.many().unicast().onBackpressureBuffer(eventQueue); + this.httpClient = Objects.requireNonNull(httpClient, "'httpClient' cannot be null."); + this.websocketUri = VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, this.options); + } + + Mono connect() { + return Mono.defer(() -> { + if (!state.compareAndSet(State.NEW, State.CONNECTING)) { + return Mono.error(new IllegalStateException("The voice-agent session has already been started.")); + } + + TokenRequestContext tokenContext = VoiceAgentWebSocketUtils.createTokenRequestContext(options); + return configuration.getCredential().getToken(tokenContext).map(AccessToken::getToken).flatMap(token -> { + Disposable connection = openWebSocket(token).subscribe(unused -> { + }, this::terminateWithError, this::terminateNormally); + connectionOperation.set(connection); + return ready.asMono().timeout(options.getHandshakeTimeout()); + }) + .doOnError(this::terminateWithError) + .doOnCancel(() -> terminateWithError(new CancellationException("WebSocket connection cancelled."))); + }); + } + + /** + * Gets the WebSocket endpoint used by this session. + * + * @return the WebSocket endpoint. + */ + public URI getEndpoint() { + return websocketUri; + } + + /** + * Determines whether the WebSocket session is open. + * + * @return {@code true} when the session is open. + */ + public boolean isOpen() { + Channel current = channel.get(); + return state.get() == State.OPEN && current != null && current.isActive(); + } + + /** + * Gets the peer close code after the session closes. + * + * @return the close code, or {@code null} when no close frame has been received. + */ + public Integer getCloseCode() { + return closeCode; + } + + /** + * Gets the peer close reason after the session closes. + * + * @return the close reason, or {@code null} when no close frame has been received. + */ + public String getCloseReason() { + return closeReason; + } + + /** + * Receives typed server events in wire order. Only one subscriber is supported per session. + * + * @return the server event stream. + */ + public Flux receiveEvents() { + return Flux.defer(() -> receiveClaimed.compareAndSet(false, true) + ? events.asFlux() + : Flux.error(new IllegalStateException("Only one receiveEvents subscriber is supported per session."))); + } + + /** + * Sends a typed realtime client event. + * + * @param event the event to send. + * @return a completion signal emitted after the frame is written. + */ + public Mono sendEvent(RealtimeClientEvent event) { + Objects.requireNonNull(event, "'event' cannot be null."); + return Mono.defer(() -> { + Channel current = requireOpenChannel(); + if (!sendPermits.tryAcquire()) { + return Mono.error(new IllegalStateException("Too many voice-agent WebSocket sends are outstanding.")); + } + + final String json; + try { + json = serialize(event); + } catch (IOException error) { + sendPermits.release(); + return Mono + .error(new IllegalArgumentException("Failed to serialize the realtime client event.", error)); + } + + return Mono.create(sink -> current.writeAndFlush(new TextWebSocketFrame(json)).addListener(result -> { + sendPermits.release(); + if (result.isSuccess()) { + sink.success(); + } else { + sink.error(result.cause()); + } + })); + }); + } + + /** + * Sends a JSON object, including event types and fields unknown to this SDK. + * @param event the complete JSON event. + * @return completion after the frame is written, or an error for invalid JSON or a closed session. + */ + public Mono sendEvent(BinaryData event) { + Objects.requireNonNull(event, "'event' cannot be null."); + return Mono.defer(() -> { + String json = VoiceAgentWebSocketUtils.validateEvent(event); + Channel current = requireOpenChannel(); + if (!sendPermits.tryAcquire()) { + return Mono.error(new IllegalStateException("Too many voice-agent WebSocket sends are outstanding.")); + } + return Mono.create(sink -> current.writeAndFlush(new TextWebSocketFrame(json)).addListener(result -> { + sendPermits.release(); + if (result.isSuccess()) { + sink.success(); + } else { + sink.error(result.cause()); + } + })); + }); + } + + /** + * Adds an item to the session conversation. + * + * @param item the item to add. + * @return a completion signal emitted after the event is written. + */ + public Mono createConversationItem(RealtimeConversationItem item) { + return createConversationItem(item, null); + } + + /** + * Adds an item after a specific conversation item. + * + * @param item the item to add. + * @param previousItemId the preceding item identifier, or {@code null} to append. + * @return a completion signal emitted after the event is written. + */ + public Mono createConversationItem(RealtimeConversationItem item, String previousItemId) { + Objects.requireNonNull(item, "'item' cannot be null."); + return sendEvent(new RealtimeConversationItemCreateEvent(item).setPreviousItemId(previousItemId)); + } + + /** + * Adds a user text message to the session conversation. + * + * @param text the user message. + * @return a completion signal emitted after the event is written. + */ + public Mono sendText(String text) { + Objects.requireNonNull(text, "'text' cannot be null."); + Content content = Content.builder().type(Content.Type.INPUT_TEXT).text(text).build(); + return createConversationItem(new RealtimeConversationItemUserMessage(Collections.singletonList(content))); + } + + /** + * Appends PCM or encoded audio bytes to the input audio buffer. + * + * @param audio the bytes in the input format configured by the voice agent. + * @return a completion signal emitted after the event is written. + */ + public Mono appendInputAudio(BinaryData audio) { + Objects.requireNonNull(audio, "'audio' cannot be null."); + String encoded = Base64.getEncoder().encodeToString(audio.toBytes()); + return sendEvent(new RealtimeInputAudioBufferAppendEvent(encoded)); + } + + /** + * Clears the input audio buffer. + * + * @return a completion signal emitted after the event is written. + */ + public Mono clearInputAudio() { + return sendEvent(new RealtimeInputAudioBufferClearEvent()); + } + + /** + * Commits the input audio buffer. + * + * @return a completion signal emitted after the event is written. + */ + public Mono commitInputAudio() { + return sendEvent(new RealtimeInputAudioBufferCommitEvent()); + } + + /** + * Requests a response using the persisted voice-agent configuration. + * + * @return a completion signal emitted after the event is written. + */ + public Mono createResponse() { + return sendEvent(new RealtimeResponseCreateEvent()); + } + + /** + * Requests a response with per-response options. + * + * @param responseOptions the response options. + * @return a completion signal emitted after the event is written. + */ + public Mono createResponse(VoiceAgentResponseCreateOptions responseOptions) { + Objects.requireNonNull(responseOptions, "'responseOptions' cannot be null."); + return sendEvent(new RealtimeResponseCreateEvent().setResponse(responseOptions)); + } + + /** + * Cancels the response currently writing to the default conversation. + * + * @return a completion signal emitted after the event is written. + */ + public Mono cancelResponse() { + return sendEvent(new RealtimeResponseCancelEvent()); + } + + /** + * Cancels a specific response. + * + * @param responseId the response identifier. + * @return a completion signal emitted after the event is written. + */ + public Mono cancelResponse(String responseId) { + Objects.requireNonNull(responseId, "'responseId' cannot be null."); + return sendEvent(new RealtimeResponseCancelEvent().setResponseId(responseId)); + } + + /** + * Sends a function-call result and requests the next response. + * + * @param callId the function call identifier. + * @param output the serialized function result. + * @return a completion signal emitted after both events are written. + */ + public Mono sendFunctionCallOutput(String callId, String output) { + RealtimeConversationItemFunctionCallOutput item + = new RealtimeConversationItemFunctionCallOutput(callId, output); + return createConversationItem(item).then(createResponse()); + } + + /** + * Closes the WebSocket session. + * + * @return a completion signal for closing the session. + */ + @Override + public Mono closeAsync() { + return closeAsync(1000, ""); + } + + /** + * Closes the session with an application-selected WebSocket close frame. + * The first requested close frame wins when close is called more than once. + * @param code a valid WebSocket close code. + * @param reason non-null reason of at most 123 UTF-8 bytes. + * @return a completion signal, or an error if the code or reason is invalid. + */ + public Mono closeAsync(int code, String reason) { + try { + VoiceAgentWebSocketUtils.validateClose(code, reason); + } catch (IllegalArgumentException exception) { + return Mono.error(exception); + } + Mono existing = closeOperation.get(); + if (existing != null) { + return existing; + } + + Mono created = Mono.defer(() -> { + State current = state.get(); + if (current == State.CLOSED || current == State.NEW) { + state.set(State.CLOSED); + events.tryEmitComplete(); + return Mono.empty(); + } + state.set(State.CLOSING); + WebsocketOutbound currentOutbound = outbound.get(); + Channel currentChannel = channel.get(); + Mono graceful = currentOutbound == null ? Mono.empty() : currentOutbound.sendClose(code, reason); + Mono disposed = currentChannel == null ? Mono.empty() : Connection.from(currentChannel).onDispose(); + return graceful.then(disposed).timeout(options.getCloseTimeout()).onErrorResume(error -> { + if (currentChannel != null) { + currentChannel.close(); + } + return Mono.empty(); + }).doFinally(signal -> terminateNormally()); + }).cache(); + + if (closeOperation.compareAndSet(null, created)) { + return created; + } + return closeOperation.get(); + } + + /** + * Closes the WebSocket session synchronously. + */ + @Override + public void close() { + closeAsync().block(options.getCloseTimeout().plusSeconds(1)); + } + + private Mono openWebSocket(String token) { + HttpClient client = configureProxy(httpClient).followRedirect(false) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, toConnectTimeoutMillis(options.getHandshakeTimeout())) + .doOnConnected(connection -> connection.addHandlerLast("voiceAgentHandshakeResponseObserver", + new VoiceAgentWebSocketHandshakeHandler(this::terminateWithError))) + .headers(headers -> { + for (HttpHeader header : VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token)) { + headers.set(header.getName(), header.getValue()); + } + }); + WebsocketClientSpec spec = WebsocketClientSpec.builder() + .protocols(VoiceAgentWebSocketUtils.SUBPROTOCOL) + .maxFramePayloadLength(options.getMaxMessageSize()) + .build(); + + return client.websocket(spec).uri(websocketUri.toString()).connect().flatMap(connection -> { + if (!(connection instanceof WebsocketInbound) || !(connection instanceof WebsocketOutbound)) { + return Mono.error(new IllegalStateException("The WebSocket transport returned an invalid connection.")); + } + return handleConnection((WebsocketInbound) connection, (WebsocketOutbound) connection); + }); + } + + private static int toConnectTimeoutMillis(Duration timeout) { + if (timeout.compareTo(Duration.ofMillis(Integer.MAX_VALUE)) >= 0) { + return Integer.MAX_VALUE; + } + return Math.toIntExact(Math.max(1L, timeout.toMillis())); + } + + private Mono handleConnection(WebsocketInbound inbound, WebsocketOutbound outbound) { + this.outbound.set(outbound); + inbound.withConnection(connection -> channel.set(connection.channel())); + state.set(State.OPEN); + ready.tryEmitEmpty(); + + inbound.receiveCloseStatus().subscribe(status -> { + closeCode = status.code(); + closeReason = status.reasonText(); + }, error -> LOGGER.atVerbose().addKeyValue("error", error.getMessage()).log("Close status unavailable.")); + + Disposable receive = inbound.aggregateFrames(options.getMaxMessageSize()) + .receiveFrames() + .subscribe(this::handleFrame, this::terminateWithError, this::terminateNormally); + lifecycle.set(receive); + return closeSignal.asMono(); + } + + private void handleFrame(WebSocketFrame frame) { + if (frame instanceof TextWebSocketFrame || frame instanceof BinaryWebSocketFrame) { + try { + byte[] bytes = new byte[frame.content().readableBytes()]; + frame.content().getBytes(frame.content().readerIndex(), bytes); + RealtimeServerEvent event + = VoiceAgentWebSocketUtils.deserializeEvent(VoiceAgentWebSocketUtils.decodeEvent(bytes)); + Sinks.EmitResult result = events.tryEmitNext(event); + if (result == Sinks.EmitResult.FAIL_OVERFLOW || result == Sinks.EmitResult.FAIL_ZERO_SUBSCRIBER) { + switch (options.getOverflowStrategy()) { + case DROP_LATEST: + return; + + case DROP_OLDEST: + eventQueue.poll(); + result = events.tryEmitNext(event); + break; + + default: + break; + } + } + if (result.isFailure()) { + terminateWithError(new IllegalStateException("Voice-agent event emission failed: " + result)); + } + } catch (IOException | RuntimeException error) { + Throwable failure = error; + if (options.getMalformedEventHandler() != null) { + try { + options.getMalformedEventHandler().accept(error); + return; + } catch (RuntimeException callbackError) { + failure = callbackError; + } + } + closeWithProtocolError(1007, "Invalid JSON event", failure); + } + } + } + + private void closeWithProtocolError(int code, String reason, Throwable error) { + WebsocketOutbound current = outbound.get(); + terminateWithError(error); + if (current != null) { + current.sendClose(code, reason).subscribe(unused -> { + }, ignored -> { + }); + } + } + + private void terminateWithError(Throwable error) { + Throwable mappedError = mapHandshakeError(error); + State previous = state.getAndSet(State.CLOSED); + if (previous == State.CLOSED) { + return; + } + ready.tryEmitError(mappedError); + events.tryEmitError(mappedError); + closeSignal.tryEmitError(mappedError); + disposeReceive(); + } + + private Throwable mapHandshakeError(Throwable error) { + Throwable current = error; + while (current != null && !(current instanceof WebSocketClientHandshakeException)) { + current = current.getCause(); + } + if (current == null) { + return error; + } + WebSocketClientHandshakeException handshakeError = (WebSocketClientHandshakeException) current; + if (handshakeError.response() == null) { + return error; + } + VoiceAgentWebSocketHttpResponse response + = new VoiceAgentWebSocketHttpResponse(websocketUri, handshakeError.response()); + String message = "Voice-agent WebSocket handshake failed with status " + response.getStatusCode() + "."; + switch (response.getStatusCode()) { + case 401: + return new ClientAuthenticationException(message, response, error); + + case 404: + return new ResourceNotFoundException(message, response, error); + + case 409: + return new ResourceModifiedException(message, response, error); + + default: + return new HttpResponseException(message, response, error); + } + } + + private void terminateNormally() { + State previous = state.getAndSet(State.CLOSED); + if (previous == State.CLOSED) { + return; + } + if (previous == State.CONNECTING) { + ready.tryEmitError(new IllegalStateException("The WebSocket closed before the handshake completed.")); + } + events.tryEmitComplete(); + closeSignal.tryEmitEmpty(); + disposeReceive(); + } + + private void disposeReceive() { + Disposable connection = connectionOperation.getAndSet(null); + if (connection != null && !connection.isDisposed()) { + connection.dispose(); + } + Disposable receive = lifecycle.getAndSet(null); + if (receive != null && !receive.isDisposed()) { + receive.dispose(); + } + } + + private Channel requireOpenChannel() { + Channel current = channel.get(); + if (state.get() != State.OPEN || current == null || !current.isActive()) { + throw LOGGER + .logExceptionAsError(new IllegalStateException("The voice-agent WebSocket session is not open.")); + } + return current; + } + + private HttpClient configureProxy(HttpClient client) { + ProxyOptions proxy = configuration.getProxyOptions(); + if (proxy == null) { + return client; + } + return client.proxy(typeSpec -> { + ProxyProvider.Proxy proxyType; + switch (proxy.getType()) { + case SOCKS4: + proxyType = ProxyProvider.Proxy.SOCKS4; + break; + + case SOCKS5: + proxyType = ProxyProvider.Proxy.SOCKS5; + break; + + default: + proxyType = ProxyProvider.Proxy.HTTP; + break; + } + ProxyProvider.Builder builder = typeSpec.type(proxyType).socketAddress(proxy.getAddress()); + if (proxy.getUsername() != null) { + builder.username(proxy.getUsername()).password(ignored -> proxy.getPassword()); + } + if (proxy.getNonProxyHosts() != null) { + builder.nonProxyHosts(proxy.getNonProxyHosts()); + } + }); + } + + private static String serialize(RealtimeClientEvent event) throws IOException { + return event.toJsonString(); + } + + private enum State { + NEW, CONNECTING, OPEN, CLOSING, CLOSED + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java new file mode 100644 index 000000000000..525217e2249a --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/BetaVoiceAgentWebSocketSessionClient.java @@ -0,0 +1,613 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHttpResponse; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketUtils; +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.ai.agents.models.RealtimeClientEvent; +import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; +import com.azure.ai.agents.models.RealtimeInputAudioBufferAppendEvent; +import com.azure.ai.agents.models.RealtimeInputAudioBufferClearEvent; +import com.azure.ai.agents.models.RealtimeInputAudioBufferCommitEvent; +import com.azure.ai.agents.models.RealtimeResponseCancelEvent; +import com.azure.ai.agents.models.RealtimeResponseCreateEvent; +import com.azure.ai.agents.models.RealtimeConversationItem; +import com.azure.ai.agents.models.RealtimeConversationItemFunctionCallOutput; +import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.VoiceAgentResponseCreateOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.core.exception.ClientAuthenticationException; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.exception.ResourceModifiedException; +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.core.http.HttpHeader; +import com.azure.core.http.ProxyOptions; +import com.azure.core.util.BinaryData; +import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; +import com.azure.core.util.IterableStream; +import com.azure.core.util.logging.ClientLogger; +import java.io.IOException; +import java.net.Proxy; +import java.net.URI; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import okhttp3.Credentials; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; +import okio.ByteString; + +/** + * A synchronous bidirectional realtime session connected to a Foundry voice agent. + */ +@Beta(warningText = "This class is in preview and may change in future releases.") +public final class BetaVoiceAgentWebSocketSessionClient implements AutoCloseable { + private static final ClientLogger LOGGER = new ClientLogger(BetaVoiceAgentWebSocketSessionClient.class); + private final URI websocketUri; + private final VoiceAgentWebSocketConnectionOptions options; + private final OkHttpClient httpClient; + private final BlockingQueue events; + private final int receiveBufferCapacity; + private final CountDownLatch handshakeCompleted = new CountDownLatch(1); + private final CountDownLatch closeCompleted = new CountDownLatch(1); + private final AtomicReference connectionError = new AtomicReference<>(); + private final AtomicBoolean handshakeSucceeded = new AtomicBoolean(); + private final AtomicBoolean receiveClaimed = new AtomicBoolean(); + private final AtomicBoolean open = new AtomicBoolean(); + private final AtomicBoolean closed = new AtomicBoolean(); + private final AtomicBoolean clientShutdown = new AtomicBoolean(); + + private volatile WebSocket webSocket; + private volatile Integer closeCode; + private volatile String closeReason; + + private BetaVoiceAgentWebSocketSessionClient(VoiceAgentWebSocketClientConfiguration configuration, String agentName, + VoiceAgentWebSocketConnectionOptions options) { + this.options = options; + this.receiveBufferCapacity = options.getReceiveBufferCapacity(); + this.events = new ArrayBlockingQueue<>(receiveBufferCapacity + 1); + this.websocketUri = VoiceAgentWebSocketUtils.buildWebSocketUri(configuration, agentName, options); + String token = configuration.getCredential() + .getTokenSync(VoiceAgentWebSocketUtils.createTokenRequestContext(options)) + .getToken(); + this.httpClient = createHttpClient(configuration, options); + Request.Builder request = new Request.Builder().url(websocketUri.toString()) + .header("Sec-WebSocket-Protocol", VoiceAgentWebSocketUtils.SUBPROTOCOL); + for (HttpHeader header : VoiceAgentWebSocketUtils.buildHeaders(configuration, options, token)) { + request.header(header.getName(), header.getValue()); + } + this.webSocket = httpClient.newWebSocket(request.build(), new Listener()); + } + + static BetaVoiceAgentWebSocketSessionClient connect(VoiceAgentWebSocketClientConfiguration configuration, + String agentName, VoiceAgentWebSocketConnectionOptions options) { + BetaVoiceAgentWebSocketSessionClient session + = new BetaVoiceAgentWebSocketSessionClient(configuration, agentName, options); + try { + session.awaitHandshake(); + return session; + } catch (RuntimeException error) { + session.webSocket.cancel(); + session.shutdownHttpClient(); + throw error; + } + } + + /** + * Gets the WebSocket endpoint used by this session. + * + * @return the WebSocket endpoint. + */ + public URI getEndpoint() { + return websocketUri; + } + + /** + * Determines whether the session is open. + * + * @return {@code true} when the session is open. + */ + public boolean isOpen() { + return open.get() && !closed.get(); + } + + /** + * Gets the peer close code. + * + * @return the close code, or {@code null}. + */ + public Integer getCloseCode() { + return closeCode; + } + + /** + * Gets the peer close reason. + * + * @return the close reason, or {@code null}. + */ + public String getCloseReason() { + return closeReason; + } + + /** + * Receives typed server events in wire order. The returned stream may be iterated once. + * + * @throws IllegalStateException if the event stream has already been claimed. + * @return the server event stream. + */ + public IterableStream receiveEvents() { + return receiveEvents(null); + } + + /** + * Receives events with a timeout for each wait. A timeout leaves the session open and the iterator can be retried. + * @param timeout positive per-event timeout, or null to wait indefinitely. + * @return the single-consumer event stream. + * @throws IllegalArgumentException if timeout is zero or negative. + * @throws IllegalStateException if another receiver exists or a wait times out (with a TimeoutException cause). + */ + public IterableStream receiveEvents(Duration timeout) { + if (timeout != null && (timeout.isZero() || timeout.isNegative())) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Receive timeout must be positive.")); + } + if (!receiveClaimed.compareAndSet(false, true)) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("Only one receiveEvents iterator is supported per session.")); + } + return IterableStream.of(() -> new EventIterator(events, timeout)); + } + + /** + * Sends a typed realtime client event. + * + * @param event the event to send. + * @throws IllegalArgumentException if the event cannot be serialized. + * @throws IllegalStateException if the session is closed or cannot accept another event. + */ + public void sendEvent(RealtimeClientEvent event) { + Objects.requireNonNull(event, "'event' cannot be null."); + ensureOpen(); + final String json; + try { + json = event.toJsonString(); + } catch (IOException error) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException("Failed to serialize the realtime client event.", error)); + } + if (!webSocket.send(json)) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("The voice-agent WebSocket send queue is full or closed.")); + } + } + + /** + * Sends a JSON object, including event types and fields unknown to this SDK. + * @param event the complete JSON event. + * @throws IllegalArgumentException if the event is not a JSON object. + * @throws IllegalStateException if the session cannot accept the event. + */ + public void sendEvent(BinaryData event) { + String json = VoiceAgentWebSocketUtils.validateEvent(event); + ensureOpen(); + if (!webSocket.send(json)) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("The voice-agent WebSocket send queue is full or closed.")); + } + } + + /** + * Adds a conversation item. + * + * @param item the item to add. + */ + public void createConversationItem(RealtimeConversationItem item) { + createConversationItem(item, null); + } + + /** + * Adds a conversation item after another item. + * + * @param item the item to add. + * @param previousItemId the preceding item identifier. + */ + public void createConversationItem(RealtimeConversationItem item, String previousItemId) { + Objects.requireNonNull(item, "'item' cannot be null."); + sendEvent(new RealtimeConversationItemCreateEvent(item).setPreviousItemId(previousItemId)); + } + + /** + * Adds a user text message. + * + * @param text the user message. + */ + public void sendText(String text) { + Objects.requireNonNull(text, "'text' cannot be null."); + Content content = Content.builder().type(Content.Type.INPUT_TEXT).text(text).build(); + createConversationItem(new RealtimeConversationItemUserMessage(Collections.singletonList(content))); + } + + /** + * Appends audio to the input buffer. + * + * @param audio the audio bytes. + */ + public void appendInputAudio(BinaryData audio) { + Objects.requireNonNull(audio, "'audio' cannot be null."); + sendEvent(new RealtimeInputAudioBufferAppendEvent(Base64.getEncoder().encodeToString(audio.toBytes()))); + } + + /** Clears the input audio buffer. */ + public void clearInputAudio() { + sendEvent(new RealtimeInputAudioBufferClearEvent()); + } + + /** Commits the input audio buffer. */ + public void commitInputAudio() { + sendEvent(new RealtimeInputAudioBufferCommitEvent()); + } + + /** Requests a response using the voice agent's configuration. */ + public void createResponse() { + sendEvent(new RealtimeResponseCreateEvent()); + } + + /** + * Requests a response with per-response options. + * + * @param responseOptions response options. + */ + public void createResponse(VoiceAgentResponseCreateOptions responseOptions) { + Objects.requireNonNull(responseOptions, "'responseOptions' cannot be null."); + sendEvent(new RealtimeResponseCreateEvent().setResponse(responseOptions)); + } + + /** Cancels the active response. */ + public void cancelResponse() { + sendEvent(new RealtimeResponseCancelEvent()); + } + + /** + * Cancels a specific response. + * + * @param responseId the response identifier. + */ + public void cancelResponse(String responseId) { + Objects.requireNonNull(responseId, "'responseId' cannot be null."); + sendEvent(new RealtimeResponseCancelEvent().setResponseId(responseId)); + } + + /** + * Sends a function-call result and requests the next response. + * + * @param callId the function call identifier. + * @param output the serialized function output. + */ + public void sendFunctionCallOutput(String callId, String output) { + createConversationItem(new RealtimeConversationItemFunctionCallOutput(callId, output)); + createResponse(); + } + + /** Closes the session. */ + @Override + public void close() { + close(1000, ""); + } + + /** + * Closes the session with an application-selected WebSocket close frame. + * @param code a valid WebSocket close code. + * @param reason non-null reason of at most 123 UTF-8 bytes. + * @throws IllegalArgumentException if the code or reason is invalid. + */ + public void close(int code, String reason) { + VoiceAgentWebSocketUtils.validateClose(code, reason); + if (!closed.compareAndSet(false, true)) { + shutdownHttpClient(); + return; + } + open.set(false); + if (!webSocket.close(code, reason)) { + webSocket.cancel(); + closeCompleted.countDown(); + } + try { + if (!closeCompleted.await(options.getCloseTimeout().toMillis(), TimeUnit.MILLISECONDS)) { + webSocket.cancel(); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + webSocket.cancel(); + } finally { + signal(EventSignal.complete()); + shutdownHttpClient(); + } + } + + private void awaitHandshake() { + try { + if (!handshakeCompleted.await(options.getHandshakeTimeout().toMillis(), TimeUnit.MILLISECONDS)) { + webSocket.cancel(); + throw LOGGER + .logExceptionAsError(new IllegalStateException("Voice-agent WebSocket handshake timed out.")); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + webSocket.cancel(); + throw LOGGER.logExceptionAsError( + new IllegalStateException("Interrupted while opening the voice-agent WebSocket session.", error)); + } + if (handshakeSucceeded.get()) { + return; + } + Throwable error = connectionError.get(); + if (error instanceof RuntimeException) { + throw LOGGER.logExceptionAsError((RuntimeException) error); + } + if (error != null) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("Failed to open the voice-agent WebSocket session.", error)); + } + } + + private void ensureOpen() { + if (!isOpen()) { + throw LOGGER + .logExceptionAsError(new IllegalStateException("The voice-agent WebSocket session is not open.")); + } + } + + private void fail(Throwable error, Response response) { + Throwable mapped; + try { + mapped = response == null ? error : mapHandshakeError(error, response); + } finally { + if (response != null) { + response.close(); + } + } + connectionError.compareAndSet(null, mapped); + open.set(false); + closed.set(true); + handshakeCompleted.countDown(); + signal(EventSignal.error(mapped)); + closeCompleted.countDown(); + shutdownHttpClient(); + } + + private Throwable mapHandshakeError(Throwable error, Response response) { + VoiceAgentWebSocketHttpResponse azureResponse = new VoiceAgentWebSocketHttpResponse(websocketUri, response); + String message = "Voice-agent WebSocket handshake failed with status " + response.code() + "."; + switch (response.code()) { + case 401: + return new ClientAuthenticationException(message, azureResponse, error); + + case 404: + return new ResourceNotFoundException(message, azureResponse, error); + + case 409: + return new ResourceModifiedException(message, azureResponse, error); + + default: + return new HttpResponseException(message, azureResponse, error); + } + } + + private synchronized void signal(EventSignal signal) { + if (signal.event != null && events.size() >= receiveBufferCapacity) { + switch (options.getOverflowStrategy()) { + case DROP_LATEST: + return; + + case DROP_OLDEST: + events.poll(); + break; + + default: + break; + } + } + if ((signal.event != null && events.size() >= receiveBufferCapacity) || !events.offer(signal)) { + events.clear(); + events.add(EventSignal.error(new IllegalStateException("Voice-agent receive buffer overflow."))); + WebSocket current = webSocket; + if (current != null) { + current.cancel(); + } + open.set(false); + closed.set(true); + shutdownHttpClient(); + } + } + + private void shutdownHttpClient() { + if (clientShutdown.compareAndSet(false, true)) { + httpClient.dispatcher().executorService().shutdown(); + httpClient.connectionPool().evictAll(); + } + } + + private static OkHttpClient createHttpClient(VoiceAgentWebSocketClientConfiguration configuration, + VoiceAgentWebSocketConnectionOptions options) { + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + builder.connectTimeout(options.getHandshakeTimeout().toMillis(), TimeUnit.MILLISECONDS) + .readTimeout(0, TimeUnit.MILLISECONDS) + .followRedirects(false); + ProxyOptions proxyOptions = configuration.getProxyOptions(); + if (proxyOptions != null) { + Proxy.Type proxyType = proxyOptions.getType() == ProxyOptions.Type.SOCKS4 + || proxyOptions.getType() == ProxyOptions.Type.SOCKS5 ? Proxy.Type.SOCKS : Proxy.Type.HTTP; + builder.proxy(new Proxy(proxyType, proxyOptions.getAddress())); + if (proxyOptions.getUsername() != null) { + builder.proxyAuthenticator((route, response) -> response.request() + .newBuilder() + .header("Proxy-Authorization", + Credentials.basic(proxyOptions.getUsername(), proxyOptions.getPassword())) + .build()); + } + } + return builder.build(); + } + + private final class Listener extends WebSocketListener { + @Override + public void onOpen(WebSocket webSocket, Response response) { + handshakeSucceeded.set(true); + open.set(true); + handshakeCompleted.countDown(); + } + + @Override + public void onMessage(WebSocket webSocket, String text) { + if (text.getBytes(StandardCharsets.UTF_8).length > options.getMaxMessageSize()) { + fail(new IllegalArgumentException("Voice-agent message exceeds the configured size limit."), null); + webSocket.close(1009, "Message too large"); + return; + } + try { + signal(EventSignal.event(VoiceAgentWebSocketUtils.deserializeEvent(text))); + } catch (IOException | RuntimeException error) { + malformedEvent(webSocket, error); + } + } + + @Override + public void onMessage(WebSocket webSocket, ByteString bytes) { + if (bytes.size() > options.getMaxMessageSize()) { + fail(new IllegalArgumentException("Voice-agent message exceeds the configured size limit."), null); + webSocket.close(1009, "Message too large"); + return; + } + try { + onMessage(webSocket, VoiceAgentWebSocketUtils.decodeEvent(bytes.toByteArray())); + } catch (CharacterCodingException error) { + malformedEvent(webSocket, error); + } + } + + private void malformedEvent(WebSocket webSocket, Throwable error) { + if (options.getMalformedEventHandler() != null) { + try { + options.getMalformedEventHandler().accept(error); + return; + } catch (RuntimeException callbackError) { + error = callbackError; + } + } + fail(new IllegalArgumentException("Invalid JSON event.", error), null); + webSocket.close(1007, "Invalid JSON event"); + } + + @Override + public void onClosing(WebSocket webSocket, int code, String reason) { + closeCode = code; + closeReason = reason; + webSocket.close(code == 1005 ? 1000 : code, reason); + } + + @Override + public void onClosed(WebSocket webSocket, int code, String reason) { + closeCode = code; + closeReason = reason; + open.set(false); + closed.set(true); + signal(EventSignal.complete()); + handshakeCompleted.countDown(); + closeCompleted.countDown(); + shutdownHttpClient(); + } + + @Override + public void onFailure(WebSocket webSocket, Throwable error, Response response) { + fail(error, response); + } + } + + private static final class EventSignal { + private final RealtimeServerEvent event; + private final Throwable error; + private final boolean complete; + + private EventSignal(RealtimeServerEvent event, Throwable error, boolean complete) { + this.event = event; + this.error = error; + this.complete = complete; + } + + private static EventSignal event(RealtimeServerEvent event) { + return new EventSignal(event, null, false); + } + + private static EventSignal error(Throwable error) { + return new EventSignal(null, error, false); + } + + private static EventSignal complete() { + return new EventSignal(null, null, true); + } + } + + private static final class EventIterator implements Iterator { + private final BlockingQueue events; + private final Duration timeout; + private EventSignal next; + + private EventIterator(BlockingQueue events, Duration timeout) { + this.events = events; + this.timeout = timeout; + } + + @Override + public boolean hasNext() { + if (next == null) { + try { + next = timeout == null ? events.take() : events.poll(timeout.toNanos(), TimeUnit.NANOSECONDS); + if (next == null) { + throw new IllegalStateException("Timed out waiting for a voice-agent event.", + new TimeoutException()); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for a voice-agent event.", error); + } + } + if (next.error != null) { + Throwable error = next.error; + next = null; + if (error instanceof RuntimeException) { + throw (RuntimeException) error; + } + throw new IllegalStateException("Voice-agent event stream failed.", error); + } + return !next.complete; + } + + @Override + public RealtimeServerEvent next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + RealtimeServerEvent event = next.event; + next = null; + return event; + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java new file mode 100644 index 000000000000..d123c4d96630 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketClientConfiguration.java @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +import com.azure.core.credential.TokenCredential; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.ProxyOptions; + +import java.net.URI; +import java.util.Objects; + +/** + * Immutable configuration used by the voice-agent WebSocket clients. + */ +public final class VoiceAgentWebSocketClientConfiguration { + private final URI endpoint; + private final TokenCredential credential; + private final String apiVersion; + private final String userAgent; + private final HttpHeaders headers; + private final ProxyOptions proxyOptions; + + /** + * Creates the connection configuration. + * + * @param endpoint the Foundry project endpoint. + * @param credential the credential used for authentication. + * @param apiVersion the service API version. + * @param userAgent the SDK user agent. + * @param headers safe additional handshake headers. + * @param proxyOptions proxy settings loaded from configuration. + */ + public VoiceAgentWebSocketClientConfiguration(URI endpoint, TokenCredential credential, String apiVersion, + String userAgent, HttpHeaders headers, ProxyOptions proxyOptions) { + this.endpoint = Objects.requireNonNull(endpoint, "'endpoint' cannot be null."); + this.credential = Objects.requireNonNull(credential, "'credential' cannot be null."); + this.apiVersion = Objects.requireNonNull(apiVersion, "'apiVersion' cannot be null."); + this.userAgent = Objects.requireNonNull(userAgent, "'userAgent' cannot be null."); + this.headers = headers; + this.proxyOptions = proxyOptions; + } + + /** + * Gets the endpoint. + * + * @return the endpoint. + */ + public URI getEndpoint() { + return endpoint; + } + + /** + * Gets the credential. + * + * @return the credential. + */ + public TokenCredential getCredential() { + return credential; + } + + /** + * Gets the API version. + * + * @return the API version. + */ + public String getApiVersion() { + return apiVersion; + } + + /** + * Gets the user agent. + * + * @return the user agent. + */ + public String getUserAgent() { + return userAgent; + } + + /** + * Gets safe additional headers. + * + * @return safe additional headers, or {@code null}. + */ + public HttpHeaders getHeaders() { + return headers; + } + + /** + * Gets proxy settings. + * + * @return proxy settings, or {@code null}. + */ + public ProxyOptions getProxyOptions() { + return proxyOptions; + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketHandshakeHandler.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketHandshakeHandler.java new file mode 100644 index 000000000000..70d5dec2212f --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketHandshakeHandler.java @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.HttpContent; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http.websocketx.WebSocketClientHandshakeException; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Observes the HTTP response used for a WebSocket upgrade and reports rejected handshakes. + */ +public final class VoiceAgentWebSocketHandshakeHandler extends ChannelInboundHandlerAdapter { + private final Consumer errorConsumer; + private ByteArrayOutputStream responseBody; + private HttpResponse rejectedResponse; + + /** + * Creates a handshake response observer. + * + * @param errorConsumer consumer invoked when the server rejects the upgrade. + */ + public VoiceAgentWebSocketHandshakeHandler(Consumer errorConsumer) { + this.errorConsumer = Objects.requireNonNull(errorConsumer, "'errorConsumer' cannot be null."); + } + + @Override + public void channelRead(ChannelHandlerContext context, Object message) throws IOException { + if (message instanceof HttpResponse) { + HttpResponse response = (HttpResponse) message; + if (!HttpResponseStatus.SWITCHING_PROTOCOLS.equals(response.status())) { + rejectedResponse = response; + } + } + if (rejectedResponse != null && message instanceof HttpContent) { + HttpContent content = (HttpContent) message; + ByteBuf byteBuf = content.content(); + if (byteBuf != null && byteBuf.isReadable()) { + if (responseBody == null) { + responseBody = new ByteArrayOutputStream(); + } + byteBuf.readBytes(responseBody, byteBuf.readableBytes()); + } + if (message instanceof LastHttpContent) { + byte[] body = responseBody == null ? new byte[0] : responseBody.toByteArray(); + DefaultFullHttpResponse response = new DefaultFullHttpResponse(rejectedResponse.protocolVersion(), + rejectedResponse.status(), Unpooled.wrappedBuffer(body)); + response.headers().set(rejectedResponse.headers()); + errorConsumer + .accept(new WebSocketClientHandshakeException("Voice-agent WebSocket handshake failed.", response)); + context.close(); + } + } + context.fireChannelRead(message); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketHttpResponse.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketHttpResponse.java new file mode 100644 index 000000000000..7183a02aaf31 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketHttpResponse.java @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import io.netty.buffer.ByteBufUtil; +import io.netty.handler.codec.http.FullHttpResponse; +import okhttp3.ResponseBody; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; + +/** + * Azure Core response adapter for a rejected WebSocket handshake. + */ +public final class VoiceAgentWebSocketHttpResponse extends HttpResponse { + private final int statusCode; + private final HttpHeaders headers; + private final byte[] body; + + /** + * Creates a response adapter. + * + * @param endpoint the WebSocket endpoint. + * @param response the rejected Netty handshake response. + */ + public VoiceAgentWebSocketHttpResponse(URI endpoint, io.netty.handler.codec.http.HttpResponse response) { + super(new HttpRequest(HttpMethod.GET, toHttpUrl(endpoint))); + this.statusCode = response.status().code(); + this.headers = new HttpHeaders(); + for (Map.Entry header : response.headers()) { + this.headers.set(com.azure.core.http.HttpHeaderName.fromString(header.getKey()), header.getValue()); + } + this.body = response instanceof FullHttpResponse + ? ByteBufUtil.getBytes(((FullHttpResponse) response).content()) + : new byte[0]; + } + + /** + * Creates a response adapter. + * + * @param endpoint the WebSocket endpoint. + * @param response the rejected OkHttp handshake response. + */ + public VoiceAgentWebSocketHttpResponse(URI endpoint, okhttp3.Response response) { + super(new HttpRequest(HttpMethod.GET, toHttpUrl(endpoint))); + this.statusCode = response.code(); + this.headers = new HttpHeaders(); + for (String name : response.headers().names()) { + for (String value : response.headers(name)) { + this.headers.add(HttpHeaderName.fromString(name), value); + } + } + this.body = readBody(response.body()); + } + + private static byte[] readBody(ResponseBody responseBody) { + if (responseBody == null) { + return new byte[0]; + } + try { + return responseBody.bytes(); + } catch (IOException error) { + throw new UncheckedIOException("Failed to read the WebSocket handshake response body.", error); + } + } + + private static String toHttpUrl(URI endpoint) { + String endpointUrl = endpoint.toString(); + if ("wss".equalsIgnoreCase(endpoint.getScheme())) { + return "https" + endpointUrl.substring(endpoint.getScheme().length()); + } + if ("ws".equalsIgnoreCase(endpoint.getScheme())) { + return "http" + endpointUrl.substring(endpoint.getScheme().length()); + } + return endpointUrl; + } + + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + @SuppressWarnings("deprecation") + public String getHeaderValue(String name) { + return headers.getValue(HttpHeaderName.fromString(name)); + } + + @Override + public HttpHeaders getHeaders() { + return headers; + } + + @Override + public Flux getBody() { + return body.length == 0 ? Flux.empty() : Flux.just(ByteBuffer.wrap(Arrays.copyOf(body, body.length))); + } + + @Override + public Mono getBodyAsByteArray() { + return Mono.just(Arrays.copyOf(body, body.length)); + } + + @Override + public Mono getBodyAsString() { + return getBodyAsString(StandardCharsets.UTF_8); + } + + @Override + public Mono getBodyAsString(Charset charset) { + return Mono.just(new String(body, charset)); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java new file mode 100644 index 000000000000..ea921f92d37c --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/implementation/realtime/VoiceAgentWebSocketUtils.java @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.implementation.realtime; + +import com.azure.ai.agents.models.RawRealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.VoiceAgentTransport; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.core.credential.TokenRequestContext; +import com.azure.core.http.HttpHeader; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.BinaryData; +import com.azure.core.util.UrlBuilder; +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; +import com.azure.json.JsonToken; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.Objects; + +public final class VoiceAgentWebSocketUtils { + public static String decodeEvent(byte[] bytes) throws CharacterCodingException { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)) + .toString(); + } + + public static String validateEvent(BinaryData event) { + String json = Objects.requireNonNull(event, "'event' cannot be null.").toString(); + try (JsonReader reader = JsonProviders.createReader(json)) { + if (reader.nextToken() != JsonToken.START_OBJECT) { + throw new IllegalArgumentException("A realtime event must be a JSON object."); + } + reader.readUntyped(); + if (reader.nextToken() != JsonToken.END_DOCUMENT) { + throw new IllegalArgumentException("A realtime event must contain only one JSON object."); + } + return json; + } catch (IOException error) { + throw new IllegalArgumentException("Invalid realtime JSON event.", error); + } + } + + public static RealtimeServerEvent deserializeEvent(String json) throws IOException { + validateEvent(BinaryData.fromString(json)); + RawRealtimeServerEvent raw = new RawRealtimeServerEvent(BinaryData.fromString(json)); + if (raw.getType() == null) { + return raw; + } + try (JsonReader reader = JsonProviders.createReader(json)) { + RealtimeServerEvent event = RealtimeServerEvent.fromJson(reader); + return event.getClass() == RealtimeServerEvent.class ? raw : event; + } + } + + private static final String TOKEN_SCOPE = "https://ai.azure.com/.default"; + public static final String SUBPROTOCOL = "realtime"; + + private VoiceAgentWebSocketUtils() { + } + + public static void validateClose(int code, String reason) { + if (code < 1000 + || code >= 5000 + || code == 1004 + || code == 1005 + || code == 1006 + || (code >= 1015 && code < 3000)) { + throw new IllegalArgumentException("Invalid WebSocket close code: " + code); + } + if (reason == null || reason.getBytes(StandardCharsets.UTF_8).length > 123) { + throw new IllegalArgumentException("Close reason must be non-null and at most 123 UTF-8 bytes."); + } + } + + private static boolean isProtectedHeader(String name) { + String lower = name.toLowerCase(Locale.ROOT); + return "authorization".equals(lower) + || "host".equals(lower) + || "upgrade".equals(lower) + || "connection".equals(lower) + || "foundry-features".equals(lower) + || lower.startsWith("sec-websocket-"); + } + + public static URI buildWebSocketUri(VoiceAgentWebSocketClientConfiguration configuration, String agentName, + VoiceAgentWebSocketConnectionOptions options) { + URI endpoint = configuration.getEndpoint(); + String scheme; + if ("https".equalsIgnoreCase(endpoint.getScheme()) || "wss".equalsIgnoreCase(endpoint.getScheme())) { + scheme = "wss"; + } else { + throw new IllegalArgumentException( + "Voice-agent WebSocket endpoints must use https or wss to protect credentials."); + } + if (endpoint.getHost() == null || endpoint.getRawUserInfo() != null || endpoint.getRawFragment() != null) { + throw new IllegalArgumentException( + "The project endpoint must have a host and no user information or fragment."); + } + + String basePath = endpoint.getRawPath() == null ? "" : endpoint.getRawPath().replaceAll("/$", ""); + String path = basePath + "/agents/" + encode(agentName) + "/endpoint/protocols/voice"; + URI baseUri = URI.create(scheme + "://" + endpoint.getRawAuthority() + path); + if (options.getConnectionUrl() != null) { + baseUri = options.getConnectionUrl(); + int endpointPort = endpoint.getPort() == -1 ? 443 : endpoint.getPort(); + int overridePort = baseUri.getPort() == -1 ? 443 : baseUri.getPort(); + if (!"wss".equalsIgnoreCase(baseUri.getScheme()) + || baseUri.getHost() == null + || !baseUri.getHost().equalsIgnoreCase(endpoint.getHost()) + || endpointPort != overridePort + || baseUri.getRawUserInfo() != null + || baseUri.getRawFragment() != null) { + throw new IllegalArgumentException( + "Connection URL must be a wss URL on the project endpoint's host and port, without user information or a fragment."); + } + } + UrlBuilder url = UrlBuilder.parse(baseUri.toString()); + url.setQueryParameter("api-version", + encode(options.getApiVersion() == null ? configuration.getApiVersion() : options.getApiVersion())); + url.setQueryParameter("x-ms-client-sdk", encode(configuration.getUserAgent())); + VoiceAgentTransport transport = options.getTransport(); + if (transport != null) { + url.setQueryParameter("transport", encode(transport.toString())); + } + if (options.isStoreEnabled() != null) { + url.setQueryParameter("store", options.isStoreEnabled().toString()); + } + if (options.getAgentVersionOverride() != null) { + url.setQueryParameter("x-agent-version-override", encode(options.getAgentVersionOverride())); + } + if (options.getAgentSessionId() != null) { + url.setQueryParameter("agent_session_id", encode(options.getAgentSessionId())); + } + options.getExtraQuery().forEach((name, value) -> url.setQueryParameter(encode(name), encode(value))); + return URI.create(url.toString()); + } + + public static TokenRequestContext createTokenRequestContext(VoiceAgentWebSocketConnectionOptions options) { + return options.getCredentialScopes() == null || options.getCredentialScopes().isEmpty() + ? new TokenRequestContext().addScopes(TOKEN_SCOPE) + : new TokenRequestContext().setScopes(options.getCredentialScopes()); + } + + public static HttpHeaders buildHeaders(VoiceAgentWebSocketClientConfiguration configuration, + VoiceAgentWebSocketConnectionOptions options, String token) { + HttpHeaders headers = new HttpHeaders().set(HttpHeaderName.USER_AGENT, configuration.getUserAgent()); + if (configuration.getHeaders() != null) { + for (HttpHeader header : configuration.getHeaders()) { + if (!isProtectedHeader(header.getName())) { + headers.set(HttpHeaderName.fromString(header.getName()), header.getValue()); + } + } + } + headers.set(HttpHeaderName.fromString("Foundry-Features"), options.getFoundryFeatures()); + if (options.getStructuredInputs() != null) { + headers.set(HttpHeaderName.fromString("x-ms-voice-structured-inputs"), options.getStructuredInputs()); + } + options.getExtraHeaders().forEach((name, value) -> { + if (!isProtectedHeader(name) || "Foundry-Features".equalsIgnoreCase(name)) { + headers.set(HttpHeaderName.fromString(name), value); + } + }); + return headers.set(HttpHeaderName.AUTHORIZATION, "Bearer " + token); + } + + private static String encode(String value) { + try { + return URLEncoder.encode(value, StandardCharsets.UTF_8.name()).replace("+", "%20"); + } catch (UnsupportedEncodingException error) { + throw new IllegalStateException("UTF-8 encoding is unavailable.", error); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/RawRealtimeServerEvent.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/RawRealtimeServerEvent.java new file mode 100644 index 000000000000..47e4dd5af1ad --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/RawRealtimeServerEvent.java @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.models; + +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.core.util.BinaryData; +import com.azure.json.JsonReader; +import com.azure.json.JsonWriter; +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +/** A server event whose complete JSON payload is retained for forward compatibility. */ +@Beta(warningText = "Preview API. VoiceAgents=V1Preview") +public final class RawRealtimeServerEvent extends RealtimeServerEvent { + private final BinaryData rawEvent; + private final RealtimeServerEventType type; + + /** + * Creates an event from a JSON object. + * @param rawEvent the complete event payload. + */ + public RawRealtimeServerEvent(BinaryData rawEvent) { + this.rawEvent + = BinaryData.fromString(Objects.requireNonNull(rawEvent, "'rawEvent' cannot be null.").toString()); + Object value = this.rawEvent.toObject(Map.class).get("type"); + this.type = value instanceof String ? RealtimeServerEventType.fromString((String) value) : null; + } + + /** + * Gets the complete event, including fields unknown to this SDK. + * @return the JSON payload. + */ + public BinaryData getRawEvent() { + return rawEvent; + } + + @Override + public RealtimeServerEventType getType() { + return type; + } + + @Override + public JsonWriter toJson(JsonWriter writer) throws IOException { + return writer.writeRawValue(rawEvent.toString()); + } + + /** + * Reads a raw event without discarding unknown properties. + * @param reader the JSON reader. + * @return the event, or null for JSON null. + * @throws IOException if the JSON cannot be read. + */ + public static RawRealtimeServerEvent fromJson(JsonReader reader) throws IOException { + Object payload = reader.readUntyped(); + return payload == null ? null : new RawRealtimeServerEvent(BinaryData.fromObject(payload)); + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketConnectionOptions.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketConnectionOptions.java new file mode 100644 index 000000000000..23d29798891b --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketConnectionOptions.java @@ -0,0 +1,422 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.models; + +import com.azure.ai.agents.implementation.utils.Beta; +import com.azure.core.annotation.Fluent; +import com.azure.core.util.logging.ClientLogger; +import java.net.URI; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Options used when opening a realtime voice-agent WebSocket session. + */ +@Beta(warningText = "Preview API. VoiceAgents=V1Preview") +@Fluent +public final class VoiceAgentWebSocketConnectionOptions { + private static final ClientLogger LOGGER = new ClientLogger(VoiceAgentWebSocketConnectionOptions.class); + private int receiveBufferCapacity = 256; + private int maxMessageSize = 32 * 1024 * 1024; + private VoiceAgentWebSocketOverflowStrategy overflowStrategy = VoiceAgentWebSocketOverflowStrategy.ERROR; + private Consumer malformedEventHandler; + + /** + * Gets the maximum number of queued events. + * @return the capacity, default 256. + */ + public int getReceiveBufferCapacity() { + return receiveBufferCapacity; + } + + /** + * Sets the bounded receive queue capacity. Configure before connecting. + * @param capacity number of events, between 1 and 65536. + * @return this options instance. + * @throws IllegalArgumentException if capacity is outside the supported range. + */ + public VoiceAgentWebSocketConnectionOptions setReceiveBufferCapacity(int capacity) { + if (capacity < 1 || capacity > 65536) { + throw LOGGER + .logExceptionAsError(new IllegalArgumentException("Receive capacity must be between 1 and 65536.")); + } + this.receiveBufferCapacity = capacity; + return this; + } + + /** + * Gets the maximum accepted JSON message size. + * @return the size in bytes, default 32 MiB. + */ + public int getMaxMessageSize() { + return maxMessageSize; + } + + /** + * Sets the maximum accepted JSON message size. Oversized messages terminate the connection. + * @param bytes positive size in bytes. + * @return this options instance. + * @throws IllegalArgumentException if bytes is not positive. + */ + public VoiceAgentWebSocketConnectionOptions setMaxMessageSize(int bytes) { + if (bytes <= 0) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException("Message size must be positive.")); + } + this.maxMessageSize = bytes; + return this; + } + + /** + * Gets the receive queue overflow action. + * @return the strategy, default ERROR. + */ + public VoiceAgentWebSocketOverflowStrategy getOverflowStrategy() { + return overflowStrategy; + } + + /** + * Sets the receive queue overflow action. Drop strategies explicitly permit data loss. + * @param strategy the non-null strategy. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy strategy) { + this.overflowStrategy = Objects.requireNonNull(strategy, "'strategy' cannot be null."); + return this; + } + + /** + * Gets the callback for skipping malformed events. + * @return the callback, or null to terminate on malformed events. + */ + public Consumer getMalformedEventHandler() { + return malformedEventHandler; + } + + /** + * Sets a callback that reports and skips malformed events without terminating reception. + * The callback runs on the receive thread and must not block. If it throws, the session terminates. + * This does not recover from transport errors or oversized messages. + * @param handler callback, or null to terminate on malformed events (the default). + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setMalformedEventHandler(Consumer handler) { + this.malformedEventHandler = handler; + return this; + } + + private VoiceAgentTransport transport = VoiceAgentTransport.WEBSOCKET; + private Boolean store; + private String agentVersionOverride; + private Duration handshakeTimeout = Duration.ofSeconds(30); + private Duration closeTimeout = Duration.ofSeconds(10); + private String agentSessionId; + private String structuredInputs; + private URI connectionUrl; + private String apiVersion; + private String foundryFeatures = "VoiceAgents=V1Preview"; + private List credentialScopes; + private Map extraQuery = Collections.emptyMap(); + private Map extraHeaders = Collections.emptyMap(); + + /** + * Creates options for opening a realtime voice-agent WebSocket session. + */ + public VoiceAgentWebSocketConnectionOptions() { + } + + /** + * Creates a copy of the supplied connection options. + * + * @param source the options to copy. + * @throws NullPointerException if {@code source} is null. + */ + public VoiceAgentWebSocketConnectionOptions(VoiceAgentWebSocketConnectionOptions source) { + Objects.requireNonNull(source, "'source' cannot be null."); + this.receiveBufferCapacity = source.receiveBufferCapacity; + this.maxMessageSize = source.maxMessageSize; + this.overflowStrategy = source.overflowStrategy; + this.malformedEventHandler = source.malformedEventHandler; + this.transport = source.transport; + this.store = source.store; + this.agentVersionOverride = source.agentVersionOverride; + this.handshakeTimeout = source.handshakeTimeout; + this.closeTimeout = source.closeTimeout; + this.agentSessionId = source.agentSessionId; + this.structuredInputs = source.structuredInputs; + this.connectionUrl = source.connectionUrl; + this.apiVersion = source.apiVersion; + this.foundryFeatures = source.foundryFeatures; + this.credentialScopes = source.credentialScopes == null + ? null + : Collections.unmodifiableList(new ArrayList<>(source.credentialScopes)); + this.extraQuery = Collections.unmodifiableMap(new LinkedHashMap<>(source.extraQuery)); + this.extraHeaders = Collections.unmodifiableMap(new LinkedHashMap<>(source.extraHeaders)); + } + + /** + * Gets the session correlation identifier. + * @return the session identifier, or null. + */ + public String getAgentSessionId() { + return agentSessionId; + } + + /** + * Sets the session correlation identifier. + * @param agentSessionId the session identifier, or null. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setAgentSessionId(String agentSessionId) { + this.agentSessionId = agentSessionId; + return this; + } + + /** + * Gets the structured inputs JSON object sent in the handshake header. + * @return the structured inputs, or null. + */ + public String getStructuredInputs() { + return structuredInputs; + } + + /** + * Sets the structured inputs JSON object sent in the handshake header. + * @param structuredInputs the JSON object, or null. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setStructuredInputs(String structuredInputs) { + this.structuredInputs = structuredInputs; + return this; + } + + /** + * Gets the full WebSocket URL override. + * @return the URL override, or null. + */ + public URI getConnectionUrl() { + return connectionUrl; + } + + /** + * Sets a full WebSocket URL override. It must use wss and the project endpoint's host and port. + * User information and fragments are not supported. Existing query parameters are preserved unless overridden. + * @param connectionUrl the URL override, or null to use the agent route. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setConnectionUrl(URI connectionUrl) { + this.connectionUrl = connectionUrl; + return this; + } + + /** + * Gets the handshake API version override. + * @return the API version, or null. + */ + public String getApiVersion() { + return apiVersion; + } + + /** + * Sets the handshake API version override. + * @param apiVersion the API version, or null to use the client's version. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + return this; + } + + /** + * Gets the preview feature header value. + * @return the preview feature header value. + */ + public String getFoundryFeatures() { + return foundryFeatures; + } + + /** + * Sets the preview feature header value. + * @param foundryFeatures comma-separated preview features, or an empty string to suppress opt-in. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setFoundryFeatures(String foundryFeatures) { + this.foundryFeatures = Objects.requireNonNull(foundryFeatures, "'foundryFeatures' cannot be null."); + return this; + } + + /** + * Gets credential scopes for the handshake. + * @return an unmodifiable list, or null to use the default Foundry scope. + */ + public List getCredentialScopes() { + return credentialScopes; + } + + /** + * Sets credential scopes for the handshake. + * @param credentialScopes the scopes, or null to use the default Foundry scope. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setCredentialScopes(List credentialScopes) { + this.credentialScopes + = credentialScopes == null ? null : Collections.unmodifiableList(new ArrayList<>(credentialScopes)); + return this; + } + + /** + * Gets additional handshake query parameters. + * @return an unmodifiable map of query parameters. + */ + public Map getExtraQuery() { + return extraQuery; + } + + /** + * Sets additional handshake query parameters, taking precedence over defaults. + * @param extraQuery unencoded query names and values, or null to clear. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setExtraQuery(Map extraQuery) { + this.extraQuery = extraQuery == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(extraQuery)); + return this; + } + + /** + * Gets additional handshake headers. + * @return an unmodifiable map of headers. + */ + public Map getExtraHeaders() { + return extraHeaders; + } + + /** + * Sets additional handshake headers. Authorization, host, connection, upgrade, and WebSocket protocol headers + * remain transport-controlled. Other headers override defaults case-insensitively, including empty values. + * @param extraHeaders the additional headers, or null to clear. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setExtraHeaders(Map extraHeaders) { + this.extraHeaders = extraHeaders == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(extraHeaders)); + return this; + } + + /** + * Gets the session transport. + * + * @return the session transport. + */ + public VoiceAgentTransport getTransport() { + return transport; + } + + /** + * Sets the session transport. WebRTC transport performs signaling only; the SDK does not provide a WebRTC media + * implementation. + * + * @param transport the session transport. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setTransport(VoiceAgentTransport transport) { + this.transport = transport; + return this; + } + + /** + * Gets whether the conversation is persisted for this session. + * + * @return whether the conversation is persisted, or {@code null} to use the agent definition. + */ + public Boolean isStoreEnabled() { + return store; + } + + /** + * Sets whether the conversation is persisted for this session. + * + * @param store whether the conversation is persisted, or {@code null} to use the agent definition. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setStoreEnabled(Boolean store) { + this.store = store; + return this; + } + + /** + * Gets the agent version override. + * + * @return the agent version override. + */ + public String getAgentVersionOverride() { + return agentVersionOverride; + } + + /** + * Sets the agent version override. + * + * @param agentVersionOverride the agent version to use for this session. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setAgentVersionOverride(String agentVersionOverride) { + this.agentVersionOverride = agentVersionOverride; + return this; + } + + /** + * Gets the WebSocket handshake timeout. + * + * @return the WebSocket handshake timeout. + */ + public Duration getHandshakeTimeout() { + return handshakeTimeout; + } + + /** + * Sets the WebSocket handshake timeout. + * + * @param handshakeTimeout the positive handshake timeout. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setHandshakeTimeout(Duration handshakeTimeout) { + validatePositive(handshakeTimeout, "handshakeTimeout"); + this.handshakeTimeout = handshakeTimeout; + return this; + } + + /** + * Gets the graceful close timeout. + * + * @return the graceful close timeout. + */ + public Duration getCloseTimeout() { + return closeTimeout; + } + + /** + * Sets the graceful close timeout. + * + * @param closeTimeout the positive close timeout. + * @return this options instance. + */ + public VoiceAgentWebSocketConnectionOptions setCloseTimeout(Duration closeTimeout) { + validatePositive(closeTimeout, "closeTimeout"); + this.closeTimeout = closeTimeout; + return this; + } + + private static void validatePositive(Duration duration, String name) { + if (duration == null || duration.isZero() || duration.isNegative()) { + throw new IllegalArgumentException("'" + name + "' must be positive."); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketOverflowStrategy.java b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketOverflowStrategy.java new file mode 100644 index 000000000000..39f3dc0acb61 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/main/java/com/azure/ai/agents/models/VoiceAgentWebSocketOverflowStrategy.java @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.models; + +import com.azure.ai.agents.implementation.utils.Beta; + +/** Action taken when a voice-agent session's bounded receive queue fills. */ +@Beta(warningText = "Preview API. VoiceAgents=V1Preview") +public enum VoiceAgentWebSocketOverflowStrategy { + /** Terminate the connection with an error. No overflow is silently ignored. */ + ERROR, + /** Discard the oldest buffered event to accept the new event. This loses data. */ + DROP_OLDEST, + /** Discard the incoming event. This loses data. */ + DROP_LATEST +} diff --git a/sdk/ai/azure-ai-agents/src/main/java/module-info.java b/sdk/ai/azure-ai-agents/src/main/java/module-info.java index ef7e2a9cd86e..06498120ce9e 100644 --- a/sdk/ai/azure-ai-agents/src/main/java/module-info.java +++ b/sdk/ai/azure-ai-agents/src/main/java/module-info.java @@ -4,8 +4,17 @@ module com.azure.ai.agents { requires transitive com.azure.core; - requires transitive openai.java.client.okhttp; requires transitive openai.java.core; + requires transitive openai.java.client.okhttp; + requires reactor.netty.http; + requires reactor.netty.core; + requires io.netty.codec.http; + requires io.netty.transport; + requires io.netty.common; + requires io.netty.codec; + requires io.netty.buffer; + requires okhttp3; + requires okio; exports com.azure.ai.agents; exports com.azure.ai.agents.models; diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java index 40ee3dfdf99e..4baeeb806c16 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/ReadmeSamples.java @@ -10,10 +10,15 @@ import com.azure.ai.agents.models.FixedRatioVersionSelectionRule; import com.azure.ai.agents.models.PromptAgentDefinition; import com.azure.ai.agents.models.ProtocolConfiguration; +import com.azure.ai.agents.models.RawRealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeServerEvent; import com.azure.ai.agents.models.ResponsesProtocolConfiguration; import com.azure.ai.agents.models.SessionLogEvent; import com.azure.ai.agents.models.UpdateAgentDetailsOptions; import com.azure.ai.agents.models.VersionSelector; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketOverflowStrategy; +import com.azure.core.util.BinaryData; import com.azure.core.util.IterableStream; import com.azure.identity.AuthenticationUtil; import com.azure.identity.DefaultAzureCredentialBuilder; @@ -28,6 +33,27 @@ import com.openai.services.blocking.ConversationService; public final class ReadmeSamples { + public void realtimeForwardCompatibility(BetaVoiceAgentWebSocketClient realtimeClient, String agentName) { + // BEGIN: com.azure.ai.agents.realtime_forward_compatibility + VoiceAgentWebSocketConnectionOptions options + = new VoiceAgentWebSocketConnectionOptions() + .setReceiveBufferCapacity(512) + .setMaxMessageSize(8 * 1024 * 1024) + .setOverflowStrategy(VoiceAgentWebSocketOverflowStrategy.ERROR); + try (BetaVoiceAgentWebSocketSessionClient session = realtimeClient.openWebSocketSession(agentName, options)) { + session.sendEvent(BinaryData.fromString( + "{\"type\":\"response.create\",\"event_id\":\"response-1\"}")); + for (RealtimeServerEvent event : session.receiveEvents()) { + if (event instanceof RawRealtimeServerEvent) { + BinaryData payload + = ((RawRealtimeServerEvent) event).getRawEvent(); + System.out.println("Received an unrecognized event with " + payload.getLength() + " bytes."); + } + } + } + // END: com.azure.ai.agents.realtime_forward_compatibility + } + public void readmeSamples() { String endpoint = "my-resource-url"; String model = "model"; diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java new file mode 100644 index 000000000000..d5ee1ea3e8e8 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentKind; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceModelType; +import reactor.core.publisher.Mono; + +/** + * Demonstrates the asynchronous voice-agent lifecycle. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_MODEL} - Optional. The voice model or deployment name. Defaults to {@code gpt-realtime}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL_TYPE} - Optional. The voice model type. Defaults to {@code managed}.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name. Defaults to {@code voice-agent-async-java}.
    • + *
+ */ +public class VoiceAgentBasicAsyncSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "voice-agent-async-java"); + + AgentsAsyncClient client = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true) + .buildAgentsAsyncClient(); + + client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a friendly voice assistant. Keep replies short and natural."))) + .doOnNext(created -> System.out.printf("Created voice agent %s, version %s%n", + created.getName(), created.getVersion())) + .then(client.getAgent(agentName)) + .doOnNext(agent -> System.out.printf("Retrieved voice agent %s, state %s%n", + agent.getName(), agent.getState())) + .thenMany(client.listAgents(AgentKind.VOICE, null, null, null, null)) + .doOnNext(agent -> System.out.println("Voice agent: " + agent.getName())) + .then(client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a friendly voice assistant. Always greet the caller warmly.")) + .setDescription("Updated voice-agent instructions."))) + .doOnNext(updated -> System.out.println("Created updated version: " + updated.getVersion())) + .then(client.disableAgent(agentName)) + .then(client.enableAgent(agentName)) + .then(client.deleteAgent(agentName) + .doOnSuccess(ignored -> System.out.println("Deleted agent after successful completion: " + agentName))) + .onErrorResume(error -> client.deleteAgent(agentName) + .doOnSuccess(ignored -> System.out.println("Deleted agent during error cleanup: " + agentName)) + .onErrorResume(cleanupError -> Mono.empty()) + .then(Mono.error(error))) + .block(); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java new file mode 100644 index 000000000000..e03ab1996238 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentDetails; +import com.azure.ai.agents.models.AgentKind; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceModelType; + +/** + * Demonstrates the synchronous voice-agent lifecycle. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_MODEL} - Optional. The voice model or deployment name. Defaults to {@code gpt-realtime}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL_TYPE} - Optional. The voice model type. Defaults to {@code managed}.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name. Defaults to {@code voice-agent-java}.
    • + *
+ */ +public class VoiceAgentBasicSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "voice-agent-java"); + + AgentsClient client = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true) + .buildAgentsClient(); + try { + AgentVersionDetails created = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a friendly voice assistant. Keep replies short and natural."))); + System.out.printf("Created voice agent %s, version %s%n", created.getName(), created.getVersion()); + + AgentDetails agent = client.getAgent(agentName); + System.out.printf("Retrieved voice agent %s, state %s%n", agent.getName(), agent.getState()); + for (AgentDetails item : client.listAgents(AgentKind.VOICE, null, null, null, null)) { + System.out.println("Voice agent: " + item.getName()); + } + + AgentVersionDetails updated = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a friendly voice assistant. Always greet the caller warmly.")) + .setDescription("Updated voice-agent instructions.")); + System.out.println("Created updated version: " + updated.getVersion()); + client.disableAgent(agentName); + System.out.println("Disabled voice agent"); + client.enableAgent(agentName); + System.out.println("Enabled voice agent"); + } finally { + client.deleteAgent(agentName); + System.out.println("Deleted voice agent: " + agentName); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java new file mode 100644 index 000000000000..d24846311b2a --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaAgentsClient; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentDetails; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.core.util.BinaryData; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Demonstrates guided authoring of a voice agent through the agent generation API. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name. Defaults to {@code generated-voice-agent-java}.
    • + *
+ */ +public class VoiceAgentGenerateSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "generated-voice-agent-java"); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsClient client = builder.buildAgentsClient(); + BetaAgentsClient betaClient = builder.beta().buildBetaAgentsClient(); + + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + AgentDetails generated = betaClient.createAgentFromPrompt(BinaryData.fromObject(request)); + try { + System.out.println("Generated voice agent: " + generated.getName()); + AgentVersionDetails latest = generated.getVersions().getLatest(); + if (latest != null && latest.getDefinition() instanceof VoiceAgentDefinition) { + VoiceAgentDefinition definition = (VoiceAgentDefinition) latest.getDefinition(); + System.out.println("Instructions: " + definition.getInstructions()); + } + } finally { + client.deleteAgent(generated.getName()); + System.out.println("Deleted agent: " + generated.getName()); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java new file mode 100644 index 000000000000..2a80cc060fba --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java @@ -0,0 +1,375 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsConversationsAsyncClient; +import com.azure.ai.agents.BetaAgentsAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeConversationItemInputAudioTranscriptionCompletedEvent; +import com.azure.ai.agents.models.RealtimeInputAudioBufferSpeechStartedEvent; +import com.azure.ai.agents.models.RealtimeErrorEvent; +import com.azure.ai.agents.models.RealtimeResponseAudioDeltaEvent; +import com.azure.ai.agents.models.RealtimeResponseAudioTranscriptDoneEvent; +import com.azure.ai.agents.models.RealtimeResponseCreatedEvent; +import com.azure.ai.agents.models.RealtimeResponseDoneEvent; +import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.core.scheduler.Schedulers; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.SourceDataLine; +import javax.sound.sampled.TargetDataLine; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Demonstrates an asynchronous hands-free voice conversation using Java Sound and server-side VAD. + * + *

To end the call, focus the terminal running the sample and press Enter. The sample then closes the WebSocket, + * stops the microphone and speaker, reads the persisted conversation, and deletes the agent unless + * {@code FOUNDRY_KEEP_VOICE_AGENT} is set to {@code true}.

+ * + *

Disconnection or an audio failure also stops the call and releases the audio devices. Up to 60 seconds of PCM + * audio can wait for playback so that faster-than-realtime responses do not block WebSocket reception. Exceeding + * that limit ends the call rather than dropping speech or growing memory without a bound. This sample must be the + * only reader of standard input.

+ * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to + * {@code sample-live-audio-conversation-agent-async-java}.
    • + *
  • {@code FOUNDRY_KEEP_VOICE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. + * Defaults to {@code false}.
    • + *
+ */ +public class VoiceAgentLiveAudioConversationAsyncSample { + private static final Duration SEND_TIMEOUT = Duration.ofSeconds(10); + + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", + "sample-live-audio-conversation-agent-async-java"); + boolean keepAgent + = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_VOICE_AGENT", "false")); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsAsyncClient agents = builder.buildAgentsAsyncClient(); + BetaAgentsAsyncClient betaAgents = builder.beta().buildBetaAgentsAsyncClient(); + BetaVoiceAgentWebSocketAsyncClient realtime = builder.beta().buildBetaVoiceAgentWebSocketAsyncClient(); + BetaVoiceAgentsConversationsAsyncClient conversations + = builder.beta().buildBetaVoiceAgentsConversationsAsyncClient(); + + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + AtomicReference conversationId = new AtomicReference<>(); + + betaAgents.createAgentFromPrompt(BinaryData.fromObject(request)) + .flatMap(generated -> { + VoiceAgentDefinition definition + = (VoiceAgentDefinition) generated.getVersions().getLatest().getDefinition(); + return agents.createAgentVersion(agentName, + new CreateAgentVersionInput(definition.setStore(true))); + }) + .then(Mono.usingWhen(realtime.openWebSocketSession(agentName), + session -> runConversation(session, conversationId), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, + (session, error) -> session.closeAsync(), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync)) + .then(Mono.defer(() -> conversationId.get() == null + ? Mono.fromRunnable(() -> System.out.println("No persisted conversation ID was returned.")) + : VoiceAgentRealtimeSampleUtils.readConversation(conversations, agentName, conversationId.get()))) + .then(Mono.defer(() -> cleanupAgent(agents, agentName, keepAgent))) + .onErrorResume(error -> Mono.defer(() -> cleanupAgent(agents, agentName, keepAgent)) + .onErrorResume(cleanupError -> Mono.empty()) + .then(Mono.error(error))) + .block(); + } + + private static Mono cleanupAgent(AgentsAsyncClient agents, String agentName, boolean keepAgent) { + if (keepAgent) { + return Mono.fromRunnable(() -> System.out.println("Kept voice agent: " + agentName)); + } + return agents.deleteAgent(agentName) + .doOnSuccess(ignored -> System.out.println("Deleted voice agent: " + agentName)); + } + + private static Mono runConversation(BetaVoiceAgentWebSocketSessionAsyncClient session, + AtomicReference conversationId) { + AudioProcessor processor = new AudioProcessor(session); + AtomicBoolean responseActive = new AtomicBoolean(); + Mono receive = session.receiveEvents().concatMap(event -> { + if (event instanceof RealtimeSessionCreatedEvent) { + String id = ((RealtimeSessionCreatedEvent) event).getConversationId(); + if (id != null) { + conversationId.set(id); + } + } else if (event instanceof RealtimeInputAudioBufferSpeechStartedEvent) { + if (responseActive.get()) { + processor.skipPendingAudio(); + System.out.println("(listening...)"); + return session.cancelResponse().timeout(SEND_TIMEOUT); + } + } else if (event instanceof RealtimeConversationItemInputAudioTranscriptionCompletedEvent) { + System.out.println("You: " + + ((RealtimeConversationItemInputAudioTranscriptionCompletedEvent) event) + .getTranscript().trim()); + } else if (event instanceof RealtimeResponseCreatedEvent) { + responseActive.set(true); + } else if (event instanceof RealtimeResponseDoneEvent) { + responseActive.set(false); + } else if (event instanceof RealtimeResponseAudioDeltaEvent) { + processor.queueAudio(((RealtimeResponseAudioDeltaEvent) event).getDelta()); + } else if (event instanceof RealtimeResponseAudioTranscriptDoneEvent) { + System.out.println("Agent: " + + ((RealtimeResponseAudioTranscriptDoneEvent) event).getTranscript()); + } else if (event instanceof RealtimeErrorEvent) { + RealtimeErrorEvent error + = (RealtimeErrorEvent) event; + System.out.println("Session error: " + error.getError().message()); + } + return Mono.empty(); + }).then(); + return runConversation(receive, processor, System.in); + } + + static Mono runConversation(Mono receive, AudioProcessor processor, InputStream input) { + return Mono.usingWhen(Mono.fromSupplier(() -> processor), audio -> { + System.out.println("Speak now; talk over the agent to interrupt it. Press Enter to end the session."); + return Mono.fromRunnable(processor::start) + .subscribeOn(Schedulers.boundedElastic()) + .then(Mono.firstWithSignal(receive, processor.failure.asMono(), waitForEnter(input))); + }, VoiceAgentLiveAudioConversationAsyncSample::closeAudio, + (audio, error) -> closeAudio(audio), VoiceAgentLiveAudioConversationAsyncSample::closeAudio); + } + + private static Mono closeAudio(AudioProcessor processor) { + return Mono.fromRunnable(processor::close).subscribeOn(Schedulers.boundedElastic()); + } + + static Mono waitForEnter(InputStream input) { + return Flux.interval(Duration.ZERO, Duration.ofMillis(100), Schedulers.boundedElastic()) + .handle((tick, sink) -> { + try { + int available = input.available(); + for (int remaining = available; remaining > 0; remaining--) { + int next = input.read(); + if (next == '\n' || next == '\r' || next == -1) { + sink.complete(); + return; + } + } + } catch (IOException error) { + sink.error(error); + } + }).then(); + } + + static final class AudioProcessor implements AutoCloseable { + private static final int CHUNK_BYTES = 2400; + static final int MAX_PLAYBACK_BYTES = VoiceAgentRealtimeSampleUtils.SAMPLE_RATE * 2 * 60; + private static final byte[] STOP = new byte[0]; + private final BetaVoiceAgentWebSocketSessionAsyncClient session; + private final AudioFormat format = new AudioFormat(VoiceAgentRealtimeSampleUtils.SAMPLE_RATE, 16, 1, true, false); + private final BlockingQueue playback = new LinkedBlockingQueue<>(MAX_PLAYBACK_BYTES / 2); + private int queuedPlaybackBytes; + private final AtomicBoolean running = new AtomicBoolean(); + private final Sinks.Empty failure = Sinks.empty(); + private boolean closed; + private TargetDataLine microphone; + private SourceDataLine speaker; + private Thread captureThread; + private Thread playbackThread; + + AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session) { + this(session, null, null); + } + + AudioProcessor(BetaVoiceAgentWebSocketSessionAsyncClient session, TargetDataLine microphone, SourceDataLine speaker) { + this.session = session; + this.microphone = microphone; + this.speaker = speaker; + } + + synchronized void start() { + if (closed) { + throw new IllegalStateException("Audio processor is already closed."); + } + try { + if (microphone == null) { + microphone = AudioSystem.getTargetDataLine(format); + } + microphone.open(format, CHUNK_BYTES * 4); + if (speaker == null) { + speaker = AudioSystem.getSourceDataLine(format); + } + speaker.open(format); + microphone.start(); + speaker.start(); + } catch (LineUnavailableException | IllegalArgumentException error) { + close(); + throw new IllegalStateException("A 24-kHz mono PCM16 microphone and speaker are required.", error); + } + + running.set(true); + captureThread = new Thread(this::capture, "voice-agent-microphone"); + playbackThread = new Thread(this::playback, "voice-agent-speaker"); + captureThread.setDaemon(true); + playbackThread.setDaemon(true); + captureThread.start(); + playbackThread.start(); + } + + private void capture() { + byte[] buffer = new byte[CHUNK_BYTES]; + try { + while (running.get()) { + int read = microphone.read(buffer, 0, buffer.length); + if (read > 0 && running.get()) { + session.appendInputAudio(BinaryData.fromBytes(Arrays.copyOf(buffer, read))).block(SEND_TIMEOUT); + } + } + } catch (RuntimeException error) { + fail(error); + } + } + + private void playback() { + try { + while (running.get()) { + byte[] pcm = playback.take(); + if (pcm == STOP) { + break; + } + synchronized (playback) { + queuedPlaybackBytes -= pcm.length; + } + speaker.write(pcm, 0, pcm.length); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + fail(error); + } catch (RuntimeException error) { + fail(error); + } + } + + void queueAudio(byte[] pcm) { + if (pcm == null || pcm.length == 0) { + return; + } + if (pcm.length % 2 != 0) { + fail(new IllegalArgumentException("PCM16 audio must contain complete two-byte samples.")); + return; + } + synchronized (playback) { + if (!running.get()) { + return; + } + if (pcm.length <= MAX_PLAYBACK_BYTES - queuedPlaybackBytes && playback.offer(pcm)) { + queuedPlaybackBytes += pcm.length; + return; + } + } + fail(new IllegalStateException("Audio playback backlog exceeded 60 seconds.")); + } + + private void fail(Throwable error) { + if (running.compareAndSet(true, false)) { + failure.tryEmitError(error); + } + } + + synchronized void skipPendingAudio() { + clearPlayback(); + if (speaker != null) { + speaker.flush(); + } + } + + @Override + public synchronized void close() { + if (closed) { + return; + } + closed = true; + running.set(false); + clearPlayback(); + playback.offer(STOP); + try { + closeLine(microphone); + } finally { + try { + closeLine(speaker); + } finally { + if (captureThread != null) { + captureThread.interrupt(); + } + if (playbackThread != null) { + playbackThread.interrupt(); + } + join(captureThread); + join(playbackThread); + } + } + } + + private void clearPlayback() { + synchronized (playback) { + byte[] discarded; + while ((discarded = playback.poll()) != null) { + queuedPlaybackBytes -= discarded.length; + } + } + } + + private static void closeLine(javax.sound.sampled.DataLine line) { + if (line != null) { + try { + line.stop(); + } finally { + line.close(); + } + } + } + + private static void join(Thread thread) { + if (thread != null && thread != Thread.currentThread()) { + try { + thread.join(SEND_TIMEOUT.toMillis()); + if (thread.isAlive()) { + System.err.println("Audio thread did not stop: " + thread.getName()); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java new file mode 100644 index 000000000000..f51801608fb3 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentWebSocketClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeConversationItem; +import com.azure.ai.agents.models.RealtimeConversationItemType; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeErrorEvent; +import com.azure.ai.agents.models.RealtimeResponseDoneEvent; +import com.azure.ai.agents.models.RealtimeResponseFunctionCallArgumentsDoneEvent; +import com.azure.ai.agents.models.RealtimeResponseTextDoneEvent; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceAgentFunctionTool; +import com.azure.ai.agents.models.VoiceAgentTool; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; + +import java.time.Duration; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Demonstrates executing a client-side function tool during a live voice-agent session. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to + * {@code sample-voice-agent-function-tool-java}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL} - Optional. The voice model. Defaults to {@code gpt-realtime}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL_TYPE} - Optional. The voice model type. Defaults to {@code managed}.
    • + *
+ */ +public class VoiceAgentLiveFunctionToolSample { + private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(45); + + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", + "sample-voice-agent-function-tool-java"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsClient agents = builder.buildAgentsClient(); + BetaVoiceAgentWebSocketClient realtime = builder.beta().buildBetaVoiceAgentWebSocketClient(); + + Map cityProperty = new LinkedHashMap<>(); + cityProperty.put("type", "string"); + cityProperty.put("description", "City name, for example Seattle."); + Map properties = new LinkedHashMap<>(); + properties.put("city", cityProperty); + Map schema = new LinkedHashMap<>(); + schema.put("type", "object"); + schema.put("properties", properties); + schema.put("required", Collections.singletonList("city")); + + VoiceAgentFunctionTool weatherTool = new VoiceAgentFunctionTool("get_weather") + .setDescription("Get the current weather for a city.") + .setParameters(BinaryData.fromObject(schema)); + VoiceAgentDefinition definition = new VoiceAgentDefinition() + .setModelType(modelType) + .setModel(model) + .setInstructions("Use the get_weather tool when asked about weather, then answer using its result.") + .setOutputModalities(Collections.singletonList(VoiceOutputModality.TEXT)) + .setTools(Collections.singletonList(weatherTool)); + + try { + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition)); + System.out.println("Created voice agent: " + agentName); + try (BetaVoiceAgentWebSocketSessionClient session = realtime.openWebSocketSession(agentName)) { + ExecutorService receiver = Executors.newSingleThreadExecutor(); + Future response = receiver.submit(() -> receiveResponse(session)); + try { + session.sendText("What's the weather like in Seattle right now?"); + session.createResponse(); + response.get(RESPONSE_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException error) { + System.out.println("Timed out waiting for the agent's reply; cancelling the active response."); + session.cancelResponse(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } catch (java.util.concurrent.ExecutionException error) { + throw new IllegalStateException("The realtime receive loop failed.", error.getCause()); + } finally { + response.cancel(true); + receiver.shutdownNow(); + } + } + } finally { + agents.deleteAgent(agentName); + System.out.println("Deleted voice agent: " + agentName); + } + } + + private static void receiveResponse(BetaVoiceAgentWebSocketSessionClient session) { + for (RealtimeServerEvent event : session.receiveEvents()) { + if (event instanceof RealtimeResponseFunctionCallArgumentsDoneEvent) { + RealtimeResponseFunctionCallArgumentsDoneEvent call + = (RealtimeResponseFunctionCallArgumentsDoneEvent) event; + session.sendFunctionCallOutput(call.getCallId(), executeTool(call)); + } else if (event instanceof RealtimeResponseTextDoneEvent) { + System.out.println("Agent: " + ((RealtimeResponseTextDoneEvent) event).getText()); + } else if (event instanceof RealtimeResponseDoneEvent) { + if (!containsFunctionCall((RealtimeResponseDoneEvent) event)) { + return; + } + } else if (event instanceof RealtimeErrorEvent) { + RealtimeErrorEvent error + = (RealtimeErrorEvent) event; + System.out.println("Session error: " + error.getError().message()); + return; + } + } + } + + @SuppressWarnings("unchecked") + private static String executeTool(RealtimeResponseFunctionCallArgumentsDoneEvent call) { + Map arguments = BinaryData.fromString(call.getArguments()).toObject(Map.class); + System.out.printf("Tool call: %s(%s)%n", call.getName(), arguments); + Map result = new LinkedHashMap<>(); + if ("get_weather".equals(call.getName())) { + result.put("city", arguments.get("city")); + result.put("condition", "sunny"); + result.put("temperature_f", 72); + } else { + result.put("error", "Unknown tool: " + call.getName()); + } + return BinaryData.fromObject(result).toString(); + } + + private static boolean containsFunctionCall(RealtimeResponseDoneEvent event) { + List output = event.getResponse().getOutput(); + if (output == null) { + return false; + } + for (RealtimeConversationItem item : output) { + if (item.getType() == RealtimeConversationItemType.FUNCTION_CALL) { + return true; + } + } + return false; + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java new file mode 100644 index 000000000000..c054427a5cc3 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsConversationsAsyncClient; +import com.azure.ai.agents.BetaAgentsAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import reactor.core.Disposable; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.core.scheduler.Schedulers; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Scanner; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Demonstrates an asynchronous, typed, multi-turn realtime conversation with a persisted voice agent. + * + *

To end the call, submit a blank line or enter {@code exit} or {@code quit}. The sample then closes the WebSocket, + * reads the persisted conversation, and deletes the agent unless {@code FOUNDRY_KEEP_VOICE_AGENT} is set to + * {@code true}.

+ * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to + * {@code sample-live-text-conversation-agent-async-java}.
    • + *
  • {@code FOUNDRY_KEEP_VOICE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. + * Defaults to {@code false}.
    • + *
+ */ +public class VoiceAgentLiveTextConversationAsyncSample { + private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(45); + + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", + "sample-live-text-conversation-agent-async-java"); + boolean keepAgent + = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_VOICE_AGENT", "false")); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsAsyncClient agents = builder.buildAgentsAsyncClient(); + BetaAgentsAsyncClient betaAgents = builder.beta().buildBetaAgentsAsyncClient(); + BetaVoiceAgentWebSocketAsyncClient realtime = builder.beta().buildBetaVoiceAgentWebSocketAsyncClient(); + BetaVoiceAgentsConversationsAsyncClient conversations + = builder.beta().buildBetaVoiceAgentsConversationsAsyncClient(); + + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + + AtomicReference conversationId = new AtomicReference<>(); + VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer(); + Scanner scanner = new Scanner(System.in); + + betaAgents.createAgentFromPrompt(BinaryData.fromObject(request)) + .flatMap(generated -> { + VoiceAgentDefinition definition + = (VoiceAgentDefinition) generated.getVersions().getLatest().getDefinition(); + return agents.createAgentVersion(agentName, + new CreateAgentVersionInput(definition.setStore(true))); + }) + .then(Mono.usingWhen(realtime.openWebSocketSession(agentName), + session -> runConversation(session, scanner, conversationId, player), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, + (session, error) -> session.closeAsync(), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync)) + .then(Mono.defer(() -> conversationId.get() == null + ? Mono.fromRunnable(() -> System.out.println("No persisted conversation ID was returned.")) + : VoiceAgentRealtimeSampleUtils.readConversation(conversations, agentName, conversationId.get()))) + .then(Mono.defer(() -> cleanupAgent(agents, agentName, keepAgent))) + .onErrorResume(error -> Mono.defer(() -> cleanupAgent(agents, agentName, keepAgent)) + .onErrorResume(cleanupError -> Mono.empty()) + .then(Mono.error(error))) + .doFinally(signal -> { + scanner.close(); + player.close(); + }) + .block(); + } + + private static Mono cleanupAgent(AgentsAsyncClient agents, String agentName, boolean keepAgent) { + if (keepAgent) { + return Mono.fromRunnable(() -> System.out.println("Kept voice agent: " + agentName)); + } + return agents.deleteAgent(agentName) + .doOnSuccess(ignored -> System.out.println("Deleted voice agent: " + agentName)); + } + + private static Mono runConversation(BetaVoiceAgentWebSocketSessionAsyncClient session, Scanner scanner, + AtomicReference conversationId, VoiceAgentRealtimeSampleUtils.SpeakerPlayer player) { + AtomicReference> responseCompleted = new AtomicReference<>(); + Disposable receiver = session.receiveEvents().subscribe(event -> { + if (VoiceAgentRealtimeSampleUtils.handleResponseEvent(event, conversationId, player)) { + Sinks.One completion = responseCompleted.getAndSet(null); + if (completion != null) { + completion.tryEmitEmpty(); + } + } + }, error -> { + Sinks.One completion = responseCompleted.getAndSet(null); + if (completion != null) { + completion.tryEmitError(error); + } + }); + + System.out.println("Type a message and press Enter. Blank line (or 'exit') ends the session."); + return prompt(session, scanner, responseCompleted) + .doFinally(signal -> { + receiver.dispose(); + System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), + player.isEnabled() ? " and played it" : ""); + }); + } + + private static Mono prompt(BetaVoiceAgentWebSocketSessionAsyncClient session, Scanner scanner, + AtomicReference> responseCompleted) { + return Mono.fromCallable(() -> { + System.out.print("You: "); + return scanner.nextLine().trim(); + }).subscribeOn(Schedulers.boundedElastic()).flatMap(text -> { + String normalized = text.toLowerCase(Locale.ROOT); + if (text.isEmpty() || "exit".equals(normalized) || "quit".equals(normalized)) { + return Mono.empty(); + } + + Sinks.One completion = Sinks.one(); + responseCompleted.set(completion); + return session.sendText(text) + .then(session.createResponse()) + .then(completion.asMono().timeout(RESPONSE_TIMEOUT)) + .onErrorResume(java.util.concurrent.TimeoutException.class, error -> { + System.out.println("Timed out waiting for the agent's reply; cancelling the active response."); + return session.cancelResponse() + .then(completion.asMono().timeout(Duration.ofSeconds(10))) + .onErrorMap(cancelError -> new IllegalStateException( + "Unable to cancel the active response.", cancelError)); + }) + .then(Mono.defer(() -> prompt(session, scanner, responseCompleted))); + }); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java new file mode 100644 index 000000000000..6b9ce05b1c00 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsConversationsClient; +import com.azure.ai.agents.BetaAgentsClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.models.AgentDetails; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Scanner; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Demonstrates a synchronous, typed, multi-turn realtime conversation with a persisted voice agent. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to + * {@code sample-live-text-conversation-agent-java}.
    • + *
+ */ +public class VoiceAgentLiveTextConversationSample { + private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(45); + + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", + "sample-live-text-conversation-agent-java"); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsClient agents = builder.buildAgentsClient(); + BetaAgentsClient betaAgents = builder.beta().buildBetaAgentsClient(); + BetaVoiceAgentWebSocketClient realtime = builder.beta().buildBetaVoiceAgentWebSocketClient(); + BetaVoiceAgentsConversationsClient conversations = builder.beta().buildBetaVoiceAgentsConversationsClient(); + + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + AgentDetails generated = betaAgents.createAgentFromPrompt(BinaryData.fromObject(request)); + + try { + AgentVersionDetails latest = generated.getVersions().getLatest(); + VoiceAgentDefinition definition = (VoiceAgentDefinition) latest.getDefinition(); + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition.setStore(true))); + + AtomicReference conversationId = new AtomicReference<>(); + try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player + = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer(); + BetaVoiceAgentWebSocketSessionClient session = realtime.openWebSocketSession(agentName); + Scanner scanner = new Scanner(System.in)) { + AtomicReference> responseCompleted = new AtomicReference<>(); + ExecutorService receiver = Executors.newSingleThreadExecutor(); + receiver.submit(() -> { + try { + for (RealtimeServerEvent event : session.receiveEvents()) { + if (VoiceAgentRealtimeSampleUtils.handleResponseEvent(event, conversationId, player)) { + CompletableFuture completion = responseCompleted.getAndSet(null); + if (completion != null) { + completion.complete(null); + } + } + } + } catch (RuntimeException error) { + CompletableFuture completion = responseCompleted.getAndSet(null); + if (completion != null) { + completion.completeExceptionally(error); + } + } + }); + + try { + System.out.println("Type a message and press Enter. Blank line (or 'exit') ends the session."); + while (true) { + System.out.print("You: "); + String prompt = scanner.nextLine().trim(); + String normalized = prompt.toLowerCase(Locale.ROOT); + if (prompt.isEmpty() || "exit".equals(normalized) || "quit".equals(normalized)) { + break; + } + + CompletableFuture completion = new CompletableFuture<>(); + responseCompleted.set(completion); + session.sendText(prompt); + session.createResponse(); + if (!awaitResponse(session, completion)) { + break; + } + } + } finally { + receiver.shutdownNow(); + } + System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), + player.isEnabled() ? " and played it" : ""); + } + + if (conversationId.get() != null) { + VoiceAgentRealtimeSampleUtils.readConversation(conversations, agentName, conversationId.get()); + } else { + System.out.println("No persisted conversation ID was returned."); + } + } finally { + agents.deleteAgent(agentName); + System.out.println("Deleted voice agent: " + agentName); + } + } + + private static boolean awaitResponse(BetaVoiceAgentWebSocketSessionClient session, + CompletableFuture completion) { + try { + completion.get(RESPONSE_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + return true; + } catch (TimeoutException error) { + System.out.println("Timed out waiting for the agent's reply; cancelling the active response."); + session.cancelResponse(); + try { + completion.get(10, TimeUnit.SECONDS); + return true; + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException | TimeoutException ignored) { + return false; + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException error) { + throw new IllegalStateException("The realtime receive loop failed.", error.getCause()); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java new file mode 100644 index 000000000000..431e30c5213b --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaVoiceAgentsConversationsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.VoiceAudioItem; +import com.azure.ai.agents.models.VoiceRecording; +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +/** + * Demonstrates downloading whole-call and item-level audio from a persisted voice conversation. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name.
    • + *
  • {@code FOUNDRY_VOICE_CONVERSATION_ID} - The persisted voice conversation ID.
    • + *
+ */ +public class VoiceAgentReadConversationAudioSample { + public static void main(String[] args) throws IOException { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME"); + String conversationId = configuration.get("FOUNDRY_VOICE_CONVERSATION_ID"); + BetaVoiceAgentsConversationsClient conversations = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .beta() + .buildBetaVoiceAgentsConversationsClient(); + + VoiceRecording recording = conversations.getAgentConversationAudio(agentName, conversationId); + System.out.printf("Recording: format=%s, rate=%d, channels=%d, duration=%s%n", + recording.getFormat(), recording.getSampleRate(), recording.getChannels(), recording.getDurationMs()); + if (recording.getBlobUri() != null) { + System.out.println("Recording is stored in customer storage: " + recording.getBlobUri()); + } else { + Path output = Files.createTempFile(conversationId + "-", ".wav"); + Files.write(output, conversations.downloadAgentConversationAudio(agentName, conversationId).toBytes()); + System.out.println("Wrote merged recording: " + output); + } + + for (BinaryData itemData : conversations.listAgentConversationItems(agentName, conversationId, + new RequestOptions())) { + @SuppressWarnings("unchecked") + Map item = itemData.toObject(Map.class); + String itemId = (String) item.get("id"); + if (itemId == null) { + continue; + } + try { + VoiceAudioItem metadata = conversations.getAgentConversationAudioItem( + agentName, conversationId, itemId); + if (metadata.getBlobUri() != null) { + System.out.println("Item audio is stored in customer storage: " + metadata.getBlobUri()); + } else { + Path output = Files.createTempFile(conversationId + "-" + itemId + "-", ".wav"); + Files.write(output, conversations.downloadAgentConversationAudioItem( + agentName, conversationId, itemId).toBytes()); + System.out.println("Wrote item audio: " + output); + } + break; + } catch (ResourceNotFoundException ignored) { + // This transcript item has no persisted audio. + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java new file mode 100644 index 000000000000..86ce493b8341 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaVoiceAgentsConversationsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.RealtimeConversationItem; +import com.azure.ai.agents.models.VoiceConversation; +import com.azure.ai.agents.models.VoiceResponse; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; + +import java.util.Map; + +/** + * Demonstrates reading a persisted voice conversation, its responses, and transcript items. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name.
    • + *
  • {@code FOUNDRY_VOICE_CONVERSATION_ID} - The persisted voice conversation ID.
    • + *
+ */ +public class VoiceAgentReadConversationSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME"); + String conversationId = configuration.get("FOUNDRY_VOICE_CONVERSATION_ID"); + BetaVoiceAgentsConversationsClient conversations = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .beta() + .buildBetaVoiceAgentsConversationsClient(); + + VoiceConversation conversation = conversations.getAgentConversation(agentName, conversationId); + System.out.printf("Conversation %s: status=%s, created=%s, usage=%s%n", + conversation.getId(), conversation.getStatus(), conversation.getCreatedAt(), conversation.getUsage()); + + for (VoiceResponse response : conversations.listAgentConversationResponses(agentName, conversationId)) { + VoiceResponse detail = conversations.getAgentConversationResponse(agentName, conversationId, + response.getId()); + System.out.printf("Response %s: status=%s, usage=%s%n", + detail.getId(), detail.getStatus(), detail.getUsage()); + for (RealtimeConversationItem item : conversations.listAgentConversationResponseItems( + agentName, conversationId, response.getId())) { + System.out.println(" Response item type: " + item.getType()); + } + } + + for (BinaryData itemData : conversations.listAgentConversationItems(agentName, conversationId, + new RequestOptions())) { + @SuppressWarnings("unchecked") + Map item = itemData.toObject(Map.class); + String itemId = (String) item.get("id"); + System.out.printf("Transcript item: type=%s, id=%s%n", item.get("type"), itemId); + if (itemId != null) { + RealtimeConversationItem fetched = conversations.getAgentConversationItem(agentName, + conversationId, itemId); + System.out.println(" Fetched item type: " + fetched.getType()); + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java new file mode 100644 index 000000000000..6ce3f33a22d5 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaVoiceAgentsConversationsAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsConversationsClient; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeErrorEvent; +import com.azure.ai.agents.models.RealtimeResponseAudioDeltaEvent; +import com.azure.ai.agents.models.RealtimeResponseAudioTranscriptDoneEvent; +import com.azure.ai.agents.models.RealtimeResponseDoneEvent; +import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; +import com.azure.ai.agents.models.VoiceConversation; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.util.BinaryData; +import reactor.core.publisher.Mono; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.SourceDataLine; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +final class VoiceAgentRealtimeSampleUtils { + static final int SAMPLE_RATE = 24000; + + private VoiceAgentRealtimeSampleUtils() { + } + + static boolean handleResponseEvent(RealtimeServerEvent event, AtomicReference conversationId, + SpeakerPlayer player) { + if (event instanceof RealtimeSessionCreatedEvent) { + String id = ((RealtimeSessionCreatedEvent) event).getConversationId(); + if (id != null) { + conversationId.set(id); + } + } else if (event instanceof RealtimeResponseAudioDeltaEvent) { + player.play(((RealtimeResponseAudioDeltaEvent) event).getDelta()); + } else if (event instanceof RealtimeResponseAudioTranscriptDoneEvent) { + System.out.println("Agent: " + + ((RealtimeResponseAudioTranscriptDoneEvent) event).getTranscript()); + } else if (event instanceof RealtimeErrorEvent) { + RealtimeErrorEvent error + = (RealtimeErrorEvent) event; + System.out.println("Session error: " + error.getError().message()); + return true; + } + return event instanceof RealtimeResponseDoneEvent; + } + + static void readConversation(BetaVoiceAgentsConversationsClient conversations, String agentName, + String conversationId) { + VoiceConversation conversation = conversations.getAgentConversation(agentName, conversationId); + System.out.printf("Conversation %s: status=%s, created=%s%n", conversation.getId(), + conversation.getStatus(), conversation.getCreatedAt()); + for (BinaryData item : conversations.listAgentConversationItems(agentName, conversationId, + new RequestOptions())) { + printConversationItem(item); + } + } + + static Mono readConversation(BetaVoiceAgentsConversationsAsyncClient conversations, String agentName, + String conversationId) { + return conversations.getAgentConversation(agentName, conversationId) + .doOnNext(conversation -> System.out.printf("Conversation %s: status=%s, created=%s%n", + conversation.getId(), conversation.getStatus(), conversation.getCreatedAt())) + .thenMany(conversations.listAgentConversationItems(agentName, conversationId, new RequestOptions())) + .doOnNext(VoiceAgentRealtimeSampleUtils::printConversationItem) + .then(); + } + + @SuppressWarnings("unchecked") + private static void printConversationItem(BinaryData itemData) { + Map item = itemData.toObject(Map.class); + System.out.printf(" - %s id=%s%n", item.get("role") == null ? item.get("type") : item.get("role"), + item.get("id")); + Object contentValue = item.get("content"); + if (!(contentValue instanceof List)) { + return; + } + StringBuilder transcript = new StringBuilder(); + for (Object partValue : (List) contentValue) { + if (partValue instanceof Map) { + Map part = (Map) partValue; + Object text = part.get("transcript") == null ? part.get("text") : part.get("transcript"); + if (text != null && !text.toString().trim().isEmpty()) { + if (transcript.length() > 0) { + transcript.append(' '); + } + transcript.append(text.toString().trim()); + } + } + } + if (transcript.length() > 0) { + System.out.println(" " + transcript); + } + } + + static final class SpeakerPlayer implements AutoCloseable { + private SourceDataLine line; + private long bytesReceived; + + SpeakerPlayer() { + AudioFormat format = new AudioFormat(SAMPLE_RATE, 16, 1, true, false); + try { + line = AudioSystem.getSourceDataLine(format); + line.open(format); + line.start(); + } catch (LineUnavailableException | IllegalArgumentException error) { + line = null; + System.out.println("(speaker playback unavailable; audio will be counted but not played)"); + } + } + + synchronized void play(byte[] pcm) { + if (pcm == null) { + return; + } + bytesReceived += pcm.length; + if (line != null) { + line.write(pcm, 0, pcm.length); + } + } + + synchronized void discardQueuedAudio() { + if (line != null) { + line.flush(); + } + } + + double getSecondsReceived() { + return bytesReceived / 2.0 / SAMPLE_RATE; + } + + boolean isEnabled() { + return line != null; + } + + @Override + public synchronized void close() { + if (line != null) { + line.drain(); + line.stop(); + line.close(); + line = null; + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java new file mode 100644 index 000000000000..2b37d1c0b1e0 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; + +import java.util.Collections; + +final class VoiceAgentSampleUtils { + private VoiceAgentSampleUtils() { + } + + static VoiceAgentDefinition createDefinition(VoiceModelType modelType, String model, String instructions) { + VoiceAgentAudioOutputConfiguration output = new VoiceAgentAudioOutputConfiguration() + .setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD); + return new VoiceAgentDefinition() + .setModelType(modelType) + .setModel(model) + .setInstructions(instructions) + .setAudio(new VoiceAgentAudioConfiguration().setOutput(output)) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setStore(true); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java new file mode 100644 index 000000000000..724b553e3075 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceModelType; + +/** + * Demonstrates released and draft voice-agent versions. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_MODEL} - Optional. The voice model or deployment name. Defaults to {@code gpt-realtime}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL_TYPE} - Optional. The voice model type. Defaults to {@code managed}.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name. Defaults to {@code versioned-voice-agent-java}.
    • + *
+ */ +public class VoiceAgentVersionsSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "versioned-voice-agent-java"); + + AgentsClient client = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true) + .buildAgentsClient(); + try { + AgentVersionDetails first = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a helpful voice assistant."))); + AgentVersionDetails released = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a helpful voice assistant. Greet the caller by name.")) + .setDescription("Added a personalized greeting.")); + AgentVersionDetails draft = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are an experimental voice assistant.")) + .setDescription("Candidate persona under review.") + .setDraft(true)); + System.out.printf("Created versions %s, %s and draft %s%n", + first.getVersion(), released.getVersion(), draft.getVersion()); + + System.out.println("Released versions:"); + for (AgentVersionDetails version : client.listAgentVersions(agentName)) { + System.out.printf(" %s (draft=%s)%n", version.getVersion(), version.isDraft()); + } + System.out.println("All versions including drafts:"); + for (AgentVersionDetails version : client.listAgentVersions(agentName, null, null, null, null, true)) { + System.out.printf(" %s (draft=%s)%n", version.getVersion(), version.isDraft()); + } + AgentVersionDetails fetched = client.getAgentVersionDetails(agentName, released.getVersion()); + System.out.println("Fetched version: " + fetched.getVersion()); + } finally { + client.deleteAgent(agentName); + System.out.println("Deleted agent: " + agentName); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java new file mode 100644 index 000000000000..7386f7daf3d7 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimePcmAudioFormat; +import com.openai.models.realtime.RealtimeAudioFormats.AudioPcm.Rate; +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioInputConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceAgentFunctionTool; +import com.azure.ai.agents.models.VoiceAgentInputTranscription; +import com.azure.ai.agents.models.VoiceAgentInputTranscriptionModel; +import com.azure.ai.agents.models.VoiceAgentServerVadTurnDetection; +import com.azure.ai.agents.models.VoiceAgentEndConversationSystemTool; +import com.azure.ai.agents.models.VoiceAgentTool; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; +import com.azure.ai.agents.models.VoiceModelType; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Demonstrates a voice-agent definition with audio processing, transcription, and tools. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_MODEL} - Optional. The voice model or deployment name. Defaults to {@code gpt-realtime}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL_TYPE} - Optional. The voice model type. Defaults to {@code managed}.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name. Defaults to {@code voice-agent-with-tools-java}.
    • + *
+ */ +public class VoiceAgentWithToolsSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "voice-agent-with-tools-java"); + + AgentsClient client = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true) + .buildAgentsClient(); + + RealtimePcmAudioFormat pcm = new RealtimePcmAudioFormat() + .setRate(Rate._24000); + VoiceAgentAudioInputConfiguration input = new VoiceAgentAudioInputConfiguration() + .setFormat(pcm) + .setTurnDetection(new VoiceAgentServerVadTurnDetection() + .setThreshold(0.5) + .setPrefixPaddingMs(300L) + .setSilenceDurationMs(500L)) + .setTranscription(new VoiceAgentInputTranscription(VoiceAgentInputTranscriptionModel.WHISPER_1)); + VoiceAgentAudioOutputConfiguration output = new VoiceAgentAudioOutputConfiguration() + .setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD); + Map cityProperty = new LinkedHashMap<>(); + cityProperty.put("type", "string"); + cityProperty.put("description", "City name, for example Seattle."); + Map properties = new LinkedHashMap<>(); + properties.put("city", cityProperty); + Map parameters = new LinkedHashMap<>(); + parameters.put("type", "object"); + parameters.put("properties", properties); + parameters.put("required", Collections.singletonList("city")); + VoiceAgentFunctionTool weather = new VoiceAgentFunctionTool("get_weather") + .setDescription("Get the current weather for a city.") + .setParameters(BinaryData.fromObject(parameters)); + VoiceAgentEndConversationSystemTool endCall = new VoiceAgentEndConversationSystemTool(); + VoiceAgentDefinition definition = new VoiceAgentDefinition() + .setModelType(modelType) + .setModel(model) + .setInstructions("Use tools when they help answer the caller.") + .setAudio(new VoiceAgentAudioConfiguration().setInput(input).setOutput(output)) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setTools(Arrays.asList(weather, endCall)) + .setStore(true); + + boolean agentCreated = false; + try { + AgentVersionDetails created = client.createAgentVersion(agentName, + new CreateAgentVersionInput(definition)); + agentCreated = true; + AgentVersionDetails fetched = client.getAgentVersionDetails(agentName, created.getVersion()); + VoiceAgentDefinition fetchedDefinition = (VoiceAgentDefinition) fetched.getDefinition(); + System.out.println("Configured voice tools: " + fetchedDefinition.getTools().size()); + for (VoiceAgentTool tool : fetchedDefinition.getTools()) { + System.out.printf(" %s%n", tool.getType()); + } + } finally { + if (agentCreated) { + client.deleteAgent(agentName); + System.out.println("Deleted agent: " + agentName); + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/models/VoiceAgentDefinitionSerializationTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/models/VoiceAgentDefinitionSerializationTests.java index 1dc171e00cae..53c59ec5bc3f 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/models/VoiceAgentDefinitionSerializationTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/models/VoiceAgentDefinitionSerializationTests.java @@ -3,15 +3,25 @@ package com.azure.ai.agents.models; +import com.azure.core.util.BinaryData; +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; +import com.azure.json.JsonWriter; import com.openai.models.responses.ToolChoiceFunction; import com.openai.models.responses.ToolChoiceMcp; import com.openai.models.responses.ToolChoiceOptions; +import com.openai.models.realtime.RealtimeAudioFormats.AudioPcm.Rate; import org.junit.jupiter.api.Test; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class VoiceAgentDefinitionSerializationTests { @@ -69,4 +79,109 @@ private VoiceAgentDefinition roundTrip(VoiceAgentDefinition value) throws IOExce String json = UnionTypeSerializationTestUtils.serialize(value); return UnionTypeSerializationTestUtils.deserialize(json, VoiceAgentDefinition::fromJson); } + + @Test + public void fullVoiceDefinitionRoundTrips() throws IOException { + RealtimePcmAudioFormat pcm = new RealtimePcmAudioFormat().setRate(Rate._24000); + VoiceAgentAudioInputConfiguration input = new VoiceAgentAudioInputConfiguration().setFormat(pcm) + .setTurnDetection(new VoiceAgentServerVadTurnDetection().setThreshold(0.5) + .setPrefixPaddingMs(300L) + .setSilenceDurationMs(500L)) + .setTranscription(new VoiceAgentInputTranscription(VoiceAgentInputTranscriptionModel.WHISPER_1)); + VoiceAgentAudioOutputConfiguration output = new VoiceAgentAudioOutputConfiguration().setFormat(pcm) + .setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD); + VoiceAgentFunctionTool functionTool + = new VoiceAgentFunctionTool("get_weather").setDescription("Get weather for a city.") + .setParameters(BinaryData.fromString("{}")); + VoiceAgentSystemTool systemTool = new VoiceAgentEndConversationSystemTool(); + + VoiceAgentDefinition original = new VoiceAgentDefinition().setModelType(VoiceModelType.MANAGED) + .setModel("gpt-realtime") + .setInstructions("Keep replies short and natural.") + .setAudio(new VoiceAgentAudioConfiguration().setInput(input).setOutput(output)) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setTools(Arrays.asList(functionTool, systemTool)) + .setStore(true); + + String json = serialize(original); + assertTrue(json.contains("\"kind\":\"voice\"")); + assertTrue(json.contains("\"model_type\":\"managed\"")); + assertTrue(json.contains("\"model\":\"gpt-realtime\"")); + assertTrue(json.contains("\"voice\":\"en-US-AvaNeural\"")); + assertTrue(json.contains("\"voice_type\":\"azure-standard\"")); + assertTrue(json.contains("\"rate\":24000")); + assertTrue(json.contains("\"type\":\"server_vad\"")); + assertTrue(json.contains("\"model\":\"whisper-1\"")); + assertTrue(json.contains("\"output_modalities\":[\"audio\"]")); + assertTrue(json.contains("\"store\":true")); + assertTrue(json.contains("\"name\":\"get_weather\"")); + assertTrue(json.contains("\"name\":\"end_conversation\"")); + + AgentDefinition deserialized; + try (JsonReader reader = JsonProviders.createReader(json)) { + deserialized = AgentDefinition.fromJson(reader); + } + assertInstanceOf(VoiceAgentDefinition.class, deserialized); + VoiceAgentDefinition voice = (VoiceAgentDefinition) deserialized; + assertEquals(AgentKind.VOICE, voice.getKind()); + assertEquals(VoiceModelType.MANAGED, voice.getModelType()); + assertEquals("gpt-realtime", voice.getModel()); + assertEquals("Keep replies short and natural.", voice.getInstructions()); + assertEquals(Boolean.TRUE, voice.isStore()); + assertEquals(VoiceOutputModality.AUDIO, voice.getOutputModalities().get(0)); + + VoiceAgentAudioInputConfiguration deserializedInput = voice.getAudio().getInput(); + RealtimePcmAudioFormat deserializedInputFormat + = assertInstanceOf(RealtimePcmAudioFormat.class, deserializedInput.getFormat()); + assertEquals(pcm.getRate(), deserializedInputFormat.getRate()); + VoiceAgentServerVadTurnDetection deserializedVad + = assertInstanceOf(VoiceAgentServerVadTurnDetection.class, deserializedInput.getTurnDetection()); + VoiceAgentServerVadTurnDetection originalVad = (VoiceAgentServerVadTurnDetection) input.getTurnDetection(); + assertEquals(originalVad.getThreshold(), deserializedVad.getThreshold()); + assertEquals(originalVad.getPrefixPaddingMs(), deserializedVad.getPrefixPaddingMs()); + assertEquals(originalVad.getSilenceDurationMs(), deserializedVad.getSilenceDurationMs()); + assertEquals(input.getTranscription().getModel(), deserializedInput.getTranscription().getModel()); + + VoiceAgentAudioOutputConfiguration deserializedOutput = voice.getAudio().getOutput(); + RealtimePcmAudioFormat deserializedOutputFormat + = assertInstanceOf(RealtimePcmAudioFormat.class, deserializedOutput.getFormat()); + assertEquals(pcm.getRate(), deserializedOutputFormat.getRate()); + assertEquals(output.getVoice(), deserializedOutput.getVoice()); + assertEquals(output.getVoiceType(), deserializedOutput.getVoiceType()); + + assertEquals(2, voice.getTools().size()); + VoiceAgentFunctionTool deserializedFunction + = assertInstanceOf(VoiceAgentFunctionTool.class, voice.getTools().get(0)); + assertEquals(functionTool.getName(), deserializedFunction.getName()); + assertEquals(functionTool.getDescription(), deserializedFunction.getDescription()); + VoiceAgentEndConversationSystemTool deserializedSystem + = assertInstanceOf(VoiceAgentEndConversationSystemTool.class, voice.getTools().get(1)); + assertEquals(systemTool.getName(), deserializedSystem.getName()); + } + + @Test + public void selfDeployedVoiceDefinitionRoundTrips() throws IOException { + VoiceAgentDefinition original = new VoiceAgentDefinition().setModelType(VoiceModelType.SELF_DEPLOYED) + .setModel("customer-realtime-deployment") + .setInstructions("Use the customer deployment."); + + String json = serialize(original); + VoiceAgentDefinition deserialized; + try (JsonReader reader = JsonProviders.createReader(json)) { + deserialized = VoiceAgentDefinition.fromJson(reader); + } + + assertEquals(VoiceModelType.SELF_DEPLOYED, deserialized.getModelType()); + assertEquals("customer-realtime-deployment", deserialized.getModel()); + assertEquals("Use the customer deployment.", deserialized.getInstructions()); + } + + private static String serialize(VoiceAgentDefinition definition) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (JsonWriter writer = JsonProviders.createWriter(output)) { + definition.toJson(writer); + } + return output.toString("UTF-8"); + } } diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsAsyncTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsAsyncTests.java new file mode 100644 index 000000000000..6d684658132a --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsAsyncTests.java @@ -0,0 +1,346 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.models.VoiceConversation; +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsConversationsAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeResponseDoneEvent; +import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; +import com.azure.ai.agents.models.VoiceConversationStatus; +import com.azure.ai.agents.models.VoiceAudioItem; +import com.azure.ai.agents.models.VoiceRecording; +import com.azure.ai.agents.models.VoiceResponse; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.test.utils.MockTokenCredential; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Mono; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Persisted voice REST coverage, separate from native OpenAI conversation tests. + * Deterministic cases use scripted HTTP responses, not service recordings. The live parity case requires + * AZURE_TEST_MODE=LIVE, FOUNDRY_PROJECT_ENDPOINT, FOUNDRY_VOICE_MODEL_NAME and DefaultAzureCredential authentication. + * It creates and deletes its own agent and conversation; no microphone or speaker is required. + * Interrupted/generated audio is excluded, matching the Python parity test. + */ +public class VoiceAgentConversationsAsyncTests { + private static final Duration TIMEOUT = Duration.ofSeconds(30); + private static final String AGENT = "test-conversations-read-agent-async-java"; + private static final String CONVERSATION = "conversation-1"; + private static final String ROOT = "/agents/" + AGENT + "/endpoint/protocols/voice/conversations"; + private static final String PATH = ROOT + "/" + CONVERSATION; + private static final String ENVELOPE + = "{\"id\":\"conversation-1\",\"status\":\"completed\",\"created_at\":1700000000}"; + private static final String RESPONSE = "{\"id\":\"response-1\",\"status\":\"completed\"}"; + private static final String USER_ITEM = "{\"id\":\"user-1\",\"type\":\"message\",\"role\":\"user\",\"content\":[]}"; + private static final String ASSISTANT_ITEM + = "{\"id\":\"assistant-1\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[]}"; + + @Test + @EnabledIfEnvironmentVariable(named = "AZURE_TEST_MODE", matches = "LIVE") + public void readLivePersistedConversation() { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL_NAME"); + assertNotNull(endpoint, "FOUNDRY_PROJECT_ENDPOINT is required for live parity testing."); + assertNotNull(model, "FOUNDRY_VOICE_MODEL_NAME is required for live parity testing."); + String agentName = "test-voice-read-" + UUID.randomUUID(); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + AgentsAsyncClient agents = builder.buildAgentsAsyncClient(); + BetaVoiceAgentsConversationsAsyncClient conversations + = builder.beta().buildBetaVoiceAgentsConversationsAsyncClient(); + VoiceAgentDefinition definition = new VoiceAgentDefinition().setModelType(VoiceModelType.MANAGED) + .setModel(model) + .setInstructions("You are a helpful voice assistant. Keep replies short.") + .setAudio(new VoiceAgentAudioConfiguration() + .setOutput(new VoiceAgentAudioOutputConfiguration().setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD))) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setStore(true); + AtomicReference conversationId = new AtomicReference<>(); + boolean created = false; + boolean reading = false; + try { + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition)).block(TIMEOUT); + created = true; + Mono.usingWhen(builder.beta().buildBetaVoiceAgentWebSocketAsyncClient().openWebSocketSession(agentName), + session -> session.receiveEvents().index().concatMap(indexed -> { + if (indexed.getT1() == 0) { + assertTrue(indexed.getT2() instanceof RealtimeSessionCreatedEvent, + "The first event must be session.created."); + conversationId.set(((RealtimeSessionCreatedEvent) indexed.getT2()).getConversationId()); + assertNotNull(conversationId.get(), "store=True must return a conversation ID."); + return session.sendText("Say hello.") + .then(session.createResponse()) + .thenReturn(indexed.getT2()); + } + return Mono.just(indexed.getT2()); + }) + .filter(RealtimeResponseDoneEvent.class::isInstance) + .next() + .switchIfEmpty(Mono.error(new AssertionError("Session ended without response.done."))) + .timeout(Duration.ofSeconds(45)) + .then(), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync).block(Duration.ofSeconds(90)); + Mono.delay(Duration.ofSeconds(30)).block(Duration.ofSeconds(35)); + reading = true; + assertPersistedConversation(conversations, agentName, conversationId.get()); + } finally { + try { + if (!reading && conversationId.get() != null) { + conversations.deleteAgentConversation(agentName, conversationId.get()).block(TIMEOUT); + } + } finally { + if (created) { + agents.deleteAgent(agentName).block(TIMEOUT); + } + } + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void readPersistedConversation(boolean blobStorage) { + ScriptedTransport transport = new ScriptedTransport(); + enqueueTranscript(transport, ENVELOPE); + String blob = blobStorage ? ",\"blob_uri\":\"https://storage.example/recording.wav\"" : ""; + transport.get(PATH + "/audio", 200, "{\"conversation_id\":\"conversation-1\",\"format\":\"wav\"," + + "\"sample_rate\":24000,\"channels\":2,\"channel_layout\":{},\"duration_ms\":1000" + blob + "}"); + if (!blobStorage) { + transport.audio(PATH + "/audio/content"); + } + transport.get(PATH + "/items/user-1/audio", 404, + "{\"error\":{\"code\":\"NotFound\",\"message\":\"No audio\"}}"); + transport.get(PATH + "/items/assistant-1/audio", 200, + "{\"conversation_id\":\"conversation-1\",\"item_id\":\"assistant-1\",\"role\":\"assistant\"" + blob + "}"); + if (!blobStorage) { + transport.audio(PATH + "/items/assistant-1/audio/content"); + } + transport.delete(PATH); + + assertPersistedConversation(client(transport), AGENT, CONVERSATION); + transport.assertComplete(); + } + + @Test + public void incompleteConversationFailsInsteadOfSkippingAudioAndStillDeletes() { + ScriptedTransport transport = new ScriptedTransport(); + enqueueTranscript(transport, ENVELOPE.replace("completed", "in_progress")); + transport.delete(PATH); + assertThrows(AssertionError.class, () -> assertPersistedConversation(client(transport), AGENT, CONVERSATION)); + transport.assertComplete(); + } + + @Test + public void nonNotFoundItemAudioErrorPropagatesAndStillDeletes() { + ScriptedTransport transport = new ScriptedTransport(); + enqueueTranscript(transport, ENVELOPE); + transport.get(PATH + "/audio", 200, + "{\"conversation_id\":\"conversation-1\",\"format\":\"wav\"," + + "\"sample_rate\":24000,\"channels\":2,\"channel_layout\":{},\"duration_ms\":1000," + + "\"blob_uri\":\"https://storage.example/recording.wav\"}"); + transport.get(PATH + "/items/user-1/audio", 403, + "{\"error\":{\"code\":\"Forbidden\",\"message\":\"Audio access denied\"}}"); + transport.delete(PATH); + HttpResponseException error = assertThrows(HttpResponseException.class, + () -> assertPersistedConversation(client(transport), AGENT, CONVERSATION)); + assertEquals(403, error.getResponse().getStatusCode()); + transport.assertComplete(); + } + + private static void enqueueTranscript(ScriptedTransport transport, String envelope) { + transport.get(ROOT, 200, page(envelope)); + transport.get(PATH, 200, envelope); + transport.get(PATH + "/responses", 200, page(RESPONSE)); + transport.get(PATH + "/responses/response-1", 200, RESPONSE); + transport.get(PATH + "/responses/response-1/items", 200, page()); + transport.get(PATH + "/items", 200, page(USER_ITEM, ASSISTANT_ITEM)); + transport.get(PATH + "/items/user-1", 200, USER_ITEM); + } + + private static BetaVoiceAgentsConversationsAsyncClient client(HttpClient transport) { + return new AgentsClientBuilder().endpoint("https://localhost") + .credential(new MockTokenCredential()) + .httpClient(transport) + .allowPreview(true) + .beta() + .buildBetaVoiceAgentsConversationsAsyncClient(); + } + + private static void assertPersistedConversation(BetaVoiceAgentsConversationsAsyncClient client, String agentName, + String conversationId) { + try { + assertEquals(Boolean.TRUE, + client.listAgentConversations(agentName) + .any(conversation -> conversationId.equals(conversation.getId())) + .block(TIMEOUT)); + VoiceConversation conversation = client.getAgentConversation(agentName, conversationId).block(TIMEOUT); + assertNotNull(conversation); + assertEquals(conversationId, conversation.getId()); + assertTrue( + Arrays + .asList(VoiceConversationStatus.IN_PROGRESS, VoiceConversationStatus.COMPLETED, + VoiceConversationStatus.FAILED) + .contains(conversation.getStatus())); + assertNotNull(conversation.getCreatedAt()); + + List responses + = client.listAgentConversationResponses(agentName, conversationId).collectList().block(TIMEOUT); + assertNotNull(responses); + assertFalse(responses.isEmpty()); + String responseId = responses.get(0).getId(); + VoiceResponse response + = client.getAgentConversationResponse(agentName, conversationId, responseId).block(TIMEOUT); + assertNotNull(response); + assertEquals(responseId, response.getId()); + client.listAgentConversationResponseItems(agentName, conversationId, responseId, new RequestOptions()) + .collectList() + .block(TIMEOUT); + + List items = client.listAgentConversationItems(agentName, conversationId, new RequestOptions()) + .collectList() + .block(TIMEOUT); + assertNotNull(items); + assertFalse(items.isEmpty()); + String firstId = itemId(items.get(0)); + assertNotNull(firstId); + assertFalse(firstId.isEmpty()); + BinaryData fetched + = client.getAgentConversationItemWithResponse(agentName, conversationId, firstId, new RequestOptions()) + .block(TIMEOUT) + .getValue(); + assertEquals(firstId, itemId(fetched)); + + assertEquals(VoiceConversationStatus.COMPLETED, conversation.getStatus(), + "Audio assertions require a finalized conversation."); + VoiceRecording recording = client.getAgentConversationAudio(agentName, conversationId).block(TIMEOUT); + assertNotNull(recording); + assertNotNull(recording.getFormat()); + if (recording.getBlobUri() == null || recording.getBlobUri().isEmpty()) { + assertAudio(client.downloadAgentConversationAudio(agentName, conversationId)); + } + for (BinaryData item : items) { + String id = itemId(item); + if (id == null || id.isEmpty()) { + continue; + } + VoiceAudioItem audio; + try { + audio = client.getAgentConversationAudioItem(agentName, conversationId, id).block(TIMEOUT); + } catch (HttpResponseException error) { + if (error.getResponse().getStatusCode() == 404) { + continue; + } + throw error; + } + assertNotNull(audio); + assertNotNull(audio.getRole()); + if (audio.getBlobUri() == null || audio.getBlobUri().isEmpty()) { + assertAudio(client.downloadAgentConversationAudioItem(agentName, conversationId, id)); + } + break; + } + } finally { + client.deleteAgentConversation(agentName, conversationId).block(TIMEOUT); + } + } + + private static String itemId(BinaryData item) { + return (String) item.toObject(Map.class).get("id"); + } + + private static void assertAudio(Mono content) { + BinaryData audio = content.block(TIMEOUT); + assertNotNull(audio); + assertTrue(audio.toBytes().length > 0); + } + + private static String page(String... entries) { + return "{\"data\":[" + String.join(",", entries) + "],\"has_more\":false}"; + } + + private static final class ScriptedTransport implements HttpClient { + private final Deque> requests = new ArrayDeque<>(); + + void get(String path, int status, String json) { + expect(HttpMethod.GET, path, status, "application/json", BinaryData.fromString(json).toBytes()); + } + + void audio(String path) { + expect(HttpMethod.GET, path, 200, "audio/wav", new byte[] { 82, 73, 70, 70, 0, 1, 2, 3 }); + } + + void delete(String path) { + expect(HttpMethod.DELETE, path, 204, "application/json", new byte[0]); + } + + private void expect(HttpMethod method, String path, int status, String contentType, byte[] body) { + requests.add(request -> { + assertEquals(method, request.getHttpMethod()); + assertEquals(path, request.getUrl().getPath()); + return new MockHttpResponse(request, status, + new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, contentType), body); + }); + } + + @Override + public Mono send(HttpRequest request) { + return Mono.fromSupplier(() -> { + assertFalse(requests.isEmpty(), "Unexpected request: " + request.getUrl()); + return requests.removeFirst().apply(request); + }); + } + + @Override + public HttpResponse sendSync(HttpRequest request, Context context) { + throw new AssertionError("The async client must not use synchronous HTTP."); + } + + void assertComplete() { + assertTrue(requests.isEmpty(), "Not all expected voice REST operations were called."); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsTests.java new file mode 100644 index 000000000000..590ed1b4ccea --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentConversationsTests.java @@ -0,0 +1,341 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsConversationsClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeResponseDoneEvent; +import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceConversation; +import com.azure.ai.agents.models.VoiceConversationStatus; +import com.azure.ai.agents.models.VoiceAudioItem; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceRecording; +import com.azure.ai.agents.models.VoiceResponse; +import com.azure.ai.agents.models.VoiceType; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.test.utils.MockTokenCredential; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.core.util.Context; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Mono; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Synchronous parity for Python test_voice_agent_conversations.py. + * Deterministic cases use scripted HTTP responses, not service recordings. The live case requires + * AZURE_TEST_MODE=LIVE, FOUNDRY_PROJECT_ENDPOINT, FOUNDRY_VOICE_MODEL_NAME and DefaultAzureCredential authentication. + * It creates and deletes its own agent and conversation without a microphone or speaker. + * Interrupted/generated audio is excluded, matching the Python test. + */ +public class VoiceAgentConversationsTests { + private static final Duration TIMEOUT = Duration.ofSeconds(30); + private static final String AGENT = "test-conversations-read-agent-java"; + private static final String CONVERSATION = "conversation-1"; + private static final String ROOT = "/agents/" + AGENT + "/endpoint/protocols/voice/conversations"; + private static final String PATH = ROOT + "/" + CONVERSATION; + private static final String ENVELOPE + = "{\"id\":\"conversation-1\",\"status\":\"completed\",\"created_at\":1700000000}"; + private static final String RESPONSE = "{\"id\":\"response-1\",\"status\":\"completed\"}"; + private static final String USER_ITEM = "{\"id\":\"user-1\",\"type\":\"message\",\"role\":\"user\",\"content\":[]}"; + private static final String ASSISTANT_ITEM + = "{\"id\":\"assistant-1\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[]}"; + + @Test + @EnabledIfEnvironmentVariable(named = "AZURE_TEST_MODE", matches = "LIVE") + public void readLivePersistedConversation() throws InterruptedException { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL_NAME"); + assertNotNull(endpoint, "FOUNDRY_PROJECT_ENDPOINT is required for live parity testing."); + assertNotNull(model, "FOUNDRY_VOICE_MODEL_NAME is required for live parity testing."); + String agentName = "test-voice-read-sync-" + UUID.randomUUID(); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + AgentsClient agents = builder.buildAgentsClient(); + BetaVoiceAgentsConversationsClient conversations = builder.beta().buildBetaVoiceAgentsConversationsClient(); + VoiceAgentDefinition definition = new VoiceAgentDefinition().setModelType(VoiceModelType.MANAGED) + .setModel(model) + .setInstructions("You are a helpful voice assistant. Keep replies short.") + .setAudio(new VoiceAgentAudioConfiguration() + .setOutput(new VoiceAgentAudioOutputConfiguration().setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD))) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setStore(true); + String conversationId = null; + boolean created = false; + boolean reading = false; + try { + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition)); + created = true; + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession(agentName)) { + Iterator events = session.receiveEvents(TIMEOUT).iterator(); + assertTrue(events.hasNext(), "Expected session.created."); + RealtimeServerEvent first = events.next(); + assertTrue(first instanceof RealtimeSessionCreatedEvent, "The first event must be session.created."); + conversationId = ((RealtimeSessionCreatedEvent) first).getConversationId(); + assertNotNull(conversationId, "store=True must return a conversation ID."); + session.sendText("Say hello."); + session.createResponse(); + long deadline = System.nanoTime() + Duration.ofSeconds(45).toNanos(); + boolean responseDone = false; + while (System.nanoTime() < deadline && events.hasNext()) { + if (events.next() instanceof RealtimeResponseDoneEvent) { + responseDone = true; + break; + } + } + assertTrue(responseDone, "Session ended or timed out without response.done."); + } + Thread.sleep(TIMEOUT.toMillis()); + reading = true; + assertPersistedConversation(conversations, agentName, conversationId); + } finally { + try { + if (!reading && conversationId != null) { + conversations.deleteAgentConversation(agentName, conversationId); + } + } finally { + if (created) { + agents.deleteAgent(agentName); + } + } + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void readPersistedConversation(boolean blobStorage) { + ScriptedTransport transport = new ScriptedTransport(); + enqueueTranscript(transport, ENVELOPE); + String blob = blobStorage ? ",\"blob_uri\":\"https://storage.example/recording.wav\"" : ""; + transport.get(PATH + "/audio", 200, "{\"conversation_id\":\"conversation-1\",\"format\":\"wav\"," + + "\"sample_rate\":24000,\"channels\":2,\"channel_layout\":{},\"duration_ms\":1000" + blob + "}"); + if (!blobStorage) { + transport.audio(PATH + "/audio/content"); + } + transport.get(PATH + "/items/user-1/audio", 404, + "{\"error\":{\"code\":\"NotFound\",\"message\":\"No audio\"}}"); + transport.get(PATH + "/items/assistant-1/audio", 200, + "{\"conversation_id\":\"conversation-1\",\"item_id\":\"assistant-1\",\"role\":\"assistant\"" + blob + "}"); + if (!blobStorage) { + transport.audio(PATH + "/items/assistant-1/audio/content"); + } + transport.delete(PATH); + + assertPersistedConversation(client(transport), AGENT, CONVERSATION); + transport.assertComplete(); + } + + @Test + public void incompleteConversationFailsInsteadOfSkippingAudioAndStillDeletes() { + ScriptedTransport transport = new ScriptedTransport(); + enqueueTranscript(transport, ENVELOPE.replace("completed", "in_progress")); + transport.delete(PATH); + assertThrows(AssertionError.class, () -> assertPersistedConversation(client(transport), AGENT, CONVERSATION)); + transport.assertComplete(); + } + + @Test + public void nonNotFoundItemAudioErrorPropagatesAndStillDeletes() { + ScriptedTransport transport = new ScriptedTransport(); + enqueueTranscript(transport, ENVELOPE); + transport.get(PATH + "/audio", 200, + "{\"conversation_id\":\"conversation-1\",\"format\":\"wav\"," + + "\"sample_rate\":24000,\"channels\":2,\"channel_layout\":{},\"duration_ms\":1000," + + "\"blob_uri\":\"https://storage.example/recording.wav\"}"); + transport.get(PATH + "/items/user-1/audio", 403, + "{\"error\":{\"code\":\"Forbidden\",\"message\":\"Audio access denied\"}}"); + transport.delete(PATH); + HttpResponseException error = assertThrows(HttpResponseException.class, + () -> assertPersistedConversation(client(transport), AGENT, CONVERSATION)); + assertEquals(403, error.getResponse().getStatusCode()); + transport.assertComplete(); + } + + private static void enqueueTranscript(ScriptedTransport transport, String envelope) { + transport.get(ROOT, 200, page(envelope)); + transport.get(PATH, 200, envelope); + transport.get(PATH + "/responses", 200, page(RESPONSE)); + transport.get(PATH + "/responses/response-1", 200, RESPONSE); + transport.get(PATH + "/responses/response-1/items", 200, page()); + transport.get(PATH + "/items", 200, page(USER_ITEM, ASSISTANT_ITEM)); + transport.get(PATH + "/items/user-1", 200, USER_ITEM); + } + + private static BetaVoiceAgentsConversationsClient client(HttpClient transport) { + return new AgentsClientBuilder().endpoint("https://localhost") + .credential(new MockTokenCredential()) + .httpClient(transport) + .allowPreview(true) + .beta() + .buildBetaVoiceAgentsConversationsClient(); + } + + private static void assertPersistedConversation(BetaVoiceAgentsConversationsClient client, String agentName, + String conversationId) { + try { + assertTrue(client.listAgentConversations(agentName) + .stream() + .anyMatch(conversation -> conversationId.equals(conversation.getId()))); + VoiceConversation conversation = client.getAgentConversation(agentName, conversationId); + assertNotNull(conversation); + assertEquals(conversationId, conversation.getId()); + assertTrue( + Arrays + .asList(VoiceConversationStatus.IN_PROGRESS, VoiceConversationStatus.COMPLETED, + VoiceConversationStatus.FAILED) + .contains(conversation.getStatus())); + assertNotNull(conversation.getCreatedAt()); + + List responses = client.listAgentConversationResponses(agentName, conversationId) + .stream() + .collect(Collectors.toList()); + assertFalse(responses.isEmpty()); + String responseId = responses.get(0).getId(); + VoiceResponse response = client.getAgentConversationResponse(agentName, conversationId, responseId); + assertNotNull(response); + assertEquals(responseId, response.getId()); + client.listAgentConversationResponseItems(agentName, conversationId, responseId, new RequestOptions()) + .stream() + .collect(Collectors.toList()); + + List items = client.listAgentConversationItems(agentName, conversationId, new RequestOptions()) + .stream() + .collect(Collectors.toList()); + assertFalse(items.isEmpty()); + String firstId = itemId(items.get(0)); + assertNotNull(firstId); + assertFalse(firstId.isEmpty()); + BinaryData fetched + = client.getAgentConversationItemWithResponse(agentName, conversationId, firstId, new RequestOptions()) + .getValue(); + assertEquals(firstId, itemId(fetched)); + + assertEquals(VoiceConversationStatus.COMPLETED, conversation.getStatus(), + "Audio assertions require a finalized conversation."); + VoiceRecording recording = client.getAgentConversationAudio(agentName, conversationId); + assertNotNull(recording); + assertNotNull(recording.getFormat()); + if (recording.getBlobUri() == null || recording.getBlobUri().isEmpty()) { + assertAudio(client.downloadAgentConversationAudio(agentName, conversationId)); + } + for (BinaryData item : items) { + String id = itemId(item); + if (id == null || id.isEmpty()) { + continue; + } + VoiceAudioItem audio; + try { + audio = client.getAgentConversationAudioItem(agentName, conversationId, id); + } catch (HttpResponseException error) { + if (error.getResponse().getStatusCode() == 404) { + continue; + } + throw error; + } + assertNotNull(audio); + assertNotNull(audio.getRole()); + if (audio.getBlobUri() == null || audio.getBlobUri().isEmpty()) { + assertAudio(client.downloadAgentConversationAudioItem(agentName, conversationId, id)); + } + break; + } + } finally { + client.deleteAgentConversation(agentName, conversationId); + } + } + + private static String itemId(BinaryData item) { + return (String) item.toObject(Map.class).get("id"); + } + + private static void assertAudio(BinaryData audio) { + assertNotNull(audio); + assertTrue(audio.toBytes().length > 0); + } + + private static String page(String... entries) { + return "{\"data\":[" + String.join(",", entries) + "],\"has_more\":false}"; + } + + private static final class ScriptedTransport implements HttpClient { + private final Deque> requests = new ArrayDeque<>(); + + void get(String path, int status, String json) { + expect(HttpMethod.GET, path, status, "application/json", BinaryData.fromString(json).toBytes()); + } + + void audio(String path) { + expect(HttpMethod.GET, path, 200, "audio/wav", new byte[] { 82, 73, 70, 70, 0, 1, 2, 3 }); + } + + void delete(String path) { + expect(HttpMethod.DELETE, path, 204, "application/json", new byte[0]); + } + + private void expect(HttpMethod method, String path, int status, String contentType, byte[] body) { + requests.add(request -> { + assertEquals(method, request.getHttpMethod()); + assertEquals(path, request.getUrl().getPath()); + return new MockHttpResponse(request, status, + new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, contentType), body); + }); + } + + @Override + public Mono send(HttpRequest request) { + throw new AssertionError("The sync client must not use asynchronous HTTP."); + } + + @Override + public HttpResponse sendSync(HttpRequest request, Context context) { + assertFalse(requests.isEmpty(), "Unexpected request: " + request.getUrl()); + return requests.removeFirst().apply(request); + } + + void assertComplete() { + assertTrue(requests.isEmpty(), "Not all expected voice REST operations were called."); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentCrudAsyncTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentCrudAsyncTests.java new file mode 100644 index 000000000000..335a8cd63a91 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentCrudAsyncTests.java @@ -0,0 +1,290 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.AgentDetails; +import com.azure.ai.agents.models.AgentState; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.test.utils.MockTokenCredential; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.core.util.Context; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Parity for Python test_voice_agent_crud_async.py: versioned CRUD, disable/enable and guided generation. + * Offline tests use scripted HTTP responses, not recordings. Live tests require AZURE_TEST_MODE=LIVE, + * FOUNDRY_PROJECT_ENDPOINT and DefaultAzureCredential authentication. CRUD and state tests additionally + * require FOUNDRY_VOICE_MODEL_NAME. Every scenario deletes only its own uniquely named live agent. + */ +public class VoiceAgentCrudAsyncTests { + private static final Duration TIMEOUT = Duration.ofMinutes(2); + private static final String AGENT = "voice-agent-crud-async-java"; + private static final String MODEL = "voice-model"; + private static final String INSTRUCTIONS = "You are a helpful voice assistant."; + private static final String UPDATED_INSTRUCTIONS = INSTRUCTIONS + " Always greet the caller by name."; + + enum Scenario { + CRUD, DISABLE_ENABLE, GENERATE + } + + @ParameterizedTest + @EnumSource(Scenario.class) + public void voiceAgentOperations(Scenario scenario) { + ScriptedTransport transport = new ScriptedTransport(); + String path = "/agents/" + AGENT; + Map first = version("1", INSTRUCTIONS); + switch (scenario) { + case CRUD: + Map second = version("2", UPDATED_INSTRUCTIONS); + transport.expect(HttpMethod.POST, path + "/versions", createBody(INSTRUCTIONS), first); + transport.expect(HttpMethod.POST, path + "/versions", createBody(UPDATED_INSTRUCTIONS), second); + transport.expect(HttpMethod.GET, path, null, agent(second, "enabled")); + transport.expect(HttpMethod.GET, path + "/versions/1", null, first); + transport.expect(HttpMethod.GET, path + "/versions", null, + object("data", java.util.Arrays.asList(first, second), "has_more", false)); + break; + + case DISABLE_ENABLE: + transport.expect(HttpMethod.POST, path + "/versions", createBody(INSTRUCTIONS), first); + transport.expect(HttpMethod.POST, path + ":disable", null, null, 204); + transport.expect(HttpMethod.GET, path, null, agent(first, "disabled")); + transport.expect(HttpMethod.POST, path + ":enable", null, null, 204); + transport.expect(HttpMethod.GET, path, null, agent(first, "enabled")); + break; + + case GENERATE: + transport.expect(HttpMethod.POST, "/agents:generate", object("kind", "voice", "name", AGENT), + agent(first, "enabled")); + break; + + default: + throw new AssertionError("Unexpected scenario: " + scenario); + } + transport.expect(HttpMethod.DELETE, path, null, object("deleted", true, "name", AGENT)); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost") + .credential(new MockTokenCredential()) + .httpClient(transport) + .allowPreview(true); + StepVerifier.create(runScenario(builder, AGENT, MODEL, scenario)).expectComplete().verify(TIMEOUT); + assertTrue(transport.requests.isEmpty(), "All expected REST operations must be exercised."); + } + + @ParameterizedTest + @EnumSource(Scenario.class) + @EnabledIfEnvironmentVariable(named = "AZURE_TEST_MODE", matches = "LIVE") + public void voiceAgentOperationsLive(Scenario scenario) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL_NAME"); + assertNotNull(endpoint, "FOUNDRY_PROJECT_ENDPOINT is required for live tests."); + if (scenario != Scenario.GENERATE) { + assertNotNull(model, "FOUNDRY_VOICE_MODEL_NAME is required for this live test."); + } + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + StepVerifier.create(runScenario(builder, "test-voice-crud-" + UUID.randomUUID(), model, scenario)) + .expectComplete() + .verify(TIMEOUT); + } + + private static Mono runScenario(AgentsClientBuilder builder, String name, String model, Scenario scenario) { + AgentsAsyncClient client = builder.buildAgentsAsyncClient(); + Mono create = scenario == Scenario.GENERATE + ? builder.beta() + .buildBetaAgentsAsyncClient() + .createAgentFromPrompt(BinaryData.fromObject(object("kind", "voice", "name", name))) + : client.createAgentVersion(name, new CreateAgentVersionInput(definition(model, INSTRUCTIONS))); + return Mono.usingWhen(create, created -> { + if (scenario == Scenario.GENERATE) { + AgentDetails generated = (AgentDetails) created; + validateAgent(generated, name, null); + VoiceAgentDefinition voice + = assertInstanceOf(VoiceAgentDefinition.class, generated.getVersions().getLatest().getDefinition()); + assertNotNull(voice.getInstructions()); + assertFalse(voice.getInstructions().isEmpty()); + return Mono.empty(); + } + AgentVersionDetails first = (AgentVersionDetails) created; + validateVersion(first, name, null); + validateDefinition(first, model, INSTRUCTIONS); + if (scenario == Scenario.DISABLE_ENABLE) { + return client.disableAgent(name) + .then(client.getAgent(name)) + .doOnNext(agent -> assertEquals(AgentState.DISABLED, agent.getState())) + .then(client.enableAgent(name)) + .then(client.getAgent(name)) + .doOnNext(agent -> assertEquals(AgentState.ENABLED, agent.getState())) + .then(); + } + return client.createAgentVersion(name, new CreateAgentVersionInput(definition(model, UPDATED_INSTRUCTIONS))) + .flatMap(second -> { + validateVersion(second, name, null); + validateDefinition(second, model, UPDATED_INSTRUCTIONS); + assertNotEquals(first.getVersion(), second.getVersion()); + return client.getAgent(name) + .doOnNext(agent -> validateAgent(agent, name, second.getVersion())) + .then(client.getAgentVersionDetails(name, first.getVersion())) + .doOnNext(version -> { + validateVersion(version, name, first.getVersion()); + validateDefinition(version, model, INSTRUCTIONS); + }) + .thenMany(client.listAgentVersions(name)) + .collectList() + .doOnNext(versions -> { + assertTrue(versions.size() >= 2); + versions.forEach(version -> validateVersion(version, name, null)); + assertTrue( + versions.stream().anyMatch(version -> first.getVersion().equals(version.getVersion()))); + assertTrue(versions.stream() + .anyMatch(version -> second.getVersion().equals(version.getVersion()))); + }) + .then(); + }); + }, created -> client.deleteAgent(name), (created, error) -> client.deleteAgent(name), + created -> client.deleteAgent(name)); + } + + private static VoiceAgentDefinition definition(String model, String instructions) { + return new VoiceAgentDefinition().setModelType(VoiceModelType.MANAGED) + .setModel(model) + .setInstructions(instructions) + .setAudio(new VoiceAgentAudioConfiguration() + .setOutput(new VoiceAgentAudioOutputConfiguration().setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD))) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)); + } + + private static void validateDefinition(AgentVersionDetails version, String model, String instructions) { + VoiceAgentDefinition voice = assertInstanceOf(VoiceAgentDefinition.class, version.getDefinition()); + assertEquals(VoiceModelType.MANAGED, voice.getModelType()); + assertEquals(model, voice.getModel()); + assertEquals(instructions, voice.getInstructions()); + assertNotNull(voice.getAudio()); + assertNotNull(voice.getAudio().getOutput()); + assertEquals("en-US-AvaNeural", voice.getAudio().getOutput().getVoice()); + assertEquals(VoiceType.AZURE_STANDARD, voice.getAudio().getOutput().getVoiceType()); + assertEquals(Collections.singletonList(VoiceOutputModality.AUDIO), voice.getOutputModalities()); + } + + private static void validateVersion(AgentVersionDetails version, String name, String expectedVersion) { + assertNotNull(version); + assertNotNull(version.getId()); + assertEquals(name, version.getName()); + assertNotNull(version.getVersion()); + assertFalse(version.getVersion().isEmpty()); + assertNotNull(version.getCreatedAt()); + assertInstanceOf(VoiceAgentDefinition.class, version.getDefinition()); + if (expectedVersion != null) { + assertEquals(expectedVersion, version.getVersion()); + } + } + + private static void validateAgent(AgentDetails agent, String name, String expectedVersion) { + assertNotNull(agent); + assertNotNull(agent.getId()); + assertEquals(name, agent.getName()); + assertNotNull(agent.getVersions()); + validateVersion(agent.getVersions().getLatest(), name, expectedVersion); + } + + private static Map createBody(String instructions) { + return object("definition", + object("kind", "voice", "model_type", "managed", "model", MODEL, "instructions", instructions, "audio", + object("output", object("voice", "en-US-AvaNeural", "voice_type", "azure-standard")), + "output_modalities", Collections.singletonList("audio"))); + } + + private static Map version(String version, String instructions) { + return object("id", AGENT + ":" + version, "name", AGENT, "version", version, "object", "agent.version", + "created_at", 1700000000, "definition", createBody(instructions).get("definition")); + } + + private static Map agent(Map latest, String state) { + return object("id", AGENT, "name", AGENT, "object", "agent", "state", state, "versions", + object("latest", latest)); + } + + private static Map object(Object... entries) { + Map result = new LinkedHashMap<>(); + for (int index = 0; index < entries.length; index += 2) { + result.put((String) entries[index], entries[index + 1]); + } + return result; + } + + private static final class ScriptedTransport implements HttpClient { + private final Deque>> requests = new ArrayDeque<>(); + + void expect(HttpMethod method, String path, Map body, Map response) { + expect(method, path, body, response, 200); + } + + void expect(HttpMethod method, String path, Map body, Map response, + int status) { + requests.add(request -> { + assertEquals(method, request.getHttpMethod()); + assertEquals(path, request.getUrl().getPath()); + Mono check = body == null + ? Mono.empty() + : BinaryData.fromFlux(request.getBody()) + .doOnNext(actual -> assertEquals(body, actual.toObject(Map.class))) + .then(); + return check.thenReturn(new MockHttpResponse(request, status, + new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "application/json"), + response == null ? new byte[0] : BinaryData.fromObject(response).toBytes())); + }); + } + + @Override + public Mono send(HttpRequest request) { + return Mono.defer(() -> { + assertFalse(requests.isEmpty(), "Unexpected request: " + request.getUrl()); + return requests.removeFirst().apply(request); + }); + } + + @Override + public HttpResponse sendSync(HttpRequest request, Context context) { + throw new AssertionError("The async client must not use synchronous HTTP."); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentCrudTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentCrudTests.java new file mode 100644 index 000000000000..f502e686873b --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentCrudTests.java @@ -0,0 +1,277 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.AgentDetails; +import com.azure.ai.agents.models.AgentState; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.test.utils.MockTokenCredential; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.core.util.Context; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import reactor.core.publisher.Mono; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Synchronous parity for Python test_voice_agent_crud.py: versioned CRUD, disable/enable and guided generation. + * Offline tests use scripted HTTP responses, not recordings. Live tests require AZURE_TEST_MODE=LIVE, + * FOUNDRY_PROJECT_ENDPOINT and DefaultAzureCredential authentication. CRUD and state tests additionally + * require FOUNDRY_VOICE_MODEL_NAME. Every scenario deletes only its own uniquely named live agent. + */ +public class VoiceAgentCrudTests { + private static final String AGENT = "voice-agent-crud-java"; + private static final String MODEL = "voice-model"; + private static final String INSTRUCTIONS = "You are a helpful voice assistant."; + private static final String UPDATED_INSTRUCTIONS = INSTRUCTIONS + " Always greet the caller by name."; + + enum Scenario { + CRUD, DISABLE_ENABLE, GENERATE + } + + @ParameterizedTest + @EnumSource(Scenario.class) + public void voiceAgentOperations(Scenario scenario) { + ScriptedTransport transport = new ScriptedTransport(); + String path = "/agents/" + AGENT; + Map first = version("1", INSTRUCTIONS); + switch (scenario) { + case CRUD: + Map second = version("2", UPDATED_INSTRUCTIONS); + transport.expect(HttpMethod.POST, path + "/versions", createBody(INSTRUCTIONS), first); + transport.expect(HttpMethod.POST, path + "/versions", createBody(UPDATED_INSTRUCTIONS), second); + transport.expect(HttpMethod.GET, path, null, agent(second, "enabled")); + transport.expect(HttpMethod.GET, path + "/versions/1", null, first); + transport.expect(HttpMethod.GET, path + "/versions", null, + object("data", Arrays.asList(first, second), "has_more", false)); + break; + + case DISABLE_ENABLE: + transport.expect(HttpMethod.POST, path + "/versions", createBody(INSTRUCTIONS), first); + transport.expect(HttpMethod.POST, path + ":disable", null, null, 204); + transport.expect(HttpMethod.GET, path, null, agent(first, "disabled")); + transport.expect(HttpMethod.POST, path + ":enable", null, null, 204); + transport.expect(HttpMethod.GET, path, null, agent(first, "enabled")); + break; + + case GENERATE: + transport.expect(HttpMethod.POST, "/agents:generate", object("kind", "voice", "name", AGENT), + agent(first, "enabled")); + break; + + default: + throw new AssertionError("Unexpected scenario: " + scenario); + } + transport.expect(HttpMethod.DELETE, path, null, object("deleted", true, "name", AGENT)); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost") + .credential(new MockTokenCredential()) + .httpClient(transport) + .allowPreview(true); + runScenario(builder, AGENT, MODEL, scenario); + assertTrue(transport.requests.isEmpty(), "All expected REST operations must be exercised."); + } + + @ParameterizedTest + @EnumSource(Scenario.class) + @EnabledIfEnvironmentVariable(named = "AZURE_TEST_MODE", matches = "LIVE") + public void voiceAgentOperationsLive(Scenario scenario) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL_NAME"); + assertNotNull(endpoint, "FOUNDRY_PROJECT_ENDPOINT is required for live tests."); + if (scenario != Scenario.GENERATE) { + assertNotNull(model, "FOUNDRY_VOICE_MODEL_NAME is required for this live test."); + } + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + runScenario(builder, "test-voice-crud-sync-" + UUID.randomUUID(), model, scenario); + } + + private static void runScenario(AgentsClientBuilder builder, String name, String model, Scenario scenario) { + AgentsClient client = builder.buildAgentsClient(); + boolean created = false; + try { + if (scenario == Scenario.GENERATE) { + AgentDetails generated = builder.beta() + .buildBetaAgentsClient() + .createAgentFromPrompt(BinaryData.fromObject(object("kind", "voice", "name", name))); + created = true; + validateAgent(generated, name, null); + VoiceAgentDefinition voice + = assertInstanceOf(VoiceAgentDefinition.class, generated.getVersions().getLatest().getDefinition()); + assertNotNull(voice.getInstructions()); + assertFalse(voice.getInstructions().isEmpty()); + return; + } + AgentVersionDetails first + = client.createAgentVersion(name, new CreateAgentVersionInput(definition(model, INSTRUCTIONS))); + created = true; + validateVersion(first, name, null); + validateDefinition(first, model, INSTRUCTIONS); + if (scenario == Scenario.DISABLE_ENABLE) { + client.disableAgent(name); + assertEquals(AgentState.DISABLED, client.getAgent(name).getState()); + client.enableAgent(name); + assertEquals(AgentState.ENABLED, client.getAgent(name).getState()); + return; + } + AgentVersionDetails second + = client.createAgentVersion(name, new CreateAgentVersionInput(definition(model, UPDATED_INSTRUCTIONS))); + validateVersion(second, name, null); + validateDefinition(second, model, UPDATED_INSTRUCTIONS); + assertNotEquals(first.getVersion(), second.getVersion()); + validateAgent(client.getAgent(name), name, second.getVersion()); + AgentVersionDetails retrieved = client.getAgentVersionDetails(name, first.getVersion()); + validateVersion(retrieved, name, first.getVersion()); + validateDefinition(retrieved, model, INSTRUCTIONS); + List versions = client.listAgentVersions(name).stream().collect(Collectors.toList()); + assertTrue(versions.size() >= 2); + versions.forEach(version -> validateVersion(version, name, null)); + assertTrue(versions.stream().anyMatch(version -> first.getVersion().equals(version.getVersion()))); + assertTrue(versions.stream().anyMatch(version -> second.getVersion().equals(version.getVersion()))); + } finally { + if (created) { + client.deleteAgent(name); + } + } + } + + private static VoiceAgentDefinition definition(String model, String instructions) { + return new VoiceAgentDefinition().setModelType(VoiceModelType.MANAGED) + .setModel(model) + .setInstructions(instructions) + .setAudio(new VoiceAgentAudioConfiguration() + .setOutput(new VoiceAgentAudioOutputConfiguration().setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD))) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)); + } + + private static void validateDefinition(AgentVersionDetails version, String model, String instructions) { + VoiceAgentDefinition voice = assertInstanceOf(VoiceAgentDefinition.class, version.getDefinition()); + assertEquals(VoiceModelType.MANAGED, voice.getModelType()); + assertEquals(model, voice.getModel()); + assertEquals(instructions, voice.getInstructions()); + assertNotNull(voice.getAudio()); + assertNotNull(voice.getAudio().getOutput()); + assertEquals("en-US-AvaNeural", voice.getAudio().getOutput().getVoice()); + assertEquals(VoiceType.AZURE_STANDARD, voice.getAudio().getOutput().getVoiceType()); + assertEquals(Collections.singletonList(VoiceOutputModality.AUDIO), voice.getOutputModalities()); + } + + private static void validateVersion(AgentVersionDetails version, String name, String expectedVersion) { + assertNotNull(version); + assertNotNull(version.getId()); + assertEquals(name, version.getName()); + assertNotNull(version.getVersion()); + assertFalse(version.getVersion().isEmpty()); + assertNotNull(version.getCreatedAt()); + assertInstanceOf(VoiceAgentDefinition.class, version.getDefinition()); + if (expectedVersion != null) { + assertEquals(expectedVersion, version.getVersion()); + } + } + + private static void validateAgent(AgentDetails agent, String name, String expectedVersion) { + assertNotNull(agent); + assertNotNull(agent.getId()); + assertEquals(name, agent.getName()); + assertNotNull(agent.getVersions()); + validateVersion(agent.getVersions().getLatest(), name, expectedVersion); + } + + private static Map createBody(String instructions) { + return object("definition", + object("kind", "voice", "model_type", "managed", "model", MODEL, "instructions", instructions, "audio", + object("output", object("voice", "en-US-AvaNeural", "voice_type", "azure-standard")), + "output_modalities", Collections.singletonList("audio"))); + } + + private static Map version(String version, String instructions) { + return object("id", AGENT + ":" + version, "name", AGENT, "version", version, "object", "agent.version", + "created_at", 1700000000, "definition", createBody(instructions).get("definition")); + } + + private static Map agent(Map latest, String state) { + return object("id", AGENT, "name", AGENT, "object", "agent", "state", state, "versions", + object("latest", latest)); + } + + private static Map object(Object... entries) { + Map result = new LinkedHashMap<>(); + for (int index = 0; index < entries.length; index += 2) { + result.put((String) entries[index], entries[index + 1]); + } + return result; + } + + private static final class ScriptedTransport implements HttpClient { + private final Deque> requests = new ArrayDeque<>(); + + void expect(HttpMethod method, String path, Map body, Map response) { + expect(method, path, body, response, 200); + } + + void expect(HttpMethod method, String path, Map body, Map response, + int status) { + requests.add(request -> { + assertEquals(method, request.getHttpMethod()); + assertEquals(path, request.getUrl().getPath()); + if (body != null) { + assertNotNull(request.getBodyAsBinaryData()); + assertEquals(body, request.getBodyAsBinaryData().toObject(Map.class)); + } + return new MockHttpResponse(request, status, + new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "application/json"), + response == null ? new byte[0] : BinaryData.fromObject(response).toBytes()); + }); + } + + @Override + public Mono send(HttpRequest request) { + throw new AssertionError("The sync client must not use asynchronous HTTP."); + } + + @Override + public HttpResponse sendSync(HttpRequest request, Context context) { + assertFalse(requests.isEmpty(), "Unexpected request: " + request.getUrl()); + return requests.removeFirst().apply(request); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSampleTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSampleTests.java new file mode 100644 index 000000000000..0b77049b84e3 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSampleTests.java @@ -0,0 +1,226 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeConversationCreatedEvent; +import com.azure.core.util.BinaryData; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import javax.sound.sampled.SourceDataLine; +import javax.sound.sampled.TargetDataLine; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Mono; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class VoiceAgentLiveAudioConversationAsyncSampleTests { + private static final Duration TIMEOUT = Duration.ofSeconds(5); + + @Test + public void conversationCreatedModelIsAvailable() { + RealtimeServerEvent event = BinaryData + .fromString("{\"type\":\"conversation.created\"," + + "\"conversation\":{\"id\":\"test\",\"object\":\"realtime.conversation\"}}") + .toObject(RealtimeServerEvent.class); + assertTrue(event instanceof RealtimeConversationCreatedEvent); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void receiveTerminationClosesAudio(boolean failReceive) { + FakeAudio audio = new FakeAudio(); + IllegalStateException error = new IllegalStateException("Disconnected"); + Mono receive = failReceive ? Mono.error(error) : Mono.empty(); + Mono conversation + = VoiceAgentLiveAudioConversationAsyncSample.runConversation(receive, audio.processor(), emptyInput()); + if (failReceive) { + assertSame(error, assertThrows(IllegalStateException.class, () -> conversation.block(TIMEOUT))); + } else { + conversation.block(TIMEOUT); + } + audio.assertClosed(); + } + + @Test + public void microphoneFailureClosesAudio() { + FakeAudio audio = new FakeAudio(); + audio.readFailure = new IllegalStateException("Microphone failed"); + Mono conversation + = VoiceAgentLiveAudioConversationAsyncSample.runConversation(Mono.never(), audio.processor(), emptyInput()); + assertSame(audio.readFailure, assertThrows(IllegalStateException.class, () -> conversation.block(TIMEOUT))); + audio.assertClosed(); + } + + @Test + public void playbackAcceptsBurstsAndEnforcesByteLimitWithoutBlocking() throws Exception { + FakeAudio audio = new FakeAudio(); + VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor processor = audio.processor(); + CountDownLatch receiving = new CountDownLatch(1); + CompletableFuture conversation + = VoiceAgentLiveAudioConversationAsyncSample + .runConversation(Mono.never().doOnSubscribe(subscription -> receiving.countDown()), processor, + emptyInput()) + .toFuture(); + try { + assertTrue(receiving.await(5, TimeUnit.SECONDS)); + processor.queueAudio(new byte[2]); + assertTrue(audio.writing.await(5, TimeUnit.SECONDS)); + assertTimeoutPreemptively(TIMEOUT, () -> { + for (int chunk = 0; chunk < 100; chunk++) { + processor.queueAudio(new byte[2400]); + } + }); + assertFalse(conversation.isDone()); + processor.skipPendingAudio(); + processor + .queueAudio(new byte[VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor.MAX_PLAYBACK_BYTES]); + assertFalse(conversation.isDone()); + assertTimeoutPreemptively(TIMEOUT, () -> processor.queueAudio(new byte[2])); + ExecutionException error + = assertThrows(ExecutionException.class, () -> conversation.get(5, TimeUnit.SECONDS)); + assertEquals("Audio playback backlog exceeded 60 seconds.", error.getCause().getMessage()); + audio.assertClosed(); + } finally { + conversation.cancel(true); + processor.close(); + } + } + + @Test + public void cancellationClosesAudioAndCancelsReceive() throws Exception { + FakeAudio audio = new FakeAudio(); + CountDownLatch cancelled = new CountDownLatch(1); + CountDownLatch receiving = new CountDownLatch(1); + VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor processor = audio.processor(); + CompletableFuture conversation = VoiceAgentLiveAudioConversationAsyncSample.runConversation( + Mono.never().doOnSubscribe(subscription -> receiving.countDown()).doOnCancel(cancelled::countDown), + processor, emptyInput()).toFuture(); + try { + assertTrue(receiving.await(5, TimeUnit.SECONDS)); + conversation.cancel(true); + assertTrue(audio.closed.await(5, TimeUnit.SECONDS)); + assertTrue(cancelled.await(5, TimeUnit.SECONDS)); + } finally { + conversation.cancel(true); + processor.close(); + } + audio.assertClosed(); + } + + @Test + public void closedProcessorCannotRestart() { + FakeAudio audio = new FakeAudio(); + VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor processor = audio.processor(); + processor.close(); + assertThrows(IllegalStateException.class, processor::start); + audio.assertClosed(); + } + + @Test + public void enterStopsConversationWithoutClosingStandardInput() { + FakeAudio audio = new FakeAudio(); + AtomicBoolean inputClosed = new AtomicBoolean(); + InputStream input = new ByteArrayInputStream("end\n".getBytes(StandardCharsets.UTF_8)) { + @Override + public void close() { + inputClosed.set(true); + } + }; + VoiceAgentLiveAudioConversationAsyncSample.runConversation(Mono.never(), audio.processor(), input) + .block(TIMEOUT); + assertFalse(inputClosed.get()); + audio.assertClosed(); + } + + @Test + public void inputPollingNeverReadsUnavailableBytes() { + InputStream input = new InputStream() { + @Override + public int read() { + throw new AssertionError("A read with no available bytes can block indefinitely."); + } + }; + Mono.firstWithSignal(VoiceAgentLiveAudioConversationAsyncSample.waitForEnter(input), + Mono.delay(Duration.ofMillis(300)).then()).block(TIMEOUT); + } + + private static InputStream emptyInput() { + return new ByteArrayInputStream(new byte[0]); + } + + private static final class FakeAudio implements InvocationHandler { + private final CountDownLatch closed = new CountDownLatch(2); + private final CountDownLatch writing = new CountDownLatch(1); + private final AtomicReference captureThread = new AtomicReference<>(); + private final AtomicReference playbackThread = new AtomicReference<>(); + private RuntimeException readFailure; + + VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor processor() { + TargetDataLine microphone = (TargetDataLine) Proxy.newProxyInstance(getClass().getClassLoader(), + new Class[] { TargetDataLine.class }, this); + SourceDataLine speaker = (SourceDataLine) Proxy.newProxyInstance(getClass().getClassLoader(), + new Class[] { SourceDataLine.class }, this); + return new VoiceAgentLiveAudioConversationAsyncSample.AudioProcessor(null, microphone, speaker); + } + + @Override + public Object invoke(Object proxy, Method method, Object[] arguments) throws InterruptedException { + switch (method.getName()) { + case "read": + captureThread.set(Thread.currentThread()); + if (readFailure != null) { + throw readFailure; + } + closed.await(); + return 0; + + case "write": + playbackThread.set(Thread.currentThread()); + writing.countDown(); + closed.await(); + return arguments[2]; + + case "close": + closed.countDown(); + return null; + + default: + return null; + } + } + + void assertClosed() { + assertEquals(0, closed.getCount()); + assertTimeoutPreemptively(TIMEOUT, () -> { + if (captureThread.get() != null) { + captureThread.get().join(); + assertFalse(captureThread.get().isAlive()); + } + if (playbackThread.get() != null) { + playbackThread.get().join(); + assertFalse(playbackThread.get().isAlive()); + } + }); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentRealtimeLiveTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentRealtimeLiveTests.java new file mode 100644 index 000000000000..9b05562145d2 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentRealtimeLiveTests.java @@ -0,0 +1,393 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeClientEvent; +import com.azure.ai.agents.models.RealtimeConversationItemCreateEvent; +import com.azure.ai.agents.models.RealtimeResponseCreateEvent; +import com.azure.ai.agents.models.RealtimeConversationItemFunctionCallOutput; +import com.azure.ai.agents.models.RealtimeConversationItemUserMessage; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeErrorEvent; +import com.azure.ai.agents.models.RealtimeResponseAudioDeltaEvent; +import com.azure.ai.agents.models.RealtimeResponseAudioTranscriptDoneEvent; +import com.azure.ai.agents.models.RealtimeResponseDoneEvent; +import com.azure.ai.agents.models.RealtimeResponseFunctionCallArgumentsDoneEvent; +import com.azure.ai.agents.models.RealtimeResponseTextDoneEvent; +import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceAgentFunctionTool; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.openai.models.realtime.RealtimeConversationItemUserMessage.Content; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Sync/async parity for the Python live realtime suites. Live cases require AZURE_TEST_MODE=LIVE, + * FOUNDRY_PROJECT_ENDPOINT, FOUNDRY_VOICE_MODEL_NAME and DefaultAzureCredential authentication. + * They create and delete real agents but require no audio hardware. Offline cases validate the + * same event assertions with synthetic payloads, not service recordings. + */ +public class VoiceAgentRealtimeLiveTests { + private static final Duration EVENT_TIMEOUT = Duration.ofSeconds(30); + private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(45); + private static final String SESSION = "{\"type\":\"session.created\",\"session\":{}}"; + private static final String DONE = "{\"type\":\"response.done\",\"response\":{\"output\":[]}}"; + private static final String TOOL_DONE = "{\"type\":\"response.done\",\"response\":{\"output\":[" + + "{\"type\":\"function_call\",\"name\":\"get_weather\",\"call_id\":\"call-1\",\"arguments\":\"{}\"}]}}"; + + enum Scenario { + LIFECYCLE, AUDIO, FUNCTION + } + + @ParameterizedTest + @EnumSource(Scenario.class) + @EnabledIfEnvironmentVariable(named = "AZURE_TEST_MODE", matches = "LIVE") + public void realtimeLive(Scenario scenario) { + AgentsClientBuilder builder = liveBuilder(); + AgentsClient agents = builder.buildAgentsClient(); + String agentName = "test-realtime-sync-" + UUID.randomUUID(); + boolean created = false; + try { + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition(scenario))); + created = true; + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession(agentName)) { + Turn turn = new Turn(scenario); + Iterator events = session.receiveEvents(EVENT_TIMEOUT).iterator(); + turn.accept(events.next()).forEach(session::sendEvent); + long deadline = System.nanoTime() + RESPONSE_TIMEOUT.toNanos(); + while (!turn.done && System.nanoTime() < deadline && events.hasNext()) { + turn.accept(events.next()).forEach(session::sendEvent); + } + turn.assertComplete(); + } + } finally { + if (created) { + agents.deleteAgent(agentName); + } + } + } + + @ParameterizedTest + @EnumSource(Scenario.class) + @EnabledIfEnvironmentVariable(named = "AZURE_TEST_MODE", matches = "LIVE") + public void realtimeLiveAsync(Scenario scenario) { + AgentsClientBuilder builder = liveBuilder(); + AgentsAsyncClient agents = builder.buildAgentsAsyncClient(); + String agentName = "test-realtime-async-" + UUID.randomUUID(); + boolean created = false; + try { + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition(scenario))) + .block(EVENT_TIMEOUT); + created = true; + Turn turn = new Turn(scenario); + Mono.usingWhen(builder.beta().buildBetaVoiceAgentWebSocketAsyncClient().openWebSocketSession(agentName), + session -> session.receiveEvents() + .timeout(EVENT_TIMEOUT) + .concatMap(event -> Flux.fromIterable(turn.accept(event)) + .concatMap(session::sendEvent) + .then(Mono.just(turn.done))) + .filter(Boolean::booleanValue) + .next() + .switchIfEmpty(Mono.error(new AssertionError("Session ended before the turn completed."))) + .timeout(RESPONSE_TIMEOUT) + .doOnNext(ignored -> turn.assertComplete()) + .then(), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync, (session, error) -> session.closeAsync(), + BetaVoiceAgentWebSocketSessionAsyncClient::closeAsync).block(Duration.ofSeconds(90)); + } finally { + if (created) { + agents.deleteAgent(agentName).block(EVENT_TIMEOUT); + } + } + } + + @ParameterizedTest + @EnumSource(Scenario.class) + public void syntheticEventsExerciseLiveAssertions(Scenario scenario) { + Turn turn = new Turn(scenario); + List initial = turn.accept(event(SESSION)); + assertEquals(scenario == Scenario.LIFECYCLE ? 0 : 2, initial.size()); + if (scenario != Scenario.LIFECYCLE) { + assertInstanceOf(RealtimeConversationItemCreateEvent.class, initial.get(0)); + assertInstanceOf(RealtimeResponseCreateEvent.class, initial.get(1)); + } + if (scenario == Scenario.AUDIO) { + turn.accept(event("{\"type\":\"response.output_audio.delta\",\"delta\":\"AQID\"}")); + turn.accept(event("{\"type\":\"response.output_audio_transcript.done\",\"transcript\":\"Hello\"}")); + turn.accept(event(DONE)); + } else if (scenario == Scenario.FUNCTION) { + assertTrue(turn.accept(functionCall("call-1")).isEmpty()); + assertTrue(turn.accept(functionCall("call-2")).isEmpty()); + assertFalse(turn.done); + List outputs = turn.accept(event(TOOL_DONE)); + assertEquals(3, outputs.size()); + for (int index = 0; index < 2; index++) { + RealtimeConversationItemCreateEvent create + = assertInstanceOf(RealtimeConversationItemCreateEvent.class, outputs.get(index)); + RealtimeConversationItemFunctionCallOutput output + = assertInstanceOf(RealtimeConversationItemFunctionCallOutput.class, create.getItem()); + assertEquals("call-" + (index + 1), output.getCallId()); + Map result = BinaryData.fromString(output.getOutput()).toObject(Map.class); + assertEquals("Seattle", result.get("city")); + assertEquals("sunny", result.get("condition")); + assertEquals(72, result.get("temperature_f")); + } + assertInstanceOf(RealtimeResponseCreateEvent.class, outputs.get(2)); + assertFalse(turn.done); + turn.accept(event("{\"type\":\"response.output_text.done\",\"text\":\"Sunny in Seattle.\"}")); + turn.accept(event(DONE)); + } + turn.assertComplete(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "missing-audio", + "empty-audio", + "missing-transcript", + "empty-transcript", + "duplicate-transcript", + "missing-done" }) + public void incompleteAudioTurnsFail(String omission) { + Turn turn = new Turn(Scenario.AUDIO); + turn.accept(event(SESSION)); + assertThrows(AssertionError.class, () -> { + if (!"missing-audio".equals(omission)) { + String delta = "empty-audio".equals(omission) ? "" : "AQID"; + turn.accept(event("{\"type\":\"response.output_audio.delta\",\"delta\":\"" + delta + "\"}")); + } + if (!"missing-transcript".equals(omission)) { + String transcript = "empty-transcript".equals(omission) ? " " : "Hello"; + RealtimeServerEvent transcriptEvent = event( + "{\"type\":\"response.output_audio_transcript.done\"," + "\"transcript\":\"" + transcript + "\"}"); + turn.accept(transcriptEvent); + if ("duplicate-transcript".equals(omission)) { + turn.accept(transcriptEvent); + } + } + if (!"missing-done".equals(omission)) { + turn.accept(event(DONE)); + } + turn.assertComplete(); + }); + } + + @Test + public void missingHandshakeServiceErrorsAndMissingToolResultFail() { + assertThrows(AssertionError.class, () -> new Turn(Scenario.LIFECYCLE).accept(event(DONE))); + Turn audio = new Turn(Scenario.AUDIO); + audio.accept(event(SESSION)); + assertThrows(AssertionError.class, () -> audio.accept(event( + "{\"type\":\"error\",\"error\":{\"type\":\"server_error\",\"message\":\"boom\",\"code\":\"failed\"}}"))); + Turn tool = new Turn(Scenario.FUNCTION); + tool.accept(event(SESSION)); + tool.accept(event(TOOL_DONE)); + assertFalse(tool.done); + assertThrows(AssertionError.class, tool::assertComplete); + tool.accept(event("{\"type\":\"response.output_text.done\",\"text\":\"Unverified answer\"}")); + tool.accept(event(DONE)); + assertThrows(AssertionError.class, tool::assertComplete); + } + + private static AgentsClientBuilder liveBuilder() { + String endpoint = Configuration.getGlobalConfiguration().get("FOUNDRY_PROJECT_ENDPOINT"); + assertNotNull(endpoint, "FOUNDRY_PROJECT_ENDPOINT is required for live testing."); + return new AgentsClientBuilder().endpoint(endpoint) + .allowPreview(true) + .credential(new DefaultAzureCredentialBuilder().build()); + } + + private static VoiceAgentDefinition definition(Scenario scenario) { + String model = Configuration.getGlobalConfiguration().get("FOUNDRY_VOICE_MODEL_NAME"); + assertNotNull(model, "FOUNDRY_VOICE_MODEL_NAME is required for live testing."); + return definition(scenario, model); + } + + @Test + public void functionToolParametersSerializeAsObject() { + Map request + = BinaryData.fromObject(new CreateAgentVersionInput(definition(Scenario.FUNCTION, "test-model"))) + .toObject(Map.class); + Map agentDefinition = assertInstanceOf(Map.class, request.get("definition")); + List tools = assertInstanceOf(List.class, agentDefinition.get("tools")); + assertEquals(1, tools.size()); + Map tool = assertInstanceOf(Map.class, tools.get(0)); + assertEquals("get_weather", tool.get("name")); + Map parameters = assertInstanceOf(Map.class, tool.get("parameters")); + assertEquals("object", parameters.get("type")); + assertEquals(Collections.singletonList("city"), parameters.get("required")); + Map properties = assertInstanceOf(Map.class, parameters.get("properties")); + Map city = assertInstanceOf(Map.class, properties.get("city")); + assertEquals("string", city.get("type")); + assertEquals("City name, e.g. Seattle.", city.get("description")); + } + + private static VoiceAgentDefinition definition(Scenario scenario, String model) { + VoiceAgentDefinition definition = new VoiceAgentDefinition().setModelType(VoiceModelType.MANAGED) + .setModel(model) + .setInstructions("You are a helpful voice assistant. Keep replies short."); + if (scenario == Scenario.FUNCTION) { + Map citySchema = new LinkedHashMap<>(); + citySchema.put("type", "string"); + citySchema.put("description", "City name, e.g. Seattle."); + Map parameters = new LinkedHashMap<>(); + parameters.put("type", "object"); + parameters.put("properties", Collections.singletonMap("city", citySchema)); + parameters.put("required", Collections.singletonList("city")); + definition + .setInstructions("You are a helpful voice assistant. Use the get_weather tool when the " + + "caller asks about the weather, then answer using its result.") + .setOutputModalities(Collections.singletonList(VoiceOutputModality.TEXT)) + .setTools(Collections.singletonList( + new VoiceAgentFunctionTool("get_weather").setDescription("Get the current weather for a city.") + .setParameters(BinaryData.fromObject(parameters)))); + } else { + definition.setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setAudio(new VoiceAgentAudioConfiguration() + .setOutput(new VoiceAgentAudioOutputConfiguration().setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD))); + } + return definition; + } + + private static RealtimeServerEvent event(String json) { + return BinaryData.fromString(json).toObject(RealtimeServerEvent.class); + } + + private static RealtimeServerEvent functionCall(String callId) { + return event("{\"type\":\"response.function_call_arguments.done\",\"name\":\"get_weather\"," + "\"call_id\":\"" + + callId + "\",\"arguments\":\"{\\\"city\\\":\\\"Seattle\\\"}\"}"); + } + + private static final class Turn { + private final Scenario scenario; + private final List pending = new ArrayList<>(); + private boolean started; + private boolean done; + private int audioDeltas; + private long audioBytes; + private int transcripts; + private int toolCalls; + private String finalText; + + private Turn(Scenario scenario) { + this.scenario = scenario; + } + + private List accept(RealtimeServerEvent event) { + if (event instanceof RealtimeErrorEvent) { + fail("Session error: " + ((RealtimeErrorEvent) event).getError().message()); + } + if (!started) { + assertInstanceOf(RealtimeSessionCreatedEvent.class, event, "The first event must be session.created."); + assertEquals("session.created", event.getType().toString()); + started = true; + done = scenario == Scenario.LIFECYCLE; + if (!done) { + String prompt = scenario == Scenario.FUNCTION + ? "What's the weather like in Seattle right now?" + : "Say the word 'hello' and nothing else."; + return Arrays.asList( + new RealtimeConversationItemCreateEvent(new RealtimeConversationItemUserMessage(Collections + .singletonList(Content.builder().type(Content.Type.INPUT_TEXT).text(prompt).build()))), + new RealtimeResponseCreateEvent()); + } + } else if (event instanceof RealtimeResponseAudioDeltaEvent) { + audioDeltas++; + byte[] delta = ((RealtimeResponseAudioDeltaEvent) event).getDelta(); + assertNotNull(delta); + audioBytes += delta.length; + } else if (event instanceof RealtimeResponseAudioTranscriptDoneEvent) { + transcripts++; + String transcript = ((RealtimeResponseAudioTranscriptDoneEvent) event).getTranscript(); + assertNotNull(transcript); + assertFalse(transcript.trim().isEmpty()); + } else if (event instanceof RealtimeResponseFunctionCallArgumentsDoneEvent) { + RealtimeResponseFunctionCallArgumentsDoneEvent call + = (RealtimeResponseFunctionCallArgumentsDoneEvent) event; + assertEquals("get_weather", call.getName()); + Map arguments = BinaryData.fromString(call.getArguments()).toObject(Map.class); + String city = assertInstanceOf(String.class, arguments.get("city")); + assertNotNull(call.getCallId()); + Map result = new LinkedHashMap<>(); + result.put("city", city); + result.put("condition", "sunny"); + result.put("temperature_f", 72); + pending.add(new RealtimeConversationItemCreateEvent(new RealtimeConversationItemFunctionCallOutput( + call.getCallId(), BinaryData.fromObject(result).toString()))); + toolCalls++; + } else if (event instanceof RealtimeResponseTextDoneEvent) { + finalText = ((RealtimeResponseTextDoneEvent) event).getText(); + } else if (event instanceof RealtimeResponseDoneEvent) { + if (!pending.isEmpty()) { + List outputs = new ArrayList<>(pending); + pending.clear(); + outputs.add(new RealtimeResponseCreateEvent()); + return outputs; + } + RealtimeResponseDoneEvent response = (RealtimeResponseDoneEvent) event; + assertNotNull(response.getResponse()); + done = scenario != Scenario.FUNCTION + || response.getResponse().getOutput() == null + || response.getResponse() + .getOutput() + .stream() + .noneMatch(item -> "function_call".equals(item.getType().toString())); + } + return Collections.emptyList(); + } + + private void assertComplete() { + assertTrue(started, "Did not receive session.created."); + assertTrue(done, "Did not receive the final response.done within the timeout."); + if (scenario == Scenario.AUDIO) { + assertTrue(audioDeltas > 0, "Expected at least one audio delta."); + assertTrue(audioBytes > 0, "Expected non-empty streamed audio."); + assertEquals(1, transcripts, "Expected exactly one audio-transcript-done event."); + } else if (scenario == Scenario.FUNCTION) { + assertTrue(toolCalls > 0, "Expected at least one get_weather call."); + assertNotNull(finalText); + assertFalse(finalText.trim().isEmpty(), "Expected a non-empty final text reply."); + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentTelephonyLiveTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentTelephonyLiveTests.java new file mode 100644 index 000000000000..56f57c25325f --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentTelephonyLiveTests.java @@ -0,0 +1,315 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsTelephonyClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.CreateTelephonyCallJobInput; +import com.azure.ai.agents.models.CreateTwilioTelephonyBindingInput; +import com.azure.ai.agents.models.PstnTelephonyTransferDestination; +import com.azure.ai.agents.models.TelephonyBinding; +import com.azure.ai.agents.models.TelephonyBindingListItem; +import com.azure.ai.agents.models.TelephonyBindingStatus; +import com.azure.ai.agents.models.TelephonyCallJobSchedule; +import com.azure.ai.agents.models.TelephonyCallRecord; +import com.azure.ai.agents.models.TelephonyCallJob; +import com.azure.ai.agents.models.TelephonyCallSummary; +import com.azure.ai.agents.models.TelephonyOutboundDestination; +import com.azure.ai.agents.models.TelephonyOutboundDestinationType; +import com.azure.ai.agents.models.TelephonyProvider; +import com.azure.ai.agents.models.TelephonyTransferTarget; +import com.azure.ai.agents.models.TelephonyTransferTargets; +import com.azure.ai.agents.models.UpdateTelephonyBindingInput; +import com.azure.ai.agents.models.VoiceAgentAudioConfiguration; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfiguration; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.rest.Response; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Live Twilio validation for voice-agent telephony. This test places a real PSTN call and may incur provider charges. + * It runs only when AZURE_TEST_MODE=LIVE, requires FOUNDRY_VOICE_MODEL_NAME, and uses + * DefaultAzureCredential authentication. FOUNDRY_PROJECT_ENDPOINT, FOUNDRY_TELEPHONY_CONNECTION_1, + * FOUNDRY_TELEPHONY_CONNECTION_2, FOUNDRY_TELEPHONY_NUMBER_1, and FOUNDRY_TELEPHONY_NUMBER_2 can override the test + * project defaults. + */ +@Execution(ExecutionMode.SAME_THREAD) +public class VoiceAgentTelephonyLiveTests { + private static final String DEFAULT_ENDPOINT + = "https://voice-first-agents-df-tip.services.ai.azure.com/api/projects/voice-first-agents-df-tip"; + private static final String DEFAULT_CONNECTION_1 = "twilio-sdk-testing-1"; + private static final String DEFAULT_CONNECTION_2 = "twilio-sdk-testing-2"; + private static final String DEFAULT_NUMBER_1 = "+13853864628"; + private static final String DEFAULT_NUMBER_2 = "+18509702029"; + private static final Duration CALL_TIMEOUT = Duration.ofMinutes(2); + private static final Duration POLL_INTERVAL = Duration.ofSeconds(2); + + @Test + @EnabledIfEnvironmentVariable(named = "AZURE_TEST_MODE", matches = "LIVE") + public void bindingLifecycleLive() { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT", DEFAULT_ENDPOINT); + String model = configuration.get("FOUNDRY_VOICE_MODEL_NAME"); + assertNotNull(model, "FOUNDRY_VOICE_MODEL_NAME is required for live telephony testing."); + String connection = configuration.get("FOUNDRY_TELEPHONY_CONNECTION_1", DEFAULT_CONNECTION_1); + String number = e164(configuration, "FOUNDRY_TELEPHONY_NUMBER_1", DEFAULT_NUMBER_1); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + AgentsClient agents = builder.buildAgentsClient(); + BetaVoiceAgentsTelephonyClient telephony = builder.beta().buildBetaVoiceAgentsTelephonyClient(); + String agentName = "test-telephony-binding-" + shortId(); + boolean agentCreated = false; + try { + agents.createAgentVersion(agentName, + new CreateAgentVersionInput(definition(model, "Greet the caller briefly, then say goodbye."))); + agentCreated = true; + TelephonyBinding binding = telephony.createTelephonyBinding(agentName, + new CreateTwilioTelephonyBindingInput(connection, number).setLabel("Java SDK live test")); + TelephonyBindingListItem listedBinding = findBinding(telephony, agentName, binding.getId()); + assertNotNull(listedBinding.getETag()); + + TelephonyBinding retrieved = telephony.getTelephonyBinding(agentName, binding.getId()); + assertEquals(binding.getId(), retrieved.getId()); + TelephonyBinding updated = telephony.updateTelephonyBinding(agentName, binding.getId(), + listedBinding.getETag(), new UpdateTelephonyBindingInput().setLabel("Updated Java SDK live test")); + assertEquals("Updated Java SDK live test", updated.getLabel()); + + String updatedEtag = findBinding(telephony, agentName, binding.getId()).getETag(); + assertNotNull(updatedEtag); + telephony.deleteTelephonyBinding(agentName, binding.getId(), updatedEtag); + assertTrue(telephony.listTelephonyBindings(agentName) + .stream() + .noneMatch(item -> binding.getId().equals(item.getId()))); + } finally { + if (agentCreated) { + safeCleanup("delete binding test agent", () -> agents.deleteAgent(agentName)); + } + } + } + + @Test + @EnabledIfEnvironmentVariable(named = "AZURE_TEST_MODE", matches = "LIVE") + public void twilioBindingAndOutboundCallLive() throws InterruptedException { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT", DEFAULT_ENDPOINT); + String model = configuration.get("FOUNDRY_VOICE_MODEL_NAME"); + assertNotNull(model, "FOUNDRY_VOICE_MODEL_NAME is required for live telephony testing."); + String connection1 = configuration.get("FOUNDRY_TELEPHONY_CONNECTION_1", DEFAULT_CONNECTION_1); + String connection2 = configuration.get("FOUNDRY_TELEPHONY_CONNECTION_2", DEFAULT_CONNECTION_2); + String number1 = e164(configuration, "FOUNDRY_TELEPHONY_NUMBER_1", DEFAULT_NUMBER_1); + String number2 = e164(configuration, "FOUNDRY_TELEPHONY_NUMBER_2", DEFAULT_NUMBER_2); + + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint) + .credential(new DefaultAzureCredentialBuilder().build()) + .allowPreview(true); + AgentsClient agents = builder.buildAgentsClient(); + BetaVoiceAgentsTelephonyClient telephony = builder.beta().buildBetaVoiceAgentsTelephonyClient(); + String suffix = UUID.randomUUID().toString(); + String inboundAgent = "test-telephony-inbound-" + suffix; + String outboundAgent = "test-telephony-outbound-" + suffix; + String callJobId = null; + String scheduledCallJobId = null; + String inboundCallId = null; + boolean inboundAgentCreated = false; + boolean outboundAgentCreated = false; + try { + agents.createAgentVersion(inboundAgent, + new CreateAgentVersionInput(definition(model, "Greet the caller briefly, then say goodbye."))); + inboundAgentCreated = true; + agents.createAgentVersion(outboundAgent, + new CreateAgentVersionInput(definition(model, "Say hello, wait for one reply, then say goodbye."))); + outboundAgentCreated = true; + + TelephonyBinding binding = telephony.createTelephonyBinding(inboundAgent, + new CreateTwilioTelephonyBindingInput(connection1, number1).setLabel("Java SDK live test")); + assertNotNull(binding.getId()); + assertEquals(TelephonyProvider.TWILIO, binding.getProvider()); + assertEquals(TelephonyBindingStatus.ACTIVE, binding.getStatus()); + assertNotNull(binding.getIncomingCallUrl()); + + Response initialTargetsResponse + = telephony.getTelephonyTransferTargetsWithResponse(inboundAgent, new RequestOptions()); + TelephonyTransferTargets initialTargets + = initialTargetsResponse.getValue().toObject(TelephonyTransferTargets.class); + assertTrue(initialTargets.getTransferTargets().isEmpty()); + TelephonyTransferTarget transferTarget = new TelephonyTransferTarget("test_number_2", + "Java SDK live test target", new PstnTelephonyTransferDestination(number2)); + TelephonyTransferTargets replacedTargets = telephony.replaceTelephonyTransferTargets(inboundAgent, + requireEtag(initialTargetsResponse, "telephony transfer targets"), + Collections.singletonList(transferTarget)); + assertEquals(1, replacedTargets.getTransferTargets().size()); + + CreateTelephonyCallJobInput request = new CreateTelephonyCallJobInput( + new TelephonyOutboundDestination(TelephonyOutboundDestinationType.PHONE_NUMBER, number1), connection2, + number2).setPurpose("Java SDK live telephony validation"); + TelephonyCallJob job + = telephony.createTelephonyCallJob(outboundAgent, UUID.randomUUID().toString(), request); + callJobId = job.getId(); + assertNotNull(callJobId); + assertEquals(outboundAgent, job.getAgentName()); + assertEquals(connection2, job.getConnectionName()); + assertEquals(number2, job.getSource()); + + TelephonyCallSummary inboundCall = waitForInboundCall(telephony, inboundAgent); + inboundCallId = inboundCall.getId(); + assertNotNull(inboundCallId); + assertEquals(TelephonyProvider.TWILIO, inboundCall.getProvider()); + assertEquals(number2, inboundCall.getCallerNumber()); + assertEquals(number1, inboundCall.getProviderNumber()); + + TelephonyCallRecord callRecord = telephony.getTelephonyCall(inboundAgent, inboundCallId); + assertEquals(inboundCallId, callRecord.getId()); + TelephonyCallRecord transferredCall + = telephony.transferTelephonyCall(inboundAgent, inboundCallId, "test_number_2"); + assertEquals(inboundCallId, transferredCall.getId()); + inboundCallId = null; + + waitForDispatchedCallJob(telephony, outboundAgent, callJobId); + + OffsetDateTime notBefore = OffsetDateTime.now().plusMinutes(10); + CreateTelephonyCallJobInput scheduledRequest = new CreateTelephonyCallJobInput( + new TelephonyOutboundDestination(TelephonyOutboundDestinationType.PHONE_NUMBER, number1), connection2, + number2).setPurpose("Java SDK live cancellation validation") + .setSchedule( + new TelephonyCallJobSchedule().setNotBefore(notBefore).setExpiresAt(notBefore.plusMinutes(10))); + TelephonyCallJob scheduledJob + = telephony.createTelephonyCallJob(outboundAgent, UUID.randomUUID().toString(), scheduledRequest); + scheduledCallJobId = scheduledJob.getId(); + TelephonyCallJob cancelledJob = telephony.cancelTelephonyCallJob(outboundAgent, scheduledCallJobId, + Long.toString(scheduledJob.getRevision())); + assertNotNull(cancelledJob.getCancellation()); + scheduledCallJobId = null; + + telephony.replaceTelephonyTransferTargets(inboundAgent, getTransferTargetsEtag(telephony, inboundAgent), + Collections.emptyList()); + } finally { + if (inboundCallId != null) { + String callId = inboundCallId; + safeCleanup("end inbound call", () -> telephony.endTelephonyCall(inboundAgent, callId)); + } + if (callJobId != null) { + String jobId = callJobId; + safeCleanup("cancel outbound call job", () -> { + TelephonyCallJob currentJob = telephony.getTelephonyCallJob(outboundAgent, jobId); + telephony.cancelTelephonyCallJob(outboundAgent, jobId, Long.toString(currentJob.getRevision())); + }); + } + if (scheduledCallJobId != null) { + String jobId = scheduledCallJobId; + safeCleanup("cancel scheduled outbound call job", () -> { + TelephonyCallJob scheduledJob = telephony.getTelephonyCallJob(outboundAgent, jobId); + telephony.cancelTelephonyCallJob(outboundAgent, jobId, Long.toString(scheduledJob.getRevision())); + }); + } + if (inboundAgentCreated) { + safeCleanup("clear telephony transfer targets", + () -> telephony.replaceTelephonyTransferTargets(inboundAgent, + getTransferTargetsEtag(telephony, inboundAgent), Collections.emptyList())); + } + if (outboundAgentCreated) { + safeCleanup("delete outbound agent", () -> agents.deleteAgent(outboundAgent)); + } + if (inboundAgentCreated) { + safeCleanup("delete inbound agent", () -> agents.deleteAgent(inboundAgent)); + } + } + } + + private static TelephonyCallSummary waitForInboundCall(BetaVoiceAgentsTelephonyClient telephony, String agentName) + throws InterruptedException { + long deadline = System.nanoTime() + CALL_TIMEOUT.toNanos(); + while (System.nanoTime() < deadline) { + for (TelephonyCallSummary call : telephony.listTelephonyCalls(agentName)) { + return call; + } + Thread.sleep(POLL_INTERVAL.toMillis()); + } + throw new AssertionError("No inbound Twilio call arrived within " + CALL_TIMEOUT + "."); + } + + private static void waitForDispatchedCallJob(BetaVoiceAgentsTelephonyClient telephony, String agentName, + String callJobId) throws InterruptedException { + long deadline = System.nanoTime() + CALL_TIMEOUT.toNanos(); + while (System.nanoTime() < deadline) { + TelephonyCallJob callJob = telephony.getTelephonyCallJob(agentName, callJobId); + if (callJob.getAttemptCount() > 0) { + return; + } + Thread.sleep(POLL_INTERVAL.toMillis()); + } + throw new AssertionError("The outbound call job did not create an attempt within " + CALL_TIMEOUT + "."); + } + + private static VoiceAgentDefinition definition(String model, String instructions) { + return new VoiceAgentDefinition().setModelType(VoiceModelType.MANAGED) + .setModel(model) + .setInstructions(instructions) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setAudio(new VoiceAgentAudioConfiguration() + .setOutput(new VoiceAgentAudioOutputConfiguration().setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD))); + } + + private static String e164(Configuration configuration, String name, String defaultValue) { + String value = configuration.get(name, defaultValue); + assertNotNull(value, name + " is required for live telephony testing."); + value = value.trim(); + assertTrue(value.matches("^\\+[1-9]\\d{7,14}$"), name + " must be an E.164 number such as +14255550123."); + return value; + } + + private static String shortId() { + return UUID.randomUUID().toString().replace("-", "").substring(0, 12); + } + + private static TelephonyBindingListItem findBinding(BetaVoiceAgentsTelephonyClient telephony, String agentName, + String bindingId) { + return telephony.listTelephonyBindings(agentName) + .stream() + .filter(item -> bindingId.equals(item.getId())) + .findFirst() + .orElseThrow(() -> new AssertionError("Created binding was not listed.")); + } + + private static String getTransferTargetsEtag(BetaVoiceAgentsTelephonyClient telephony, String agentName) { + return requireEtag(telephony.getTelephonyTransferTargetsWithResponse(agentName, new RequestOptions()), + "telephony transfer targets"); + } + + private static String requireEtag(Response response, String resource) { + String etag = response.getHeaders().getValue(HttpHeaderName.ETAG); + assertNotNull(etag, "The service did not return an ETag for " + resource + "."); + return etag; + } + + private static void safeCleanup(String action, Runnable cleanup) { + try { + cleanup.run(); + } catch (RuntimeException exception) { + System.err.printf("Failed to %s: %s%n", action, exception.getMessage()); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentTelephonyTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentTelephonyTests.java new file mode 100644 index 000000000000..1505651aa71d --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentTelephonyTests.java @@ -0,0 +1,415 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentsTelephonyAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentsTelephonyClient; +import com.azure.ai.agents.models.CreateTelephonyCallJobInput; +import com.azure.ai.agents.models.CreateTwilioTelephonyBindingInput; +import com.azure.ai.agents.models.TelephonyBinding; +import com.azure.ai.agents.models.TelephonyBindingListItem; +import com.azure.ai.agents.models.TelephonyBindingStatus; +import com.azure.ai.agents.models.TelephonyCallRecord; +import com.azure.ai.agents.models.TelephonyCallJob; +import com.azure.ai.agents.models.TelephonyCallJobStatus; +import com.azure.ai.agents.models.TelephonyCallSummary; +import com.azure.ai.agents.models.TelephonyOutboundDestination; +import com.azure.ai.agents.models.TelephonyOutboundDestinationType; +import com.azure.ai.agents.models.TelephonyTransferTargets; +import com.azure.ai.agents.models.UpdateTelephonyBindingInput; +import com.azure.core.exception.HttpResponseException; +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.test.utils.MockTokenCredential; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Mono; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Sync/async parity for the Python telephony suites. These are HTTP contract tests, not recordings or live calls. + * Python currently skips these scenarios for service routing/version issues. No provider account is needed here. + */ +public class VoiceAgentTelephonyTests { + private static final String AGENT = "voice-telephony-test"; + private static final String MISSING = "nonexistent-id"; + private static final String ROOT = "/agents/" + AGENT + "/telephony"; + private static final String CONNECTION_1 = "twilio-sdk-testing-1"; + private static final String CONNECTION_2 = "twilio-sdk-testing-2"; + private static final String NUMBER_1 = "+13853864628"; + private static final String NUMBER_2 = "+18509702029"; + private static final HttpHeaderName IDEMPOTENCY_KEY = HttpHeaderName.fromString("Idempotency-Key"); + private static final Duration TIMEOUT = Duration.ofSeconds(10); + private static final String EMPTY_PAGE = "{\"data\":[],\"has_more\":false}"; + private static final String EMPTY_TARGETS = "{\"transfer_targets\":[]}"; + private static final String TARGETS = "{\"transfer_targets\":[{\"name\":\"sales_desk\"," + + "\"description\":\"Transfers to the sales desk for pricing questions.\"," + + "\"destination\":{\"kind\":\"pstn\",\"value\":\"+14255550123\"}}]}"; + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void createsTwilioBindingAndOutboundCallJob(boolean async) { + String bindingRequest = "{\"connection_name\":\"" + CONNECTION_1 + "\",\"label\":\"Java SDK test\"," + + "\"phone_number\":\"" + NUMBER_1 + "\",\"provider\":\"twilio\"}"; + String bindingResponse = "{\"provider\":\"twilio\",\"id\":\"binding-1\",\"label\":\"Java SDK test\"," + + "\"status\":\"active\",\"incoming_call_url\":\"https://example.test/incoming\"," + + "\"connection_name\":\"" + CONNECTION_1 + "\",\"phone_number\":\"" + NUMBER_1 + "\"}"; + String jobRequest = "{\"destination\":{\"type\":\"phone_number\",\"value\":\"" + NUMBER_1 + "\"}," + + "\"connection_name\":\"" + CONNECTION_2 + "\",\"source\":\"" + NUMBER_2 + "\"," + + "\"purpose\":\"Java SDK telephony validation\"}"; + String jobResponse = "{\"destination\":{\"type\":\"phone_number\",\"value\":\"" + NUMBER_1 + "\"}," + + "\"connection_name\":\"" + CONNECTION_2 + "\",\"source\":\"" + NUMBER_2 + "\"," + + "\"purpose\":\"Java SDK telephony validation\",\"id\":\"job-1\"," + + "\"object\":\"telephony.call_job\",\"agent_name\":\"" + AGENT + "\",\"status\":\"accepted\"," + + "\"retry_policy\":{\"max_attempts\":1},\"attempt_count\":0,\"revision\":1," + + "\"created_at\":1,\"updated_at\":1}"; + String cancelledJobResponse = jobResponse.replace("\"status\":\"accepted\"", "\"status\":\"cancelled\"") + .replace("\"revision\":1", "\"revision\":2"); + ScriptedTransport transport = new ScriptedTransport(async); + transport.expect(HttpMethod.POST, ROOT + "/bindings", bindingRequest, 201, bindingResponse); + transport.expect(HttpMethod.POST, ROOT + "/call_jobs", jobRequest, + header(IDEMPOTENCY_KEY, "offline-idempotency-key"), 202, jobResponse, new HttpHeaders()); + transport.expect(HttpMethod.GET, ROOT + "/call_jobs/job-1", null, 200, jobResponse); + transport.expect(HttpMethod.POST, ROOT + "/call_jobs/job-1:cancel", null, header(HttpHeaderName.IF_MATCH, "1"), + 200, cancelledJobResponse, new HttpHeaders()); + AgentsClientBuilder builder = builder(transport); + BetaVoiceAgentsTelephonyClient syncClient = builder.beta().buildBetaVoiceAgentsTelephonyClient(); + BetaVoiceAgentsTelephonyAsyncClient asyncClient = builder.beta().buildBetaVoiceAgentsTelephonyAsyncClient(); + + CreateTwilioTelephonyBindingInput bindingRequestModel + = new CreateTwilioTelephonyBindingInput(CONNECTION_1, NUMBER_1).setLabel("Java SDK test"); + TelephonyBinding binding = call(async, () -> syncClient.createTelephonyBinding(AGENT, bindingRequestModel), + () -> asyncClient.createTelephonyBinding(AGENT, bindingRequestModel)); + assertEquals("binding-1", binding.getId()); + assertEquals(TelephonyBindingStatus.ACTIVE, binding.getStatus()); + + CreateTelephonyCallJobInput jobRequestModel = new CreateTelephonyCallJobInput( + new TelephonyOutboundDestination(TelephonyOutboundDestinationType.PHONE_NUMBER, NUMBER_1), CONNECTION_2, + NUMBER_2).setPurpose("Java SDK telephony validation"); + TelephonyCallJob job + = call(async, () -> syncClient.createTelephonyCallJob(AGENT, "offline-idempotency-key", jobRequestModel), + () -> asyncClient.createTelephonyCallJob(AGENT, "offline-idempotency-key", jobRequestModel)); + assertEquals("job-1", job.getId()); + assertEquals(TelephonyCallJobStatus.ACCEPTED, job.getStatus()); + assertEquals(1L, job.getRevision()); + assertEquals("job-1", call(async, () -> syncClient.getTelephonyCallJob(AGENT, "job-1"), + () -> asyncClient.getTelephonyCallJob(AGENT, "job-1")).getId()); + TelephonyCallJob cancelled = call(async, () -> syncClient.cancelTelephonyCallJob(AGENT, "job-1", "1"), + () -> asyncClient.cancelTelephonyCallJob(AGENT, "job-1", "1")); + assertEquals(TelephonyCallJobStatus.CANCELLED, cancelled.getStatus()); + assertEquals(2L, cancelled.getRevision()); + transport.assertComplete(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void bindingTransferTargetAndCallLifecycle(boolean async) { + String bindingPath = ROOT + "/bindings/binding-1"; + String binding = "{\"provider\":\"twilio\",\"id\":\"binding-1\",\"label\":\"Java SDK test\"," + + "\"status\":\"active\",\"incoming_call_url\":\"https://example.test/incoming\"," + + "\"connection_name\":\"" + CONNECTION_1 + "\",\"phone_number\":\"" + NUMBER_1 + "\"}"; + String listedBinding = binding.substring(0, binding.length() - 1) + ",\"etag\":\"binding-etag\"}"; + String updatedBinding = binding.replace("Java SDK test", "Updated Java SDK test"); + String callPath = ROOT + "/calls/call-1"; + String activeCall = callRecord("in_progress", "bridging"); + String endedCall = callRecord("success", "completed"); + ScriptedTransport transport = new ScriptedTransport(async); + transport.expect(HttpMethod.GET, bindingPath, null, 200, binding); + transport.expect(HttpMethod.GET, ROOT + "/bindings", null, 200, + "{\"data\":[" + listedBinding + "],\"has_more\":false}"); + transport.expect(HttpMethod.PATCH, bindingPath, "{\"status\":\"active\",\"label\":\"Updated Java SDK test\"}", + header(HttpHeaderName.IF_MATCH, "*"), 200, updatedBinding, new HttpHeaders()); + transport.expect(HttpMethod.GET, ROOT + "/transfer_targets", null, Collections.emptyMap(), 200, EMPTY_TARGETS, + new HttpHeaders().set(HttpHeaderName.ETAG, "targets-etag")); + transport.expect(HttpMethod.PUT, ROOT + "/transfer_targets", TARGETS, + header(HttpHeaderName.IF_MATCH, "targets-etag"), 200, TARGETS, + new HttpHeaders().set(HttpHeaderName.ETAG, "updated-targets-etag")); + transport.expect(HttpMethod.GET, ROOT + "/calls", null, 200, + "{\"data\":[" + activeCall + "],\"has_more\":false}"); + transport.expect(HttpMethod.GET, callPath, null, 200, activeCall); + transport.expect(HttpMethod.POST, callPath + ":transfer", "{\"target\":\"sales_desk\"}", 200, activeCall); + transport.expect(HttpMethod.POST, callPath + ":end", null, 200, endedCall); + transport.expect(HttpMethod.DELETE, bindingPath, null, header(HttpHeaderName.IF_MATCH, "*"), 204, null, + new HttpHeaders()); + AgentsClientBuilder builder = builder(transport); + BetaVoiceAgentsTelephonyClient syncClient = builder.beta().buildBetaVoiceAgentsTelephonyClient(); + BetaVoiceAgentsTelephonyAsyncClient asyncClient = builder.beta().buildBetaVoiceAgentsTelephonyAsyncClient(); + + assertEquals("binding-1", call(async, () -> syncClient.getTelephonyBinding(AGENT, "binding-1"), + () -> asyncClient.getTelephonyBinding(AGENT, "binding-1")).getId()); + TelephonyBindingListItem listed = async + ? asyncClient.listTelephonyBindings(AGENT).blockFirst(TIMEOUT) + : syncClient.listTelephonyBindings(AGENT).iterator().next(); + assertNotNull(listed); + assertEquals("binding-etag", listed.getETag()); + UpdateTelephonyBindingInput update = new UpdateTelephonyBindingInput().setStatus(TelephonyBindingStatus.ACTIVE) + .setLabel("Updated Java SDK test"); + assertEquals("Updated Java SDK test", + call(async, () -> syncClient.updateTelephonyBinding(AGENT, "binding-1", "*", update), + () -> asyncClient.updateTelephonyBinding(AGENT, "binding-1", "*", update)).getLabel()); + assertTrue(call(async, () -> syncClient.getTelephonyTransferTargets(AGENT), + () -> asyncClient.getTelephonyTransferTargets(AGENT)).getTransferTargets().isEmpty()); + TelephonyTransferTargets desired = BinaryData.fromString(TARGETS).toObject(TelephonyTransferTargets.class); + assertTargets(call(async, + () -> syncClient.replaceTelephonyTransferTargets(AGENT, "targets-etag", desired.getTransferTargets()), + () -> asyncClient.replaceTelephonyTransferTargets(AGENT, "targets-etag", desired.getTransferTargets()))); + TelephonyCallSummary summary = async + ? asyncClient.listTelephonyCalls(AGENT).blockFirst(TIMEOUT) + : syncClient.listTelephonyCalls(AGENT).iterator().next(); + assertNotNull(summary); + assertEquals("call-1", summary.getId()); + assertEquals("call-1", call(async, () -> syncClient.getTelephonyCall(AGENT, "call-1"), + () -> asyncClient.getTelephonyCall(AGENT, "call-1")).getId()); + TelephonyCallRecord transferred + = call(async, () -> syncClient.transferTelephonyCall(AGENT, "call-1", "sales_desk"), + () -> asyncClient.transferTelephonyCall(AGENT, "call-1", "sales_desk")); + assertEquals("call-1", transferred.getId()); + assertEquals("success", call(async, () -> syncClient.endTelephonyCall(AGENT, "call-1"), + () -> asyncClient.endTelephonyCall(AGENT, "call-1")).getStatus().toString()); + call(async, () -> { + syncClient.deleteTelephonyBinding(AGENT, "binding-1", "*"); + return null; + }, () -> asyncClient.deleteTelephonyBinding(AGENT, "binding-1", "*")); + transport.assertComplete(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void bindingsAndTransferTargets(boolean async) { + ScriptedTransport transport = new ScriptedTransport(async); + transport.expect(HttpMethod.GET, ROOT + "/bindings", null, 200, EMPTY_PAGE); + transport.expect(HttpMethod.GET, ROOT + "/transfer_targets", null, Collections.emptyMap(), 200, EMPTY_TARGETS, + new HttpHeaders().set(HttpHeaderName.ETAG, "targets-etag-1")); + transport.expect(HttpMethod.PUT, ROOT + "/transfer_targets", TARGETS, + header(HttpHeaderName.IF_MATCH, "targets-etag-1"), 200, TARGETS, + new HttpHeaders().set(HttpHeaderName.ETAG, "targets-etag-2")); + transport.expect(HttpMethod.GET, ROOT + "/transfer_targets", null, Collections.emptyMap(), 200, TARGETS, + new HttpHeaders().set(HttpHeaderName.ETAG, "targets-etag-2")); + transport.expect(HttpMethod.PUT, ROOT + "/transfer_targets", EMPTY_TARGETS, + header(HttpHeaderName.IF_MATCH, "targets-etag-2"), 200, EMPTY_TARGETS, + new HttpHeaders().set(HttpHeaderName.ETAG, "targets-etag-3")); + transport.notFound(HttpMethod.GET, ROOT + "/bindings/" + MISSING, null); + transport.notFound(HttpMethod.PATCH, ROOT + "/bindings/" + MISSING, "{\"status\":\"suspended\"}"); + transport.notFound(HttpMethod.DELETE, ROOT + "/bindings/" + MISSING, null); + AgentsClientBuilder builder = builder(transport); + BetaVoiceAgentsTelephonyClient syncClient = builder.beta().buildBetaVoiceAgentsTelephonyClient(); + BetaVoiceAgentsTelephonyAsyncClient asyncClient = builder.beta().buildBetaVoiceAgentsTelephonyAsyncClient(); + assertEquals(0L, + async + ? asyncClient.listTelephonyBindings(AGENT).count().block(TIMEOUT).longValue() + : syncClient.listTelephonyBindings(AGENT).stream().count()); + assertTrue(call(async, () -> syncClient.getTelephonyTransferTargets(AGENT), + () -> asyncClient.getTelephonyTransferTargets(AGENT)).getTransferTargets().isEmpty()); + TelephonyTransferTargets desired = BinaryData.fromString(TARGETS).toObject(TelephonyTransferTargets.class); + TelephonyTransferTargets replaced = call(async, + () -> syncClient.replaceTelephonyTransferTargets(AGENT, "targets-etag-1", desired.getTransferTargets()), + () -> asyncClient.replaceTelephonyTransferTargets(AGENT, "targets-etag-1", desired.getTransferTargets())); + assertTargets(replaced); + assertTargets(call(async, () -> syncClient.getTelephonyTransferTargets(AGENT), + () -> asyncClient.getTelephonyTransferTargets(AGENT))); + assertTrue(call(async, + () -> syncClient.replaceTelephonyTransferTargets(AGENT, "targets-etag-2", Collections.emptyList()), + () -> asyncClient.replaceTelephonyTransferTargets(AGENT, "targets-etag-2", Collections.emptyList())) + .getTransferTargets() + .isEmpty()); + assertNotFound(() -> call(async, () -> syncClient.getTelephonyBinding(AGENT, MISSING), + () -> asyncClient.getTelephonyBinding(AGENT, MISSING)), true); + UpdateTelephonyBindingInput update + = new UpdateTelephonyBindingInput().setStatus(TelephonyBindingStatus.SUSPENDED); + assertNotFound(() -> call(async, () -> syncClient.updateTelephonyBinding(AGENT, MISSING, null, update), + () -> asyncClient.updateTelephonyBinding(AGENT, MISSING, null, update)), true); + assertNotFound(() -> call(async, () -> { + syncClient.deleteTelephonyBinding(AGENT, MISSING, null); + return null; + }, () -> asyncClient.deleteTelephonyBinding(AGENT, MISSING, null)), true); + transport.assertComplete(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void callsNotFound(boolean async) { + ScriptedTransport transport = new ScriptedTransport(async); + transport.expect(HttpMethod.GET, ROOT + "/calls", null, 200, EMPTY_PAGE); + transport.notFound(HttpMethod.GET, ROOT + "/calls/" + MISSING, null); + transport.notFound(HttpMethod.POST, ROOT + "/calls/" + MISSING + ":transfer", + "{\"target\":\"nonexistent-target\"}"); + transport.notFound(HttpMethod.POST, ROOT + "/calls/" + MISSING + ":end", null); + AgentsClientBuilder builder = builder(transport); + BetaVoiceAgentsTelephonyClient syncClient = builder.beta().buildBetaVoiceAgentsTelephonyClient(); + BetaVoiceAgentsTelephonyAsyncClient asyncClient = builder.beta().buildBetaVoiceAgentsTelephonyAsyncClient(); + assertEquals(0L, + async + ? asyncClient.listTelephonyCalls(AGENT).count().block(TIMEOUT).longValue() + : syncClient.listTelephonyCalls(AGENT).stream().count()); + assertNotFound(() -> call(async, () -> syncClient.getTelephonyCall(AGENT, MISSING), + () -> asyncClient.getTelephonyCall(AGENT, MISSING)), true); + assertNotFound(() -> call(async, () -> syncClient.transferTelephonyCall(AGENT, MISSING, "nonexistent-target"), + () -> asyncClient.transferTelephonyCall(AGENT, MISSING, "nonexistent-target")), false); + assertNotFound(() -> call(async, () -> syncClient.endTelephonyCall(AGENT, MISSING), + () -> asyncClient.endTelephonyCall(AGENT, MISSING)), false); + transport.assertComplete(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void generatedAudioNotFound(boolean async) { + ScriptedTransport transport = new ScriptedTransport(async); + String path = "/agents/" + AGENT + "/endpoint/protocols/voice/conversations/" + MISSING + "/items/" + MISSING + + "/audio/generated"; + transport.notFound(HttpMethod.GET, path, null); + transport.notFound(HttpMethod.GET, path + "/content", null); + AgentsClientBuilder builder = builder(transport); + assertNotFound(() -> call(async, + () -> builder.beta() + .buildBetaVoiceAgentsConversationsClient() + .getAgentConversationGeneratedAudioItem(AGENT, MISSING, MISSING), + () -> builder.beta() + .buildBetaVoiceAgentsConversationsAsyncClient() + .getAgentConversationGeneratedAudioItem(AGENT, MISSING, MISSING)), + true); + assertNotFound(() -> call(async, + () -> builder.beta() + .buildBetaVoiceAgentsConversationsClient() + .downloadAgentConversationGeneratedAudioItem(AGENT, MISSING, MISSING), + () -> builder.beta() + .buildBetaVoiceAgentsConversationsAsyncClient() + .downloadAgentConversationGeneratedAudioItem(AGENT, MISSING, MISSING)), + false); + transport.assertComplete(); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void callJobNotFound(boolean async) { + ScriptedTransport transport = new ScriptedTransport(async); + transport.notFound(HttpMethod.GET, ROOT + "/call_jobs/" + MISSING, null); + transport.notFound(HttpMethod.POST, ROOT + "/call_jobs/" + MISSING + ":cancel", null); + AgentsClientBuilder builder = builder(transport); + BetaVoiceAgentsTelephonyClient syncClient = builder.beta().buildBetaVoiceAgentsTelephonyClient(); + BetaVoiceAgentsTelephonyAsyncClient asyncClient = builder.beta().buildBetaVoiceAgentsTelephonyAsyncClient(); + assertNotFound(() -> call(async, () -> syncClient.getTelephonyCallJob(AGENT, MISSING), + () -> asyncClient.getTelephonyCallJob(AGENT, MISSING)), true); + assertNotFound(() -> call(async, () -> syncClient.cancelTelephonyCallJob(AGENT, MISSING, null), + () -> asyncClient.cancelTelephonyCallJob(AGENT, MISSING, null)), true); + transport.assertComplete(); + } + + private static void assertTargets(TelephonyTransferTargets targets) { + assertNotNull(targets); + assertEquals(1, targets.getTransferTargets().size()); + assertEquals("sales_desk", targets.getTransferTargets().get(0).getName()); + assertEquals("pstn", targets.getTransferTargets().get(0).getDestination().getKind().toString()); + } + + private static Map header(HttpHeaderName name, String value) { + return Collections.singletonMap(name, value); + } + + private static String callRecord(String status, String phase) { + return "{\"id\":\"call-1\",\"provider\":\"twilio\",\"status\":\"" + status + "\",\"phase\":\"" + phase + + "\",\"started_at\":1,\"events\":[],\"events_truncated\":false,\"caller_number\":\"" + NUMBER_2 + + "\",\"provider_number\":\"" + NUMBER_1 + "\"}"; + } + + private static T call(boolean async, Supplier syncCall, Supplier> asyncCall) { + return async ? asyncCall.get().block(TIMEOUT) : syncCall.get(); + } + + private static void assertNotFound(Runnable operation, boolean typed) { + HttpResponseException error = assertThrows(HttpResponseException.class, operation::run); + assertEquals(404, error.getResponse().getStatusCode()); + if (typed) { + assertInstanceOf(ResourceNotFoundException.class, error); + } + } + + private static AgentsClientBuilder builder(HttpClient transport) { + return new AgentsClientBuilder().endpoint("https://localhost") + .credential(new MockTokenCredential()) + .httpClient(transport) + .allowPreview(true); + } + + private static final class ScriptedTransport implements HttpClient { + private final boolean async; + private final Deque> requests = new ArrayDeque<>(); + + ScriptedTransport(boolean async) { + this.async = async; + } + + void notFound(HttpMethod method, String path, String body) { + expect(method, path, body, 404, "{\"error\":{\"code\":\"NotFound\",\"message\":\"Resource not found\"}}"); + } + + void expect(HttpMethod method, String path, String body, int status, String response) { + expect(method, path, body, Collections.emptyMap(), status, response, new HttpHeaders()); + } + + void expect(HttpMethod method, String path, String body, Map expectedHeaders, + int status, String response, HttpHeaders responseHeaders) { + requests.add(request -> { + assertEquals(method, request.getHttpMethod()); + assertEquals(path, request.getUrl().getPath()); + for (Map.Entry header : expectedHeaders.entrySet()) { + assertEquals(header.getValue(), request.getHeaders().getValue(header.getKey())); + } + if (!expectedHeaders.containsKey(HttpHeaderName.IF_MATCH)) { + assertNull(request.getHeaders().getValue(HttpHeaderName.IF_MATCH)); + } + if (body != null) { + assertEquals(BinaryData.fromString(body).toObject(Map.class), + request.getBodyAsBinaryData().toObject(Map.class)); + } + responseHeaders.set(HttpHeaderName.CONTENT_TYPE, "application/json"); + byte[] responseBody = response == null ? new byte[0] : BinaryData.fromString(response).toBytes(); + return new MockHttpResponse(request, status, responseHeaders, responseBody); + }); + } + + private HttpResponse respond(HttpRequest request) { + assertFalse(requests.isEmpty(), "Unexpected request: " + request.getUrl()); + return requests.removeFirst().apply(request); + } + + @Override + public Mono send(HttpRequest request) { + assertTrue(async, "Sync test must use synchronous HTTP."); + return Mono.fromSupplier(() -> respond(request)); + } + + @Override + public HttpResponse sendSync(HttpRequest request, Context context) { + assertFalse(async, "Async test must use asynchronous HTTP."); + return respond(request); + } + + void assertComplete() { + assertTrue(requests.isEmpty(), "All telephony operations must be called."); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionTests.java new file mode 100644 index 000000000000..b6d8e4ec31c3 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/voice/VoiceAgentWebSocketSessionTests.java @@ -0,0 +1,1111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentWebSocketAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHttpResponse; +import com.azure.ai.agents.models.RawRealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeResponseCreateEvent; +import com.azure.ai.agents.models.RealtimeSessionCreatedEvent; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.VoiceAgentWarningEvent; +import com.azure.ai.agents.models.VoiceAgentTransport; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.ai.agents.models.VoiceAgentWebSocketOverflowStrategy; +import com.azure.core.credential.AccessToken; +import com.azure.core.credential.TokenCredential; +import com.azure.core.exception.ResourceModifiedException; +import com.azure.core.util.BinaryData; +import com.azure.core.util.ClientOptions; +import com.azure.core.util.Configuration; +import com.azure.core.util.Header; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame; +import io.netty.handler.codec.http.websocketx.ContinuationWebSocketFrame; +import io.netty.handler.codec.http.websocketx.PingWebSocketFrame; +import io.netty.handler.codec.http.websocketx.PongWebSocketFrame; +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; +import io.netty.handler.codec.http.websocketx.WebSocketFrame; +import java.io.InputStream; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.netty.DisposableServer; +import reactor.netty.http.Http11SslContextSpec; +import reactor.netty.http.server.HttpServer; +import reactor.netty.http.server.WebsocketServerSpec; +import reactor.test.StepVerifier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class VoiceAgentWebSocketSessionTests { + private static final String TRUST_STORE_PROPERTY = "javax.net.ssl.trustStore"; + private static final String TRUST_STORE_PASSWORD_PROPERTY = "javax.net.ssl.trustStorePassword"; + private static final String TRUST_STORE_TYPE_PROPERTY = "javax.net.ssl.trustStoreType"; + private static final String ORIGINAL_TRUST_STORE = System.getProperty(TRUST_STORE_PROPERTY); + private static final String ORIGINAL_TRUST_STORE_PASSWORD = System.getProperty(TRUST_STORE_PASSWORD_PROPERTY); + private static final String ORIGINAL_TRUST_STORE_TYPE = System.getProperty(TRUST_STORE_TYPE_PROPERTY); + private static final SSLContext ORIGINAL_SSL_CONTEXT = getDefaultSslContext(); + private static final TestCertificate TLS_CERTIFICATE = TestCertificate.create(); + + static { + TLS_CERTIFICATE.installTrustStore(); + } + + private DisposableServer server; + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void handshakeOverridesPreserveQueryAndSingleUserAgent(boolean async) { + AtomicReference requestUri = new AtomicReference<>(); + AtomicReference headers = new AtomicReference<>(); + server = tlsServer().host("localhost").port(0).handle((request, response) -> { + requestUri.set(request.uri()); + headers.set(request.requestHeaders().copy()); + return response.sendWebsocket((inbound, outbound) -> inbound.receive().then(), + WebsocketServerSpec.builder().protocols("realtime").build()); + }).bindNow(); + AgentsClientBuilder builder + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project/") + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))) + .configuration(Configuration.NONE); + for (String userAgentHeader : new String[] { "", "User-Agent", "user-agent" }) { + Map extra = new LinkedHashMap<>(); + extra.put("X-Custom", "custom-value"); + extra.put("Authorization", "must-not-override-token"); + extra.put("Sec-WebSocket-Protocol", "other"); + VoiceAgentWebSocketConnectionOptions options + = tlsOptions().setExtraQuery(Collections.singletonMap("foo", "bar value")); + if (!userAgentHeader.isEmpty()) { + extra.put(userAgentHeader, "custom-user-agent"); + options.setConnectionUrl(URI.create("wss://localhost:" + server.port() + "/custom?sig=abc")); + } + options.setExtraHeaders(extra); + if (async) { + BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent name", options) + .block(Duration.ofSeconds(5)); + session.closeAsync().block(Duration.ofSeconds(5)); + assertFalse(session.isOpen()); + } else { + BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent name", options); + session.close(); + assertFalse(session.isOpen()); + } + assertEquals("Bearer test-token", headers.get().get(HttpHeaderNames.AUTHORIZATION)); + assertEquals("realtime", headers.get().get(HttpHeaderNames.SEC_WEBSOCKET_PROTOCOL)); + assertEquals("VoiceAgents=V1Preview", headers.get().get("Foundry-Features")); + assertEquals("custom-value", headers.get().get("X-Custom")); + assertEquals(1, headers.get().getAll(HttpHeaderNames.USER_AGENT).size()); + String userAgent = headers.get().get(HttpHeaderNames.USER_AGENT); + String uri = decode(requestUri.get()); + assertTrue(uri.contains("api-version=v1")); + assertTrue(uri.contains("foo=bar value")); + assertEquals(1, requestUri.get().chars().filter(character -> character == '?').count()); + if (userAgentHeader.isEmpty()) { + assertTrue(userAgent.startsWith("azsdk-java-azure-ai-agents/"), userAgent); + assertTrue(uri.contains("x-ms-client-sdk=" + userAgent)); + assertTrue(uri.startsWith("/api/projects/project/agents/agent name/endpoint/protocols/voice?")); + } else { + assertEquals("custom-user-agent", userAgent); + assertTrue(uri.startsWith("/custom?")); + assertTrue(uri.contains("sig=abc")); + assertTrue(uri.contains("x-ms-client-sdk=azsdk-java-azure-ai-agents/"), uri); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void typedStringAndMappingSendsRejectInvalidJson(boolean async) { + List messages = new CopyOnWriteArrayList<>(); + server = startServer(messages, new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), + new AtomicReference<>(), new AtomicReference<>(), false); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); + String raw = "{\"type\": \"response.create\"}"; + BinaryData mapping = BinaryData.fromObject(Collections.singletonMap("type", "response.cancel")); + if (async) { + BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent", tlsOptions()) + .block(Duration.ofSeconds(5)); + try { + StepVerifier.create(session.sendEvent(BinaryData.fromString("not valid json"))) + .expectError(IllegalArgumentException.class) + .verify(Duration.ofSeconds(5)); + session.sendEvent(new RealtimeResponseCreateEvent()) + .then(session.sendEvent(BinaryData.fromString(raw))) + .then(session.sendEvent(mapping)) + .block(Duration.ofSeconds(5)); + StepVerifier.create(session.receiveEvents().take(3)).expectNextCount(3).verifyComplete(); + } finally { + session.close(); + } + } else { + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", tlsOptions())) { + assertThrows(IllegalArgumentException.class, + () -> session.sendEvent(BinaryData.fromString("not valid json"))); + session.sendEvent(new RealtimeResponseCreateEvent()); + session.sendEvent(BinaryData.fromString(raw)); + session.sendEvent(mapping); + Iterator events = session.receiveEvents(Duration.ofSeconds(5)).iterator(); + for (int index = 0; index < 3; index++) { + assertWarningEvent(events.next()); + } + } + } + assertEquals(3, messages.size()); + assertEquals("response.create", BinaryData.fromString(messages.get(0)).toObject(Map.class).get("type")); + assertEquals(raw, messages.get(1)); + assertEquals(mapping.toObject(Map.class), BinaryData.fromString(messages.get(2)).toObject(Map.class)); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void pingPongFramesAreNotApplicationEvents(boolean async) { + Flux frames = Flux.defer(() -> Flux.just(new PingWebSocketFrame(), new PongWebSocketFrame(), + new TextWebSocketFrame("{\"type\":\"session.created\",\"session\":{}}"), + new TextWebSocketFrame("{\"type\":\"future.event\",\"foo\":\"bar\"}"))); + server = tlsServer().host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket( + (inbound, outbound) -> outbound.sendObject(frames).then(inbound.receive().then()), + WebsocketServerSpec.builder().protocols("realtime").build())) + .bindNow(); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); + List events = new ArrayList<>(); + if (async) { + BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent", tlsOptions()) + .block(Duration.ofSeconds(5)); + try { + events.addAll(session.receiveEvents().take(2).collectList().block(Duration.ofSeconds(5))); + } finally { + session.close(); + } + } else { + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", tlsOptions())) { + Iterator iterator = session.receiveEvents(Duration.ofSeconds(5)).iterator(); + events.add(iterator.next()); + events.add(iterator.next()); + } + } + assertEquals(2, events.size()); + assertInstanceOf(RealtimeSessionCreatedEvent.class, events.get(0)); + RawRealtimeServerEvent unknown = assertInstanceOf(RawRealtimeServerEvent.class, events.get(1)); + assertEquals("bar", unknown.getRawEvent().toObject(Map.class).get("foo")); + } + + @Test + public void explicitDefaultPortOverrideIsTrustedBeforeAuthentication() { + AtomicInteger tokens = new AtomicInteger(); + IllegalStateException tokenError = new IllegalStateException("Stop before network access."); + AgentsClientBuilder builder + = new AgentsClientBuilder().endpoint("https://example.com/api/projects/project").credential(request -> { + tokens.incrementAndGet(); + return Mono.error(tokenError); + }); + VoiceAgentWebSocketConnectionOptions options + = new VoiceAgentWebSocketConnectionOptions().setConnectionUrl(URI.create("wss://example.com:443/custom")); + assertEquals(tokenError, assertThrows(IllegalStateException.class, + () -> builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", options))); + StepVerifier + .create(builder.beta().buildBetaVoiceAgentWebSocketAsyncClient().openWebSocketSession("agent", options)) + .expectErrorSatisfies(error -> assertEquals(tokenError, error)) + .verify(Duration.ofSeconds(5)); + assertEquals(2, tokens.get()); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void malformedEventsCanBeReportedAndSkipped(boolean async) { + AtomicInteger failures = new AtomicInteger(); + server = frameWebSocketServer(Flux.defer(() -> Flux.just(new TextWebSocketFrame("{broken"), + new BinaryWebSocketFrame(Unpooled.wrappedBuffer(new byte[] { (byte) 0xc3, 0x28 })), + new BinaryWebSocketFrame(Unpooled.copiedBuffer(warningJson(), StandardCharsets.UTF_8)), + new TextWebSocketFrame(warningJson())))); + VoiceAgentWebSocketConnectionOptions options + = tlsOptions().setMalformedEventHandler(error -> failures.incrementAndGet()); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); + if (async) { + BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent", options) + .block(Duration.ofSeconds(5)); + StepVerifier.create(session.receiveEvents()) + .assertNext(this::assertWarningEvent) + .assertNext(this::assertWarningEvent) + .verifyComplete(); + session.close(); + } else { + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", options)) { + Iterator events = session.receiveEvents(Duration.ofSeconds(5)).iterator(); + assertWarningEvent(events.next()); + assertWarningEvent(events.next()); + assertFalse(events.hasNext()); + } + } + assertEquals(2, failures.get()); + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void boundedQueuesHonorOverflowPolicies(boolean async) { + for (VoiceAgentWebSocketOverflowStrategy strategy : VoiceAgentWebSocketOverflowStrategy.values()) { + server = frameWebSocketServer(Flux.range(0, 4) + .map(index -> new TextWebSocketFrame("{\"type\":\"future.event\",\"index\":" + index + "}"))); + VoiceAgentWebSocketConnectionOptions options + = tlsOptions().setReceiveBufferCapacity(2).setOverflowStrategy(strategy); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); + List received = new ArrayList<>(); + boolean overflowError = strategy == VoiceAgentWebSocketOverflowStrategy.ERROR; + if (async) { + BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent", options) + .block(Duration.ofSeconds(5)); + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + while (session.isOpen()) { + Thread.yield(); + } + }); + if (overflowError) { + StepVerifier.create(session.receiveEvents()) + .expectNextCount(2) + .expectError(IllegalStateException.class) + .verify(); + } else { + received.addAll(session.receiveEvents().collectList().block(Duration.ofSeconds(5))); + } + session.close(); + } else { + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", options)) { + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + while (session.isOpen()) { + Thread.yield(); + } + }); + Iterator events = session.receiveEvents(Duration.ofSeconds(5)).iterator(); + if (overflowError) { + assertThrows(IllegalStateException.class, events::hasNext); + } else { + events.forEachRemaining(received::add); + } + } + } + if (!overflowError) { + assertEquals(2, received.size()); + int first = strategy == VoiceAgentWebSocketOverflowStrategy.DROP_OLDEST ? 2 : 0; + for (int index = 0; index < received.size(); index++) { + assertEquals(first + index, + ((RawRealtimeServerEvent) received.get(index)).getRawEvent().toObject(Map.class).get("index")); + } + } + server.disposeNow(); + } + } + + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void messageSizeLimitCannotBeBypassedByRecoveryHandler(boolean async) { + server = oneShotWebSocketServer(warningJson()); + AtomicBoolean recovered = new AtomicBoolean(); + VoiceAgentWebSocketConnectionOptions options + = tlsOptions().setMaxMessageSize(16).setMalformedEventHandler(error -> recovered.set(true)); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); + if (async) { + StepVerifier.create(builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent", options) + .flatMapMany(BetaVoiceAgentWebSocketSessionAsyncClient::receiveEvents)).expectError().verify(); + } else { + assertThrows(RuntimeException.class, () -> { + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", options)) { + session.receiveEvents(Duration.ofSeconds(5)).iterator().next(); + } + }); + } + assertFalse(recovered.get()); + } + + @Test + public void rawEventRoundTripsAndOptionsValidateBounds() throws Exception { + BinaryData payload = BinaryData.fromString("{\"type\":\"future.event\",\"nested\":{\"value\":42}}"); + RawRealtimeServerEvent event = new RawRealtimeServerEvent(payload); + RawRealtimeServerEvent copy = BinaryData.fromObject(event).toObject(RawRealtimeServerEvent.class); + assertEquals(payload.toObject(Map.class), copy.getRawEvent().toObject(Map.class)); + server = oneShotWebSocketServer("{\"type\":42,\"value\":1}"); + BetaVoiceAgentWebSocketSessionAsyncClient session + = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(Duration.ofSeconds(5)); + try { + StepVerifier.create(session.receiveEvents()) + .assertNext(received -> assertInstanceOf(RawRealtimeServerEvent.class, received)) + .verifyComplete(); + } finally { + session.close(); + } + VoiceAgentWebSocketConnectionOptions options = new VoiceAgentWebSocketConnectionOptions(); + assertThrows(IllegalArgumentException.class, () -> options.setReceiveBufferCapacity(0)); + assertThrows(IllegalArgumentException.class, () -> options.setReceiveBufferCapacity(65537)); + assertThrows(IllegalArgumentException.class, () -> options.setMaxMessageSize(0)); + assertThrows(NullPointerException.class, () -> options.setOverflowStrategy(null)); + } + + @Test + public void insecureEndpointsAreRejectedBeforeAuthentication() { + AtomicBoolean requested = new AtomicBoolean(); + TokenCredential credential = context -> { + requested.set(true); + return Mono.error(new AssertionError("Token retrieval must not run.")); + }; + for (String endpoint : new String[] { + "http://example.com", + "ws://example.com", + "https://user@example.com", + "https://example.com/#fragment" }) { + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint(endpoint).credential(credential); + assertThrows(IllegalArgumentException.class, + () -> builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent")); + StepVerifier.create(builder.beta().buildBetaVoiceAgentWebSocketAsyncClient().openWebSocketSession("agent")) + .expectError(IllegalArgumentException.class) + .verify(); + } + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://example.com").credential(credential); + for (String override : new String[] { + "ws://example.com", + "wss://other.example.com", + "wss://example.com:8443" }) { + VoiceAgentWebSocketConnectionOptions options + = new VoiceAgentWebSocketConnectionOptions().setConnectionUrl(URI.create(override)); + assertThrows(IllegalArgumentException.class, + () -> builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", options)); + StepVerifier + .create(builder.beta().buildBetaVoiceAgentWebSocketAsyncClient().openWebSocketSession("agent", options)) + .expectError(IllegalArgumentException.class) + .verify(); + } + assertFalse(requested.get()); + } + + private static HttpServer tlsServer() { + return HttpServer.create() + .secure(ssl -> ssl.sslContext(Http11SslContextSpec.forServer(TLS_CERTIFICATE.keyManagerFactory))); + } + + private static VoiceAgentWebSocketConnectionOptions tlsOptions() { + return new VoiceAgentWebSocketConnectionOptions(); + } + + @Test + public void rawEventsUseCustomizedTlsTransports() { + List messages = new CopyOnWriteArrayList<>(); + server = tlsServer().host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket( + (inbound, outbound) -> outbound.sendString(inbound.receive().asString().doOnNext(messages::add)).then(), + WebsocketServerSpec.builder().protocols("realtime").build())) + .bindNow(); + AgentsClientBuilder builder = new AgentsClientBuilder().endpoint("https://localhost:" + server.port()) + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); + BinaryData payload = BinaryData.fromString("{\"type\":\"future.event\",\"value\":42}"); + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", tlsOptions())) { + assertThrows(IllegalArgumentException.class, () -> session.sendEvent(BinaryData.fromString("[]"))); + assertThrows(IllegalArgumentException.class, () -> session.sendEvent(BinaryData.fromString("{} {}"))); + session.sendEvent(payload); + RawRealtimeServerEvent received = assertInstanceOf(RawRealtimeServerEvent.class, + session.receiveEvents(Duration.ofSeconds(5)).iterator().next()); + assertEquals(payload.toObject(Map.class), received.getRawEvent().toObject(Map.class)); + } + BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent", tlsOptions()) + .block(Duration.ofSeconds(5)); + StepVerifier.create(session.sendEvent(BinaryData.fromString("[]"))) + .expectError(IllegalArgumentException.class) + .verify(); + StepVerifier.create(session.receiveEvents().take(1)) + .then(() -> session.sendEvent(payload).block(Duration.ofSeconds(5))) + .assertNext(event -> assertEquals(42, + ((RawRealtimeServerEvent) event).getRawEvent().toObject(Map.class).get("value"))) + .verifyComplete(); + session.close(); + assertEquals(2, messages.size()); + } + + @Test + public void customCloseFrameAndReceiveTimeout() { + server = startServer(new CopyOnWriteArrayList<>(), new AtomicReference<>(), new AtomicReference<>(), + new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), false); + AgentsClientBuilder builder + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1)))); + try (BetaVoiceAgentWebSocketSessionClient session + = builder.beta().buildBetaVoiceAgentWebSocketClient().openWebSocketSession("agent", tlsOptions())) { + Iterator iterator = session.receiveEvents(Duration.ofMillis(20)).iterator(); + IllegalStateException timeout = assertThrows(IllegalStateException.class, iterator::hasNext); + assertInstanceOf(TimeoutException.class, timeout.getCause()); + assertTrue(session.isOpen()); + assertThrows(IllegalArgumentException.class, () -> session.close(1005, "invalid")); + session.close(4001, "finished"); + assertEquals(4001, session.getCloseCode()); + assertEquals("finished", session.getCloseReason()); + } + BetaVoiceAgentWebSocketSessionAsyncClient session = builder.beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("agent", tlsOptions()) + .block(Duration.ofSeconds(5)); + assertNotNull(session); + StepVerifier.create(session.closeAsync(1006, "invalid")).expectError(IllegalArgumentException.class).verify(); + session.closeAsync(4002, "done").block(Duration.ofSeconds(5)); + assertEquals(4002, session.getCloseCode()); + assertEquals("done", session.getCloseReason()); + } + + @AfterEach + public void disposeServer() { + if (server != null) { + server.disposeNow(); + } + } + + @AfterAll + public static void deleteTlsCertificate() { + SSLContext.setDefault(ORIGINAL_SSL_CONTEXT); + restoreProperty(TRUST_STORE_PROPERTY, ORIGINAL_TRUST_STORE); + restoreProperty(TRUST_STORE_PASSWORD_PROPERTY, ORIGINAL_TRUST_STORE_PASSWORD); + restoreProperty(TRUST_STORE_TYPE_PROPERTY, ORIGINAL_TRUST_STORE_TYPE); + TLS_CERTIFICATE.delete(); + } + + @Test + public void asyncSessionNegotiatesHandshakeAndExchangesTypedEvents() { + List clientMessages = new CopyOnWriteArrayList<>(); + AtomicReference requestUri = new AtomicReference<>(); + AtomicReference authorization = new AtomicReference<>(); + AtomicReference foundryFeatures = new AtomicReference<>(); + AtomicReference userAgent = new AtomicReference<>(); + AtomicReference customHeader = new AtomicReference<>(); + server + = startServer(clientMessages, requestUri, authorization, foundryFeatures, userAgent, customHeader, false); + AtomicReference> requestedScopes = new AtomicReference<>(); + TokenCredential credential = request -> { + requestedScopes.set(request.getScopes()); + return Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + }; + VoiceAgentWebSocketConnectionOptions options = tlsOptions().setTransport(VoiceAgentTransport.WEBSOCKET) + .setStoreEnabled(true) + .setAgentVersionOverride("version 2"); + BetaVoiceAgentWebSocketAsyncClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .clientOptions(new ClientOptions().setApplicationId("test-app") + .setHeaders(Collections.singletonList(new Header("X-Test-Header", "test-value")))) + .beta() + .buildBetaVoiceAgentWebSocketAsyncClient(); + + Mono sessionMono + = client.openWebSocketSession("agent name", options); + options.setStoreEnabled(false).setAgentVersionOverride("mutated"); + BetaVoiceAgentWebSocketSessionAsyncClient session = sessionMono.block(); + assertTrue(session.isOpen()); + + StepVerifier.create(session.receiveEvents().take(4)) + .then(() -> session.sendText("hello").block()) + .assertNext(this::assertWarningEvent) + .then(() -> session.appendInputAudio(BinaryData.fromBytes(new byte[] { 1, 2, 3 })).block()) + .assertNext(this::assertWarningEvent) + .then(() -> session.createResponse().block()) + .assertNext(this::assertWarningEvent) + .then(() -> session.cancelResponse("response-1").block()) + .assertNext(this::assertWarningEvent) + .verifyComplete(); + + assertEquals(Collections.singletonList("https://ai.azure.com/.default"), requestedScopes.get()); + assertEquals("Bearer test-token", authorization.get()); + assertEquals("VoiceAgents=V1Preview", foundryFeatures.get()); + assertTrue(userAgent.get().startsWith("test-app azsdk-java-")); + assertEquals("test-value", customHeader.get()); + String decodedUri = decode(requestUri.get()); + assertTrue(decodedUri.contains("/agents/agent name/endpoint/protocols/voice")); + assertTrue(decodedUri.contains("api-version=v1")); + assertTrue(decodedUri.contains("transport=websocket")); + assertTrue(decodedUri.contains("store=true")); + assertTrue(decodedUri.contains("x-agent-version-override=version 2")); + assertTrue(decodedUri.contains("x-ms-client-sdk=test-app azsdk-java-")); + assertEquals(4, clientMessages.size()); + assertTrue(clientMessages.get(0).contains("\"type\":\"conversation.item.create\"")); + assertTrue(clientMessages.get(0).contains("\"role\":\"user\"")); + assertTrue(clientMessages.get(0).contains("\"text\":\"hello\"")); + assertTrue(clientMessages.get(1).contains("\"audio\":\"AQID\"")); + assertTrue(clientMessages.get(2).contains("\"type\":\"response.create\"")); + assertTrue(clientMessages.get(3).contains("\"response_id\":\"response-1\"")); + + StepVerifier.create(session.receiveEvents()) + .expectErrorMatches( + error -> error instanceof IllegalStateException && error.getMessage().contains("Only one")) + .verify(); + session.close(); + assertFalse(session.isOpen()); + } + + @Test + public void syncConnectRejectsNullArguments() { + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("https://localhost/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketClient(); + + NullPointerException agentNameException + = assertThrows(NullPointerException.class, () -> client.openWebSocketSession(null, tlsOptions())); + assertEquals("'agentName' cannot be null.", agentNameException.getMessage()); + + NullPointerException optionsException + = assertThrows(NullPointerException.class, () -> client.openWebSocketSession("agent", null)); + assertEquals("'options' cannot be null.", optionsException.getMessage()); + } + + @Test + public void tokenFailureOccursBeforeNetworkAccess() { + AtomicBoolean connected = new AtomicBoolean(); + server = tlsServer().host("localhost").port(0).handle((request, response) -> { + connected.set(true); + return response.send(); + }).bindNow(); + TokenCredential credential = request -> Mono.error(new IllegalStateException("token unavailable")); + BetaVoiceAgentWebSocketAsyncClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketAsyncClient(); + + StepVerifier.create(client.openWebSocketSession("agent", tlsOptions())) + .expectErrorMatches( + error -> error instanceof IllegalStateException && error.getMessage().contains("token unavailable")) + .verify(); + assertFalse(connected.get()); + } + + @Test + public void tokenAcquisitionDoesNotUseHandshakeTimeout() { + server = startServer(new CopyOnWriteArrayList<>(), new AtomicReference<>(), new AtomicReference<>(), + new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), false); + TokenCredential credential = request -> Mono.delay(Duration.ofMillis(1500)) + .map(ignored -> new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketAsyncClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketAsyncClient(); + VoiceAgentWebSocketConnectionOptions options = tlsOptions().setHandshakeTimeout(Duration.ofSeconds(1)); + + StepVerifier.withVirtualTime(() -> client.openWebSocketSession("agent", options).flatMap(session -> { + assertTrue(session.isOpen()); + return session.closeAsync(); + })).thenAwait(Duration.ofMillis(1500)).verifyComplete(); + } + + @Test + public void syncTokenFailureOccursBeforeNetworkAccess() { + AtomicBoolean connected = new AtomicBoolean(); + server = tlsServer().host("localhost").port(0).handle((request, response) -> { + connected.set(true); + return response.send(); + }).bindNow(); + TokenCredential credential = request -> Mono.error(new IllegalStateException("token unavailable")); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketClient(); + + IllegalStateException exception + = assertThrows(IllegalStateException.class, () -> client.openWebSocketSession("agent", tlsOptions())); + assertTrue(exception.getMessage().contains("token unavailable")); + assertFalse(connected.get()); + } + + @Test + public void rejectedHandshakeMapsConflictToAzureException() { + server = tlsServer().host("localhost") + .port(0) + .handle( + (request, response) -> response.status(HttpResponseStatus.CONFLICT).sendString(Mono.just("conflict"))) + .bindNow(); + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketAsyncClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketAsyncClient(); + + StepVerifier.create(client.openWebSocketSession("disabled-agent", tlsOptions())).expectErrorSatisfies(error -> { + ResourceModifiedException exception = assertInstanceOf(ResourceModifiedException.class, error); + assertEquals(409, exception.getResponse().getStatusCode()); + }).verify(); + } + + @Test + public void nettyHandshakeResponseExposesBufferedBody() { + DefaultFullHttpResponse nettyResponse = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, + HttpResponseStatus.CONFLICT, Unpooled.copiedBuffer("conflict", StandardCharsets.UTF_8)); + VoiceAgentWebSocketHttpResponse response + = new VoiceAgentWebSocketHttpResponse(URI.create("wss://example.com"), nettyResponse); + + assertEquals("conflict", response.getBodyAsString().block()); + } + + @Test + public void syncRejectedHandshakeMapsConflictToAzureException() { + server = tlsServer().host("localhost") + .port(0) + .handle( + (request, response) -> response.status(HttpResponseStatus.CONFLICT).sendString(Mono.just("conflict"))) + .bindNow(); + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketClient(); + + ResourceModifiedException exception = assertThrows(ResourceModifiedException.class, + () -> client.openWebSocketSession("disabled-agent", tlsOptions())); + assertEquals(409, exception.getResponse().getStatusCode()); + assertEquals("conflict", exception.getResponse().getBodyAsString().block()); + } + + @Test + public void syncClientRejectsEmptyAgentName() { + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client = new AgentsClientBuilder().endpoint("https://example.com") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketClient(); + + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> client.openWebSocketSession("", tlsOptions())); + assertEquals("'agentName' cannot be empty.", exception.getMessage()); + } + + @Test + public void cancellingAsyncConnectCancelsTokenRequest() { + AtomicBoolean tokenRequestCancelled = new AtomicBoolean(); + TokenCredential credential + = request -> Mono.never().doOnCancel(() -> tokenRequestCancelled.set(true)); + BetaVoiceAgentWebSocketAsyncClient client = new AgentsClientBuilder().endpoint("https://example.com") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketAsyncClient(); + + StepVerifier.create(client.openWebSocketSession("agent", tlsOptions())).thenCancel().verify(); + assertTrue(tokenRequestCancelled.get()); + } + + @Test + public void unknownEventFallsBackToRealtimeServerEvent() { + server = oneShotWebSocketServer("{\"type\":\"future.event\",\"value\":42}"); + BetaVoiceAgentWebSocketSessionAsyncClient session + = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); + + StepVerifier.create(session.receiveEvents()).assertNext(event -> { + assertEquals("future.event", event.getType().toString()); + RawRealtimeServerEvent raw = assertInstanceOf(RawRealtimeServerEvent.class, event); + assertEquals(42, raw.getRawEvent().toObject(Map.class).get("value")); + }).verifyComplete(); + session.close(); + } + + @Test + public void fragmentedTextFrameIsAggregated() { + String message = warningJson(); + int split = message.length() / 2; + Flux frames = Flux.just(new TextWebSocketFrame(false, 0, message.substring(0, split)), + new ContinuationWebSocketFrame(true, 0, message.substring(split))); + server = frameWebSocketServer(frames); + BetaVoiceAgentWebSocketSessionAsyncClient session + = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); + + StepVerifier.create(session.receiveEvents()).assertNext(this::assertWarningEvent).verifyComplete(); + session.close(); + } + + @Test + public void binaryJsonFrameIsParsed() { + server = frameWebSocketServer( + Mono.just(new BinaryWebSocketFrame(Unpooled.copiedBuffer(warningJson(), StandardCharsets.UTF_8)))); + BetaVoiceAgentWebSocketSessionAsyncClient session + = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); + + StepVerifier.create(session.receiveEvents()).assertNext(this::assertWarningEvent).verifyComplete(); + session.close(); + } + + @Test + public void malformedJsonTerminatesReceiveStream() { + server = oneShotWebSocketServer("{not-json"); + BetaVoiceAgentWebSocketSessionAsyncClient session + = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); + + StepVerifier.create(session.receiveEvents()).expectError().verify(); + assertFalse(session.isOpen()); + } + + @Test + public void syncReceiveBufferOverflowFailsTheEventStream() { + Flux frames = Flux.range(0, 257).map(index -> new TextWebSocketFrame(warningJson())); + server = frameWebSocketServer(frames); + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketClient(); + + try (BetaVoiceAgentWebSocketSessionClient session = client.openWebSocketSession("agent", tlsOptions())) { + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + while (session.isOpen()) { + Thread.yield(); + } + }); + IllegalStateException exception + = assertThrows(IllegalStateException.class, () -> session.receiveEvents().iterator().hasNext()); + assertEquals("Voice-agent receive buffer overflow.", exception.getMessage()); + } + } + + @Test + public void syncOrderlyClosePreservesFullReceiveBuffer() { + Flux frames = Flux.range(0, 256).map(index -> new TextWebSocketFrame(warningJson())); + server = frameWebSocketServer(frames); + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketClient(); + + try (BetaVoiceAgentWebSocketSessionClient session = client.openWebSocketSession("agent", tlsOptions())) { + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + Iterator events = session.receiveEvents().iterator(); + int eventCount = 0; + while (events.hasNext()) { + events.next(); + eventCount++; + } + assertEquals(256, eventCount); + }); + } + } + + @Test + public void closeIsIdempotentAndSendAfterCloseFails() { + List clientMessages = new CopyOnWriteArrayList<>(); + server = startServer(clientMessages, new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), + new AtomicReference<>(), new AtomicReference<>(), false); + BetaVoiceAgentWebSocketSessionAsyncClient session + = createAsyncClient(server.port()).openWebSocketSession("agent", tlsOptions()).block(); + + StepVerifier.create(session.closeAsync().then(session.closeAsync())).verifyComplete(); + StepVerifier.create(session.sendText("after close")) + .expectErrorMatches( + error -> error instanceof IllegalStateException && error.getMessage().contains("not open")) + .verify(); + } + + @Test + public void secureSessionUsesWssAndReceivesTypedEvent() throws Exception { + Http11SslContextSpec serverSsl = Http11SslContextSpec.forServer(TLS_CERTIFICATE.keyManagerFactory); + WebsocketServerSpec websocketSpec = WebsocketServerSpec.builder().protocols("realtime").build(); + server = tlsServer().host("localhost") + .port(0) + .secure(ssl -> ssl.sslContext(serverSsl)) + .handle((request, response) -> response.sendWebsocket( + (inbound, outbound) -> outbound.sendString(Mono.just(warningJson()), StandardCharsets.UTF_8) + .then(outbound.sendClose()), + websocketSpec)) + .bindNow(); + + TokenCredential credential + = request -> Mono.just(new AccessToken("tls-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketSessionAsyncClient session + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .beta() + .buildBetaVoiceAgentWebSocketAsyncClient() + .openWebSocketSession("secure-agent", tlsOptions()) + .block(Duration.ofSeconds(5)); + + assertEquals("wss", session.getEndpoint().getScheme()); + StepVerifier.create(session.receiveEvents()).assertNext(this::assertWarningEvent).verifyComplete(); + session.close(); + } + + @Test + public void syncSessionReceivesTypedEventAndCloses() { + List clientMessages = new CopyOnWriteArrayList<>(); + AtomicReference requestUri = new AtomicReference<>(); + AtomicReference authorization = new AtomicReference<>(); + AtomicReference foundryFeatures = new AtomicReference<>(); + AtomicReference userAgent = new AtomicReference<>(); + server = startServer(clientMessages, requestUri, authorization, foundryFeatures, userAgent, + new AtomicReference<>(), true); + TokenCredential credential + = request -> Mono.just(new AccessToken("sync-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("https://localhost:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketClient(); + + try (BetaVoiceAgentWebSocketSessionClient session = client.openWebSocketSession("sync-agent", tlsOptions())) { + Iterator events = session.receiveEvents().iterator(); + assertWarningEvent(events.next()); + session.sendFunctionCallOutput("call-1", "{\"temperature\":72}"); + assertWarningEvent(events.next()); + assertWarningEvent(events.next()); + + assertEquals(2, clientMessages.size()); + Map functionOutput = BinaryData.fromString(clientMessages.get(0)).toObject(Map.class); + assertEquals("conversation.item.create", functionOutput.get("type")); + Map item = (Map) functionOutput.get("item"); + assertEquals("function_call_output", item.get("type")); + assertEquals("call-1", item.get("call_id")); + assertEquals("{\"temperature\":72}", item.get("output")); + Map responseCreate = BinaryData.fromString(clientMessages.get(1)).toObject(Map.class); + assertEquals("response.create", responseCreate.get("type")); + assertTrue(session.isOpen()); + } + } + + private BetaVoiceAgentWebSocketAsyncClient createAsyncClient(int port) { + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + return new AgentsClientBuilder().endpoint("https://localhost:" + port + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .beta() + .buildBetaVoiceAgentWebSocketAsyncClient(); + } + + private DisposableServer frameWebSocketServer(org.reactivestreams.Publisher frames) { + WebsocketServerSpec spec = WebsocketServerSpec.builder().protocols("realtime").build(); + return tlsServer().host("localhost") + .port(0) + .handle((request, response) -> response + .sendWebsocket((inbound, outbound) -> outbound.sendObject(frames).then(outbound.sendClose()), spec)) + .bindNow(); + } + + private DisposableServer oneShotWebSocketServer(String message) { + WebsocketServerSpec spec = WebsocketServerSpec.builder().protocols("realtime").build(); + return tlsServer().host("localhost") + .port(0) + .handle((request, response) -> response.sendWebsocket((inbound, + outbound) -> outbound.sendString(Mono.just(message), StandardCharsets.UTF_8).then(outbound.sendClose()), + spec)) + .bindNow(); + } + + private DisposableServer startServer(List clientMessages, AtomicReference requestUri, + AtomicReference authorization, AtomicReference foundryFeatures, + AtomicReference userAgent, AtomicReference customHeader, boolean sendInitialEvent) { + WebsocketServerSpec spec = WebsocketServerSpec.builder().protocols("realtime").build(); + return tlsServer().host("localhost").port(0).handle((request, response) -> { + requestUri.set(request.uri()); + authorization.set(request.requestHeaders().get(HttpHeaderNames.AUTHORIZATION)); + foundryFeatures.set(request.requestHeaders().get("Foundry-Features")); + userAgent.set(request.requestHeaders().get(HttpHeaderNames.USER_AGENT)); + customHeader.set(request.requestHeaders().get("X-Test-Header")); + return response.sendWebsocket((inbound, outbound) -> { + Flux replies = inbound.receive() + .asString(StandardCharsets.UTF_8) + .doOnNext(clientMessages::add) + .map(ignored -> warningJson()); + if (sendInitialEvent) { + replies = replies.startWith(warningJson()); + } + return outbound.sendString(replies, StandardCharsets.UTF_8).then(); + }, spec); + }).bindNow(); + } + + private void assertWarningEvent(RealtimeServerEvent event) { + VoiceAgentWarningEvent warning = assertInstanceOf(VoiceAgentWarningEvent.class, event); + assertEquals("loopback warning", warning.getWarning().getMessage()); + assertEquals("test_warning", warning.getWarning().getCode()); + } + + private static String warningJson() { + return "{\"type\":\"warning\",\"event_id\":\"event-1\",\"warning\":{" + + "\"message\":\"loopback warning\",\"code\":\"test_warning\"}}"; + } + + private static final class TestCertificate { + private final Path path; + private final String password; + private final KeyStore keyStore; + private final KeyManagerFactory keyManagerFactory; + + private TestCertificate(Path path, String password, KeyStore keyStore, KeyManagerFactory keyManagerFactory) { + this.path = path; + this.password = password; + this.keyStore = keyStore; + this.keyManagerFactory = keyManagerFactory; + } + + private static TestCertificate create() { + try { + Path path = Files.createTempFile("voice-agent-websocket-", ".p12"); + Files.delete(path); + String password = UUID.randomUUID().toString(); + String executable + = Paths + .get(System.getProperty("java.home"), "bin", + System.getProperty("os.name").startsWith("Windows") ? "keytool.exe" : "keytool") + .toString(); + Process process = new ProcessBuilder(executable, "-genkeypair", "-alias", "localhost", "-keyalg", "RSA", + "-keysize", "2048", "-validity", "1", "-dname", "CN=localhost", "-ext", "SAN=dns:localhost", + "-storetype", "PKCS12", "-keystore", path.toString(), "-storepass", password, "-keypass", password, + "-noprompt").redirectErrorStream(true).start(); + if (process.waitFor() != 0) { + throw new IllegalStateException("keytool failed to generate the test certificate."); + } + KeyStore store = KeyStore.getInstance("PKCS12"); + try (InputStream input = Files.newInputStream(path)) { + store.load(input, password.toCharArray()); + } + KeyManagerFactory keyManagerFactory + = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(store, password.toCharArray()); + return new TestCertificate(path, password, store, keyManagerFactory); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + + private void installTrustStore() { + try { + System.setProperty(TRUST_STORE_PROPERTY, path.toString()); + System.setProperty(TRUST_STORE_PASSWORD_PROPERTY, password); + System.setProperty(TRUST_STORE_TYPE_PROPERTY, "PKCS12"); + TrustManagerFactory trustManagerFactory + = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(keyStore); + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), null); + SSLContext.setDefault(sslContext); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + + private void delete() { + try { + Files.deleteIfExists(path); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + } + + private static SSLContext getDefaultSslContext() { + try { + return SSLContext.getDefault(); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } + + private static void restoreProperty(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } + + private static String decode(String value) { + try { + return URLDecoder.decode(value, StandardCharsets.UTF_8.name()); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } +}