From 5e0db5fafe44454af4e1b9b6d4d03c963edf9045 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 11 Sep 2026 09:10:10 +0800 Subject: [PATCH 1/2] Add realtime voice agent samples --- sdk/ai/azure-ai-agents/README.md | 123 +++++++- .../voice/VoiceAgentBasicAsyncSample.java | 64 +++++ .../agents/voice/VoiceAgentBasicSample.java | 67 +++++ .../voice/VoiceAgentGenerateSample.java | 56 ++++ ...AgentLiveAudioConversationAsyncSample.java | 265 ++++++++++++++++++ .../VoiceAgentLiveFunctionToolSample.java | 165 +++++++++++ ...eAgentLiveTextConversationAsyncSample.java | 159 +++++++++++ .../VoiceAgentLiveTextConversationSample.java | 157 +++++++++++ ...VoiceAgentReadConversationAudioSample.java | 78 ++++++ .../VoiceAgentReadConversationSample.java | 67 +++++ .../voice/VoiceAgentRealtimeSampleUtils.java | 152 ++++++++++ .../agents/voice/VoiceAgentSampleUtils.java | 31 ++ .../voice/VoiceAgentVersionsSample.java | 69 +++++ .../voice/VoiceAgentWithToolsSample.java | 111 ++++++++ 14 files changed, 1561 insertions(+), 3 deletions(-) create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java create mode 100644 sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java diff --git a/sdk/ai/azure-ai-agents/README.md b/sdk/ai/azure-ai-agents/README.md index aee2c3bc97e2..3736914cc7c7 100644 --- a/sdk/ai/azure-ai-agents/README.md +++ b/sdk/ai/azure-ai-agents/README.md @@ -68,6 +68,8 @@ The Agents client library has the following sub-clients which group the differen - `ResponsesClient` / `ResponsesAsyncClient`: Handle responses operations. See the [OpenAI's Responses API documentation][openai_responses_api_docs] for more information. - `BetaMemoryStoresClient` / `BetaMemoryStoresAsyncClient` **(preview)**: Manage memory stores and individual memory items for agents. - `ToolboxesClient` / `ToolboxesAsyncClient`: Manage toolboxes and toolbox versions. +- `BetaVoiceAgentWebSocketClient` / `BetaVoiceAgentWebSocketAsyncClient` **(preview)**: Open typed realtime WebSocket sessions with voice agents. +- `BetaAgentEndpointConversationsClient` / `BetaAgentEndpointConversationsAsyncClient` **(preview)**: Read persisted voice-agent conversations, transcripts, and audio metadata. Conversation operations are accessed through the [OpenAI Official Java SDK][openai_java_sdk]'s `ConversationService`. See the [OpenAI's Conversation API documentation][openai_conversations_api_docs] for more information. @@ -184,9 +186,17 @@ Build clients whose names start with `Beta` from `AgentsClientBuilder.beta()`. T |---|---| | `BetaAgentsClient` | `WorkflowAgents=V1Preview,ExternalAgents=V1Preview,DraftAgents=V1Preview,AgentsOptimization=V2Preview` | | `BetaMemoryStoresClient` | `MemoryStores=V1Preview` | +| `BetaVoiceAgentWebSocketClient` | `VoiceAgents=V1Preview` | +| `BetaAgentEndpointConversationsClient` | `VoiceAgents=V1Preview` | The async `Beta*AsyncClient` counterparts follow the same behavior. +### Realtime voice-agent sessions + +Use `BetaVoiceAgentWebSocketClient` or `BetaVoiceAgentWebSocketAsyncClient` to open a typed, bidirectional session with an existing voice agent. The client acquires a token for `https://ai.azure.com/.default`, negotiates the `realtime` WebSocket subprotocol, and sends the required `VoiceAgents=V1Preview` feature header automatically. + +The session API supports text and PCM16 audio input, typed streaming server events, response cancellation, client-executed function tools, and persisted conversations. See [Realtime voice-agent WebSocket examples](#realtime-voice-agent-websocket-examples-preview) for a walkthrough and complete samples. + ### Agent optimization The preview `BetaAgentsClient` and `BetaAgentsAsyncClient` can create and monitor agent optimization jobs. These jobs @@ -198,9 +208,9 @@ and [AgentOptimizationAsyncSample.java](https://github.com/Azure/azure-sdk-for-j ### Memory item management -`BetaMemoryStoresClient` and `BetaMemoryStoresAsyncClient` manage memory stores and individual memory items. In addition to store-level operations, use `createMemory`, `updateMemory`, `listMemories`, `getMemory`, and `deleteMemory` to manage individual memories. `ListMemoriesOptions` supports filtering by scope and `MemoryItemKind`, including `MemoryItemKind.PROCEDURAL`. See `MemoryStoreItemsSample` and `MemoryStoreItemsAsyncSample` for complete examples. +`BetaMemoryStoresClient` and `BetaMemoryStoresAsyncClient` manage memory stores and individual memory items. In addition to store-level operations, use `createMemory`, `updateMemory`, `listMemories`, `getMemory`, and `deleteMemory` to manage individual memories. `ListMemoriesOptions` supports filtering by scope and `MemoryItemKind`, including `MemoryItemKind.PROCEDURAL`. See [MemoryStoreItemsSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/memory/MemoryStoreItemsSample.java) and [MemoryStoreItemsAsyncSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/memory/MemoryStoreItemsAsyncSample.java) for complete examples. -For conversational memory workflows, use `beginUpdateMemories` to extract memories from conversation items, `searchMemories` to retrieve relevant memories, and `deleteScope` to remove all memories for a scope. See `MemoryStoreAdvancedSample` and `MemoryStoreAdvancedAsyncSample` for complete synchronous and asynchronous examples. +For conversational memory workflows, use `beginUpdateMemories` to extract memories from conversation items, `searchMemories` to retrieve relevant memories, and `deleteScope` to remove all memories for a scope. See [MemoryStoreAdvancedSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/memory/MemoryStoreAdvancedSample.java) and [MemoryStoreAdvancedAsyncSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/memory/MemoryStoreAdvancedAsyncSample.java) for complete synchronous and asynchronous examples. ### Using OpenAI's official library @@ -465,7 +475,7 @@ MemorySearchPreviewTool tool = new MemorySearchPreviewTool(memoryStore.getName() .setUpdateDelaySeconds(1); ``` -See the full sample in [MemorySearchSync.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/tools/MemorySearchSync.java) showing how to create an agent with a memory store and use it across multiple conversations. +See the full samples in [MemorySearchSync.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/tools/MemorySearchSync.java) and [MemorySearchAsync.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/tools/MemorySearchAsync.java), which show how to create an agent with a memory store and use it across multiple conversations. --- @@ -904,6 +914,113 @@ See the full sample in [CreateResponseWithStructuredInput.java](https://github.c --- +### Voice agent samples (preview) + +The [voice-agent samples](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice) cover agent management and persisted conversations. + +| Scenario | Samples | +|---|---| +| Lifecycle | [VoiceAgentBasicSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java) and [VoiceAgentBasicAsyncSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java) create, retrieve, update, list, enable, disable, and delete voice agents. | +| Versions and drafts | [VoiceAgentVersionsSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java) creates and lists released and draft versions. | +| Guided generation | [VoiceAgentGenerateSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java) generates a voice-agent definition. | +| Audio and tools | [VoiceAgentWithToolsSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java) configures PCM audio, transcription, voice activity detection, function tools, and system tools. | +| Persisted conversations | [VoiceAgentReadConversationSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java) reads responses and transcripts, while [VoiceAgentReadConversationAudioSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java) downloads call and item audio. | + +Authenticate with `DefaultAzureCredential`. Every voice sample requires `FOUNDRY_PROJECT_ENDPOINT`. Samples that create explicit definitions optionally use `FOUNDRY_VOICE_MODEL`, `FOUNDRY_VOICE_MODEL_TYPE`, and `FOUNDRY_VOICE_AGENT_NAME`. The persisted-conversation samples require `FOUNDRY_VOICE_AGENT_NAME` and `FOUNDRY_VOICE_CONVERSATION_ID`. + +### Realtime voice-agent WebSocket examples (preview) + +Realtime WebSocket sessions provide bidirectional text and audio communication with a voice agent. Create the voice agent before opening a session; the lifecycle samples above demonstrate how to create one. + +#### Create a realtime WebSocket client + +Build a synchronous or asynchronous preview client from the same `AgentsClientBuilder`. Beta clients automatically send the required preview feature header. + +```java +AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint); + +BetaVoiceAgentWebSocketClient realtimeClient + = builder.beta().buildBetaVoiceAgentWebSocketClient(); +BetaVoiceAgentWebSocketAsyncClient realtimeAsyncClient + = builder.beta().buildBetaVoiceAgentWebSocketAsyncClient(); +``` + +#### Send a synchronous text turn + +Connect to the voice agent, add the user's text to the conversation, and request a response. Consume the typed server events until the response finishes. A session supports only one consumer of `receiveEvents()`. + +```java +try (VoiceAgentWebSocketSessionClient session = realtimeClient.connect(agentName)) { + session.sendText("Hello! Tell me about the services you provide."); + session.createResponse(); + + for (RealtimeServerEvent event : session.receiveEvents()) { + if (event instanceof RealtimeServerEventResponseTextDelta) { + System.out.print(((RealtimeServerEventResponseTextDelta) event).getDelta()); + } else if (event instanceof RealtimeServerEventRealtimeServerEventError) { + RealtimeServerEventRealtimeServerEventError error + = (RealtimeServerEventRealtimeServerEventError) event; + System.out.println("Session error: " + error.getError().getMessage()); + } else if (event instanceof RealtimeServerEventResponseDone) { + break; + } + } +} +``` + +Use `sendText` and `createResponse` again for subsequent turns while the session remains open. Call `cancelResponse` to interrupt an active response. + +#### Send an asynchronous text turn + +The asynchronous client returns a `Mono` when connecting and a `Flux` when receiving events. `Mono.usingWhen` closes the session on completion, error, or cancellation. + +```java +Mono.usingWhen( + realtimeAsyncClient.connect(agentName), + session -> session.sendText("Hello! Tell me about the services you provide.") + .then(session.createResponse()) + .thenMany(session.receiveEvents()) + .doOnNext(event -> { + if (event instanceof RealtimeServerEventResponseTextDelta) { + System.out.print(((RealtimeServerEventResponseTextDelta) event).getDelta()); + } + }) + .takeUntil(event -> event instanceof RealtimeServerEventResponseDone) + .then(), + VoiceAgentWebSocketSessionAsyncClient::closeAsync, + (session, error) -> session.closeAsync(), + VoiceAgentWebSocketSessionAsyncClient::closeAsync) + .block(); +``` + +#### Stream audio and handle function tools + +Use `appendInputAudio` to send PCM16 chunks, `commitInputAudio` to commit buffered audio when server-side voice activity detection is not configured, and `clearInputAudio` to discard pending input. Audio output arrives through `RealtimeServerEventResponseAudioDelta` events. When a `RealtimeServerEventResponseFunctionCallArgumentsDone` event requests a client-side tool, execute the function and call `sendFunctionCallOutput` with its call ID and serialized result. + +| Scenario | Complete sample | +|---|---| +| Synchronous live text | [VoiceAgentLiveTextConversationSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java) | +| Asynchronous live text | [VoiceAgentLiveTextConversationAsyncSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java) | +| Asynchronous live audio | [VoiceAgentLiveAudioConversationAsyncSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java) | +| Live function tool | [VoiceAgentLiveFunctionToolSample.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java) | + +All realtime examples require `FOUNDRY_PROJECT_ENDPOINT` and optionally use `FOUNDRY_VOICE_AGENT_NAME`. The function-tool example also optionally uses `FOUNDRY_VOICE_MODEL` and `FOUNDRY_VOICE_MODEL_TYPE`. The asynchronous text and audio examples delete their generated agents by default; set `FOUNDRY_KEEP_VOICE_AGENT=true` to retain them. + +The live audio example requires a Java Sound-compatible microphone and speaker. It streams signed, little-endian, mono PCM16 audio at 24 kHz. These examples use WebSocket transport. Although the generated protocol models include WebRTC signaling events, the Java client does not provide a WebRTC peer connection or media implementation. + +### Additional end-to-end samples + +All agent samples use `FOUNDRY_PROJECT_ENDPOINT`. Prompt-agent samples also use `FOUNDRY_MODEL_NAME`. + +- **Agent lifecycle and structured output:** [CreateAgent.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/agents/CreateAgent.java), [GetAgent.java](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/agents/GetAgent.java), and the `AgentStructuredOutput*` samples. +- **Workflow agents:** `WorkflowMultiAgentSample`, `WorkflowMultiAgentAsyncSample`, and `WorkflowMultiAgentMcpApprovalSample` demonstrate CSDL workflows and MCP approval handling. +- **Optimization jobs:** the [optimization samples](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/optimization) cover SDK polling, application-managed polling, cancellation, listing, retrieval, and deletion. +- **Advanced tools:** additional samples cover structured inputs, generated-file download, File Search streaming, non-preview Web Search, custom search, and end-to-end toolbox search. + +--- + ### Service API versions The client library targets the latest service API version by default. diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java new file mode 100644 index 000000000000..9e6c6beac707 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicAsyncSample.java @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentKind; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceModelType; +import reactor.core.publisher.Mono; + +/** + * Demonstrates the asynchronous voice-agent lifecycle. + * + *

Before running the sample, set these environment variables:

+ * + */ +public class VoiceAgentBasicAsyncSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "voice-agent-async-java"); + + AgentsAsyncClient client = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true) + .buildAgentsAsyncClient(); + + client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a friendly voice assistant. Keep replies short and natural."))) + .doOnNext(created -> System.out.printf("Created voice agent %s, version %s%n", + created.getName(), created.getVersion())) + .then(client.getAgent(agentName)) + .doOnNext(agent -> System.out.printf("Retrieved voice agent %s, state %s%n", + agent.getName(), agent.getState())) + .thenMany(client.listAgents(AgentKind.VOICE, null, null, null, null)) + .doOnNext(agent -> System.out.println("Voice agent: " + agent.getName())) + .then(client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a friendly voice assistant. Always greet the caller warmly.")) + .setDescription("Updated voice-agent instructions."))) + .doOnNext(updated -> System.out.println("Created updated version: " + updated.getVersion())) + .then(client.disableAgent(agentName)) + .then(client.enableAgent(agentName)) + .then(client.deleteAgent(agentName)) + .onErrorResume(error -> client.deleteAgent(agentName) + .onErrorResume(cleanupError -> Mono.empty()) + .then(Mono.error(error))) + .block(); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java new file mode 100644 index 000000000000..e03ab1996238 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentBasicSample.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentDetails; +import com.azure.ai.agents.models.AgentKind; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceModelType; + +/** + * Demonstrates the synchronous voice-agent lifecycle. + * + *

Before running the sample, set these environment variables:

+ * + */ +public class VoiceAgentBasicSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "voice-agent-java"); + + AgentsClient client = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true) + .buildAgentsClient(); + try { + AgentVersionDetails created = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a friendly voice assistant. Keep replies short and natural."))); + System.out.printf("Created voice agent %s, version %s%n", created.getName(), created.getVersion()); + + AgentDetails agent = client.getAgent(agentName); + System.out.printf("Retrieved voice agent %s, state %s%n", agent.getName(), agent.getState()); + for (AgentDetails item : client.listAgents(AgentKind.VOICE, null, null, null, null)) { + System.out.println("Voice agent: " + item.getName()); + } + + AgentVersionDetails updated = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a friendly voice assistant. Always greet the caller warmly.")) + .setDescription("Updated voice-agent instructions.")); + System.out.println("Created updated version: " + updated.getVersion()); + client.disableAgent(agentName); + System.out.println("Disabled voice agent"); + client.enableAgent(agentName); + System.out.println("Enabled voice agent"); + } finally { + client.deleteAgent(agentName); + System.out.println("Deleted voice agent: " + agentName); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java new file mode 100644 index 000000000000..f24bdb4a34f8 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentGenerateSample.java @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaAgentsClient; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentDetails; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.core.util.BinaryData; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Demonstrates guided authoring of a voice agent through the agent generation API. + * + *

Before running the sample, set these environment variables:

+ * + */ +public class VoiceAgentGenerateSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "generated-voice-agent-java"); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsClient client = builder.buildAgentsClient(); + BetaAgentsClient betaClient = builder.beta().buildBetaAgentsClient(); + + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + AgentDetails generated = betaClient.generateAgent(BinaryData.fromObject(request)); + try { + System.out.println("Generated voice agent: " + generated.getName()); + AgentVersionDetails latest = generated.getVersions().getLatest(); + if (latest != null && latest.getDefinition() instanceof VoiceAgentDefinition) { + VoiceAgentDefinition definition = (VoiceAgentDefinition) latest.getDefinition(); + System.out.println("Instructions: " + definition.getInstructions()); + } + } finally { + client.deleteAgent(generated.getName()); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java new file mode 100644 index 000000000000..cce325c32b0a --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaAgentEndpointConversationsAsyncClient; +import com.azure.ai.agents.BetaAgentsAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketAsyncClient; +import com.azure.ai.agents.VoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeServerEventConversationItemInputAudioTranscriptionCompleted; +import com.azure.ai.agents.models.RealtimeServerEventInputAudioBufferSpeechStarted; +import com.azure.ai.agents.models.RealtimeServerEventRealtimeServerEventError; +import com.azure.ai.agents.models.RealtimeServerEventResponseAudioDelta; +import com.azure.ai.agents.models.RealtimeServerEventResponseAudioTranscriptDone; +import com.azure.ai.agents.models.RealtimeServerEventResponseCreated; +import com.azure.ai.agents.models.RealtimeServerEventResponseDone; +import com.azure.ai.agents.models.RealtimeServerEventSessionCreated; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import reactor.core.Disposable; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.SourceDataLine; +import javax.sound.sampled.TargetDataLine; +import java.time.Duration; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Scanner; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Demonstrates an asynchronous hands-free voice conversation using Java Sound and server-side VAD. + * + *

To end the call, focus the terminal running the sample and press Enter. The sample then closes the WebSocket, + * stops the microphone and speaker, reads the persisted conversation, and deletes the agent unless + * {@code FOUNDRY_KEEP_VOICE_AGENT} is set to {@code true}.

+ * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to + * {@code sample-live-audio-conversation-agent-async-java}.
    • + *
  • {@code FOUNDRY_KEEP_VOICE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. + * Defaults to {@code false}.
    • + *
+ */ +public class VoiceAgentLiveAudioConversationAsyncSample { + private static final Duration SEND_TIMEOUT = Duration.ofSeconds(10); + + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", + "sample-live-audio-conversation-agent-async-java"); + boolean keepAgent + = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_VOICE_AGENT", "false")); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsAsyncClient agents = builder.buildAgentsAsyncClient(); + BetaAgentsAsyncClient betaAgents = builder.beta().buildBetaAgentsAsyncClient(); + BetaVoiceAgentWebSocketAsyncClient realtime = builder.buildBetaVoiceAgentWebSocketAsyncClient(); + BetaAgentEndpointConversationsAsyncClient conversations + = builder.buildBetaAgentEndpointConversationsAsyncClient(); + + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + AtomicReference conversationId = new AtomicReference<>(); + + betaAgents.generateAgent(BinaryData.fromObject(request)) + .flatMap(generated -> { + VoiceAgentDefinition definition + = (VoiceAgentDefinition) generated.getVersions().getLatest().getDefinition(); + return agents.createAgentVersion(agentName, + new CreateAgentVersionInput(definition.setStore(true))); + }) + .then(Mono.usingWhen(realtime.connect(agentName), + session -> runConversation(session, conversationId), + VoiceAgentWebSocketSessionAsyncClient::closeAsync, + (session, error) -> session.closeAsync(), + VoiceAgentWebSocketSessionAsyncClient::closeAsync)) + .then(Mono.defer(() -> conversationId.get() == null + ? Mono.fromRunnable(() -> System.out.println("No persisted conversation ID was returned.")) + : VoiceAgentRealtimeSampleUtils.readConversation(conversations, agentName, conversationId.get()))) + .then(Mono.defer(() -> cleanupAgent(agents, agentName, keepAgent))) + .onErrorResume(error -> Mono.defer(() -> cleanupAgent(agents, agentName, keepAgent)) + .onErrorResume(cleanupError -> Mono.empty()) + .then(Mono.error(error))) + .block(); + } + + private static Mono cleanupAgent(AgentsAsyncClient agents, String agentName, boolean keepAgent) { + if (keepAgent) { + return Mono.fromRunnable(() -> System.out.println("Kept voice agent: " + agentName)); + } + return agents.deleteAgent(agentName) + .doOnSuccess(ignored -> System.out.println("Deleted voice agent: " + agentName)); + } + + private static Mono runConversation(VoiceAgentWebSocketSessionAsyncClient session, + AtomicReference conversationId) { + return Mono.using(() -> new AudioProcessor(session), processor -> { + processor.start(); + AtomicBoolean responseActive = new AtomicBoolean(); + Disposable receiver = session.receiveEvents().subscribe(event -> { + if (event instanceof RealtimeServerEventSessionCreated) { + String id = ((RealtimeServerEventSessionCreated) event).getConversationId(); + if (id != null) { + conversationId.set(id); + } + } else if (event instanceof RealtimeServerEventInputAudioBufferSpeechStarted) { + if (responseActive.get()) { + processor.skipPendingAudio(); + session.cancelResponse().onErrorResume(error -> Mono.empty()).subscribe(); + System.out.println("(listening...)"); + } + } else if (event instanceof RealtimeServerEventConversationItemInputAudioTranscriptionCompleted) { + System.out.println("You: " + + ((RealtimeServerEventConversationItemInputAudioTranscriptionCompleted) event) + .getTranscript().trim()); + } else if (event instanceof RealtimeServerEventResponseCreated) { + responseActive.set(true); + } else if (event instanceof RealtimeServerEventResponseDone) { + responseActive.set(false); + } else if (event instanceof RealtimeServerEventResponseAudioDelta) { + processor.queueAudio(((RealtimeServerEventResponseAudioDelta) event).getDelta()); + } else if (event instanceof RealtimeServerEventResponseAudioTranscriptDone) { + System.out.println("Agent: " + + ((RealtimeServerEventResponseAudioTranscriptDone) event).getTranscript()); + } else if (event instanceof RealtimeServerEventRealtimeServerEventError) { + RealtimeServerEventRealtimeServerEventError error + = (RealtimeServerEventRealtimeServerEventError) event; + System.out.println("Session error: " + error.getError().getMessage()); + } + }, error -> System.err.println("Realtime session ended: " + error.getMessage())); + System.out.println("Speak now; talk over the agent to interrupt it. Press Enter to end the session."); + return Mono.fromRunnable(() -> new Scanner(System.in).nextLine()) + .subscribeOn(Schedulers.boundedElastic()) + .doFinally(signal -> receiver.dispose()) + .then(); + }, AudioProcessor::close); + } + + private static final class AudioProcessor implements AutoCloseable { + private static final int CHUNK_BYTES = 2400; + private static final byte[] STOP = new byte[0]; + private final VoiceAgentWebSocketSessionAsyncClient session; + private final AudioFormat format = new AudioFormat(VoiceAgentRealtimeSampleUtils.SAMPLE_RATE, 16, 1, true, false); + private final BlockingQueue playback = new LinkedBlockingQueue<>(); + private final AtomicBoolean running = new AtomicBoolean(); + private TargetDataLine microphone; + private SourceDataLine speaker; + private Thread captureThread; + private Thread playbackThread; + + AudioProcessor(VoiceAgentWebSocketSessionAsyncClient session) { + this.session = session; + } + + void start() { + try { + microphone = AudioSystem.getTargetDataLine(format); + microphone.open(format, CHUNK_BYTES * 4); + speaker = AudioSystem.getSourceDataLine(format); + speaker.open(format); + microphone.start(); + speaker.start(); + } catch (LineUnavailableException | IllegalArgumentException error) { + close(); + throw new IllegalStateException("A 24-kHz mono PCM16 microphone and speaker are required.", error); + } + + running.set(true); + captureThread = new Thread(this::capture, "voice-agent-microphone"); + playbackThread = new Thread(this::playback, "voice-agent-speaker"); + captureThread.setDaemon(true); + playbackThread.setDaemon(true); + captureThread.start(); + playbackThread.start(); + } + + private void capture() { + byte[] buffer = new byte[CHUNK_BYTES]; + while (running.get()) { + int read = microphone.read(buffer, 0, buffer.length); + if (read > 0) { + try { + session.appendInputAudio(BinaryData.fromBytes(Arrays.copyOf(buffer, read))).block(SEND_TIMEOUT); + } catch (RuntimeException error) { + if (running.get()) { + System.err.println("Microphone upload stopped: " + error.getMessage()); + } + running.set(false); + } + } + } + } + + private void playback() { + try { + while (running.get()) { + byte[] pcm = playback.take(); + if (pcm == STOP) { + break; + } + speaker.write(pcm, 0, pcm.length); + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } + } + + void queueAudio(byte[] pcm) { + if (pcm != null && pcm.length > 0) { + playback.offer(pcm); + } + } + + void skipPendingAudio() { + playback.clear(); + if (speaker != null) { + speaker.flush(); + } + } + + @Override + public void close() { + running.set(false); + playback.clear(); + playback.offer(STOP); + if (microphone != null) { + microphone.stop(); + microphone.close(); + microphone = null; + } + if (speaker != null) { + speaker.stop(); + speaker.close(); + speaker = null; + } + if (captureThread != null) { + captureThread.interrupt(); + } + if (playbackThread != null) { + playbackThread.interrupt(); + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java new file mode 100644 index 000000000000..ac7754883df9 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveFunctionToolSample.java @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaVoiceAgentWebSocketClient; +import com.azure.ai.agents.VoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeConversationItem; +import com.azure.ai.agents.models.RealtimeConversationItemType; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeServerEventRealtimeServerEventError; +import com.azure.ai.agents.models.RealtimeServerEventResponseDone; +import com.azure.ai.agents.models.RealtimeServerEventResponseFunctionCallArgumentsDone; +import com.azure.ai.agents.models.RealtimeServerEventResponseTextDone; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceAgentFunctionTool; +import com.azure.ai.agents.models.VoiceAgentTool; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; + +import java.time.Duration; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Demonstrates executing a client-side function tool during a live voice-agent session. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to + * {@code sample-voice-agent-function-tool-java}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL} - Optional. The voice model. Defaults to {@code gpt-realtime}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL_TYPE} - Optional. The voice model type. Defaults to {@code managed}.
    • + *
+ */ +public class VoiceAgentLiveFunctionToolSample { + private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(45); + + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", + "sample-voice-agent-function-tool-java"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsClient agents = builder.buildAgentsClient(); + BetaVoiceAgentWebSocketClient realtime = builder.buildBetaVoiceAgentWebSocketClient(); + + Map cityProperty = new LinkedHashMap<>(); + cityProperty.put("type", "string"); + cityProperty.put("description", "City name, for example Seattle."); + Map properties = new LinkedHashMap<>(); + properties.put("city", cityProperty); + Map schema = new LinkedHashMap<>(); + schema.put("type", "object"); + schema.put("properties", properties); + schema.put("required", Collections.singletonList("city")); + + VoiceAgentFunctionTool weatherTool = new VoiceAgentFunctionTool("get_weather") + .setDescription("Get the current weather for a city.") + .setParameters(BinaryData.fromObject(schema)); + VoiceAgentDefinition definition = new VoiceAgentDefinition() + .setModelType(modelType) + .setModel(model) + .setInstructions("Use the get_weather tool when asked about weather, then answer using its result.") + .setOutputModalities(Collections.singletonList(VoiceOutputModality.TEXT)) + .setTools(Collections.singletonList(weatherTool)); + + try { + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition)); + System.out.println("Created voice agent: " + agentName); + try (VoiceAgentWebSocketSessionClient session = realtime.connect(agentName)) { + ExecutorService receiver = Executors.newSingleThreadExecutor(); + Future response = receiver.submit(() -> receiveResponse(session)); + try { + session.sendText("What's the weather like in Seattle right now?"); + session.createResponse(); + response.get(RESPONSE_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException error) { + System.out.println("Timed out waiting for the agent's reply; cancelling the active response."); + session.cancelResponse(); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } catch (java.util.concurrent.ExecutionException error) { + throw new IllegalStateException("The realtime receive loop failed.", error.getCause()); + } finally { + response.cancel(true); + receiver.shutdownNow(); + } + } + } finally { + agents.deleteAgent(agentName); + System.out.println("Deleted voice agent: " + agentName); + } + } + + private static void receiveResponse(VoiceAgentWebSocketSessionClient session) { + for (RealtimeServerEvent event : session.receiveEvents()) { + if (event instanceof RealtimeServerEventResponseFunctionCallArgumentsDone) { + RealtimeServerEventResponseFunctionCallArgumentsDone call + = (RealtimeServerEventResponseFunctionCallArgumentsDone) event; + session.sendFunctionCallOutput(call.getCallId(), executeTool(call)); + } else if (event instanceof RealtimeServerEventResponseTextDone) { + System.out.println("Agent: " + ((RealtimeServerEventResponseTextDone) event).getText()); + } else if (event instanceof RealtimeServerEventResponseDone) { + if (!containsFunctionCall((RealtimeServerEventResponseDone) event)) { + return; + } + } else if (event instanceof RealtimeServerEventRealtimeServerEventError) { + RealtimeServerEventRealtimeServerEventError error + = (RealtimeServerEventRealtimeServerEventError) event; + System.out.println("Session error: " + error.getError().getMessage()); + return; + } + } + } + + @SuppressWarnings("unchecked") + private static String executeTool(RealtimeServerEventResponseFunctionCallArgumentsDone call) { + Map arguments = BinaryData.fromString(call.getArguments()).toObject(Map.class); + System.out.printf("Tool call: %s(%s)%n", call.getName(), arguments); + Map result = new LinkedHashMap<>(); + if ("get_weather".equals(call.getName())) { + result.put("city", arguments.get("city")); + result.put("condition", "sunny"); + result.put("temperature_f", 72); + } else { + result.put("error", "Unknown tool: " + call.getName()); + } + return BinaryData.fromObject(result).toString(); + } + + private static boolean containsFunctionCall(RealtimeServerEventResponseDone event) { + List output = event.getResponse().getOutput(); + if (output == null) { + return false; + } + for (RealtimeConversationItem item : output) { + if (item.getType() == RealtimeConversationItemType.FUNCTION_CALL) { + return true; + } + } + return false; + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java new file mode 100644 index 000000000000..9623167a3665 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsAsyncClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaAgentEndpointConversationsAsyncClient; +import com.azure.ai.agents.BetaAgentsAsyncClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketAsyncClient; +import com.azure.ai.agents.VoiceAgentWebSocketSessionAsyncClient; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import reactor.core.Disposable; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; +import reactor.core.scheduler.Schedulers; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Scanner; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Demonstrates an asynchronous, typed, multi-turn realtime conversation with a persisted voice agent. + * + *

To end the call, submit a blank line or enter {@code exit} or {@code quit}. The sample then closes the WebSocket, + * reads the persisted conversation, and deletes the agent unless {@code FOUNDRY_KEEP_VOICE_AGENT} is set to + * {@code true}.

+ * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to + * {@code sample-live-text-conversation-agent-async-java}.
    • + *
  • {@code FOUNDRY_KEEP_VOICE_AGENT} - Optional. Set to {@code true} to keep the agent after the sample. + * Defaults to {@code false}.
    • + *
+ */ +public class VoiceAgentLiveTextConversationAsyncSample { + private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(45); + + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", + "sample-live-text-conversation-agent-async-java"); + boolean keepAgent + = Boolean.parseBoolean(configuration.get("FOUNDRY_KEEP_VOICE_AGENT", "false")); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsAsyncClient agents = builder.buildAgentsAsyncClient(); + BetaAgentsAsyncClient betaAgents = builder.beta().buildBetaAgentsAsyncClient(); + BetaVoiceAgentWebSocketAsyncClient realtime = builder.buildBetaVoiceAgentWebSocketAsyncClient(); + BetaAgentEndpointConversationsAsyncClient conversations + = builder.buildBetaAgentEndpointConversationsAsyncClient(); + + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + + AtomicReference conversationId = new AtomicReference<>(); + VoiceAgentRealtimeSampleUtils.SpeakerPlayer player = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer(); + Scanner scanner = new Scanner(System.in); + + betaAgents.generateAgent(BinaryData.fromObject(request)) + .flatMap(generated -> { + VoiceAgentDefinition definition + = (VoiceAgentDefinition) generated.getVersions().getLatest().getDefinition(); + return agents.createAgentVersion(agentName, + new CreateAgentVersionInput(definition.setStore(true))); + }) + .then(Mono.usingWhen(realtime.connect(agentName), + session -> runConversation(session, scanner, conversationId, player), + VoiceAgentWebSocketSessionAsyncClient::closeAsync, + (session, error) -> session.closeAsync(), + VoiceAgentWebSocketSessionAsyncClient::closeAsync)) + .then(Mono.defer(() -> conversationId.get() == null + ? Mono.fromRunnable(() -> System.out.println("No persisted conversation ID was returned.")) + : VoiceAgentRealtimeSampleUtils.readConversation(conversations, agentName, conversationId.get()))) + .then(Mono.defer(() -> cleanupAgent(agents, agentName, keepAgent))) + .onErrorResume(error -> Mono.defer(() -> cleanupAgent(agents, agentName, keepAgent)) + .onErrorResume(cleanupError -> Mono.empty()) + .then(Mono.error(error))) + .doFinally(signal -> { + scanner.close(); + player.close(); + }) + .block(); + } + + private static Mono cleanupAgent(AgentsAsyncClient agents, String agentName, boolean keepAgent) { + if (keepAgent) { + return Mono.fromRunnable(() -> System.out.println("Kept voice agent: " + agentName)); + } + return agents.deleteAgent(agentName) + .doOnSuccess(ignored -> System.out.println("Deleted voice agent: " + agentName)); + } + + private static Mono runConversation(VoiceAgentWebSocketSessionAsyncClient session, Scanner scanner, + AtomicReference conversationId, VoiceAgentRealtimeSampleUtils.SpeakerPlayer player) { + AtomicReference> responseCompleted = new AtomicReference<>(); + Disposable receiver = session.receiveEvents().subscribe(event -> { + if (VoiceAgentRealtimeSampleUtils.handleResponseEvent(event, conversationId, player)) { + Sinks.One completion = responseCompleted.getAndSet(null); + if (completion != null) { + completion.tryEmitEmpty(); + } + } + }, error -> { + Sinks.One completion = responseCompleted.getAndSet(null); + if (completion != null) { + completion.tryEmitError(error); + } + }); + + System.out.println("Type a message and press Enter. Blank line (or 'exit') ends the session."); + return prompt(session, scanner, responseCompleted) + .doFinally(signal -> { + receiver.dispose(); + System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), + player.isEnabled() ? " and played it" : ""); + }); + } + + private static Mono prompt(VoiceAgentWebSocketSessionAsyncClient session, Scanner scanner, + AtomicReference> responseCompleted) { + return Mono.fromCallable(() -> { + System.out.print("You: "); + return scanner.nextLine().trim(); + }).subscribeOn(Schedulers.boundedElastic()).flatMap(text -> { + String normalized = text.toLowerCase(Locale.ROOT); + if (text.isEmpty() || "exit".equals(normalized) || "quit".equals(normalized)) { + return Mono.empty(); + } + + Sinks.One completion = Sinks.one(); + responseCompleted.set(completion); + return session.sendText(text) + .then(session.createResponse()) + .then(completion.asMono().timeout(RESPONSE_TIMEOUT)) + .onErrorResume(java.util.concurrent.TimeoutException.class, error -> { + System.out.println("Timed out waiting for the agent's reply; cancelling the active response."); + return session.cancelResponse() + .then(completion.asMono().timeout(Duration.ofSeconds(10))) + .onErrorResume(cancelError -> Mono.empty()); + }) + .then(Mono.defer(() -> prompt(session, scanner, responseCompleted))); + }); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java new file mode 100644 index 000000000000..a47e8aead3c5 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationSample.java @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.BetaAgentEndpointConversationsClient; +import com.azure.ai.agents.BetaAgentsClient; +import com.azure.ai.agents.BetaVoiceAgentWebSocketClient; +import com.azure.ai.agents.VoiceAgentWebSocketSessionClient; +import com.azure.ai.agents.models.AgentDetails; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Scanner; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Demonstrates a synchronous, typed, multi-turn realtime conversation with a persisted voice agent. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - Optional. The voice agent name. Defaults to + * {@code sample-live-text-conversation-agent-java}.
    • + *
+ */ +public class VoiceAgentLiveTextConversationSample { + private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(45); + + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", + "sample-live-text-conversation-agent-java"); + + AgentsClientBuilder builder = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true); + AgentsClient agents = builder.buildAgentsClient(); + BetaAgentsClient betaAgents = builder.beta().buildBetaAgentsClient(); + BetaVoiceAgentWebSocketClient realtime = builder.buildBetaVoiceAgentWebSocketClient(); + BetaAgentEndpointConversationsClient conversations = builder.buildBetaAgentEndpointConversationsClient(); + + Map request = new LinkedHashMap<>(); + request.put("kind", "voice"); + request.put("name", agentName); + AgentDetails generated = betaAgents.generateAgent(BinaryData.fromObject(request)); + + try { + AgentVersionDetails latest = generated.getVersions().getLatest(); + VoiceAgentDefinition definition = (VoiceAgentDefinition) latest.getDefinition(); + agents.createAgentVersion(agentName, new CreateAgentVersionInput(definition.setStore(true))); + + AtomicReference conversationId = new AtomicReference<>(); + try (VoiceAgentRealtimeSampleUtils.SpeakerPlayer player + = new VoiceAgentRealtimeSampleUtils.SpeakerPlayer(); + VoiceAgentWebSocketSessionClient session = realtime.connect(agentName); + Scanner scanner = new Scanner(System.in)) { + AtomicReference> responseCompleted = new AtomicReference<>(); + ExecutorService receiver = Executors.newSingleThreadExecutor(); + receiver.submit(() -> { + try { + for (RealtimeServerEvent event : session.receiveEvents()) { + if (VoiceAgentRealtimeSampleUtils.handleResponseEvent(event, conversationId, player)) { + CompletableFuture completion = responseCompleted.getAndSet(null); + if (completion != null) { + completion.complete(null); + } + } + } + } catch (RuntimeException error) { + CompletableFuture completion = responseCompleted.getAndSet(null); + if (completion != null) { + completion.completeExceptionally(error); + } + } + }); + + try { + System.out.println("Type a message and press Enter. Blank line (or 'exit') ends the session."); + while (true) { + System.out.print("You: "); + String prompt = scanner.nextLine().trim(); + String normalized = prompt.toLowerCase(Locale.ROOT); + if (prompt.isEmpty() || "exit".equals(normalized) || "quit".equals(normalized)) { + break; + } + + CompletableFuture completion = new CompletableFuture<>(); + responseCompleted.set(completion); + session.sendText(prompt); + session.createResponse(); + if (!awaitResponse(session, completion)) { + break; + } + } + } finally { + receiver.shutdownNow(); + } + System.out.printf("(received %.2fs of reply audio%s)%n", player.getSecondsReceived(), + player.isEnabled() ? " and played it" : ""); + } + + if (conversationId.get() != null) { + VoiceAgentRealtimeSampleUtils.readConversation(conversations, agentName, conversationId.get()); + } else { + System.out.println("No persisted conversation ID was returned."); + } + } finally { + agents.deleteAgent(agentName); + System.out.println("Deleted voice agent: " + agentName); + } + } + + private static boolean awaitResponse(VoiceAgentWebSocketSessionClient session, + CompletableFuture completion) { + try { + completion.get(RESPONSE_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + return true; + } catch (TimeoutException error) { + System.out.println("Timed out waiting for the agent's reply; cancelling the active response."); + session.cancelResponse(); + try { + completion.get(10, TimeUnit.SECONDS); + return true; + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException | TimeoutException ignored) { + return false; + } + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException error) { + throw new IllegalStateException("The realtime receive loop failed.", error.getCause()); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java new file mode 100644 index 000000000000..82572695b212 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationAudioSample.java @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaAgentEndpointConversationsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.VoiceItemAudioResponse; +import com.azure.ai.agents.models.VoiceRecordingResponse; +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +/** + * Demonstrates downloading whole-call and item-level audio from a persisted voice conversation. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name.
    • + *
  • {@code FOUNDRY_VOICE_CONVERSATION_ID} - The persisted voice conversation ID.
    • + *
+ */ +public class VoiceAgentReadConversationAudioSample { + public static void main(String[] args) throws IOException { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME"); + String conversationId = configuration.get("FOUNDRY_VOICE_CONVERSATION_ID"); + BetaAgentEndpointConversationsClient conversations = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .buildBetaAgentEndpointConversationsClient(); + + VoiceRecordingResponse recording = conversations.getAgentConversationAudio(agentName, conversationId); + System.out.printf("Recording: format=%s, rate=%d, channels=%d, duration=%s%n", + recording.getFormat(), recording.getSampleRate(), recording.getChannels(), recording.getDurationMs()); + if (recording.getBlobUri() != null) { + System.out.println("Recording is stored in customer storage: " + recording.getBlobUri()); + } else { + Path output = Files.createTempFile(conversationId + "-", ".wav"); + Files.write(output, conversations.getAgentConversationAudioContent(agentName, conversationId).toBytes()); + System.out.println("Wrote merged recording: " + output); + } + + for (BinaryData itemData : conversations.listAgentConversationItems(agentName, conversationId, + new RequestOptions())) { + @SuppressWarnings("unchecked") + Map item = itemData.toObject(Map.class); + String itemId = (String) item.get("id"); + if (itemId == null) { + continue; + } + try { + VoiceItemAudioResponse metadata = conversations.getAgentConversationItemAudio( + agentName, conversationId, itemId); + if (metadata.getBlobUri() != null) { + System.out.println("Item audio is stored in customer storage: " + metadata.getBlobUri()); + } else { + Path output = Files.createTempFile(conversationId + "-" + itemId + "-", ".wav"); + Files.write(output, conversations.getAgentConversationItemAudioContent( + agentName, conversationId, itemId).toBytes()); + System.out.println("Wrote item audio: " + output); + } + break; + } catch (ResourceNotFoundException ignored) { + // This transcript item has no persisted audio. + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java new file mode 100644 index 000000000000..d63e6f443852 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentReadConversationSample.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaAgentEndpointConversationsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.ai.agents.models.RealtimeConversationItem; +import com.azure.ai.agents.models.VoiceConversation; +import com.azure.ai.agents.models.VoiceResponse; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; + +import java.util.Map; + +/** + * Demonstrates reading a persisted voice conversation, its responses, and transcript items. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name.
    • + *
  • {@code FOUNDRY_VOICE_CONVERSATION_ID} - The persisted voice conversation ID.
    • + *
+ */ +public class VoiceAgentReadConversationSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME"); + String conversationId = configuration.get("FOUNDRY_VOICE_CONVERSATION_ID"); + BetaAgentEndpointConversationsClient conversations = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .buildBetaAgentEndpointConversationsClient(); + + VoiceConversation conversation = conversations.getAgentConversation(agentName, conversationId); + System.out.printf("Conversation %s: status=%s, created=%s, usage=%s%n", + conversation.getId(), conversation.getStatus(), conversation.getCreatedAt(), conversation.getUsage()); + + for (VoiceResponse response : conversations.listAgentConversationResponses(agentName, conversationId)) { + VoiceResponse detail = conversations.getAgentConversationResponse(agentName, conversationId, + response.getId()); + System.out.printf("Response %s: status=%s, usage=%s%n", + detail.getId(), detail.getStatus(), detail.getUsage()); + for (RealtimeConversationItem item : conversations.listAgentConversationResponseItems( + agentName, conversationId, response.getId())) { + System.out.println(" Response item type: " + item.getType()); + } + } + + for (BinaryData itemData : conversations.listAgentConversationItems(agentName, conversationId, + new RequestOptions())) { + @SuppressWarnings("unchecked") + Map item = itemData.toObject(Map.class); + String itemId = (String) item.get("id"); + System.out.printf("Transcript item: type=%s, id=%s%n", item.get("type"), itemId); + if (itemId != null) { + RealtimeConversationItem fetched = conversations.getAgentConversationItem(agentName, + conversationId, itemId); + System.out.println(" Fetched item type: " + fetched.getType()); + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java new file mode 100644 index 000000000000..5176cfeea851 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentRealtimeSampleUtils.java @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.BetaAgentEndpointConversationsAsyncClient; +import com.azure.ai.agents.BetaAgentEndpointConversationsClient; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.RealtimeServerEventRealtimeServerEventError; +import com.azure.ai.agents.models.RealtimeServerEventResponseAudioDelta; +import com.azure.ai.agents.models.RealtimeServerEventResponseAudioTranscriptDone; +import com.azure.ai.agents.models.RealtimeServerEventResponseDone; +import com.azure.ai.agents.models.RealtimeServerEventSessionCreated; +import com.azure.ai.agents.models.VoiceConversation; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.util.BinaryData; +import reactor.core.publisher.Mono; + +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.LineUnavailableException; +import javax.sound.sampled.SourceDataLine; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +final class VoiceAgentRealtimeSampleUtils { + static final int SAMPLE_RATE = 24000; + + private VoiceAgentRealtimeSampleUtils() { + } + + static boolean handleResponseEvent(RealtimeServerEvent event, AtomicReference conversationId, + SpeakerPlayer player) { + if (event instanceof RealtimeServerEventSessionCreated) { + String id = ((RealtimeServerEventSessionCreated) event).getConversationId(); + if (id != null) { + conversationId.set(id); + } + } else if (event instanceof RealtimeServerEventResponseAudioDelta) { + player.play(((RealtimeServerEventResponseAudioDelta) event).getDelta()); + } else if (event instanceof RealtimeServerEventResponseAudioTranscriptDone) { + System.out.println("Agent: " + + ((RealtimeServerEventResponseAudioTranscriptDone) event).getTranscript()); + } else if (event instanceof RealtimeServerEventRealtimeServerEventError) { + RealtimeServerEventRealtimeServerEventError error + = (RealtimeServerEventRealtimeServerEventError) event; + System.out.println("Session error: " + error.getError().getMessage()); + return true; + } + return event instanceof RealtimeServerEventResponseDone; + } + + static void readConversation(BetaAgentEndpointConversationsClient conversations, String agentName, + String conversationId) { + VoiceConversation conversation = conversations.getAgentConversation(agentName, conversationId); + System.out.printf("Conversation %s: status=%s, created=%s%n", conversation.getId(), + conversation.getStatus(), conversation.getCreatedAt()); + for (BinaryData item : conversations.listAgentConversationItems(agentName, conversationId, + new RequestOptions())) { + printConversationItem(item); + } + } + + static Mono readConversation(BetaAgentEndpointConversationsAsyncClient conversations, String agentName, + String conversationId) { + return conversations.getAgentConversation(agentName, conversationId) + .doOnNext(conversation -> System.out.printf("Conversation %s: status=%s, created=%s%n", + conversation.getId(), conversation.getStatus(), conversation.getCreatedAt())) + .thenMany(conversations.listAgentConversationItems(agentName, conversationId, new RequestOptions())) + .doOnNext(VoiceAgentRealtimeSampleUtils::printConversationItem) + .then(); + } + + @SuppressWarnings("unchecked") + private static void printConversationItem(BinaryData itemData) { + Map item = itemData.toObject(Map.class); + System.out.printf(" - %s id=%s%n", item.get("role") == null ? item.get("type") : item.get("role"), + item.get("id")); + Object contentValue = item.get("content"); + if (!(contentValue instanceof List)) { + return; + } + StringBuilder transcript = new StringBuilder(); + for (Object partValue : (List) contentValue) { + if (partValue instanceof Map) { + Map part = (Map) partValue; + Object text = part.get("transcript") == null ? part.get("text") : part.get("transcript"); + if (text != null && !text.toString().trim().isEmpty()) { + if (transcript.length() > 0) { + transcript.append(' '); + } + transcript.append(text.toString().trim()); + } + } + } + if (transcript.length() > 0) { + System.out.println(" " + transcript); + } + } + + static final class SpeakerPlayer implements AutoCloseable { + private SourceDataLine line; + private long bytesReceived; + + SpeakerPlayer() { + AudioFormat format = new AudioFormat(SAMPLE_RATE, 16, 1, true, false); + try { + line = AudioSystem.getSourceDataLine(format); + line.open(format); + line.start(); + } catch (LineUnavailableException | IllegalArgumentException error) { + line = null; + System.out.println("(speaker playback unavailable; audio will be counted but not played)"); + } + } + + synchronized void play(byte[] pcm) { + if (pcm == null) { + return; + } + bytesReceived += pcm.length; + if (line != null) { + line.write(pcm, 0, pcm.length); + } + } + + synchronized void discardQueuedAudio() { + if (line != null) { + line.flush(); + } + } + + double getSecondsReceived() { + return bytesReceived / 2.0 / SAMPLE_RATE; + } + + boolean isEnabled() { + return line != null; + } + + @Override + public synchronized void close() { + if (line != null) { + line.drain(); + line.stop(); + line.close(); + line = null; + } + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java new file mode 100644 index 000000000000..5f7a8da6ddf8 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentSampleUtils.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.models.VoiceAgentAudioConfig; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfig; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceModelType; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; + +import java.util.Collections; + +final class VoiceAgentSampleUtils { + private VoiceAgentSampleUtils() { + } + + static VoiceAgentDefinition createDefinition(VoiceModelType modelType, String model, String instructions) { + VoiceAgentAudioOutputConfig output = new VoiceAgentAudioOutputConfig() + .setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD); + return new VoiceAgentDefinition() + .setModelType(modelType) + .setModel(model) + .setInstructions(instructions) + .setAudio(new VoiceAgentAudioConfig().setOutput(output)) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setStore(true); + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java new file mode 100644 index 000000000000..a7b7c2d381f4 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentVersionsSample.java @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.VoiceModelType; + +/** + * Demonstrates released and draft voice-agent versions. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_MODEL} - Optional. The voice model or deployment name. Defaults to {@code gpt-realtime}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL_TYPE} - Optional. The voice model type. Defaults to {@code managed}.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name. Defaults to {@code versioned-voice-agent-java}.
    • + *
+ */ +public class VoiceAgentVersionsSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "versioned-voice-agent-java"); + + AgentsClient client = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true) + .buildAgentsClient(); + try { + AgentVersionDetails first = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a helpful voice assistant."))); + AgentVersionDetails released = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are a helpful voice assistant. Greet the caller by name.")) + .setDescription("Added a personalized greeting.")); + AgentVersionDetails draft = client.createAgentVersion(agentName, + new CreateAgentVersionInput(VoiceAgentSampleUtils.createDefinition(modelType, model, + "You are an experimental voice assistant.")) + .setDescription("Candidate persona under review.") + .setDraft(true)); + System.out.printf("Created versions %s, %s and draft %s%n", + first.getVersion(), released.getVersion(), draft.getVersion()); + + System.out.println("Released versions:"); + for (AgentVersionDetails version : client.listAgentVersions(agentName)) { + System.out.printf(" %s (draft=%s)%n", version.getVersion(), version.isDraft()); + } + System.out.println("All versions including drafts:"); + for (AgentVersionDetails version : client.listAgentVersions(agentName, null, null, null, null, true)) { + System.out.printf(" %s (draft=%s)%n", version.getVersion(), version.isDraft()); + } + AgentVersionDetails fetched = client.getAgentVersionDetails(agentName, released.getVersion()); + System.out.println("Fetched version: " + fetched.getVersion()); + } finally { + client.deleteAgent(agentName); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java new file mode 100644 index 000000000000..0c0dc693ea14 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentWithToolsSample.java @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.voice; + +import com.azure.ai.agents.AgentsClient; +import com.azure.ai.agents.AgentsClientBuilder; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Configuration; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.ai.agents.models.AgentVersionDetails; +import com.azure.ai.agents.models.CreateAgentVersionInput; +import com.azure.ai.agents.models.RealtimeAudioFormatsAudioPcm; +import com.azure.ai.agents.models.RealtimeAudioFormatsAudioPcmRate; +import com.azure.ai.agents.models.VoiceAgentAudioConfig; +import com.azure.ai.agents.models.VoiceAgentAudioInputConfig; +import com.azure.ai.agents.models.VoiceAgentAudioOutputConfig; +import com.azure.ai.agents.models.VoiceAgentDefinition; +import com.azure.ai.agents.models.VoiceAgentFunctionTool; +import com.azure.ai.agents.models.VoiceAgentInputTranscription; +import com.azure.ai.agents.models.VoiceAgentInputTranscriptionModel; +import com.azure.ai.agents.models.VoiceAgentServerVadTurnDetection; +import com.azure.ai.agents.models.VoiceAgentEndConversationSystemTool; +import com.azure.ai.agents.models.VoiceAgentTool; +import com.azure.ai.agents.models.VoiceOutputModality; +import com.azure.ai.agents.models.VoiceType; +import com.azure.ai.agents.models.VoiceModelType; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Demonstrates a voice-agent definition with audio processing, transcription, and tools. + * + *

Before running the sample, set these environment variables:

+ *
    + *
  • {@code FOUNDRY_PROJECT_ENDPOINT} - The Azure AI Project endpoint.
    • + *
  • {@code FOUNDRY_VOICE_MODEL} - Optional. The voice model or deployment name. Defaults to {@code gpt-realtime}.
    • + *
  • {@code FOUNDRY_VOICE_MODEL_TYPE} - Optional. The voice model type. Defaults to {@code managed}.
    • + *
  • {@code FOUNDRY_VOICE_AGENT_NAME} - The voice agent name. Defaults to {@code voice-agent-with-tools-java}.
    • + *
+ */ +public class VoiceAgentWithToolsSample { + public static void main(String[] args) { + Configuration configuration = Configuration.getGlobalConfiguration(); + String endpoint = configuration.get("FOUNDRY_PROJECT_ENDPOINT"); + String model = configuration.get("FOUNDRY_VOICE_MODEL", "gpt-realtime"); + VoiceModelType modelType = VoiceModelType.fromString(configuration.get( + "FOUNDRY_VOICE_MODEL_TYPE", VoiceModelType.MANAGED.toString())); + String agentName = configuration.get("FOUNDRY_VOICE_AGENT_NAME", "voice-agent-with-tools-java"); + + AgentsClient client = new AgentsClientBuilder() + .credential(new DefaultAzureCredentialBuilder().build()) + .endpoint(endpoint) + .allowPreview(true) + .buildAgentsClient(); + + RealtimeAudioFormatsAudioPcm pcm = new RealtimeAudioFormatsAudioPcm() + .setRate(RealtimeAudioFormatsAudioPcmRate.TWO_FOUR_ZERO_ZERO_ZERO); + VoiceAgentAudioInputConfig input = new VoiceAgentAudioInputConfig() + .setFormat(pcm) + .setTurnDetection(new VoiceAgentServerVadTurnDetection() + .setThreshold(0.5) + .setPrefixPaddingMs(300L) + .setSilenceDurationMs(500L)) + .setTranscription(new VoiceAgentInputTranscription(VoiceAgentInputTranscriptionModel.WHISPER_1)); + VoiceAgentAudioOutputConfig output = new VoiceAgentAudioOutputConfig() + .setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD); + Map cityProperty = new LinkedHashMap<>(); + cityProperty.put("type", "string"); + cityProperty.put("description", "City name, for example Seattle."); + Map properties = new LinkedHashMap<>(); + properties.put("city", cityProperty); + Map parameters = new LinkedHashMap<>(); + parameters.put("type", "object"); + parameters.put("properties", properties); + parameters.put("required", Collections.singletonList("city")); + VoiceAgentFunctionTool weather = new VoiceAgentFunctionTool("get_weather") + .setDescription("Get the current weather for a city.") + .setParameters(BinaryData.fromObject(parameters)); + VoiceAgentEndConversationSystemTool endCall = new VoiceAgentEndConversationSystemTool(); + VoiceAgentDefinition definition = new VoiceAgentDefinition() + .setModelType(modelType) + .setModel(model) + .setInstructions("Use tools when they help answer the caller.") + .setAudio(new VoiceAgentAudioConfig().setInput(input).setOutput(output)) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setTools(Arrays.asList(weather, endCall)) + .setStore(true); + + boolean agentCreated = false; + try { + AgentVersionDetails created = client.createAgentVersion(agentName, + new CreateAgentVersionInput(definition)); + agentCreated = true; + AgentVersionDetails fetched = client.getAgentVersionDetails(agentName, created.getVersion()); + VoiceAgentDefinition fetchedDefinition = (VoiceAgentDefinition) fetched.getDefinition(); + System.out.println("Configured voice tools: " + fetchedDefinition.getTools().size()); + for (VoiceAgentTool tool : fetchedDefinition.getTools()) { + System.out.printf(" %s%n", tool.getType()); + } + } finally { + if (agentCreated) { + client.deleteAgent(agentName); + } + } + } +} From b97bf9251bb30c3cf0bcf45e2d5f526551eae1be Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 11 Sep 2026 09:45:41 +0800 Subject: [PATCH 2/2] Address voice sample review feedback --- .../VoiceAgentLiveAudioConversationAsyncSample.java | 9 +++++++-- .../voice/VoiceAgentLiveTextConversationAsyncSample.java | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java index cce325c32b0a..94ec1cf88d17 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveAudioConversationAsyncSample.java @@ -159,10 +159,11 @@ private static Mono runConversation(VoiceAgentWebSocketSessionAsyncClient private static final class AudioProcessor implements AutoCloseable { private static final int CHUNK_BYTES = 2400; + private static final int MAX_PLAYBACK_CHUNKS = 8; private static final byte[] STOP = new byte[0]; private final VoiceAgentWebSocketSessionAsyncClient session; private final AudioFormat format = new AudioFormat(VoiceAgentRealtimeSampleUtils.SAMPLE_RATE, 16, 1, true, false); - private final BlockingQueue playback = new LinkedBlockingQueue<>(); + private final BlockingQueue playback = new LinkedBlockingQueue<>(MAX_PLAYBACK_CHUNKS); private final AtomicBoolean running = new AtomicBoolean(); private TargetDataLine microphone; private SourceDataLine speaker; @@ -228,7 +229,11 @@ private void playback() { void queueAudio(byte[] pcm) { if (pcm != null && pcm.length > 0) { - playback.offer(pcm); + try { + playback.put(pcm); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } } } diff --git a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java index 9623167a3665..d5d6c070f3a1 100644 --- a/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java +++ b/sdk/ai/azure-ai-agents/src/samples/java/com/azure/ai/agents/voice/VoiceAgentLiveTextConversationAsyncSample.java @@ -151,7 +151,8 @@ private static Mono prompt(VoiceAgentWebSocketSessionAsyncClient session, System.out.println("Timed out waiting for the agent's reply; cancelling the active response."); return session.cancelResponse() .then(completion.asMono().timeout(Duration.ofSeconds(10))) - .onErrorResume(cancelError -> Mono.empty()); + .onErrorMap(cancelError -> new IllegalStateException( + "Unable to cancel the active response.", cancelError)); }) .then(Mono.defer(() -> prompt(session, scanner, responseCompleted))); });