diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/ConversationsAsyncTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/ConversationsAsyncTests.java index 16e7ab8830d14..e302ecb22e5b0 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/ConversationsAsyncTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/ConversationsAsyncTests.java @@ -93,9 +93,13 @@ public void basicItemCRUDOperations(HttpClient httpClient, AgentsServiceVersion assertNotNull(conversationItem); assertNotNull(conversationItem.data()); assertFalse(conversationItem.data().isEmpty()); - assertTrue(conversationItem.data().get(0).isMessage()); - Message createdConversationItem = conversationItem.data().get(0).asMessage(); + Message createdConversationItem = conversationItem.data() + .stream() + .filter(ConversationItem::isMessage) + .map(ConversationItem::asMessage) + .findFirst() + .orElseThrow(() -> new AssertionError("Created conversation item did not contain a message.")); assertTrue(createdConversationItem.content().get(0).isInputText()); assertEquals("Hello, agent!", createdConversationItem.content().get(0).asInputText().text()); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/ConversationsTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/ConversationsTests.java index 9aa2ba4a2e6e9..13d8ac51e7131 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/ConversationsTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/ConversationsTests.java @@ -81,9 +81,13 @@ public void basicItemCRUDOperations(HttpClient httpClient, AgentsServiceVersion assertNotNull(conversationItem); assertNotNull(conversationItem.data()); assertFalse(conversationItem.data().isEmpty()); - assertTrue(conversationItem.data().get(0).isMessage()); - Message createdConversationItem = conversationItem.data().get(0).asMessage(); + Message createdConversationItem = conversationItem.data() + .stream() + .filter(ConversationItem::isMessage) + .map(ConversationItem::asMessage) + .findFirst() + .orElseThrow(() -> new AssertionError("Created conversation item did not contain a message.")); assertTrue(createdConversationItem.content().get(0).isInputText()); assertEquals("Hello, agent!", createdConversationItem.content().get(0).asInputText().text()); diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/MemoryStoresAsyncTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/MemoryStoresAsyncTests.java index e510a0de0070c..8006e219183b3 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/MemoryStoresAsyncTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/MemoryStoresAsyncTests.java @@ -42,7 +42,7 @@ public class MemoryStoresAsyncTests extends ClientTestBase { public void basicMemoryStoresCrud(HttpClient httpClient, AgentsServiceVersion serviceVersion) { BetaMemoryStoresAsyncClient memoryStoreClient = getMemoryStoresAsyncClient(httpClient, serviceVersion); - String memoryStoreName = "my_memory_store_java"; + String memoryStoreName = "my-memory-store-java"; String initialDescription = "Example memory store for conversations"; String updatedDescription = "Updated description"; @@ -104,7 +104,7 @@ public void basicMemoryStoresCrud(HttpClient httpClient, AgentsServiceVersion se public void basicMemoryStores(HttpClient httpClient, AgentsServiceVersion serviceVersion) { BetaMemoryStoresAsyncClient memoryStoreClient = getMemoryStoresAsyncClient(httpClient, serviceVersion); - String memoryStoreName = "my_memory_store"; + String memoryStoreName = "my-memory-store"; String description = "Example memory store for conversations"; String scope = "user_123"; String userMessageContent = "I prefer dark roast coffee and usually drink it in the morning"; @@ -164,7 +164,7 @@ public void basicMemoryStores(HttpClient httpClient, AgentsServiceVersion servic public void advancedMemoryStores(HttpClient httpClient, AgentsServiceVersion serviceVersion) { BetaMemoryStoresAsyncClient memoryStoreClient = getMemoryStoresAsyncClient(httpClient, serviceVersion); - String memoryStoreName = "my_memory_store"; + String memoryStoreName = "my-memory-store"; String description = "Example memory store for conversations"; String scope = "user_123"; String firstMessageContent = "I prefer dark roast coffee and usually drink it in the morning"; diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/MemoryStoresTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/MemoryStoresTests.java index 53b53cb2f62b3..b64f8b94d97f2 100644 --- a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/MemoryStoresTests.java +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/MemoryStoresTests.java @@ -33,7 +33,7 @@ public class MemoryStoresTests extends ClientTestBase { public void basicMemoryStoresCrud(HttpClient httpClient, AgentsServiceVersion serviceVersion) { BetaMemoryStoresClient memoryStoreClient = getMemoryStoresSyncClient(httpClient, serviceVersion); - String memoryStoreName = "my_memory_store_java"; + String memoryStoreName = "my-memory-store-java"; String initialDescription = "Example memory store for conversations"; String updatedDescription = "Updated description"; @@ -92,7 +92,7 @@ public void basicMemoryStoresCrud(HttpClient httpClient, AgentsServiceVersion se public void basicMemoryStores(HttpClient httpClient, AgentsServiceVersion serviceVersion) { BetaMemoryStoresClient memoryStoreClient = getMemoryStoresSyncClient(httpClient, serviceVersion); - String memoryStoreName = "my_memory_store"; + String memoryStoreName = "my-memory-store"; String description = "Example memory store for conversations"; String scope = "user_123"; String userMessageContent = "I prefer dark roast coffee and usually drink it in the morning"; @@ -160,7 +160,7 @@ public void basicMemoryStores(HttpClient httpClient, AgentsServiceVersion servic public void advancedMemoryStores(HttpClient httpClient, AgentsServiceVersion serviceVersion) { BetaMemoryStoresClient memoryStoreClient = getMemoryStoresSyncClient(httpClient, serviceVersion); - String memoryStoreName = "my_memory_store"; + String memoryStoreName = "my-memory-store"; String description = "Example memory store for conversations"; String scope = "user_123"; String firstMessageContent = "I prefer dark roast coffee and usually drink it in the morning"; diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/VoiceAgentWebSocketSessionTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/VoiceAgentWebSocketSessionTests.java new file mode 100644 index 0000000000000..28079acba82e5 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/VoiceAgentWebSocketSessionTests.java @@ -0,0 +1,539 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents; + +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketClientConfiguration; +import com.azure.ai.agents.implementation.realtime.VoiceAgentWebSocketHttpResponse; +import com.azure.ai.agents.models.RealtimeServerEvent; +import com.azure.ai.agents.models.VoiceAgentServerEventWarning; +import com.azure.ai.agents.models.VoiceAgentTransport; +import com.azure.ai.agents.models.VoiceAgentWebSocketConnectionOptions; +import com.azure.core.credential.AccessToken; +import com.azure.core.credential.TokenCredential; +import com.azure.core.exception.ResourceModifiedException; +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.BinaryData; +import com.azure.core.util.ClientOptions; +import com.azure.core.util.Configuration; +import com.azure.core.util.Header; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.http.websocketx.BinaryWebSocketFrame; +import io.netty.handler.codec.http.websocketx.ContinuationWebSocketFrame; +import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; +import io.netty.handler.codec.http.websocketx.WebSocketFrame; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.netty.DisposableServer; +import reactor.netty.http.Http11SslContextSpec; +import reactor.netty.http.client.HttpClient; +import reactor.netty.http.server.HttpServer; +import reactor.netty.http.server.WebsocketServerSpec; +import reactor.test.StepVerifier; + +import java.io.File; +import java.net.URI; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class VoiceAgentWebSocketSessionTests { + private DisposableServer server; + + @AfterEach + public void disposeServer() { + if (server != null) { + server.disposeNow(); + } + } + + @Test + public void asyncSessionNegotiatesHandshakeAndExchangesTypedEvents() { + List clientMessages = new CopyOnWriteArrayList<>(); + AtomicReference requestUri = new AtomicReference<>(); + AtomicReference authorization = new AtomicReference<>(); + AtomicReference foundryFeatures = new AtomicReference<>(); + AtomicReference userAgent = new AtomicReference<>(); + AtomicReference customHeader = new AtomicReference<>(); + server + = startServer(clientMessages, requestUri, authorization, foundryFeatures, userAgent, customHeader, false); + AtomicReference> requestedScopes = new AtomicReference<>(); + TokenCredential credential = request -> { + requestedScopes.set(request.getScopes()); + return Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + }; + VoiceAgentWebSocketConnectionOptions options + = new VoiceAgentWebSocketConnectionOptions().setTransport(VoiceAgentTransport.WEBSOCKET) + .setStoreEnabled(true) + .setAgentVersionOverride("version 2"); + BetaVoiceAgentWebSocketAsyncClient client + = new AgentsClientBuilder().endpoint("http://127.0.0.1:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .clientOptions(new ClientOptions().setApplicationId("test-app") + .setHeaders(Collections.singletonList(new Header("X-Test-Header", "test-value")))) + .beta() + .buildBetaVoiceAgentWebSocketAsyncClient(); + + VoiceAgentWebSocketSessionAsyncClient session = client.connect("agent name", options).block(); + assertTrue(session.isOpen()); + + StepVerifier.create(session.receiveEvents().take(4)) + .then(() -> session.sendText("hello").block()) + .assertNext(this::assertWarningEvent) + .then(() -> session.appendInputAudio(BinaryData.fromBytes(new byte[] { 1, 2, 3 })).block()) + .assertNext(this::assertWarningEvent) + .then(() -> session.createResponse().block()) + .assertNext(this::assertWarningEvent) + .then(() -> session.cancelResponse("response-1").block()) + .assertNext(this::assertWarningEvent) + .verifyComplete(); + + assertEquals(Collections.singletonList("https://ai.azure.com/.default"), requestedScopes.get()); + assertEquals("Bearer test-token", authorization.get()); + assertEquals("VoiceAgents=V1Preview", foundryFeatures.get()); + assertTrue(userAgent.get().startsWith("test-app azsdk-java-")); + assertEquals("test-value", customHeader.get()); + String decodedUri = decode(requestUri.get()); + assertTrue(decodedUri.contains("/agents/agent name/endpoint/protocols/voice")); + assertTrue(decodedUri.contains("api-version=v1")); + assertTrue(decodedUri.contains("transport=websocket")); + assertTrue(decodedUri.contains("store=true")); + assertTrue(decodedUri.contains("x-agent-version-override=version 2")); + assertTrue(decodedUri.contains("x-ms-client-sdk=test-app azsdk-java-")); + assertEquals(4, clientMessages.size()); + assertTrue(clientMessages.get(0).contains("\"type\":\"conversation.item.create\"")); + assertTrue(clientMessages.get(0).contains("\"role\":\"user\"")); + assertTrue(clientMessages.get(0).contains("\"text\":\"hello\"")); + assertTrue(clientMessages.get(1).contains("\"audio\":\"AQID\"")); + assertTrue(clientMessages.get(2).contains("\"type\":\"response.create\"")); + assertTrue(clientMessages.get(3).contains("\"response_id\":\"response-1\"")); + + StepVerifier.create(session.receiveEvents()) + .expectErrorMatches( + error -> error instanceof IllegalStateException && error.getMessage().contains("Only one")) + .verify(); + session.close(); + assertFalse(session.isOpen()); + } + + @Test + public void syncConnectRejectsNullArguments() { + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("https://localhost/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketClient(); + + NullPointerException agentNameException = assertThrows(NullPointerException.class, + () -> client.connect(null, new VoiceAgentWebSocketConnectionOptions())); + assertEquals("'agentName' cannot be null.", agentNameException.getMessage()); + + NullPointerException optionsException + = assertThrows(NullPointerException.class, () -> client.connect("agent", null)); + assertEquals("'options' cannot be null.", optionsException.getMessage()); + } + + @Test + public void tokenFailureOccursBeforeNetworkAccess() { + AtomicBoolean connected = new AtomicBoolean(); + server = HttpServer.create().host("127.0.0.1").port(0).handle((request, response) -> { + connected.set(true); + return response.send(); + }).bindNow(); + TokenCredential credential = request -> Mono.error(new IllegalStateException("token unavailable")); + BetaVoiceAgentWebSocketAsyncClient client + = new AgentsClientBuilder().endpoint("http://127.0.0.1:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketAsyncClient(); + + StepVerifier.create(client.connect("agent")) + .expectErrorMatches( + error -> error instanceof IllegalStateException && error.getMessage().contains("token unavailable")) + .verify(); + assertFalse(connected.get()); + } + + @Test + public void syncTokenFailureOccursBeforeNetworkAccess() { + AtomicBoolean connected = new AtomicBoolean(); + server = HttpServer.create().host("127.0.0.1").port(0).handle((request, response) -> { + connected.set(true); + return response.send(); + }).bindNow(); + TokenCredential credential = request -> Mono.error(new IllegalStateException("token unavailable")); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("http://127.0.0.1:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketClient(); + + IllegalStateException exception = assertThrows(IllegalStateException.class, () -> client.connect("agent")); + assertTrue(exception.getMessage().contains("token unavailable")); + assertFalse(connected.get()); + } + + @Test + public void rejectedHandshakeMapsConflictToAzureException() { + server = HttpServer.create() + .host("127.0.0.1") + .port(0) + .handle( + (request, response) -> response.status(HttpResponseStatus.CONFLICT).sendString(Mono.just("conflict"))) + .bindNow(); + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketAsyncClient client + = new AgentsClientBuilder().endpoint("http://127.0.0.1:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketAsyncClient(); + + StepVerifier.create(client.connect("disabled-agent")).expectErrorSatisfies(error -> { + ResourceModifiedException exception = assertInstanceOf(ResourceModifiedException.class, error); + assertEquals(409, exception.getResponse().getStatusCode()); + }).verify(); + } + + @Test + public void nettyHandshakeResponseExposesBufferedBody() { + DefaultFullHttpResponse nettyResponse = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, + HttpResponseStatus.CONFLICT, Unpooled.copiedBuffer("conflict", StandardCharsets.UTF_8)); + VoiceAgentWebSocketHttpResponse response + = new VoiceAgentWebSocketHttpResponse(URI.create("wss://example.com"), nettyResponse); + + assertEquals("conflict", response.getBodyAsString().block()); + } + + @Test + public void syncRejectedHandshakeMapsConflictToAzureException() { + server = HttpServer.create() + .host("127.0.0.1") + .port(0) + .handle( + (request, response) -> response.status(HttpResponseStatus.CONFLICT).sendString(Mono.just("conflict"))) + .bindNow(); + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("http://127.0.0.1:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketClient(); + + ResourceModifiedException exception + = assertThrows(ResourceModifiedException.class, () -> client.connect("disabled-agent")); + assertEquals(409, exception.getResponse().getStatusCode()); + assertEquals("conflict", exception.getResponse().getBodyAsString().block()); + } + + @Test + public void syncClientRejectsEmptyAgentName() { + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client = new AgentsClientBuilder().endpoint("https://example.com") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketClient(); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> client.connect("")); + assertEquals("'agentName' cannot be empty.", exception.getMessage()); + } + + @Test + public void cancellingAsyncConnectCancelsTokenRequest() { + AtomicBoolean tokenRequestCancelled = new AtomicBoolean(); + TokenCredential credential + = request -> Mono.never().doOnCancel(() -> tokenRequestCancelled.set(true)); + BetaVoiceAgentWebSocketAsyncClient client = new AgentsClientBuilder().endpoint("https://example.com") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketAsyncClient(); + + StepVerifier.create(client.connect("agent")).thenCancel().verify(); + assertTrue(tokenRequestCancelled.get()); + } + + @Test + public void unknownEventFallsBackToRealtimeServerEvent() { + server = oneShotWebSocketServer("{\"type\":\"future.event\",\"value\":42}"); + VoiceAgentWebSocketSessionAsyncClient session = createAsyncClient(server.port()).connect("agent").block(); + + StepVerifier.create(session.receiveEvents()) + .assertNext(event -> assertEquals("future.event", event.getType().toString())) + .verifyComplete(); + session.close(); + } + + @Test + public void fragmentedTextFrameIsAggregated() { + String message = warningJson(); + int split = message.length() / 2; + Flux frames = Flux.just(new TextWebSocketFrame(false, 0, message.substring(0, split)), + new ContinuationWebSocketFrame(true, 0, message.substring(split))); + server = frameWebSocketServer(frames); + VoiceAgentWebSocketSessionAsyncClient session = createAsyncClient(server.port()).connect("agent").block(); + + StepVerifier.create(session.receiveEvents()).assertNext(this::assertWarningEvent).verifyComplete(); + session.close(); + } + + @Test + public void binaryFrameTerminatesReceiveStream() { + server = frameWebSocketServer(Mono.just(new BinaryWebSocketFrame())); + VoiceAgentWebSocketSessionAsyncClient session = createAsyncClient(server.port()).connect("agent").block(); + + StepVerifier.create(session.receiveEvents()) + .expectErrorMatches( + error -> error instanceof IllegalArgumentException && error.getMessage().contains("JSON text frames")) + .verify(); + assertFalse(session.isOpen()); + } + + @Test + public void malformedJsonTerminatesReceiveStream() { + server = oneShotWebSocketServer("{not-json"); + VoiceAgentWebSocketSessionAsyncClient session = createAsyncClient(server.port()).connect("agent").block(); + + StepVerifier.create(session.receiveEvents()).expectError().verify(); + assertFalse(session.isOpen()); + } + + @Test + public void syncReceiveBufferOverflowFailsTheEventStream() { + Flux frames = Flux.range(0, 257).map(index -> new TextWebSocketFrame(warningJson())); + server = frameWebSocketServer(frames); + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("http://127.0.0.1:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketClient(); + + try (VoiceAgentWebSocketSessionClient session = client.connect("agent")) { + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + while (session.isOpen()) { + Thread.yield(); + } + }); + IllegalStateException exception + = assertThrows(IllegalStateException.class, () -> session.receiveEvents().iterator().hasNext()); + assertEquals("Voice-agent receive buffer overflow.", exception.getMessage()); + } + } + + @Test + public void syncOrderlyClosePreservesFullReceiveBuffer() { + Flux frames = Flux.range(0, 256).map(index -> new TextWebSocketFrame(warningJson())); + server = frameWebSocketServer(frames); + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("http://127.0.0.1:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketClient(); + + try (VoiceAgentWebSocketSessionClient session = client.connect("agent")) { + assertTimeoutPreemptively(Duration.ofSeconds(5), () -> { + Iterator events = session.receiveEvents().iterator(); + int eventCount = 0; + while (events.hasNext()) { + events.next(); + eventCount++; + } + assertEquals(256, eventCount); + }); + } + } + + @Test + public void closeIsIdempotentAndSendAfterCloseFails() { + List clientMessages = new CopyOnWriteArrayList<>(); + server = startServer(clientMessages, new AtomicReference<>(), new AtomicReference<>(), new AtomicReference<>(), + new AtomicReference<>(), new AtomicReference<>(), false); + VoiceAgentWebSocketSessionAsyncClient session = createAsyncClient(server.port()).connect("agent").block(); + + StepVerifier.create(session.closeAsync().then(session.closeAsync())).verifyComplete(); + StepVerifier.create(session.sendText("after close")) + .expectErrorMatches( + error -> error instanceof IllegalStateException && error.getMessage().contains("not open")) + .verify(); + } + + @Test + public void secureSessionUsesWssAndReceivesTypedEvent() throws Exception { + File certificate = resourceFile("websocket-localhost-cert.pem"); + File privateKey = resourceFile("websocket-localhost-key.pem"); + Http11SslContextSpec serverSsl = Http11SslContextSpec.forServer(certificate, privateKey); + WebsocketServerSpec websocketSpec = WebsocketServerSpec.builder().protocols("realtime").build(); + server = HttpServer.create() + .host("127.0.0.1") + .port(0) + .secure(ssl -> ssl.sslContext(serverSsl)) + .handle((request, response) -> response.sendWebsocket( + (inbound, outbound) -> outbound.sendString(Mono.just(warningJson()), StandardCharsets.UTF_8) + .then(outbound.sendClose()), + websocketSpec)) + .bindNow(); + + Http11SslContextSpec clientSsl + = Http11SslContextSpec.forClient().configure(builder -> builder.trustManager(certificate)); + HttpClient httpClient = HttpClient.create().secure(ssl -> ssl.sslContext(clientSsl)); + TokenCredential credential + = request -> Mono.just(new AccessToken("tls-token", OffsetDateTime.now().plusHours(1))); + VoiceAgentWebSocketClientConfiguration configuration = new VoiceAgentWebSocketClientConfiguration( + URI.create("https://localhost:" + server.port() + "/api/projects/project"), credential, "v1", + "azure-ai-agents-test", new HttpHeaders(), null); + VoiceAgentWebSocketSessionAsyncClient session = new VoiceAgentWebSocketSessionAsyncClient(configuration, + "secure-agent", new VoiceAgentWebSocketConnectionOptions(), httpClient); + + session.connect().block(); + assertEquals("wss", session.getEndpoint().getScheme()); + StepVerifier.create(session.receiveEvents()).assertNext(this::assertWarningEvent).verifyComplete(); + session.close(); + } + + @Test + public void syncSessionReceivesTypedEventAndCloses() { + List clientMessages = new CopyOnWriteArrayList<>(); + AtomicReference requestUri = new AtomicReference<>(); + AtomicReference authorization = new AtomicReference<>(); + AtomicReference foundryFeatures = new AtomicReference<>(); + AtomicReference userAgent = new AtomicReference<>(); + server = startServer(clientMessages, requestUri, authorization, foundryFeatures, userAgent, + new AtomicReference<>(), true); + TokenCredential credential + = request -> Mono.just(new AccessToken("sync-token", OffsetDateTime.now().plusHours(1))); + BetaVoiceAgentWebSocketClient client + = new AgentsClientBuilder().endpoint("http://127.0.0.1:" + server.port() + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketClient(); + + try (VoiceAgentWebSocketSessionClient session = client.connect("sync-agent")) { + Iterator events = session.receiveEvents().iterator(); + assertWarningEvent(events.next()); + session.sendFunctionCallOutput("call-1", "{\"temperature\":72}"); + assertWarningEvent(events.next()); + assertWarningEvent(events.next()); + + assertEquals(2, clientMessages.size()); + Map functionOutput = BinaryData.fromString(clientMessages.get(0)).toObject(Map.class); + assertEquals("conversation.item.create", functionOutput.get("type")); + Map item = (Map) functionOutput.get("item"); + assertEquals("function_call_output", item.get("type")); + assertEquals("call-1", item.get("call_id")); + assertEquals("{\"temperature\":72}", item.get("output")); + Map responseCreate = BinaryData.fromString(clientMessages.get(1)).toObject(Map.class); + assertEquals("response.create", responseCreate.get("type")); + assertTrue(session.isOpen()); + } + } + + private BetaVoiceAgentWebSocketAsyncClient createAsyncClient(int port) { + TokenCredential credential + = request -> Mono.just(new AccessToken("test-token", OffsetDateTime.now().plusHours(1))); + return new AgentsClientBuilder().endpoint("http://127.0.0.1:" + port + "/api/projects/project") + .credential(credential) + .configuration(Configuration.NONE) + .buildBetaVoiceAgentWebSocketAsyncClient(); + } + + private DisposableServer frameWebSocketServer(org.reactivestreams.Publisher frames) { + WebsocketServerSpec spec = WebsocketServerSpec.builder().protocols("realtime").build(); + return HttpServer.create() + .host("127.0.0.1") + .port(0) + .handle((request, response) -> response + .sendWebsocket((inbound, outbound) -> outbound.sendObject(frames).then(outbound.sendClose()), spec)) + .bindNow(); + } + + private DisposableServer oneShotWebSocketServer(String message) { + WebsocketServerSpec spec = WebsocketServerSpec.builder().protocols("realtime").build(); + return HttpServer.create() + .host("127.0.0.1") + .port(0) + .handle((request, response) -> response.sendWebsocket((inbound, + outbound) -> outbound.sendString(Mono.just(message), StandardCharsets.UTF_8).then(outbound.sendClose()), + spec)) + .bindNow(); + } + + private DisposableServer startServer(List clientMessages, AtomicReference requestUri, + AtomicReference authorization, AtomicReference foundryFeatures, + AtomicReference userAgent, AtomicReference customHeader, boolean sendInitialEvent) { + WebsocketServerSpec spec = WebsocketServerSpec.builder().protocols("realtime").build(); + return HttpServer.create().host("127.0.0.1").port(0).handle((request, response) -> { + requestUri.set(request.uri()); + authorization.set(request.requestHeaders().get(HttpHeaderNames.AUTHORIZATION)); + foundryFeatures.set(request.requestHeaders().get("Foundry-Features")); + userAgent.set(request.requestHeaders().get(HttpHeaderNames.USER_AGENT)); + customHeader.set(request.requestHeaders().get("X-Test-Header")); + return response.sendWebsocket((inbound, outbound) -> { + Flux replies = inbound.receive() + .asString(StandardCharsets.UTF_8) + .doOnNext(clientMessages::add) + .map(ignored -> warningJson()); + if (sendInitialEvent) { + replies = replies.startWith(warningJson()); + } + return outbound.sendString(replies, StandardCharsets.UTF_8).then(); + }, spec); + }).bindNow(); + } + + private void assertWarningEvent(RealtimeServerEvent event) { + VoiceAgentServerEventWarning warning = assertInstanceOf(VoiceAgentServerEventWarning.class, event); + assertEquals("loopback warning", warning.getWarning().getMessage()); + assertEquals("test_warning", warning.getWarning().getCode()); + } + + private static String warningJson() { + return "{\"type\":\"warning\",\"event_id\":\"event-1\",\"warning\":{" + + "\"message\":\"loopback warning\",\"code\":\"test_warning\"}}"; + } + + private static File resourceFile(String name) throws Exception { + URL resource = VoiceAgentWebSocketSessionTests.class.getClassLoader().getResource(name); + assertNotNull(resource); + return Paths.get(resource.toURI()).toFile(); + } + + private static String decode(String value) { + try { + return URLDecoder.decode(value, StandardCharsets.UTF_8.name()); + } catch (Exception error) { + throw new IllegalStateException(error); + } + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/models/VoiceAgentDefinitionSerializationTests.java b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/models/VoiceAgentDefinitionSerializationTests.java new file mode 100644 index 0000000000000..b8b87bb65d6a8 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/java/com/azure/ai/agents/models/VoiceAgentDefinitionSerializationTests.java @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.ai.agents.models; + +import com.azure.core.util.BinaryData; +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; +import com.azure.json.JsonWriter; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class VoiceAgentDefinitionSerializationTests { + + @Test + public void fullVoiceDefinitionRoundTrips() throws IOException { + RealtimeAudioFormatsAudioPcm pcm + = new RealtimeAudioFormatsAudioPcm().setRate(RealtimeAudioFormatsAudioPcmRate.TWO_FOUR_ZERO_ZERO_ZERO); + VoiceAgentAudioInputConfig input = new VoiceAgentAudioInputConfig().setFormat(pcm) + .setTurnDetection(new VoiceAgentServerVadTurnDetection().setThreshold(0.5) + .setPrefixPaddingMs(300L) + .setSilenceDurationMs(500L)) + .setTranscription(new VoiceAgentInputTranscription(VoiceAgentInputTranscriptionModel.WHISPER_1)); + VoiceAgentAudioOutputConfig output = new VoiceAgentAudioOutputConfig().setFormat(pcm) + .setVoice("en-US-AvaNeural") + .setVoiceType(VoiceType.AZURE_STANDARD); + VoiceAgentFunctionTool functionTool + = new VoiceAgentFunctionTool("get_weather").setDescription("Get weather for a city.") + .setParameters(BinaryData.fromString("{}")); + VoiceAgentSystemTool systemTool = new VoiceAgentEndConversationSystemTool(); + + VoiceAgentDefinition original = new VoiceAgentDefinition().setModelType(VoiceModelType.MANAGED) + .setModel("gpt-realtime") + .setInstructions("Keep replies short and natural.") + .setAudio(new VoiceAgentAudioConfig().setInput(input).setOutput(output)) + .setOutputModalities(Collections.singletonList(VoiceOutputModality.AUDIO)) + .setTools(Arrays.asList(functionTool, systemTool)) + .setStore(true); + + String json = serialize(original); + assertTrue(json.contains("\"kind\":\"voice\"")); + assertTrue(json.contains("\"model_type\":\"managed\"")); + assertTrue(json.contains("\"model\":\"gpt-realtime\"")); + assertTrue(json.contains("\"voice\":\"en-US-AvaNeural\"")); + assertTrue(json.contains("\"voice_type\":\"azure-standard\"")); + assertTrue(json.contains("\"rate\":24000")); + assertTrue(json.contains("\"type\":\"server_vad\"")); + assertTrue(json.contains("\"model\":\"whisper-1\"")); + assertTrue(json.contains("\"output_modalities\":[\"audio\"]")); + assertTrue(json.contains("\"store\":true")); + assertTrue(json.contains("\"name\":\"get_weather\"")); + assertTrue(json.contains("\"name\":\"end_conversation\"")); + + AgentDefinition deserialized; + try (JsonReader reader = JsonProviders.createReader(json)) { + deserialized = AgentDefinition.fromJson(reader); + } + assertInstanceOf(VoiceAgentDefinition.class, deserialized); + VoiceAgentDefinition voice = (VoiceAgentDefinition) deserialized; + assertEquals(AgentKind.VOICE, voice.getKind()); + assertEquals(VoiceModelType.MANAGED, voice.getModelType()); + assertEquals("gpt-realtime", voice.getModel()); + assertEquals("Keep replies short and natural.", voice.getInstructions()); + assertEquals(Boolean.TRUE, voice.isStore()); + assertEquals(VoiceOutputModality.AUDIO, voice.getOutputModalities().get(0)); + + VoiceAgentAudioInputConfig deserializedInput = voice.getAudio().getInput(); + RealtimeAudioFormatsAudioPcm deserializedInputFormat + = assertInstanceOf(RealtimeAudioFormatsAudioPcm.class, deserializedInput.getFormat()); + assertEquals(pcm.getRate(), deserializedInputFormat.getRate()); + VoiceAgentServerVadTurnDetection deserializedVad + = assertInstanceOf(VoiceAgentServerVadTurnDetection.class, deserializedInput.getTurnDetection()); + VoiceAgentServerVadTurnDetection originalVad = (VoiceAgentServerVadTurnDetection) input.getTurnDetection(); + assertEquals(originalVad.getThreshold(), deserializedVad.getThreshold()); + assertEquals(originalVad.getPrefixPaddingMs(), deserializedVad.getPrefixPaddingMs()); + assertEquals(originalVad.getSilenceDurationMs(), deserializedVad.getSilenceDurationMs()); + assertEquals(input.getTranscription().getModel(), deserializedInput.getTranscription().getModel()); + + VoiceAgentAudioOutputConfig deserializedOutput = voice.getAudio().getOutput(); + RealtimeAudioFormatsAudioPcm deserializedOutputFormat + = assertInstanceOf(RealtimeAudioFormatsAudioPcm.class, deserializedOutput.getFormat()); + assertEquals(pcm.getRate(), deserializedOutputFormat.getRate()); + assertEquals(output.getVoice(), deserializedOutput.getVoice()); + assertEquals(output.getVoiceType(), deserializedOutput.getVoiceType()); + + assertEquals(2, voice.getTools().size()); + VoiceAgentFunctionTool deserializedFunction + = assertInstanceOf(VoiceAgentFunctionTool.class, voice.getTools().get(0)); + assertEquals(functionTool.getName(), deserializedFunction.getName()); + assertEquals(functionTool.getDescription(), deserializedFunction.getDescription()); + VoiceAgentEndConversationSystemTool deserializedSystem + = assertInstanceOf(VoiceAgentEndConversationSystemTool.class, voice.getTools().get(1)); + assertEquals(systemTool.getName(), deserializedSystem.getName()); + } + + @Test + public void selfDeployedVoiceDefinitionRoundTrips() throws IOException { + VoiceAgentDefinition original = new VoiceAgentDefinition().setModelType(VoiceModelType.SELF_DEPLOYED) + .setModel("customer-realtime-deployment") + .setInstructions("Use the customer deployment."); + + String json = serialize(original); + VoiceAgentDefinition deserialized; + try (JsonReader reader = JsonProviders.createReader(json)) { + deserialized = VoiceAgentDefinition.fromJson(reader); + } + + assertEquals(VoiceModelType.SELF_DEPLOYED, deserialized.getModelType()); + assertEquals("customer-realtime-deployment", deserialized.getModel()); + assertEquals("Use the customer deployment.", deserialized.getInstructions()); + } + + private static String serialize(VoiceAgentDefinition definition) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (JsonWriter writer = JsonProviders.createWriter(output)) { + definition.toJson(writer); + } + return output.toString("UTF-8"); + } +} diff --git a/sdk/ai/azure-ai-agents/src/test/resources/websocket-localhost-cert.pem b/sdk/ai/azure-ai-agents/src/test/resources/websocket-localhost-cert.pem new file mode 100644 index 0000000000000..51389bf8a86e1 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/resources/websocket-localhost-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJTCCAg2gAwIBAgIUfAusxvG/l3WCuKMFyNBEfEr30rswDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDkwMzA5NTgyMloXDTM2MDgz +MTA5NTgyMlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEA3YAeebCQfQbC+BoOyHqM6EgmXxIfpixqG26ElKdCehCH +yt6FqnHhvjf/TRfbwgij/GVxIR1wO4CXkAATNMxsaFu/EbDzI/eDqZKIOQIzd0if +JHC20kVibaFnNDlI9NKC/Ywphz0d8JXCHnYVMVJP27moNYcG91/Lka8223O+qoAw +sta603tAcpsFEl9muc8y0UhwAKED03Gr0mjjGZZ6vTvCE+i3IsslZKXqtS6Fo1wm +NTUCpB/yF8i+WnnVrLetMy45D3hEjPeh2p8cjTHOsOyKkqNAH6hVB0bBzCccN+cY +1YtI0A4Umu7FO5RlAoR8eYnBdoZ06qpv1eXjwj/QZwIDAQABo28wbTAdBgNVHQ4E +FgQU6saTU+QSavalG6I49czWXwtSwY0wHwYDVR0jBBgwFoAU6saTU+QSavalG6I4 +9czWXwtSwY0wDwYDVR0TAQH/BAUwAwEB/zAaBgNVHREEEzARgglsb2NhbGhvc3SH +BH8AAAEwDQYJKoZIhvcNAQELBQADggEBAJ73cFPhCnH1IoItmWDJxhIaR7g1MIfh +o7DxnXEf8ZFw7bpzo4Epp6R6+RRH/fnbosw73vtDqEVZQfxKKjAo0NvguNJIuOoz +oISXYpAIX8eBT2ZrH6m0tJfgwyp7V0+SaHChy1+TmtnaT7rfC7N5r/rcr1abQV78 +qUK7N1+aF0dV1fGE4oP3jon+MNc7pZSagVDTz/k2qHwDnwPoVG37BXf7UZ7jbA2g +/afI/YHCt7zT8aHtjJWJMWLgHOtFTGqx1h7x1rEiLNPK/6USrSFEw+7ZQDFSbG7g +NQyC8lVm3QCVeze2q2/x1DUz20UnGaHz+o3fuK+qsDg/NtHIUMM3wj4= +-----END CERTIFICATE----- diff --git a/sdk/ai/azure-ai-agents/src/test/resources/websocket-localhost-key.pem b/sdk/ai/azure-ai-agents/src/test/resources/websocket-localhost-key.pem new file mode 100644 index 0000000000000..d1576a3c7c4d2 --- /dev/null +++ b/sdk/ai/azure-ai-agents/src/test/resources/websocket-localhost-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDdgB55sJB9BsL4 +Gg7IeozoSCZfEh+mLGobboSUp0J6EIfK3oWqceG+N/9NF9vCCKP8ZXEhHXA7gJeQ +ABM0zGxoW78RsPMj94Opkog5AjN3SJ8kcLbSRWJtoWc0OUj00oL9jCmHPR3wlcIe +dhUxUk/buag1hwb3X8uRrzbbc76qgDCy1rrTe0BymwUSX2a5zzLRSHAAoQPTcavS +aOMZlnq9O8IT6LciyyVkpeq1LoWjXCY1NQKkH/IXyL5aedWst60zLjkPeESM96Ha +nxyNMc6w7IqSo0AfqFUHRsHMJxw35xjVi0jQDhSa7sU7lGUChHx5icF2hnTqqm/V +5ePCP9BnAgMBAAECggEAASQTFmWptpRLVjpkIfWno92DRjpgGVu87C3SRLgJjOhE +CIJ6WFmyGrEnbxE5ZLMuaHxEtVY+e1JEGilagkIdBEQF23/mQwAqYZem6oxB7Qk5 +J3wu27/XdTw/dET7RMr98E74XzgaFheWPfURdym28ruBFQRbv9PgWUWdDt2/ndBY +e9XDZ0737YDGjWkZFwLZ/q6YDEUc4NhTClVvzCyTlLMVVL4xsvPzmyxlT093Hdys +ZDs/6UVJOPYsgv7Z9ww6fwv+oPi/oNtvX3dEOjEQkLvmIfXrPSvYQZVQ6Ok5K0UF +eKUP8tB2ZIrX70nhg8R8ThFjldMPb/lS2i59PWQBgQKBgQDwehtp27diMPnUo7ZR +xSPt2UAUTiRRlwQo4rFZNiR5ZMLPBAX4DP5aQDyDiTAMqBOjHqgipagra9xOapCN +uqlEINaMlsuSwMD1cxkp85V5FWab+u1MqBr3B1aq2INqrYDGmPs3kS6Po4M0N12+ +O6Bob4YWBaabIY+rEEJuBmK9QQKBgQDrzGyXVRYGarKW4tiWVeRAA5yNF/w0YW8B +u62wUXLXUXfzhND4ETCUxUpgkcjY1AICWQQnbUFXi/0WUowkKZg8Vh0zfBJmsbs2 +LPhCUEMITKB3owwJLKDCpSake+9Afxi7XB4UltsjInep1XGE6tvKKr9bAF1Sqd47 +V74dv0CbpwKBgQCkuT/l91dar2myuqG8yWmfF13Jiu1d5jA3QXFyRqAdd2PqIjtk +eqIQeEf7YhHD2a354poRgZ/8flnebSivrNkdjdDpZLH1yItkln76OZx94Kb02aGL +DOvLov8+8Ci0/jxjzY7ntU9LnRnWvsY79OQgJaSXmS9SvF6JMw4OB9nDAQKBgQCJ +kxrkbJtOISCTkkTl6bUjeDf1xkG62gInU7XyAoNrhzfiF+LIaVcb5cQQdd5mS8Pk +VMVsr30JND70sDLdwnr08RVWfZRK4HWnFTO/lQ6XIAYb50BVdflRt4PFQh4EVmM6 +pXNTdfTjGfARYdw6vcCAwtIkqSDJ4xwrKXVd68EpTwKBgDURo00XBIFSyqAbvxwF +NchLRcoNZVxxd9hHGTUiTgstfDGxEuHpWEETiVsGGnU/Mq+cTKW98P3y/mdW9Gd+ +mYRmo1L0J6lkBMP3xD5PXxbvTWkxVReuSCl0zUqfKcHjwllpsS91If0HVeg0c5ze +/C4ecakj7llQKhUYIm/dNULb +-----END PRIVATE KEY-----