Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -469,12 +469,13 @@ private ModelResponse invokeModel(ModelRequest request, UsageAccumulator usage,
AtomicReference<ToolRegistry> effectiveTools) {
ToolRegistry effective = effectiveToolSet(request);
effectiveTools.set(effective);
ChatClient selectedClient = request.getChatClient() != null ? request.getChatClient() : chatClient;
ChatRequest chatRequest = new ChatRequest(composeMessages(request), effective.definitions(),
request.getOptions(), request.getAttributes());
if (streamModel) {
return ModelResponse.of(collectModelStream(chatRequest, usage));
return ModelResponse.of(collectModelStream(selectedClient, chatRequest, usage));
}
CompletableFuture<ChatResponse> future = chatClient.call(chatRequest).toCompletableFuture();
CompletableFuture<ChatResponse> future = selectedClient.call(chatRequest).toCompletableFuture();
toolOperation.set(future);
try (AutoCloseable ignored = context.control().onLoss(() -> future.cancel(true))) {
ChatResponse response = await(future);
Expand All @@ -490,9 +491,9 @@ private ModelResponse invokeModel(ModelRequest request, UsageAccumulator usage,
}
}

private ChatResponse collectModelStream(ChatRequest request, UsageAccumulator usage) {
return collectModelStream(Objects.requireNonNull(chatClient.stream(request), "chat client returned no stream"),
usage);
private ChatResponse collectModelStream(ChatClient selectedClient, ChatRequest request, UsageAccumulator usage) {
return collectModelStream(
Objects.requireNonNull(selectedClient.stream(request), "chat client returned no stream"), usage);
}

private ChatResponse collectModelStream(Flow.Publisher<ChatEvent> publisher, UsageAccumulator usage) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import java.util.Queue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Flow;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
Expand Down Expand Up @@ -291,6 +292,149 @@ public ModelResponse interceptModel(ModelRequest request, ModelCallHandler next)
.map(ChatMessage::text).toList();
assertEquals(List.of("required-policy", "first", "reply", "second"), texts);
}

// ------------------------------------------------------------------
// issue #271: request-selected clients reach the model invocation
// ------------------------------------------------------------------

@Test
void interceptorSelectedClientOverridesTheDefaultClient() throws Exception {
ScriptedClient primary = new ScriptedClient(
List.of(response(ChatMessage.assistant("primary"), List.of(), 1)));
ScriptedClient selected = new ScriptedClient(
List.of(response(ChatMessage.assistant("selected"), List.of(), 1)));
ModelInterceptor interceptor = selectingClient(selected);

AgentOutcome outcome = ReActAgent.builder(primary).interceptors(List.of(interceptor)).build()
.invoke(request(), context())
.get(3, TimeUnit.SECONDS);

assertEquals("selected", outcome.messages().get(outcome.messages().size() - 1).text());
assertEquals(0, primary.requests.size());
assertEquals(1, selected.requests.size());
}

@Test
void streamingRunUsesTheInterceptorSelectedClient() throws Exception {
StreamingClient primary = new StreamingClient();
StreamingClient selected = new StreamingClient();
ModelInterceptor interceptor = selectingClient(selected);

ReActAgent.builder(primary).streamModel(true).interceptors(List.of(interceptor)).build()
.invoke(request(), context())
.get(3, TimeUnit.SECONDS);

assertEquals(0, primary.requests.size());
assertEquals(1, selected.requests.size());
}

@Test
void fallbackRetryUsesTheClientSelectedOnTheRetryRequest() throws Exception {
AtomicInteger primaryCalls = new AtomicInteger();
ChatClient primary = new ChatClient() {
public CompletionStage<ChatResponse> call(ChatRequest request) {
primaryCalls.incrementAndGet();
return CompletableFuture.failedFuture(new IllegalStateException("primary unavailable"));
}

public Flow.Publisher<ChatEvent> stream(ChatRequest request) {
throw new UnsupportedOperationException();
}

public ChatClientCapabilities capabilities() {
return ChatClientCapabilities.basic();
}
};
ScriptedClient fallback = new ScriptedClient(
List.of(response(ChatMessage.assistant("fallback"), List.of(), 1)));
ModelInterceptor interceptor = new ModelInterceptor() {
public String getName() {
return "fallback";
}

public ModelResponse interceptModel(ModelRequest request, ModelCallHandler next) {
try {
return next.call(request);
}
catch (RuntimeException primaryFailure) {
return next.call(ModelRequest.builder(request).chatClient(fallback).build());
}
}
};

AgentOutcome outcome = ReActAgent.builder(primary).interceptors(List.of(interceptor)).build()
.invoke(request(), context())
.get(3, TimeUnit.SECONDS);

assertEquals("fallback", outcome.messages().get(outcome.messages().size() - 1).text());
assertEquals(1, primaryCalls.get());
assertEquals(1, fallback.requests.size());
}

@Test
void allFailedCandidatesReportTheSelectedClientFailure() {
AtomicInteger primaryCalls = new AtomicInteger();
AtomicInteger fallbackCalls = new AtomicInteger();
ChatClient primary = failingClient(primaryCalls, "primary unavailable");
ChatClient fallback = failingClient(fallbackCalls, "fallback unavailable");
ModelInterceptor interceptor = new ModelInterceptor() {
public String getName() {
return "fallback";
}

public ModelResponse interceptModel(ModelRequest request, ModelCallHandler next) {
try {
return next.call(request);
}
catch (RuntimeException primaryFailure) {
return next.call(ModelRequest.builder(request).chatClient(fallback).build());
}
}
};

ExecutionException failure = assertThrows(ExecutionException.class,
() -> ReActAgent.builder(primary).interceptors(List.of(interceptor)).build()
.invoke(request(), context())
.get(3, TimeUnit.SECONDS));

assertEquals("fallback unavailable", failure.getCause().getMessage());
assertEquals(1, primaryCalls.get());
assertEquals(1, fallbackCalls.get());
}

@Test
void concurrentRunsKeepTheirClientSelectionsIsolated() throws Exception {
ScriptedClient primary = new ScriptedClient(
List.of(response(ChatMessage.assistant("primary"), List.of(), 1)));
ScriptedClient selected = new ScriptedClient(
List.of(response(ChatMessage.assistant("selected"), List.of(), 1)));
ModelInterceptor interceptor = new ModelInterceptor() {
public String getName() {
return "route-client";
}

public ModelResponse interceptModel(ModelRequest request, ModelCallHandler next) {
boolean useSelected = request.getMessages().stream()
.anyMatch(message -> "route-selected".equals(message.text()));
return useSelected
? next.call(ModelRequest.builder(request).chatClient(selected).build())
: next.call(request);
}
};
ReActAgent agent = ReActAgent.builder(primary).interceptors(List.of(interceptor)).build();

CompletableFuture<AgentOutcome> primaryRun = agent.invoke(
AgentRequest.messages(List.of(ChatMessage.user("route-primary"))), context());
CompletableFuture<AgentOutcome> selectedRun = agent.invoke(
AgentRequest.messages(List.of(ChatMessage.user("route-selected"))), context());
AgentOutcome primaryOutcome = primaryRun.get(3, TimeUnit.SECONDS);
AgentOutcome selectedOutcome = selectedRun.get(3, TimeUnit.SECONDS);

assertEquals("primary", primaryOutcome.messages().get(primaryOutcome.messages().size() - 1).text());
assertEquals("selected", selectedOutcome.messages().get(selectedOutcome.messages().size() - 1).text());
assertEquals(1, primary.requests.size());
assertEquals(1, selected.requests.size());
}

@Test
void emitsStreamingDeltasAndSupportsDynamicTools() throws Exception {
Expand Down Expand Up @@ -864,6 +1008,35 @@ public CompletionStage<ToolResult> execute(ToolCall call, ToolContext context) {
};
}

private static ModelInterceptor selectingClient(ChatClient selected) {
return new ModelInterceptor() {
public String getName() {
return "select-client";
}

public ModelResponse interceptModel(ModelRequest request, ModelCallHandler next) {
return next.call(ModelRequest.builder(request).chatClient(selected).build());
}
};
}

private static ChatClient failingClient(AtomicInteger calls, String message) {
return new ChatClient() {
public CompletionStage<ChatResponse> call(ChatRequest request) {
calls.incrementAndGet();
return CompletableFuture.failedFuture(new IllegalStateException(message));
}

public Flow.Publisher<ChatEvent> stream(ChatRequest request) {
throw new UnsupportedOperationException();
}

public ChatClientCapabilities capabilities() {
return ChatClientCapabilities.basic();
}
};
}

private static ChatResponse response(ChatMessage message, List<ToolCall> calls, long total) {
return new ChatResponse(null, "test", List.of(message), calls, FinishReason.STOP,
new Usage(1L, 1L, total), Map.of());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@
import io.github.agentic.ai.agent.AgentExecutionContext;
import io.github.agentic.ai.agent.AgentEvent;
import io.github.agentic.ai.agent.AgentRequest;
import io.github.agentic.ai.agent.interceptor.ModelCallHandler;
import io.github.agentic.ai.agent.interceptor.ModelInterceptor;
import io.github.agentic.ai.agent.interceptor.ModelRequest;
import io.github.agentic.ai.agent.interceptor.ModelResponse;
import io.github.agentic.ai.api.chat.ChatClient;
import io.github.agentic.ai.api.chat.ChatClientCapabilities;
import io.github.agentic.ai.api.chat.ChatEvent;
Expand Down Expand Up @@ -86,6 +90,42 @@ void cancelBeforeModelOnSubscribeNeverRequests() throws Exception {
assertTrue(events.isEmpty(), "a cancelled run must not deliver events");
}

@Test
void cancellationTargetsTheClientSelectedByTheInterceptor() throws Exception {
DeferredPublisher defaultModel = new DeferredPublisher(
response(ChatMessage.assistant("default"), List.of(), 1), false);
DeferredPublisher selectedModel = new DeferredPublisher(
response(ChatMessage.assistant("selected"), List.of(), 1), false);
ScriptedPublisherClient defaultClient = new ScriptedPublisherClient(List.of(defaultModel));
ScriptedPublisherClient selectedClient = new ScriptedPublisherClient(List.of(selectedModel));
ModelInterceptor interceptor = new ModelInterceptor() {
@Override
public String getName() {
return "select-client";
}

@Override
public ModelResponse interceptModel(ModelRequest request, ModelCallHandler next) {
return next.call(ModelRequest.builder(request).chatClient(selectedClient).build());
}
};
List<AgentEvent> events = new CopyOnWriteArrayList<>();
try (AgentExecutionContext context = context()) {
Flow.Subscription subscription = subscribe(ReActAgent.builder(defaultClient)
.streamModel(true)
.interceptors(List.of(interceptor))
.build(), events, context);

assertTrue(selectedModel.subscribed.await(3, TimeUnit.SECONDS), "selected publisher was never subscribed");
subscription.cancel();
selectedModel.deliverOnSubscribe();

assertTrue(defaultClient.requests.isEmpty(), "the default client must not be invoked");
assertEquals(0, selectedModel.requests.get());
assertEquals(1, selectedModel.cancels.get());
}
}

@Test
void normalStreamingCompletionRequestsOnceAndIsNotCancelled() throws Exception {
DeferredPublisher model = new DeferredPublisher(response(ChatMessage.assistant("answer"), List.of(), 1), true);
Expand Down