diff --git a/apps/web/src/app/api/openrouter/[...path]/route.test.ts b/apps/web/src/app/api/openrouter/[...path]/route.test.ts index 88d28ed773..34740a4c7d 100644 --- a/apps/web/src/app/api/openrouter/[...path]/route.test.ts +++ b/apps/web/src/app/api/openrouter/[...path]/route.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect, beforeEach, afterEach } from '@jest/globals'; +import { NextResponse } from 'next/server'; +import type { PoolEntry } from '@kilocode/auto-routing-contracts'; import type { User } from '@kilocode/db/schema'; +import type { OpenCodeSettings } from '@kilocode/db/schema-types'; +import type { OpenRouterModel } from '@/lib/organizations/organization-types'; import { getUserFromAuth } from '@/lib/user/server'; import { getBalanceAndOrgSettings } from '@/lib/organizations/organization-usage'; import { classifyAbuse } from '@/lib/ai-gateway/abuse-service'; @@ -7,6 +11,7 @@ import { getProvider } from '@/lib/ai-gateway/providers/get-provider'; import { upstreamRequest } from '@/lib/ai-gateway/providers/upstream-request'; import { getOpenRouterModelsFromRedis, + getOpenRouterModelsMetadataFromDatabase, isValidOpenRouterModelId, } from '@/lib/ai-gateway/providers/gateway-models-cache'; import { emitApiMetricsForResponse } from '@/lib/ai-gateway/o11y/api-metrics.server'; @@ -14,7 +19,13 @@ import { accountForMicrodollarUsage } from '@/lib/ai-gateway/llm-proxy-helpers'; import { redisClient } from '@/lib/redis'; import { ReasoningDetailsTransform, type Provider } from '@/lib/ai-gateway/providers/types'; import { fetchEfficientAutoDecision } from '@/lib/ai-gateway/auto-routing-decision'; -import { collectDeniedAutoRoutingModelIds } from '@/lib/ai-gateway/auto-routing-denied-models'; +import { + collectDeniedAutoRoutingModelIds, + loadEffectiveAutoRoutingPool, +} from '@/lib/ai-gateway/auto-routing-denied-models'; +import { getEnhancedOpenRouterModels } from '@/lib/ai-gateway/providers/openrouter'; +import { gatewayChatApisForModel } from '@/lib/ai-gateway/model-api-kinds'; +import { BALANCED_FALLBACK_MODEL } from '@/lib/ai-gateway/auto-model'; import { logMicrodollarUsage } from '@/lib/ai-gateway/processUsage'; import { applyResolvedAutoModel } from '@/lib/ai-gateway/auto-model/resolution'; import { getDirectByokModel } from '@/lib/ai-gateway/providers/direct-byok'; @@ -110,6 +121,15 @@ jest.mock('@/lib/utils.server', () => ({ })); jest.mock('@/lib/ai-gateway/auto-routing-denied-models', () => ({ collectDeniedAutoRoutingModelIds: jest.fn().mockResolvedValue([]), + loadEffectiveAutoRoutingPool: jest.fn().mockResolvedValue(null), +})); +jest.mock('@/lib/ai-gateway/providers/openrouter', () => ({ + ...jest.requireActual>('@/lib/ai-gateway/providers/openrouter'), + getEnhancedOpenRouterModels: jest.fn(), +})); +jest.mock('@/lib/ai-gateway/model-api-kinds', () => ({ + ...jest.requireActual>('@/lib/ai-gateway/model-api-kinds'), + gatewayChatApisForModel: jest.fn(), })); jest.mock('@/lib/ai-gateway/processUsage', () => { const actual = jest.requireActual('@/lib/ai-gateway/processUsage'); @@ -139,6 +159,10 @@ const mockedRedisGet = jest.mocked(redisClient.get); const mockedRedisSet = jest.mocked(redisClient.set); const mockedFetchEfficientAutoDecision = jest.mocked(fetchEfficientAutoDecision); const mockedCollectDeniedAutoRoutingModelIds = jest.mocked(collectDeniedAutoRoutingModelIds); +const mockedLoadEffectiveAutoRoutingPool = jest.mocked(loadEffectiveAutoRoutingPool); +const mockedGetEnhancedOpenRouterModels = jest.mocked(getEnhancedOpenRouterModels); +const mockedGatewayChatApisForModel = jest.mocked(gatewayChatApisForModel); +const mockedGetOpenRouterModelsMetadata = jest.mocked(getOpenRouterModelsMetadataFromDatabase); const mockedLogMicrodollarUsage = jest.mocked(logMicrodollarUsage); const mockedApplyResolvedAutoModel = jest.mocked(applyResolvedAutoModel); const mockedGetDirectByokModel = jest.mocked(getDirectByokModel); @@ -1233,6 +1257,428 @@ describe('kilo-auto/efficient classifier billing', () => { }); expect(mockedUpstreamRequest).not.toHaveBeenCalled(); }); + + describe('configured pool fallback', () => { + function catalogModel( + id: string, + inputModalities = ['text', 'image'], + variants?: OpenCodeSettings['variants'] + ): OpenRouterModel { + return { + id, + name: id, + created: 0, + description: '', + context_length: 100_000, + architecture: { + input_modalities: inputModalities, + output_modalities: ['text'], + tokenizer: 'Other', + }, + top_provider: { is_moderated: false }, + pricing: { prompt: '0.000001', completion: '0.000002' }, + ...(variants ? { opencode: { variants } } : {}), + }; + } + + beforeEach(() => { + mockedFetchEfficientAutoDecision.mockResolvedValue(null); + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue(null); + mockedGetEnhancedOpenRouterModels.mockResolvedValue({ data: [] }); + mockedGatewayChatApisForModel.mockReturnValue(['chat_completions', 'responses', 'messages']); + mockedGetOpenRouterModelsMetadata.mockResolvedValue({}); + }); + + it.each(['kilo-auto/efficient', 'kilo-auto/balanced'])( + '%s uses the first organization-permitted pool entry and its exact variant instead of the platform fallback', + async requestedModel => { + const pool: PoolEntry[] = [ + { model: 'openai/gpt-4o', variant: null }, + { model: 'anthropic/claude-sonnet-5', variant: 'xhigh' }, + { model: 'anthropic/claude-sonnet-5', variant: 'max' }, + { model: 'google/gemini-2.5-flash', variant: null }, + ]; + mockedGetUserFromAuth.mockResolvedValue({ + user: { + id: 'user-123', + google_user_email: 'test@example.com', + microdollars_used: 0, + } as User, + authFailedResponse: null, + organizationId: 'org-123', + }); + mockedGetBalanceAndOrgSettings.mockResolvedValue({ + balance: 1000, + settings: {}, + plan: 'enterprise', + }); + mockedGetEffectiveModelDecision.mockImplementation(async (_policy, modelId) => + modelId === 'openai/gpt-4o' + ? { allowed: false, denialSource: 'organization_model' } + : { allowed: true } + ); + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue(pool); + mockedGetEnhancedOpenRouterModels.mockResolvedValue({ + data: [ + catalogModel('google/gemini-2.5-flash'), + catalogModel('anthropic/claude-sonnet-5', undefined, { + xhigh: { reasoning: { enabled: true, effort: 'xhigh' }, verbosity: 'xhigh' }, + max: { reasoning: { enabled: true, effort: 'max' }, verbosity: 'max' }, + }), + catalogModel('openai/gpt-4o'), + ], + }); + mockedApplyResolvedAutoModel.mockImplementationOnce( + jest.requireActual<{ applyResolvedAutoModel: typeof applyResolvedAutoModel }>( + '@/lib/ai-gateway/auto-model/resolution' + ).applyResolvedAutoModel + ); + + const { POST } = await import('./route'); + const response = await POST( + makeRequest({ + ...makeBody(requestedModel), + reasoning: { enabled: false, effort: 'none' }, + verbosity: 'low', + }) as never + ); + + expect(response.status).toBe(200); + expect(mockedFetchEfficientAutoDecision).toHaveBeenCalledTimes(1); + expect(mockedFetchEfficientAutoDecision).toHaveBeenCalledWith( + expect.objectContaining({ requestedModel }) + ); + expect(mockedLoadEffectiveAutoRoutingPool).toHaveBeenCalledTimes(1); + expect(mockedLoadEffectiveAutoRoutingPool).toHaveBeenCalledWith({ + userId: 'user-123', + organizationId: 'org-123', + }); + expect(mockedGetEffectiveModelDecision).toHaveBeenCalledWith( + expect.anything(), + 'openai/gpt-4o' + ); + expect(mockedGetProvider).toHaveBeenCalledWith( + expect.objectContaining({ requestedModel: 'anthropic/claude-sonnet-5' }) + ); + expect(mockedUpstreamRequest).toHaveBeenCalledTimes(1); + const upstreamBody = mockedUpstreamRequest.mock.calls[0]?.[0].body; + expect(upstreamBody.model).toBe('anthropic/claude-sonnet-5'); + expect(upstreamBody).toHaveProperty('reasoning', { enabled: true, effort: 'xhigh' }); + expect(upstreamBody).toHaveProperty('verbosity', 'xhigh'); + expect(upstreamBody.model).not.toBe(BALANCED_FALLBACK_MODEL.model); + } + ); + + it.each<{ name: string; variants?: OpenCodeSettings['variants'] }>([ + { name: 'absent' }, + { name: 'empty', variants: {} }, + { name: 'blank-only', variants: { '': {}, ' ': {} } }, + ])( + 'uses GPT-4o with a null variant instead of the platform fallback when catalog variants are $name', + async ({ variants }) => { + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue([ + { model: 'openai/gpt-4o', variant: null }, + ]); + mockedGetEnhancedOpenRouterModels.mockResolvedValue({ + data: [catalogModel('openai/gpt-4o', undefined, variants)], + }); + mockedApplyResolvedAutoModel.mockImplementationOnce( + jest.requireActual<{ applyResolvedAutoModel: typeof applyResolvedAutoModel }>( + '@/lib/ai-gateway/auto-model/resolution' + ).applyResolvedAutoModel + ); + + const { POST } = await import('./route'); + const response = await POST(makeRequest(makeBody('kilo-auto/efficient')) as never); + + expect(response.status).toBe(200); + expect(mockedLoadEffectiveAutoRoutingPool).toHaveBeenCalledTimes(1); + expect(mockedGetProvider).toHaveBeenCalledWith( + expect.objectContaining({ requestedModel: 'openai/gpt-4o' }) + ); + expect(mockedUpstreamRequest).toHaveBeenCalledTimes(1); + const upstreamBody = mockedUpstreamRequest.mock.calls[0]?.[0].body; + expect(upstreamBody.model).toBe('openai/gpt-4o'); + expect(upstreamBody.model).not.toBe(BALANCED_FALLBACK_MODEL.model); + expect(upstreamBody).not.toHaveProperty('reasoning'); + expect(upstreamBody).not.toHaveProperty('verbosity'); + } + ); + + it.each<{ + name: string; + staleEntry: PoolEntry; + variants?: OpenCodeSettings['variants']; + }>([ + { + name: 'a null variant on a model now exposing variants', + staleEntry: { model: 'anthropic/claude-sonnet-5', variant: null }, + variants: { xhigh: { reasoning: { enabled: true, effort: 'xhigh' }, verbosity: 'xhigh' } }, + }, + { + name: 'a named variant on a model no longer exposing variants', + staleEntry: { model: 'openai/gpt-4o', variant: 'high' }, + }, + { + name: 'a family fallback variant absent from the current catalog', + staleEntry: { model: 'anthropic/claude-sonnet-5', variant: 'max' }, + variants: { xhigh: { reasoning: { enabled: true, effort: 'xhigh' }, verbosity: 'xhigh' } }, + }, + { + name: 'a blank variant key alongside a named catalog variant', + staleEntry: { model: 'anthropic/claude-sonnet-5', variant: ' ' }, + variants: { + ' ': {}, + xhigh: { reasoning: { enabled: true, effort: 'xhigh' }, verbosity: 'xhigh' }, + }, + }, + ])( + 'filters $name before selecting the next allowed pool entry', + async ({ staleEntry, variants }) => { + const allowedEntry: PoolEntry = { + model: 'mistralai/mistral-medium-3-5', + variant: 'thinking', + }; + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue([staleEntry, allowedEntry]); + mockedGetEnhancedOpenRouterModels.mockResolvedValue({ + data: [ + catalogModel(staleEntry.model, undefined, variants), + catalogModel('mistralai/mistral-medium-3-5', undefined, { + thinking: { reasoning: { enabled: true, effort: 'high' } }, + }), + ], + }); + mockedApplyResolvedAutoModel.mockImplementationOnce( + jest.requireActual<{ applyResolvedAutoModel: typeof applyResolvedAutoModel }>( + '@/lib/ai-gateway/auto-model/resolution' + ).applyResolvedAutoModel + ); + + const { POST } = await import('./route'); + const response = await POST(makeRequest(makeBody('kilo-auto/efficient')) as never); + + expect(response.status).toBe(200); + expect(mockedGetProvider).toHaveBeenCalledWith( + expect.objectContaining({ requestedModel: allowedEntry.model }) + ); + expect(mockedUpstreamRequest).toHaveBeenCalledTimes(1); + expect(mockedUpstreamRequest.mock.calls[0]?.[0].body).toMatchObject({ + model: allowedEntry.model, + reasoning: { enabled: true, effort: 'high' }, + }); + const fallbackCandidates = + mockedApplyResolvedAutoModel.mock.calls[0]?.[0].efficientFallbackCandidates; + expect(fallbackCandidates).toBeDefined(); + await expect(fallbackCandidates?.()).resolves.toEqual([allowedEntry]); + } + ); + + it.each(['chat_completions', 'responses', 'messages'] as const)( + 'filters absent, unavailable, API-incompatible and text-only entries for %s image requests', + async apiKind => { + const pool: PoolEntry[] = [ + { model: 'missing/model', variant: null }, + { model: 'openai/gpt-4o', variant: null }, + { model: 'qwen/qwen3-coder', variant: null }, + { model: 'google/gemma-4-26b-a4b-it:free', variant: null }, + { model: 'google/gemini-2.5-flash', variant: 'high' }, + { model: 'anthropic/claude-haiku-4', variant: 'low' }, + ]; + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue(pool); + mockedGetEnhancedOpenRouterModels.mockResolvedValue({ + data: [ + catalogModel('anthropic/claude-haiku-4', ['text', 'image_url'], { + low: { reasoning: { enabled: true, effort: 'low' }, verbosity: 'low' }, + }), + catalogModel('google/gemini-2.5-flash', undefined, { + high: { reasoning: { enabled: true, effort: 'high' } }, + }), + catalogModel('google/gemma-4-26b-a4b-it:free'), + catalogModel('qwen/qwen3-coder', ['text']), + catalogModel('openai/gpt-4o'), + ], + }); + mockedGatewayChatApisForModel.mockImplementation(modelId => + modelId === 'openai/gpt-4o' + ? apiKind === 'messages' + ? ['chat_completions'] + : ['messages'] + : ['chat_completions', 'responses', 'messages'] + ); + const imageUrl = 'data:image/png;base64,aGVsbG8='; + const body = + apiKind === 'responses' + ? { + input: [{ role: 'user', content: [{ type: 'input_image', image_url: imageUrl }] }], + } + : { + max_tokens: 1024, + messages: [ + { + role: 'user', + content: [ + apiKind === 'messages' + ? { type: 'image', source: { type: 'url', url: imageUrl } } + : { type: 'image_url', image_url: { url: imageUrl } }, + ], + }, + ], + }; + const path = apiKind === 'chat_completions' ? 'chat/completions' : apiKind; + const request = new Request(`http://localhost:3000/api/openrouter/v1/${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'x-forwarded-for': '127.0.0.1' }, + body: JSON.stringify({ model: 'kilo-auto/efficient', ...body }), + }); + + const { POST } = await import('./route'); + const response = await POST(request as never); + + expect(response.status).toBe(200); + expect(mockedLoadEffectiveAutoRoutingPool).not.toHaveBeenCalled(); + const fallbackCandidates = + mockedApplyResolvedAutoModel.mock.calls[0]?.[0].efficientFallbackCandidates; + expect(fallbackCandidates).toBeDefined(); + await expect(fallbackCandidates?.()).resolves.toEqual(pool.slice(4)); + expect(mockedLoadEffectiveAutoRoutingPool).toHaveBeenCalledTimes(1); + expect(mockedLoadEffectiveAutoRoutingPool).toHaveBeenCalledWith({ + userId: 'user-123', + organizationId: null, + }); + } + ); + + it('skips pool entries whose context window is too small for the prompt', async () => { + const pool: PoolEntry[] = [ + { model: 'openai/gpt-4o', variant: null }, + { model: 'anthropic/claude-haiku-4', variant: 'low' }, + ]; + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue(pool); + mockedGetEnhancedOpenRouterModels.mockResolvedValue({ + data: [ + { ...catalogModel(pool[0].model), context_length: 8_000 }, + catalogModel('anthropic/claude-haiku-4', undefined, { + low: { reasoning: { enabled: true, effort: 'low' }, verbosity: 'low' }, + }), + ], + }); + const { POST } = await import('./route'); + await POST( + makeRequest({ + ...makeBody('kilo-auto/efficient'), + messages: [{ role: 'user', content: 'x'.repeat(80_000) }], + }) as never + ); + const fallbackCandidates = + mockedApplyResolvedAutoModel.mock.calls[0]?.[0].efficientFallbackCandidates; + + await expect(fallbackCandidates?.()).resolves.toEqual([pool[1]]); + }); + + it('filters the legacy model deny list when no group policy is available', async () => { + const pool: PoolEntry[] = [ + { model: 'openai/gpt-4o', variant: null }, + { model: 'anthropic/claude-haiku-4', variant: 'low' }, + ]; + mockedGetBalanceAndOrgSettings.mockResolvedValue({ + balance: 1000, + settings: { model_deny_list: ['openai/gpt-4o:free'] }, + plan: 'enterprise', + }); + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue(pool); + mockedGetEnhancedOpenRouterModels.mockResolvedValue({ + data: [ + catalogModel('openai/gpt-4o'), + catalogModel('anthropic/claude-haiku-4', undefined, { + low: { reasoning: { enabled: true, effort: 'low' }, verbosity: 'low' }, + }), + ], + }); + + const { POST } = await import('./route'); + const response = await POST(makeRequest(makeBody('kilo-auto/efficient')) as never); + const fallbackCandidates = + mockedApplyResolvedAutoModel.mock.calls[0]?.[0].efficientFallbackCandidates; + + expect(response.status).toBe(200); + await expect(fallbackCandidates?.()).resolves.toEqual([pool[1]]); + expect(mockedGetEffectiveModelDecision).not.toHaveBeenCalled(); + }); + + it.each([ + { name: 'missing', pool: null }, + { name: 'empty', pool: [] }, + ])('keeps the platform fallback when the configured pool is $name', async ({ pool }) => { + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue(pool); + mockedApplyResolvedAutoModel.mockImplementationOnce( + jest.requireActual<{ applyResolvedAutoModel: typeof applyResolvedAutoModel }>( + '@/lib/ai-gateway/auto-model/resolution' + ).applyResolvedAutoModel + ); + + const { POST } = await import('./route'); + const response = await POST(makeRequest(makeBody('kilo-auto/efficient')) as never); + + expect(response.status).toBe(200); + expect(mockedLoadEffectiveAutoRoutingPool).toHaveBeenCalledTimes(1); + expect(mockedGetEnhancedOpenRouterModels).not.toHaveBeenCalled(); + expect(mockedUpstreamRequest).toHaveBeenCalledTimes(1); + expect(mockedUpstreamRequest.mock.calls[0]?.[0].body).toMatchObject(BALANCED_FALLBACK_MODEL); + }); + + it('keeps the platform fallback when the fallback catalog cannot be loaded', async () => { + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue([ + { model: 'openai/gpt-4o', variant: null }, + ]); + mockedGetEnhancedOpenRouterModels.mockRejectedValue(new Error('catalog unavailable')); + mockedApplyResolvedAutoModel.mockImplementationOnce( + jest.requireActual<{ applyResolvedAutoModel: typeof applyResolvedAutoModel }>( + '@/lib/ai-gateway/auto-model/resolution' + ).applyResolvedAutoModel + ); + + const { POST } = await import('./route'); + const response = await POST(makeRequest(makeBody('kilo-auto/efficient')) as never); + + expect(response.status).toBe(200); + expect(mockedUpstreamRequest.mock.calls[0]?.[0].body).toMatchObject(BALANCED_FALLBACK_MODEL); + expect(mockedWarnExceptInTest).toHaveBeenCalledWith( + 'Unable to load the Efficient fallback model catalog' + ); + }); + + it.each(['kilo-auto/efficient', 'kilo-auto/balanced'])( + 'does not fetch the configured pool or catalog for unauthorized %s requests', + async requestedModel => { + mockedGetUserFromAuth.mockResolvedValue({ + user: null, + authFailedResponse: NextResponse.json( + { success: false as const, error: 'unauthorized' }, + { status: 401 } + ), + organizationId: undefined, + }); + mockedLoadEffectiveAutoRoutingPool.mockResolvedValue([ + { model: 'openai/gpt-4o', variant: null }, + ]); + mockedApplyResolvedAutoModel.mockImplementationOnce( + jest.requireActual<{ applyResolvedAutoModel: typeof applyResolvedAutoModel }>( + '@/lib/ai-gateway/auto-model/resolution' + ).applyResolvedAutoModel + ); + + const { POST } = await import('./route'); + const response = await POST(makeRequest(makeBody(requestedModel)) as never); + + expect(response.status).toBe(401); + expect(mockedFetchEfficientAutoDecision).not.toHaveBeenCalled(); + expect(mockedLoadEffectiveAutoRoutingPool).not.toHaveBeenCalled(); + expect(mockedGetEnhancedOpenRouterModels).not.toHaveBeenCalled(); + expect(mockedGetProvider).not.toHaveBeenCalled(); + expect(mockedUpstreamRequest).not.toHaveBeenCalled(); + } + ); + }); }); describe('auto-routing shadow classifier', () => { diff --git a/apps/web/src/app/api/openrouter/[...path]/route.ts b/apps/web/src/app/api/openrouter/[...path]/route.ts index 326544a859..c1a20928b6 100644 --- a/apps/web/src/app/api/openrouter/[...path]/route.ts +++ b/apps/web/src/app/api/openrouter/[...path]/route.ts @@ -100,7 +100,12 @@ import { } from '@/lib/ai-gateway/auto-model'; import { applyResolvedAutoModel } from '@/lib/ai-gateway/auto-model/resolution'; import { fetchEfficientAutoDecision } from '@/lib/ai-gateway/auto-routing-decision'; -import { collectDeniedAutoRoutingModelIds } from '@/lib/ai-gateway/auto-routing-denied-models'; +import { + collectDeniedAutoRoutingModelIds, + loadEffectiveAutoRoutingPool, +} from '@/lib/ai-gateway/auto-routing-denied-models'; +import { getEnhancedOpenRouterModels } from '@/lib/ai-gateway/providers/openrouter'; +import { gatewayChatApisForModel } from '@/lib/ai-gateway/model-api-kinds'; import type { MicrodollarUsageContext, MicrodollarUsageStats, @@ -110,7 +115,11 @@ import { getMaxTokens, hasMiddleOutTransform, } from '@/lib/ai-gateway/providers/openrouter/request-helpers'; -import { redactProviderHints } from '@kilocode/auto-routing-contracts'; +import { + detectRequiredInputModalities, + estimateRoutingTokens, + redactProviderHints, +} from '@kilocode/auto-routing-contracts'; import { logExceptInTest, warnExceptInTest } from '@/lib/utils.server'; import { readDb } from '@/lib/drizzle'; import { getOrganizationGroupPolicyContext } from '@/lib/organizations/organization-group-policy-context.server'; @@ -319,10 +328,6 @@ export async function POST(request: NextRequest): Promise { + const { user, authFailedResponse, organizationId } = await authPromise; + if (!user || authFailedResponse) return null; + const pool = await loadEffectiveAutoRoutingPool({ + userId: user.id, + organizationId: organizationId ?? null, + }); + if (!pool?.length) return null; + const [catalog, policy, { settings, plan }] = await Promise.all([ + getEnhancedOpenRouterModels().catch(() => { + warnExceptInTest('Unable to load the Efficient fallback model catalog'); + return null; + }), + organizationGroupPolicyPromise, + balanceAndSettingsPromise, + ]); + if (!catalog || !Array.isArray(catalog.data)) return null; + const modelsById = new Map(catalog.data.map(model => [model.id, model])); + const requiresImages = detectRequiredInputModalities(requestBodyParsed.body).includes( + 'image' + ); + const promptTokensEstimate = estimateRoutingTokens(requestBodyParsed.body); + const allowed = await Promise.all( + pool.map(async entry => { + const model = modelsById.get(entry.model); + if ( + !model || + (typeof model.context_length === 'number' && + model.context_length < promptTokensEstimate) || + isUnavailableModel(entry.model) || + isDisabledKiloExclusiveModel(entry.model) || + !gatewayChatApisForModel(entry.model).includes(requestBodyParsed.kind) || + (requiresImages && + !model.architecture.input_modalities.some( + modality => modality === 'image' || modality === 'image_url' + )) + ) { + return false; + } + const variantKeys = Object.keys(model.opencode?.variants ?? {}).filter( + key => key.trim().length > 0 + ); + if ( + variantKeys.length > 0 + ? entry.variant === null || !variantKeys.includes(entry.variant) + : entry.variant !== null + ) { + return false; + } + return policy + ? (await getEffectiveModelDecision(policy, entry.model)).allowed + : !checkOrganizationModelRestrictions({ + modelId: entry.model, + settings, + organizationPlan: plan, + }).error; + }) + ); + return pool.filter((_, index) => allowed[index]); + }, organizationContext: organizationContextPromise, isAutoFreeCandidateAllowed: async modelId => { const policy = await organizationGroupPolicyPromise; diff --git a/apps/web/src/components/auto-routing/AutoRoutingModeCard.test.ts b/apps/web/src/components/auto-routing/AutoRoutingModeCard.test.ts index efadf05d1f..dded1792ed 100644 --- a/apps/web/src/components/auto-routing/AutoRoutingModeCard.test.ts +++ b/apps/web/src/components/auto-routing/AutoRoutingModeCard.test.ts @@ -41,6 +41,7 @@ import { NOT_SAVED_ENTRY_LABEL, ORGANIZATION_EMPTY_POOL_COPY, PERSONAL_EMPTY_POOL_COPY, + POOL_FALLBACK_COPY, POOL_ROLLOUT_NOTE, removePoolEntry, resolveEditableChrome, @@ -269,6 +270,21 @@ describe('settings endpoint and query key', () => { // Empty state copy (exact strings from the card module) // --------------------------------------------------------------------------- +describe('configured pool fallback copy', () => { + it('explains saved order, organization precedence, and restricted platform fallback', () => { + expect(POOL_FALLBACK_COPY).toBe( + 'If auto routing cannot select a model, Efficient and Balanced use the first allowed, available model and variant in saved pool order. Organization pools override personal pools. If no pair is usable or no pool is configured, Kilo uses the platform fallback. Organization restrictions still apply.' + ); + }); + + it.each([ + { name: 'empty', configuredPool: null }, + { name: 'configured', configuredPool: [{ ...entryReady, unavailable: false }] }, + ])('renders fallback help for a $name pool', ({ configuredPool }) => { + expect(mountCardHtml(settings({ configuredPool }))).toContain(POOL_FALLBACK_COPY); + }); +}); + describe('empty / inherited pool copy', () => { it('uses the exact personal empty string', () => { expect(PERSONAL_EMPTY_POOL_COPY).toBe( @@ -1051,6 +1067,7 @@ describe('AutoRoutingModeCard poolSupported=false', () => { expect(html).not.toContain('Add model'); expect(html).not.toContain('Clear pool'); expect(html).not.toContain(PERSONAL_EMPTY_POOL_COPY); + expect(html).not.toContain(POOL_FALLBACK_COPY); expect(html).not.toContain('Retry benchmark'); expect(html).toContain('Routing mode'); expect(html).toContain('Save auto routing'); diff --git a/apps/web/src/components/auto-routing/AutoRoutingModeCard.tsx b/apps/web/src/components/auto-routing/AutoRoutingModeCard.tsx index dfeb154241..d84295b9dc 100644 --- a/apps/web/src/components/auto-routing/AutoRoutingModeCard.tsx +++ b/apps/web/src/components/auto-routing/AutoRoutingModeCard.tsx @@ -89,6 +89,9 @@ export const PERSONAL_EMPTY_POOL_COPY = 'No custom pool. Efficient uses the plat export const ORGANIZATION_EMPTY_POOL_COPY = 'No organization override. Members use their personal pool, or the platform model pool if they have none.'; +export const POOL_FALLBACK_COPY = + 'If auto routing cannot select a model, Efficient and Balanced use the first allowed, available model and variant in saved pool order. Organization pools override personal pools. If no pair is usable or no pool is configured, Kilo uses the platform fallback. Organization restrictions still apply.'; + export const UNAVAILABLE_ENTRY_EXPLANATION = 'This model or variant is no longer available in your catalog and cannot be used for routing.'; @@ -1033,7 +1036,8 @@ export function AutoRoutingModeCard({ organizationId, readonly = false }: Props)

Efficient model pool

{poolSupported ? (

- Up to {MAX_POOL_ENTRIES} exact model and variant pairs. Leave empty to inherit. + Up to {MAX_POOL_ENTRIES} exact model and variant pairs. Leave empty to inherit.{' '} + {POOL_FALLBACK_COPY}

) : null} diff --git a/apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts b/apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts index f43f9af2d7..32fa3593b7 100644 --- a/apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts +++ b/apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, jest } from '@jest/globals'; jest.mock('@/lib/ai-gateway/providers/gateway-models-cache', () => ({ getOpenRouterModelsFromRedis: jest.fn(async () => new Set()), + getOpenRouterModelsMetadataFromDatabase: jest.fn(async () => ({})), })); import { resolveAutoModel } from './resolution'; @@ -13,7 +14,7 @@ import { KILO_AUTO_FREE_MODEL, ORG_AUTO_MODEL, } from '@/lib/ai-gateway/auto-model'; -import type { AutoRoutingDecision } from '@kilocode/auto-routing-contracts'; +import type { AutoRoutingDecision, PoolEntry } from '@kilocode/auto-routing-contracts'; const baseParams = { model: KILO_AUTO_EFFICIENT_MODEL.id, @@ -407,6 +408,185 @@ describe('resolveAutoModel — kilo-auto/efficient branch', () => { }); }); +describe('resolveAutoModel — configured efficient fallback pool', () => { + const candidates: ReadonlyArray = [ + { model: 'mistralai/mistral-medium-3-5', variant: 'thinking' }, + { model: 'anthropic/claude-sonnet-5', variant: 'xhigh' }, + ]; + const firstCandidateResolution = { + model: 'mistralai/mistral-medium-3-5', + reasoning: { enabled: true, effort: 'high' }, + }; + + it.each([KILO_AUTO_EFFICIENT_MODEL.id, KILO_AUTO_BALANCED_MODEL.id])( + 'uses the first pool entry in saved order for %s without a decision callback', + async model => { + const efficientFallbackCandidates = jest.fn(async () => candidates); + const result = await resolveAutoModel( + { + ...baseParams, + model, + apiKind: 'chat_completions', + efficientFallbackCandidates, + }, + nullUserPromise, + zeroBalancePromise + ); + + expect(result).toEqual({ kind: 'ok', resolved: firstCandidateResolution }); + expect(efficientFallbackCandidates).toHaveBeenCalledTimes(1); + } + ); + + it.each<{ name: string; decision: AutoRoutingDecision | null }>([ + { name: 'missing worker decision', decision: null }, + { + name: 'virtual worker decision', + decision: { ...sampleDecision, model: KILO_AUTO_EFFICIENT_MODEL.id }, + }, + { + name: 'missing worker decision variant', + decision: { ...sampleDecision, model: 'anthropic/claude-sonnet-5', variant: 'thinking' }, + }, + ])('loads the configured pool after a $name', async ({ decision }) => { + const efficientDecision = jest.fn(async () => decision); + const efficientFallbackCandidates = jest.fn(async () => { + expect(efficientDecision).toHaveBeenCalledTimes(1); + return candidates; + }); + const result = await resolveAutoModel( + { + ...baseParams, + apiKind: 'chat_completions', + efficientDecision, + efficientFallbackCandidates, + }, + nullUserPromise, + zeroBalancePromise + ); + + expect(result).toEqual({ kind: 'ok', resolved: firstCandidateResolution }); + expect(efficientFallbackCandidates).toHaveBeenCalledTimes(1); + }); + + it('skips virtual pool models and missing catalog variants before the first usable entry', async () => { + const result = await resolveAutoModel( + { + ...baseParams, + apiKind: 'chat_completions', + efficientFallbackCandidates: async () => [ + { model: KILO_AUTO_EFFICIENT_MODEL.id, variant: null }, + { model: ORG_AUTO_MODEL.id, variant: null }, + { model: 'anthropic/claude-sonnet-5', variant: 'thinking' }, + { model: 'some-provider/model-without-variants', variant: 'high' }, + { model: 'mistralai/mistral-medium-3-5', variant: 'instant' }, + { model: 'anthropic/claude-sonnet-5', variant: 'max' }, + ], + }, + nullUserPromise, + zeroBalancePromise + ); + + expect(result).toEqual({ + kind: 'ok', + resolved: { + model: 'mistralai/mistral-medium-3-5', + reasoning: { enabled: false, effort: 'none' }, + }, + }); + }); + + it.each(['xhigh', 'max'])( + 'applies exact reasoning and verbosity for pool variant %s', + async variant => { + const result = await resolveAutoModel( + { + ...baseParams, + apiKind: 'chat_completions', + efficientFallbackCandidates: async () => [ + { model: 'anthropic/claude-sonnet-5', variant }, + ], + }, + nullUserPromise, + zeroBalancePromise + ); + + expect(result).toEqual({ + kind: 'ok', + resolved: { + model: 'anthropic/claude-sonnet-5', + reasoning: { enabled: true, effort: variant }, + verbosity: variant, + }, + }); + } + ); + + it.each(['openai/gpt-4o', 'meta-llama/llama-3.3-70b-instruct'])( + 'uses a validated null variant for %s regardless of family fallback variants', + async model => { + const result = await resolveAutoModel( + { + ...baseParams, + apiKind: 'chat_completions', + efficientFallbackCandidates: async () => [{ model, variant: null }, ...candidates], + }, + nullUserPromise, + zeroBalancePromise + ); + + expect(result).toEqual({ kind: 'ok', resolved: { model } }); + } + ); + + it.each([ + sampleDecision, + { ...sampleDecision, model: 'anthropic/claude-sonnet-5', variant: null }, + { ...sampleDecision, model: 'anthropic/claude-sonnet-5', variant: 'xhigh' }, + ])('does not load the fallback pool for a usable $model decision', async decision => { + const efficientFallbackCandidates = jest.fn(async () => candidates); + const result = await resolveAutoModel( + { + ...baseParams, + apiKind: 'chat_completions', + efficientDecision: async () => decision, + efficientFallbackCandidates, + }, + nullUserPromise, + zeroBalancePromise + ); + + expect(result).toMatchObject({ kind: 'ok', resolved: { model: decision.model } }); + expect(efficientFallbackCandidates).not.toHaveBeenCalled(); + }); + + it.each<{ name: string; fallbackCandidates: ReadonlyArray | null }>([ + { name: 'null', fallbackCandidates: null }, + { name: 'empty', fallbackCandidates: [] }, + { + name: 'entirely unusable', + fallbackCandidates: [ + { model: KILO_AUTO_BALANCED_MODEL.id, variant: null }, + { model: 'anthropic/claude-sonnet-5', variant: 'thinking' }, + { model: 'some-provider/model-without-variants', variant: 'high' }, + ], + }, + ])('keeps BALANCED_FALLBACK_MODEL for a $name pool', async ({ fallbackCandidates }) => { + const result = await resolveAutoModel( + { + ...baseParams, + apiKind: 'chat_completions', + efficientDecision: async () => null, + efficientFallbackCandidates: async () => fallbackCandidates, + }, + nullUserPromise, + zeroBalancePromise + ); + + expect(result).toEqual({ kind: 'ok', resolved: BALANCED_FALLBACK_MODEL }); + }); +}); + describe('resolveAutoModel — kilo-auto/free branch', () => { it('excludes candidates denied by the effective organization policy', async () => { const isAutoFreeCandidateAllowed = jest.fn( diff --git a/apps/web/src/lib/ai-gateway/auto-model/resolution.ts b/apps/web/src/lib/ai-gateway/auto-model/resolution.ts index a0bf594806..945791cc98 100644 --- a/apps/web/src/lib/ai-gateway/auto-model/resolution.ts +++ b/apps/web/src/lib/ai-gateway/auto-model/resolution.ts @@ -13,7 +13,11 @@ import type { OrganizationPlan, OrganizationSettings, } from '@/lib/organizations/organization-types'; -import { isVirtualAutoModelId, type AutoRoutingDecision } from '@kilocode/auto-routing-contracts'; +import { + isVirtualAutoModelId, + type AutoRoutingDecision, + type PoolEntry, +} from '@kilocode/auto-routing-contracts'; import { KILO_AUTO_FREE_MODEL, KILO_AUTO_SMALL_MODEL, @@ -53,6 +57,7 @@ type ResolveAutoModelParams = { isAutoFreeCandidateAllowed: ((modelId: string) => Promise) | null; // Lazily fetches the auto-routing worker's decision (route.ts owns the request-body capture). efficientDecision?: () => Promise; + efficientFallbackCandidates?: () => Promise | null>; organizationContext?: Promise<{ organizationId?: string; settings?: OrganizationSettings; @@ -230,37 +235,28 @@ async function resolveOrganizationAutoModel( }; } -/** - * Map an efficient routing decision onto a concrete model + catalog settings. - * - * Prefer canonical `variant` (complete OpenCode variant settings). When the - * variant key is absent from the model's catalog, return null so the caller - * falls back to balanced rather than serving implicit defaults. When `variant` - * is absent, preserve legacy effort-only behavior for rolling deploys. - */ -async function resolveEfficientDecisionModel( - decision: AutoRoutingDecision -): Promise { - // `variant` is only on the benchmark decision branch of the discriminated - // union; coding-plan defaults never carry it. - if ('variant' in decision && decision.variant != null) { - const variants = await getModelVariants(decision.model, true); - const variantSettings: OpenCodeVariant | undefined = variants?.[decision.variant]; +async function resolveEfficientModel(candidate: { + model: string; + variant?: string | null; + reasoningEffort?: AutoRoutingDecision['reasoningEffort']; +}): Promise { + if (candidate.variant != null) { + const variants = await getModelVariants(candidate.model, true); + const variantSettings: OpenCodeVariant | undefined = variants?.[candidate.variant]; if (!variantSettings) { return null; } return { - model: decision.model, + model: candidate.model, ...(variantSettings.reasoning ? { reasoning: { ...variantSettings.reasoning } } : {}), ...(variantSettings.verbosity ? { verbosity: variantSettings.verbosity } : {}), }; } - // Legacy effort-only decisions (old workers during rolling deploy). return { - model: decision.model, - ...('reasoningEffort' in decision && decision.reasoningEffort - ? { reasoning: { enabled: true, effort: decision.reasoningEffort } } + model: candidate.model, + ...(candidate.reasoningEffort + ? { reasoning: { enabled: true, effort: candidate.reasoningEffort } } : {}), }; } @@ -322,15 +318,21 @@ export async function resolveAutoModel( if (model === KILO_AUTO_EFFICIENT_MODEL.id || model === KILO_AUTO_BALANCED_MODEL.id) { const decision = params.efficientDecision ? await params.efficientDecision() : null; if (decision && !isVirtualAutoModelId(decision.model)) { - const resolvedFromDecision = await resolveEfficientDecisionModel(decision); + const resolvedFromDecision = await resolveEfficientModel(decision); if (resolvedFromDecision) { return { kind: 'ok', resolved: resolvedFromDecision }; } - // Exact catalog variant missing or removed: never serve the chosen model - // with implicit defaults — same balanced fallback as the no-decision path. - return { kind: 'ok', resolved: BALANCED_FALLBACK_MODEL }; } - // Static fallback when the worker is slow or unavailable. + const fallbackCandidates = await params.efficientFallbackCandidates?.(); + for (const candidate of fallbackCandidates ?? []) { + if (isVirtualAutoModelId(candidate.model)) { + continue; + } + const resolvedFromCandidate = await resolveEfficientModel(candidate); + if (resolvedFromCandidate) { + return { kind: 'ok', resolved: resolvedFromCandidate }; + } + } return { kind: 'ok', resolved: BALANCED_FALLBACK_MODEL }; } const mode = resolveMode(modeHeader, featureHeader); diff --git a/apps/web/src/lib/ai-gateway/auto-routing-admin-client.test.ts b/apps/web/src/lib/ai-gateway/auto-routing-admin-client.test.ts index d45f28ac05..9ac9f3df74 100644 --- a/apps/web/src/lib/ai-gateway/auto-routing-admin-client.test.ts +++ b/apps/web/src/lib/ai-gateway/auto-routing-admin-client.test.ts @@ -147,24 +147,28 @@ describe('auto routing admin client', () => { ); }); - it('gets routing settings using worker bearer auth', async () => { + it.each([ + { name: 'without a signal', signal: undefined }, + { name: 'with a signal', signal: new AbortController().signal }, + ])('gets routing settings using worker bearer auth $name', async ({ signal }) => { mockFetch.mockResolvedValue({ status: 200, ok: true, json: () => Promise.resolve(settingsResponse), }); - await expect(getAutoRoutingSettings({ ownerType: 'user', ownerId: 'user-1' })).resolves.toEqual( - { - status: 200, - body: settingsResponse, - } - ); + await expect( + getAutoRoutingSettings({ ownerType: 'user', ownerId: 'user-1' }, signal) + ).resolves.toEqual({ + status: 200, + body: settingsResponse, + }); expect(mockFetch).toHaveBeenCalledWith( 'https://auto-routing.example.com/admin/routing-settings?ownerType=user&ownerId=user-1', { method: 'GET', + signal, headers: { authorization: 'Bearer test-internal-secret', }, diff --git a/apps/web/src/lib/ai-gateway/auto-routing-admin-client.ts b/apps/web/src/lib/ai-gateway/auto-routing-admin-client.ts index 8737e188fc..2034a387ce 100644 --- a/apps/web/src/lib/ai-gateway/auto-routing-admin-client.ts +++ b/apps/web/src/lib/ai-gateway/auto-routing-admin-client.ts @@ -154,13 +154,17 @@ async function fetchAutoRoutingSettingsAdmin( }; } -export function getAutoRoutingSettings(owner: { - ownerType: AutoRoutingModeOwnerType; - ownerId: string; -}): Promise { +export function getAutoRoutingSettings( + owner: { + ownerType: AutoRoutingModeOwnerType; + ownerId: string; + }, + signal?: AbortSignal +): Promise { const searchParams = new URLSearchParams(owner); return fetchAutoRoutingSettingsAdmin(`/admin/routing-settings?${searchParams}`, { method: 'GET', + signal, }); } diff --git a/apps/web/src/lib/ai-gateway/auto-routing-denied-models.test.ts b/apps/web/src/lib/ai-gateway/auto-routing-denied-models.test.ts index bf9d9d0fa0..6c867e96b5 100644 --- a/apps/web/src/lib/ai-gateway/auto-routing-denied-models.test.ts +++ b/apps/web/src/lib/ai-gateway/auto-routing-denied-models.test.ts @@ -1,13 +1,73 @@ -import { describe, expect, it } from '@jest/globals'; +import { afterEach, beforeEach, describe, expect, it } from '@jest/globals'; + +jest.mock('@/lib/ai-gateway/auto-routing-admin-client', () => ({ + getAutoRoutingSettings: jest.fn(), +})); + +jest.mock('@/lib/ai-gateway/auto-routing-table-cache', () => ({ + getCachedRoutingTable: jest.fn(), +})); + +import type { + AutoRoutingModeOwnerQuery, + AutoRoutingModeResponse, + AutoRoutingSettingsResponse, + PoolEntry, + RoutingTable, +} from '@kilocode/auto-routing-contracts'; import type { EffectiveOrganizationModelPolicy } from '@/lib/organizations/effective-model-access.server'; import { MINIMAX_CURRENT_MODEL_ID } from '@/lib/ai-gateway/providers/minimax'; +import { + getAutoRoutingSettings, + type AutoRoutingSettingsWorkerResult, +} from '@/lib/ai-gateway/auto-routing-admin-client'; +import { getCachedRoutingTable } from '@/lib/ai-gateway/auto-routing-table-cache'; import { candidateModelIdsFromSources, collectDeniedAutoRoutingModelIds, deniedModelIdsForCandidates, + loadAutoRoutingCandidateModelIds, + loadEffectiveAutoRoutingPool, policyNeedsCandidateEvaluation, } from './auto-routing-denied-models'; +const mockGetAutoRoutingSettings = + jest.mocked< + ( + owner: AutoRoutingModeOwnerQuery, + signal?: AbortSignal + ) => Promise< + AutoRoutingSettingsWorkerResult | { status: number; body: AutoRoutingModeResponse } + > + >(getAutoRoutingSettings); +const mockGetCachedRoutingTable = jest.mocked(getCachedRoutingTable); + +function settingsResult( + settingsOwner: AutoRoutingModeOwnerQuery, + configuredPool: PoolEntry[] | null +): { status: number; body: AutoRoutingSettingsResponse } { + return { + status: 200, + body: { + ...settingsOwner, + mode: 'cost_per_accuracy', + configuredMode: null, + defaultMode: 'cost_per_accuracy', + configuredPool, + poolStatuses: [], + }, + }; +} + +beforeEach(() => { + mockGetAutoRoutingSettings.mockReset(); + mockGetAutoRoutingSettings.mockImplementation(async settingsOwner => + settingsResult(settingsOwner, null) + ); + mockGetCachedRoutingTable.mockReset(); + mockGetCachedRoutingTable.mockResolvedValue(null); +}); + function policy( overrides: Partial = {} ): EffectiveOrganizationModelPolicy { @@ -21,6 +81,258 @@ function policy( } const owner = { userId: 'user-1', organizationId: 'org-1' }; +const orgSettingsOwner = { + ownerType: 'org', + ownerId: owner.organizationId, +} satisfies AutoRoutingModeOwnerQuery; +const personalSettingsOwner = { + ownerType: 'user', + ownerId: owner.userId, +} satisfies AutoRoutingModeOwnerQuery; +const orgPool = [ + { model: 'openai/o3', variant: 'high' }, + { model: 'anthropic/claude', variant: null }, + { model: 'openai/o3', variant: 'low' }, +] satisfies PoolEntry[]; +const personalPool = [ + { model: 'google/gemini-2.5-flash', variant: null }, + { model: 'openai/o3', variant: 'medium' }, +] satisfies PoolEntry[]; + +describe('loadEffectiveAutoRoutingPool', () => { + beforeEach(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + jest.restoreAllMocks(); + jest.useRealTimers(); + }); + + it('returns the organization override in saved order with every variant intact', async () => { + mockGetAutoRoutingSettings + .mockResolvedValueOnce(settingsResult(orgSettingsOwner, orgPool)) + .mockResolvedValueOnce(settingsResult(personalSettingsOwner, personalPool)); + + await expect(loadEffectiveAutoRoutingPool(owner)).resolves.toEqual([ + { model: 'openai/o3', variant: 'high' }, + { model: 'anthropic/claude', variant: null }, + { model: 'openai/o3', variant: 'low' }, + ]); + expect(mockGetAutoRoutingSettings.mock.calls).toEqual([ + [orgSettingsOwner, expect.any(AbortSignal)], + [personalSettingsOwner, expect.any(AbortSignal)], + ]); + expect(mockGetAutoRoutingSettings.mock.calls[0][1]).not.toBe( + mockGetAutoRoutingSettings.mock.calls[1][1] + ); + expect(mockGetCachedRoutingTable).not.toHaveBeenCalled(); + }); + + it.each([{ configuredPool: null }, { configuredPool: [] }])( + 'inherits the personal pool when the organization pool is $configuredPool', + async ({ configuredPool }) => { + mockGetAutoRoutingSettings + .mockResolvedValueOnce(settingsResult(orgSettingsOwner, configuredPool)) + .mockResolvedValueOnce(settingsResult(personalSettingsOwner, personalPool)); + + await expect(loadEffectiveAutoRoutingPool(owner)).resolves.toEqual(personalPool); + } + ); + + it('loads only personal settings for a request without an organization', async () => { + mockGetAutoRoutingSettings.mockResolvedValueOnce( + settingsResult(personalSettingsOwner, personalPool) + ); + + await expect( + loadEffectiveAutoRoutingPool({ userId: owner.userId, organizationId: null }) + ).resolves.toEqual(personalPool); + expect(mockGetAutoRoutingSettings.mock.calls).toEqual([ + [personalSettingsOwner, expect.any(AbortSignal)], + ]); + }); + + it.each([{ configuredPool: null }, { configuredPool: [] }])( + 'returns null when both configured pools are $configuredPool', + async ({ configuredPool }) => { + mockGetAutoRoutingSettings.mockImplementation(async settingsOwner => + settingsResult(settingsOwner, configuredPool) + ); + + await expect(loadEffectiveAutoRoutingPool(owner)).resolves.toBeNull(); + } + ); + + it('inherits the personal pool when the organization settings request rejects', async () => { + mockGetAutoRoutingSettings + .mockRejectedValueOnce(new Error('Sensitive upstream failure')) + .mockResolvedValueOnce(settingsResult(personalSettingsOwner, personalPool)); + + await expect(loadEffectiveAutoRoutingPool(owner)).resolves.toEqual(personalPool); + expect(console.warn).toHaveBeenCalledTimes(1); + expect(console.warn).toHaveBeenCalledWith('Failed to load auto routing settings'); + }); + + it('keeps the organization pool when the personal settings request rejects', async () => { + mockGetAutoRoutingSettings + .mockResolvedValueOnce(settingsResult(orgSettingsOwner, orgPool)) + .mockRejectedValueOnce(new Error('Sensitive upstream failure')); + + await expect(loadEffectiveAutoRoutingPool(owner)).resolves.toEqual(orgPool); + }); + + it.each([owner, { ...owner, organizationId: null }])( + 'returns null when every settings request rejects for $organizationId', + async requestOwner => { + mockGetAutoRoutingSettings.mockRejectedValue(new Error('Sensitive upstream failure')); + + await expect(loadEffectiveAutoRoutingPool(requestOwner)).resolves.toBeNull(); + } + ); + + it.each([ + { + name: 'inherits the personal pool when the organization request times out', + requestOwner: owner, + pendingOwnerType: 'org', + expectedPool: personalPool, + }, + { + name: 'keeps the organization pool when the personal request times out', + requestOwner: owner, + pendingOwnerType: 'user', + expectedPool: orgPool, + }, + { + name: 'returns null when the personal-only request times out', + requestOwner: { ...owner, organizationId: null }, + pendingOwnerType: 'user', + expectedPool: null, + }, + ])('$name', async ({ requestOwner, pendingOwnerType, expectedPool }) => { + jest.useFakeTimers(); + const timeoutSpy = jest.spyOn(AbortSignal, 'timeout').mockImplementation(milliseconds => { + const controller = new AbortController(); + setTimeout( + () => controller.abort(new DOMException('Settings request timed out', 'TimeoutError')), + milliseconds + ); + return controller.signal; + }); + mockGetAutoRoutingSettings.mockImplementation(async (settingsOwner, signal) => { + if (settingsOwner.ownerType !== pendingOwnerType) { + return settingsResult( + settingsOwner, + settingsOwner.ownerType === 'org' ? orgPool : personalPool + ); + } + if (!signal) throw new Error('Expected a timeout signal'); + return new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }); + }); + const onResolved = jest.fn(); + const result = loadEffectiveAutoRoutingPool(requestOwner).then(pool => { + onResolved(pool); + return pool; + }); + + expect(timeoutSpy.mock.calls).toEqual( + requestOwner.organizationId ? [[2000], [2000]] : [[2000]] + ); + await jest.advanceTimersByTimeAsync(1999); + expect(onResolved).not.toHaveBeenCalled(); + await jest.advanceTimersByTimeAsync(1); + + await expect(result).resolves.toEqual(expectedPool); + expect(console.warn).toHaveBeenCalledTimes(1); + expect(console.warn).toHaveBeenCalledWith('Failed to load auto routing settings'); + }); + + describe.each([ + { + name: 'an error response', + result: { status: 503, body: { error: 'Settings unavailable' } }, + }, + { + name: 'a non-200 response containing a pool', + result: { ...settingsResult(orgSettingsOwner, orgPool), status: 503 }, + }, + { + name: 'legacy settings without configuredPool', + result: { + status: 200, + body: { + ...orgSettingsOwner, + mode: 'cost_per_accuracy', + configuredMode: null, + defaultMode: 'cost_per_accuracy', + } satisfies AutoRoutingModeResponse, + }, + }, + ])('with $name', ({ result }) => { + it('skips the organization response and inherits the personal pool', async () => { + mockGetAutoRoutingSettings + .mockResolvedValueOnce(result) + .mockResolvedValueOnce(settingsResult(personalSettingsOwner, personalPool)); + + await expect(loadEffectiveAutoRoutingPool(owner)).resolves.toEqual(personalPool); + }); + + it('returns null when no later owner has a configured pool', async () => { + mockGetAutoRoutingSettings.mockResolvedValueOnce(result); + + await expect(loadEffectiveAutoRoutingPool(owner)).resolves.toBeNull(); + }); + }); +}); + +describe('loadAutoRoutingCandidateModelIds', () => { + beforeEach(() => { + mockGetCachedRoutingTable.mockResolvedValue({ + version: 'benchmark-1', + generatedAt: '2026-08-31T00:00:00.000Z', + minAccuracy: 0.8, + switchCostFactor: 1.2, + bestAccuracySwitchThreshold: 0.05, + source: 'benchmark', + routes: { + 'implementation/code_generation': [ + { + model: 'table/only-model', + variant: null, + accuracy: 0.9, + avgCostUsd: 0.01, + meetsThreshold: true, + }, + ], + }, + } satisfies RoutingTable); + }); + + it('projects the effective pool to distinct model IDs instead of using the routing table', async () => { + mockGetAutoRoutingSettings + .mockResolvedValueOnce(settingsResult(orgSettingsOwner, orgPool)) + .mockResolvedValueOnce(settingsResult(personalSettingsOwner, personalPool)); + + await expect(loadAutoRoutingCandidateModelIds(owner)).resolves.toEqual([ + 'openai/o3', + 'anthropic/claude', + MINIMAX_CURRENT_MODEL_ID, + 'byteplus-coding/bytedance-seed-code', + ]); + expect(mockGetCachedRoutingTable).toHaveBeenCalledTimes(1); + }); + + it('falls back to routing-table model IDs when no pool is configured', async () => { + await expect(loadAutoRoutingCandidateModelIds(owner)).resolves.toEqual([ + 'table/only-model', + MINIMAX_CURRENT_MODEL_ID, + 'byteplus-coding/bytedance-seed-code', + ]); + }); +}); describe('policyNeedsCandidateEvaluation', () => { it('is false for an unrestricted policy with an inactive baseline deny list', () => { diff --git a/apps/web/src/lib/ai-gateway/auto-routing-denied-models.ts b/apps/web/src/lib/ai-gateway/auto-routing-denied-models.ts index beecc4250d..63546ff89c 100644 --- a/apps/web/src/lib/ai-gateway/auto-routing-denied-models.ts +++ b/apps/web/src/lib/ai-gateway/auto-routing-denied-models.ts @@ -1,4 +1,4 @@ -import { isVirtualAutoModelId } from '@kilocode/auto-routing-contracts'; +import { isVirtualAutoModelId, type PoolEntry } from '@kilocode/auto-routing-contracts'; import { getAutoRoutingSettings } from '@/lib/ai-gateway/auto-routing-admin-client'; import { getCachedRoutingTable } from '@/lib/ai-gateway/auto-routing-table-cache'; import { normalizeModelId } from '@/lib/ai-gateway/model-utils'; @@ -63,28 +63,39 @@ export function deniedModelIdsForCandidates( return [...denied]; } -export async function loadEffectivePoolModelIds(owner: AutoRoutingOwner): Promise { +export async function loadEffectiveAutoRoutingPool( + owner: AutoRoutingOwner +): Promise { const owners = [ ...(owner.organizationId ? [{ ownerType: 'org' as const, ownerId: owner.organizationId }] : []), { ownerType: 'user' as const, ownerId: owner.userId }, ]; - const results = await Promise.all(owners.map(getAutoRoutingSettings)); + const results = await Promise.all( + owners.map(async settingsOwner => { + try { + return await getAutoRoutingSettings(settingsOwner, AbortSignal.timeout(2000)); + } catch { + console.warn('Failed to load auto routing settings'); + return null; + } + }) + ); for (const result of results) { - if (result.status !== 200 || !('configuredPool' in result.body)) continue; + if (!result || result.status !== 200 || !('configuredPool' in result.body)) continue; const pool = result.body.configuredPool; if (pool && pool.length > 0) { - return pool.map(entry => entry.model); + return pool; } } return null; } export async function loadAutoRoutingCandidateModelIds(owner: AutoRoutingOwner): Promise { - const [table, poolModelIds] = await Promise.all([ + const [table, pool] = await Promise.all([ getCachedRoutingTable(), - loadEffectivePoolModelIds(owner), + loadEffectiveAutoRoutingPool(owner), ]); - return candidateModelIdsFromSources(table, poolModelIds); + return candidateModelIdsFromSources(table, pool?.map(entry => entry.model) ?? null); } export async function collectDeniedAutoRoutingModelIds( diff --git a/apps/web/src/lib/ai-gateway/llm-proxy-helpers.test.ts b/apps/web/src/lib/ai-gateway/llm-proxy-helpers.test.ts index 64274da1c1..4b19fb7402 100644 --- a/apps/web/src/lib/ai-gateway/llm-proxy-helpers.test.ts +++ b/apps/web/src/lib/ai-gateway/llm-proxy-helpers.test.ts @@ -38,6 +38,7 @@ import { checkOrganizationModelRestrictions, countAndStoreEditUsage, countAndStoreFimUsage, + efficientPoolBlockedResponse, extractEditPromptInfo, extractEmbeddingPromptInfo, extractHeaderAndLimitLength, @@ -47,6 +48,20 @@ import { parseTranscriptionUsageFromResponse, } from './llm-proxy-helpers'; +describe('efficientPoolBlockedResponse', () => { + it('reports the selected model is blocked without claiming the entire pool is blocked', async () => { + const response = efficientPoolBlockedResponse(); + + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ + error: 'Your organization blocks the model selected by auto routing.', + error_type: 'model_not_allowed', + message: + 'Your organization blocks the model selected by auto routing. Configure a custom Efficient model pool with allowed models, or adjust your organization model restrictions.', + }); + }); +}); + describe('checkOrganizationModelRestrictions', () => { describe('enterprise plan - model deny list restrictions', () => { it('should allow model when it is not in the deny list on enterprise plan', () => { diff --git a/apps/web/src/lib/ai-gateway/llm-proxy-helpers.ts b/apps/web/src/lib/ai-gateway/llm-proxy-helpers.ts index 3ac9529bb9..a6daaadbf2 100644 --- a/apps/web/src/lib/ai-gateway/llm-proxy-helpers.ts +++ b/apps/web/src/lib/ai-gateway/llm-proxy-helpers.ts @@ -328,7 +328,7 @@ export function modelNotAllowedResponse() { } export function efficientPoolBlockedResponse() { - const error = 'Your organization blocks every model in the auto-routing pool.'; + const error = 'Your organization blocks the model selected by auto routing.'; const message = `${error} Configure a custom Efficient model pool with allowed models, ` + `or adjust your organization model restrictions.`;