Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
448 changes: 447 additions & 1 deletion apps/web/src/app/api/openrouter/[...path]/route.test.ts

Large diffs are not rendered by default.

77 changes: 71 additions & 6 deletions apps/web/src/app/api/openrouter/[...path]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,12 @@ import {
} from '@/lib/ai-gateway/auto-model';
import { applyResolvedAutoModel } from '@/lib/ai-gateway/auto-model/resolution';
import { fetchEfficientAutoDecision } from '@/lib/ai-gateway/auto-routing-decision';
import { collectDeniedAutoRoutingModelIds } from '@/lib/ai-gateway/auto-routing-denied-models';
import {
collectDeniedAutoRoutingModelIds,
loadEffectiveAutoRoutingPool,
} from '@/lib/ai-gateway/auto-routing-denied-models';
import { getEnhancedOpenRouterModels } from '@/lib/ai-gateway/providers/openrouter';
import { gatewayChatApisForModel } from '@/lib/ai-gateway/model-api-kinds';
import type {
MicrodollarUsageContext,
MicrodollarUsageStats,
Expand All @@ -110,7 +115,11 @@ import {
getMaxTokens,
hasMiddleOutTransform,
} from '@/lib/ai-gateway/providers/openrouter/request-helpers';
import { redactProviderHints } from '@kilocode/auto-routing-contracts';
import {
detectRequiredInputModalities,
estimateRoutingTokens,
redactProviderHints,
} from '@kilocode/auto-routing-contracts';
import { logExceptInTest, warnExceptInTest } from '@/lib/utils.server';
import { readDb } from '@/lib/drizzle';
import { getOrganizationGroupPolicyContext } from '@/lib/organizations/organization-group-policy-context.server';
Expand Down Expand Up @@ -319,10 +328,6 @@ export async function POST(request: NextRequest): Promise<NextResponseType<unkno
// validation after resolution.
let routingTarget: string | null = null;
let classifierCostUsd = 0;
// Efficient/balanced requests resolve through the auto-routing pool. Kept for
// the org policy check below so a team that blocks every pool model gets
// guidance to configure a custom Efficient model pool instead of the generic
// model-not-allowed error.
let isAutoEfficientRequest = false;
if (isKiloAutoModel(requestedModelLowerCased)) {
autoModel = requestedModelLowerCased;
Expand Down Expand Up @@ -381,6 +386,66 @@ export async function POST(request: NextRequest): Promise<NextResponseType<unkno
apiKind: requestBodyParsed.kind,
clientIp: ipAddress ?? null,
efficientDecision,
efficientFallbackCandidates: async () => {
const { user, authFailedResponse, organizationId } = await authPromise;
if (!user || authFailedResponse) return null;
const pool = await loadEffectiveAutoRoutingPool({
userId: user.id,
organizationId: organizationId ?? null,
});
if (!pool?.length) return null;
const [catalog, policy, { settings, plan }] = await Promise.all([
getEnhancedOpenRouterModels().catch(() => {
warnExceptInTest('Unable to load the Efficient fallback model catalog');
return null;
}),
organizationGroupPolicyPromise,
balanceAndSettingsPromise,
]);
if (!catalog || !Array.isArray(catalog.data)) return null;
const modelsById = new Map(catalog.data.map(model => [model.id, model]));
const requiresImages = detectRequiredInputModalities(requestBodyParsed.body).includes(
'image'
);
const promptTokensEstimate = estimateRoutingTokens(requestBodyParsed.body);
const allowed = await Promise.all(
pool.map(async entry => {
const model = modelsById.get(entry.model);
if (
!model ||
(typeof model.context_length === 'number' &&
model.context_length < promptTokensEstimate) ||
isUnavailableModel(entry.model) ||
isDisabledKiloExclusiveModel(entry.model) ||
!gatewayChatApisForModel(entry.model).includes(requestBodyParsed.kind) ||
(requiresImages &&
!model.architecture.input_modalities.some(
modality => modality === 'image' || modality === 'image_url'
))
) {
return false;
}
const variantKeys = Object.keys(model.opencode?.variants ?? {}).filter(
key => key.trim().length > 0
);
if (
variantKeys.length > 0
? entry.variant === null || !variantKeys.includes(entry.variant)
: entry.variant !== null
) {
return false;
}
return policy
? (await getEffectiveModelDecision(policy, entry.model)).allowed
: !checkOrganizationModelRestrictions({
modelId: entry.model,
settings,
organizationPlan: plan,
}).error;
})
);
return pool.filter((_, index) => allowed[index]);
},
organizationContext: organizationContextPromise,
isAutoFreeCandidateAllowed: async modelId => {
const policy = await organizationGroupPolicyPromise;
Expand Down
17 changes: 17 additions & 0 deletions apps/web/src/components/auto-routing/AutoRoutingModeCard.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import {
NOT_SAVED_ENTRY_LABEL,
ORGANIZATION_EMPTY_POOL_COPY,
PERSONAL_EMPTY_POOL_COPY,
POOL_FALLBACK_COPY,
POOL_ROLLOUT_NOTE,
removePoolEntry,
resolveEditableChrome,
Expand Down Expand Up @@ -269,6 +270,21 @@ describe('settings endpoint and query key', () => {
// Empty state copy (exact strings from the card module)
// ---------------------------------------------------------------------------

describe('configured pool fallback copy', () => {
it('explains saved order, organization precedence, and restricted platform fallback', () => {
expect(POOL_FALLBACK_COPY).toBe(
'If auto routing cannot select a model, Efficient and Balanced use the first allowed, available model and variant in saved pool order. Organization pools override personal pools. If no pair is usable or no pool is configured, Kilo uses the platform fallback. Organization restrictions still apply.'
);
});

it.each([
{ name: 'empty', configuredPool: null },
{ name: 'configured', configuredPool: [{ ...entryReady, unavailable: false }] },
])('renders fallback help for a $name pool', ({ configuredPool }) => {
expect(mountCardHtml(settings({ configuredPool }))).toContain(POOL_FALLBACK_COPY);
});
});

describe('empty / inherited pool copy', () => {
it('uses the exact personal empty string', () => {
expect(PERSONAL_EMPTY_POOL_COPY).toBe(
Expand Down Expand Up @@ -1051,6 +1067,7 @@ describe('AutoRoutingModeCard poolSupported=false', () => {
expect(html).not.toContain('Add model');
expect(html).not.toContain('Clear pool');
expect(html).not.toContain(PERSONAL_EMPTY_POOL_COPY);
expect(html).not.toContain(POOL_FALLBACK_COPY);
expect(html).not.toContain('Retry benchmark');
expect(html).toContain('Routing mode');
expect(html).toContain('Save auto routing');
Expand Down
6 changes: 5 additions & 1 deletion apps/web/src/components/auto-routing/AutoRoutingModeCard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ export const PERSONAL_EMPTY_POOL_COPY = 'No custom pool. Efficient uses the plat
export const ORGANIZATION_EMPTY_POOL_COPY =
'No organization override. Members use their personal pool, or the platform model pool if they have none.';

export const POOL_FALLBACK_COPY =
'If auto routing cannot select a model, Efficient and Balanced use the first allowed, available model and variant in saved pool order. Organization pools override personal pools. If no pair is usable or no pool is configured, Kilo uses the platform fallback. Organization restrictions still apply.';

export const UNAVAILABLE_ENTRY_EXPLANATION =
'This model or variant is no longer available in your catalog and cannot be used for routing.';

Expand Down Expand Up @@ -1033,7 +1036,8 @@ export function AutoRoutingModeCard({ organizationId, readonly = false }: Props)
<h3 className="text-sm font-medium">Efficient model pool</h3>
{poolSupported ? (
<p id={poolHelpId} className="text-muted-foreground text-sm">
Up to {MAX_POOL_ENTRIES} exact model and variant pairs. Leave empty to inherit.
Up to {MAX_POOL_ENTRIES} exact model and variant pairs. Leave empty to inherit.{' '}
{POOL_FALLBACK_COPY}
</p>
) : null}
</div>
Expand Down
182 changes: 181 additions & 1 deletion apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { describe, expect, it, jest } from '@jest/globals';

jest.mock('@/lib/ai-gateway/providers/gateway-models-cache', () => ({
getOpenRouterModelsFromRedis: jest.fn(async () => new Set<string>()),
getOpenRouterModelsMetadataFromDatabase: jest.fn(async () => ({})),
}));

import { resolveAutoModel } from './resolution';
Expand All @@ -13,7 +14,7 @@ import {
KILO_AUTO_FREE_MODEL,
ORG_AUTO_MODEL,
} from '@/lib/ai-gateway/auto-model';
import type { AutoRoutingDecision } from '@kilocode/auto-routing-contracts';
import type { AutoRoutingDecision, PoolEntry } from '@kilocode/auto-routing-contracts';

const baseParams = {
model: KILO_AUTO_EFFICIENT_MODEL.id,
Expand Down Expand Up @@ -407,6 +408,185 @@ describe('resolveAutoModel — kilo-auto/efficient branch', () => {
});
});

describe('resolveAutoModel — configured efficient fallback pool', () => {
const candidates: ReadonlyArray<PoolEntry> = [
{ model: 'mistralai/mistral-medium-3-5', variant: 'thinking' },
{ model: 'anthropic/claude-sonnet-5', variant: 'xhigh' },
];
const firstCandidateResolution = {
model: 'mistralai/mistral-medium-3-5',
reasoning: { enabled: true, effort: 'high' },
};

it.each([KILO_AUTO_EFFICIENT_MODEL.id, KILO_AUTO_BALANCED_MODEL.id])(
'uses the first pool entry in saved order for %s without a decision callback',
async model => {
const efficientFallbackCandidates = jest.fn(async () => candidates);
const result = await resolveAutoModel(
{
...baseParams,
model,
apiKind: 'chat_completions',
efficientFallbackCandidates,
},
nullUserPromise,
zeroBalancePromise
);

expect(result).toEqual({ kind: 'ok', resolved: firstCandidateResolution });
expect(efficientFallbackCandidates).toHaveBeenCalledTimes(1);
}
);

it.each<{ name: string; decision: AutoRoutingDecision | null }>([
{ name: 'missing worker decision', decision: null },
{
name: 'virtual worker decision',
decision: { ...sampleDecision, model: KILO_AUTO_EFFICIENT_MODEL.id },
},
{
name: 'missing worker decision variant',
decision: { ...sampleDecision, model: 'anthropic/claude-sonnet-5', variant: 'thinking' },
},
])('loads the configured pool after a $name', async ({ decision }) => {
const efficientDecision = jest.fn(async () => decision);
const efficientFallbackCandidates = jest.fn(async () => {
expect(efficientDecision).toHaveBeenCalledTimes(1);
return candidates;
});
const result = await resolveAutoModel(
{
...baseParams,
apiKind: 'chat_completions',
efficientDecision,
efficientFallbackCandidates,
},
nullUserPromise,
zeroBalancePromise
);

expect(result).toEqual({ kind: 'ok', resolved: firstCandidateResolution });
expect(efficientFallbackCandidates).toHaveBeenCalledTimes(1);
});

it('skips virtual pool models and missing catalog variants before the first usable entry', async () => {
const result = await resolveAutoModel(
{
...baseParams,
apiKind: 'chat_completions',
efficientFallbackCandidates: async () => [
{ model: KILO_AUTO_EFFICIENT_MODEL.id, variant: null },
{ model: ORG_AUTO_MODEL.id, variant: null },
{ model: 'anthropic/claude-sonnet-5', variant: 'thinking' },
{ model: 'some-provider/model-without-variants', variant: 'high' },
{ model: 'mistralai/mistral-medium-3-5', variant: 'instant' },
{ model: 'anthropic/claude-sonnet-5', variant: 'max' },
],
},
nullUserPromise,
zeroBalancePromise
);

expect(result).toEqual({
kind: 'ok',
resolved: {
model: 'mistralai/mistral-medium-3-5',
reasoning: { enabled: false, effort: 'none' },
},
});
});

it.each(['xhigh', 'max'])(
'applies exact reasoning and verbosity for pool variant %s',
async variant => {
const result = await resolveAutoModel(
{
...baseParams,
apiKind: 'chat_completions',
efficientFallbackCandidates: async () => [
{ model: 'anthropic/claude-sonnet-5', variant },
],
},
nullUserPromise,
zeroBalancePromise
);

expect(result).toEqual({
kind: 'ok',
resolved: {
model: 'anthropic/claude-sonnet-5',
reasoning: { enabled: true, effort: variant },
verbosity: variant,
},
});
}
);

it.each(['openai/gpt-4o', 'meta-llama/llama-3.3-70b-instruct'])(
'uses a validated null variant for %s regardless of family fallback variants',
async model => {
const result = await resolveAutoModel(
{
...baseParams,
apiKind: 'chat_completions',
efficientFallbackCandidates: async () => [{ model, variant: null }, ...candidates],
},
nullUserPromise,
zeroBalancePromise
);

expect(result).toEqual({ kind: 'ok', resolved: { model } });
}
);

it.each<AutoRoutingDecision>([
sampleDecision,
{ ...sampleDecision, model: 'anthropic/claude-sonnet-5', variant: null },
{ ...sampleDecision, model: 'anthropic/claude-sonnet-5', variant: 'xhigh' },
])('does not load the fallback pool for a usable $model decision', async decision => {
const efficientFallbackCandidates = jest.fn(async () => candidates);
const result = await resolveAutoModel(
{
...baseParams,
apiKind: 'chat_completions',
efficientDecision: async () => decision,
efficientFallbackCandidates,
},
nullUserPromise,
zeroBalancePromise
);

expect(result).toMatchObject({ kind: 'ok', resolved: { model: decision.model } });
expect(efficientFallbackCandidates).not.toHaveBeenCalled();
});

it.each<{ name: string; fallbackCandidates: ReadonlyArray<PoolEntry> | null }>([
{ name: 'null', fallbackCandidates: null },
{ name: 'empty', fallbackCandidates: [] },
{
name: 'entirely unusable',
fallbackCandidates: [
{ model: KILO_AUTO_BALANCED_MODEL.id, variant: null },
{ model: 'anthropic/claude-sonnet-5', variant: 'thinking' },
{ model: 'some-provider/model-without-variants', variant: 'high' },
],
},
])('keeps BALANCED_FALLBACK_MODEL for a $name pool', async ({ fallbackCandidates }) => {
const result = await resolveAutoModel(
{
...baseParams,
apiKind: 'chat_completions',
efficientDecision: async () => null,
efficientFallbackCandidates: async () => fallbackCandidates,
},
nullUserPromise,
zeroBalancePromise
);

expect(result).toEqual({ kind: 'ok', resolved: BALANCED_FALLBACK_MODEL });
});
});

describe('resolveAutoModel — kilo-auto/free branch', () => {
it('excludes candidates denied by the effective organization policy', async () => {
const isAutoFreeCandidateAllowed = jest.fn(
Expand Down
Loading