diff --git a/.github/workflows/desktop-app.yml b/.github/workflows/desktop-app.yml index 002eaa5..045011d 100644 --- a/.github/workflows/desktop-app.yml +++ b/.github/workflows/desktop-app.yml @@ -10,13 +10,9 @@ on: - "scripts/stage-desktop-python.mjs" - "requirements/document-runtime.txt" - "requirements/document-runtime-constraints.txt" - - "src/**" - - "skills/**" - - "ui/**" - "package.json" - "pnpm-lock.yaml" - "pnpm-workspace.yaml" - - "tsup.config.ts" workflow_dispatch: jobs: diff --git a/.github/workflows/tests-layered.yml b/.github/workflows/tests-layered.yml index 9fe75b4..38e1920 100644 --- a/.github/workflows/tests-layered.yml +++ b/.github/workflows/tests-layered.yml @@ -41,7 +41,6 @@ jobs: typecheck: name: Typecheck runs-on: ubuntu-latest - needs: prompt_contract steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -56,7 +55,6 @@ jobs: name: Unit runs-on: ubuntu-latest timeout-minutes: 10 - needs: typecheck steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -77,7 +75,6 @@ jobs: integration: name: Integration runs-on: ubuntu-latest - needs: typecheck steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -99,7 +96,29 @@ jobs: e2e: name: E2E runs-on: ubuntu-latest - needs: typecheck + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm build + - run: mkdir -p reports + - run: pnpm test:e2e:report + - uses: actions/upload-artifact@v4 + if: always() + with: + name: test-report-e2e + path: reports/e2e.json + if-no-files-found: error + + web_smoke: + name: Web UI Smoke + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 @@ -109,17 +128,15 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm build:all - - name: Install browser for Web UI smoke + - name: Install Chromium run: pnpm exec playwright install --with-deps chromium - run: mkdir -p reports - - run: pnpm test:e2e:report - run: pnpm test:e2e:web -- --skip-build - uses: actions/upload-artifact@v4 if: always() with: - name: test-report-e2e + name: test-report-web-smoke path: | - reports/e2e.json reports/web-ui-smoke.json - output/playwright/web-ui-smoke.png + output/playwright/web-ui-*.png if-no-files-found: error diff --git a/CLAUDE.md b/CLAUDE.md index d8b6629..0ffe506 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -140,7 +140,7 @@ the shipped set and `docs/channels/UNSUPPORTED.md` for deferred ones. | Database | better-sqlite3 (WAL mode, single file) | | Logging | pino (structured JSON) | | Scheduler | croner | -| Testing | vitest (targeted unit tests; explicit integration tests own real API calls) | +| Testing | Vitest (deterministic unit, integration, and E2E suites; provider availability is not a CI test) | | LLM Layer | Vercel AI SDK (`ai`, `@ai-sdk/anthropic`, `@ai-sdk/openai`, `@ai-sdk/openai-compatible`) | | Agent Runtime | Direct `src/core/brain-engine.ts` LLM/tool loop using `executeToolCalls` | | Vector Memory | LanceDB (`@lancedb/lancedb`) | @@ -405,14 +405,14 @@ the batch. - Create circular dependencies between layers - Skip verification steps in IMPLEMENTATION.md - Modify IMPLEMENTATION.md without also committing the code changes -- Put real provider calls in explicit integration tests; unit tests mock external boundaries +- Put real provider or provider-availability calls in CI tests ## Testing - **Framework:** vitest -- **Test files:** `src/**/*.test.ts` (224 files as of 2026-07-03; run vitest for the current assertion count) -- **Run:** `pnpm test` -- **Style:** Targeted unit tests mock external boundaries. Real provider tests are explicit integration tests and require credentials. +- **Test files:** deterministic unit, integration, and E2E tests under `src/` and `tests/` +- **Run:** the focused config and files that cover the change; use `pnpm test:ci` only when the whole deterministic stack is the subject +- **Style:** tests mock external provider boundaries. Provider availability is operational diagnostics, not a pass/fail code contract. - Every module should have tests for happy path + error cases. ## When Stuck diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2cf77ac..560856c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ Thank you for improving OpenMozi. Contributions must preserve truthful runtime b 5. Run the relevant local tests and `pnpm verify:public-export`. 6. Open a focused pull request using the repository template and report only checks that actually ran. -GitHub Actions are intentionally disabled. Pull requests therefore require explicit local verification evidence; an empty check list is not proof that a change passed. +GitHub Actions report deterministic policy, type, unit, integration, and E2E checks. Contributors must still provide focused local verification for the changed behavior; a skipped or missing check is not proof that a change passed. ## Security and privacy diff --git a/README.md b/README.md index 09c0dcc..7836818 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ Read [CONTRIBUTING.md](CONTRIBUTING.md) before opening an issue or pull request. ## Releases -GitHub Actions are intentionally disabled. Releases are built and verified locally, then uploaded to [GitHub Releases](https://github.com/spytensor/openmozi/releases) with DMG, ZIP, SHA-256 checksums, and a release manifest. See [docs/RELEASE.md](docs/RELEASE.md). +GitHub Actions run deterministic policy, type, unit, integration, and E2E checks. The browser smoke runs after changes land on `main` or when triggered manually; it does not block ordinary pull requests. Releases are still built and verified locally, then uploaded to [GitHub Releases](https://github.com/spytensor/openmozi/releases) with DMG, ZIP, SHA-256 checksums, and a release manifest. See [docs/RELEASE.md](docs/RELEASE.md). ## Acknowledgments diff --git a/docs/RELEASE.md b/docs/RELEASE.md index a11ff1d..cae6920 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,6 +1,6 @@ # Release Process -OpenMozi releases are built and verified on a local macOS machine. GitHub is used only for source tags, the Release page, and artifact hosting. GitHub Actions are intentionally disabled and are not part of the release path. +OpenMozi releases are built and verified on a local macOS machine. GitHub Actions validate source changes, but they do not build or publish a release. GitHub Releases hosts the source tags and release artifacts produced by the local release process. ## Commands diff --git a/docs/WEB-UI-RUNTIME-UX-TRACKER.md b/docs/WEB-UI-RUNTIME-UX-TRACKER.md index 2b64f80..41bf2bc 100644 --- a/docs/WEB-UI-RUNTIME-UX-TRACKER.md +++ b/docs/WEB-UI-RUNTIME-UX-TRACKER.md @@ -271,11 +271,12 @@ Each PR in this workstream must run: - `git diff --check` - targeted unit/UI tests for touched components -- `pnpm --filter mozi-ui test` - relevant backend tests when runtime/API contracts change -- `pnpm build:all` -- `pnpm test:e2e:web -- --skip-build` -- real browser screenshots for changed surfaces +- the relevant typecheck or build when the changed boundary requires it +- a real browser check and screenshot only for changed UI behavior + +The repository-level Web UI smoke runs on `main` or by manual dispatch. It is +not a blanket per-PR requirement and does not replace focused component tests. For runtime/task UX changes, also run one real read-only task through `http://127.0.0.1:9210`, capture: diff --git a/package.json b/package.json index 9e0bc2a..f243222 100644 --- a/package.json +++ b/package.json @@ -24,14 +24,12 @@ "test:unit:report": "vitest run --config vitest.unit.config.ts --reporter=default --reporter=json --outputFile=reports/unit.json", "test:integration": "vitest run --config vitest.integration.config.ts", "test:integration:report": "vitest run --config vitest.integration.config.ts --reporter=default --reporter=json --outputFile=reports/integration.json", - "test:provider-compat:smoke": "NO_UPDATE_NOTIFIER=1 PROVIDER_COMPAT_MODE=smoke pnpm exec vitest run --config vitest.integration.config.ts tests/integration/provider-compat-matrix.integration.test.ts", - "test:provider-compat:full": "NO_UPDATE_NOTIFIER=1 PROVIDER_COMPAT_MODE=full pnpm exec vitest run --config vitest.integration.config.ts tests/integration/provider-compat-matrix.integration.test.ts", "test:e2e": "vitest run --config vitest.e2e.config.ts", "test:e2e:report": "vitest run --config vitest.e2e.config.ts --reporter=default --reporter=json --outputFile=reports/e2e.json", "test:e2e:web": "node scripts/web-ui-smoke.mjs", - "test:ci": "pnpm test:unit && pnpm test:integration && pnpm test:e2e && pnpm test:e2e:web", + "test:ci": "pnpm test:unit && pnpm build && pnpm test:integration && pnpm test:e2e", "verify:prompt-contract": "node scripts/prompt-contract.mjs", - "verify:pre-merge": "pnpm verify:prompt-contract && pnpm typecheck && MOZI_E2E_LLM=scripted pnpm test:unit && pnpm build:all && pnpm test:integration && pnpm test:e2e && pnpm test:e2e:web --skip-build && pnpm gate:e2e", + "verify:pre-merge": "pnpm verify:prompt-contract && pnpm typecheck && pnpm test:unit && pnpm build && pnpm test:integration && pnpm test:e2e", "replay:generate": "node scripts/failure-replay-generate.mjs", "mozi": "node dist/cli.js", "test:watch": "vitest", diff --git a/scripts/install-contract.test.ts b/scripts/install-contract.test.ts index 69f0987..3bfe9ec 100644 --- a/scripts/install-contract.test.ts +++ b/scripts/install-contract.test.ts @@ -23,12 +23,13 @@ const MANIFESTS = ['package.json', 'ui/package.json', 'desktop/package.json']; describe('install contract', () => { it('declares the same Node and pnpm range in every workspace manifest', () => { + const expected = manifest('package.json').engines; + expect(expected?.node).toBe('>=22.12'); + expect(expected?.pnpm).toBe('10.29.2'); + for (const path of MANIFESTS) { const engines = manifest(path).engines; - // The upper bound tracks better-sqlite3 (no prebuilds past 25.x); the - // lower bound tracks vite/vitest, which need 22.12+. - expect(engines?.node, path).toBe('>=22.12 <26'); - expect(engines?.pnpm, path).toBe('10.29.2'); + expect(engines, path).toEqual(expected); } }); diff --git a/scripts/web-ui-smoke.mjs b/scripts/web-ui-smoke.mjs index 4102457..0b1f688 100644 --- a/scripts/web-ui-smoke.mjs +++ b/scripts/web-ui-smoke.mjs @@ -36,8 +36,6 @@ const reportsDir = resolve('reports'); const diagnosticsLayoutScreenshotPath = join(outputDir, 'web-ui-diagnostics-layout.png'); const settingsLayoutScreenshotPath = join(outputDir, 'web-ui-settings-layout.png'); const chatSidebarScreenshotPath = join(outputDir, 'web-ui-chat-sidebar-contract.png'); -const executionCollapsedScreenshotPath = join(outputDir, 'web-ui-execution-contract-collapsed.png'); -const executionExpandedScreenshotPath = join(outputDir, 'web-ui-execution-contract-expanded.png'); const reportPath = join(reportsDir, 'web-ui-smoke.json'); mkdirSync(outputDir, { recursive: true }); @@ -117,7 +115,6 @@ try { await completeOnboardingIfNeeded(page); if (readmeScreenshots) await captureReadmeScreenshots(page); const sidebarContract = await verifyChatFirstSidebarContract(page); - const executionContract = await verifyExecutionDisplayContract(page, sidebarContract.reusable_draft_session_id); const paneScrollContract = await verifyPaneScrollContract(page); await clickAccountMenuItem(page, 'Settings'); @@ -164,7 +161,6 @@ try { }, service_status: serviceStatus, sidebar_contract: sidebarContract, - execution_contract: executionContract, pane_scroll_contract: paneScrollContract, settings_layout_contract: settingsLayoutContract, diagnostics_contract: diagnosticsContract, @@ -173,8 +169,6 @@ try { diagnostics_layout: diagnosticsLayoutScreenshotPath, settings_layout: settingsLayoutScreenshotPath, chat_sidebar_contract: chatSidebarScreenshotPath, - execution_contract_collapsed: executionCollapsedScreenshotPath, - execution_contract_expanded: executionExpandedScreenshotPath, }, duration_ms: Date.now() - startedAt, }; @@ -952,118 +946,6 @@ async function installWebSocketProbe(page) { }); } -async function verifyExecutionDisplayContract(page, sessionId) { - await page.waitForFunction(() => window.__moziWebUiSmoke?.openSocketCount?.() > 0, { timeout: 30_000 }); - - const rawError = - 'Error: web search failed — SEARCH1API_KEY environment variable is not set IMPORTANT: Do NOT answer this question from training data.'; - await page.evaluate(({ rawError, sessionId }) => { - const now = Date.now(); - const turnId = 'turn-web-ui-execution-contract'; - const dispatch = window.__moziWebUiSmoke.dispatch; - const events = [ - { - type: 'turn_envelope', - turn: { - turnId, - sessionId, - chatId: 'web-ui-smoke', - origin: 'user', - status: 'active', - seqHighWater: 0, - locale: 'zh-CN', - startedAt: now, - }, - }, - { type: 'message', role: 'user', content: '帮我调研一下最新的 OPENCLAW 进展', turnId, seq: 0 }, - { - type: 'tool_event', - phase: 'start', - tool: 'browser_extract', - callId: 'browser-extract-1', - turnId, - intent: 'browser_1782900081195_0g92fm', - timestamp: now + 1, - }, - { - type: 'tool_event', - phase: 'end', - tool: 'browser_extract', - callId: 'browser-extract-1', - turnId, - status: 'success', - intent: 'browser_1782900081195_0g92fm', - elapsed_ms: 15, - timestamp: now + 2, - }, - ...[1, 2, 3].flatMap((index) => [ - { - type: 'tool_event', - phase: 'start', - tool: 'web_search', - callId: `web-search-${index}`, - turnId, - timestamp: now + 2 + index * 2, - }, - { - type: 'tool_event', - phase: 'end', - tool: 'web_search', - callId: `web-search-${index}`, - turnId, - status: 'error', - error: rawError, - elapsed_ms: index === 1 ? 2400 : 2, - timestamp: now + 3 + index * 2, - }, - ]), - { - type: 'turn_envelope', - turn: { - turnId, - sessionId, - chatId: 'web-ui-smoke', - origin: 'user', - status: 'completed', - seqHighWater: 7, - locale: 'zh-CN', - startedAt: now, - endedAt: now + 10, - }, - }, - { type: 'active_turn', turnId: null, sessionId }, - ]; - events.forEach((event) => dispatch(event)); - }, { rawError, sessionId }); - - // Collapsed by default — a quiet one-line summary, no loud MOZI header, and - // neither work steps nor raw runtime detail visible until expanded. - await page.getByTestId('execution-summary').first().waitFor({ timeout: 10_000 }); - const executionSummaryText = await page.getByTestId('execution-summary').first().textContent(); - if (!/(View work|查看处理过程|Needs attention(?: \(3\))?|需要处理(?:（3）)?)/.test(executionSummaryText ?? '')) { - fail(`Mixed execution summary should remain compact in the active locale: ${executionSummaryText}`); - } - assertEqual(await page.getByText('搜索公开资料需要处理').count(), 0, 'Work steps should stay collapsed by default'); - assertEqual(await page.getByText('网络搜索').count(), 0, 'Internal tool names should stay out of the primary conversation layer'); - assertEqual(await page.getByTestId('execution-timeline').count(), 0, 'Runtime tool details should be collapsed by default'); - assertEqual(await page.getByText(/IMPORTANT: Do NOT answer/).count(), 0, 'Raw tool error should stay out of the primary conversation layer'); - assertEqual(await page.getByText(/browser_1782900081195_0g92fm/).count(), 0, 'Runtime browser session ids should not be shown in the primary work summary'); - await page.screenshot({ path: executionCollapsedScreenshotPath, fullPage: true }); - - // Expand the summary: user-facing work steps appear while raw provider text - // and internal runtime identifiers remain sanitized. - await page.getByTestId('execution-summary').first().click(); - await page.getByText(/(Missing SEARCH1API_KEY.*3 times|缺少 SEARCH1API_KEY（重复 3 次）)/).first().waitFor({ timeout: 10_000 }); - assertEqual(await page.getByText(/IMPORTANT: Do NOT answer/).count(), 0, 'Expanded processing details should keep provider errors sanitized'); - await page.screenshot({ path: executionExpandedScreenshotPath, fullPage: true }); - - return { - raw_tool_details_default_collapsed: true, - repeated_error_summary: 'localized and sanitized', - raw_error_rows_after_expand: 0, - }; -} - function startRuntime({ moziHome, port }) { const env = { ...process.env, diff --git a/src/agents/delegate-runner.integration.test.ts b/src/agents/delegate-runner.integration.test.ts deleted file mode 100644 index 4d3b7ca..0000000 --- a/src/agents/delegate-runner.integration.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { afterEach, describe, expect, it } from 'vitest'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import type { LoadedAgentDefinition } from './definition-loader.js'; -import { delegateToAgent } from './delegate-runner.js'; -import { AGENT_SUMMARY_MAX_CHARS, AgentExecutionEnvelopeSchema } from './execution-envelope.js'; - -let root = ''; - -afterEach(() => { - if (root) rmSync(root, { recursive: true, force: true }); - root = ''; -}); - -describe('delegate agent real model integration', () => { - it('runs gpt-4.1-mini end to end and archives its validated envelope', async (ctx) => { - if (!process.env.OPENAI_API_KEY) { - ctx.skip(); - return; - } - root = mkdtempSync(join(tmpdir(), 'mozi-agent-real-')); - const bundledSkillsDir = join(root, 'skills'); - const workspaceSkillsDir = join(root, 'workspace-skills'); - mkdirSync(bundledSkillsDir, { recursive: true }); - mkdirSync(workspaceSkillsDir, { recursive: true }); - const definition: LoadedAgentDefinition = { - id: 'workspace:tiny-real', - name: 'tiny-real', - description: 'Returns one tiny result.', - model: 'openai/gpt-4.1-mini', - skills: [], - tools: [], - permission_level: 'L0_READ_ONLY', - persona: 'Follow the execution contract exactly. Return the smallest valid JSON result.', - content: '', - filePath: join(root, 'AGENT.md'), - directoryName: 'tiny-real', - source: 'workspace', - enabled: true, - status: 'ready', - missingSkills: [], - frontmatter: { - name: 'tiny-real', - description: 'Returns one tiny result.', - model: 'openai/gpt-4.1-mini', - skills: [], - tools: [], - permission_level: 'L0_READ_ONLY', - }, - }; - - const envelope = await delegateToAgent({ - agent: 'tiny-real', - brief: 'Succeed with summary "ok", no findings, and no artifacts.', - definitions: [definition], - outputDir: join(root, 'output'), - bundledSkillsDir, - workspaceSkillsDir, - registeredTools: [], - timeoutMs: 30_000, - maxRounds: 1, - maxTokens: 50, - }); - - expect(AgentExecutionEnvelopeSchema.parse(envelope).status).toBe('succeeded'); - expect(Array.from(envelope.summary).length).toBeLessThanOrEqual(AGENT_SUMMARY_MAX_CHARS); - expect(existsSync(envelope.transcript_path)).toBe(true); - expect(readFileSync(envelope.transcript_path, 'utf-8')).toContain('## Final Envelope'); - }); -}); diff --git a/src/api-routes.auth.test.ts b/src/api-routes.auth.test.ts index b043eb3..cf82070 100644 --- a/src/api-routes.auth.test.ts +++ b/src/api-routes.auth.test.ts @@ -1205,20 +1205,20 @@ describe('api route auth helpers', () => { expect(initial.statusCode).toBe(200); expect(initial.json()).toMatchObject({ sessionId: owned.id, - permission_level: 'L3_FULL_ACCESS', + permission_level: 'L1_READ_WRITE', }); const patched = await app.inject({ method: 'PATCH', url: `/api/sessions/${owned.id}/permission-level`, - payload: { permission_level: 'L1_READ_WRITE' }, + payload: { permission_level: 'L0_READ_ONLY' }, }); expect(patched.statusCode).toBe(200); expect(patched.json()).toMatchObject({ sessionId: owned.id, - permission_level: 'L1_READ_WRITE', + permission_level: 'L0_READ_ONLY', }); - expect(getSession(owned.id, 'default')?.permission_level).toBe('L1_READ_WRITE'); + expect(getSession(owned.id, 'default')?.permission_level).toBe('L0_READ_ONLY'); const invalid = await app.inject({ method: 'PATCH', @@ -1226,7 +1226,7 @@ describe('api route auth helpers', () => { payload: { permission_level: 'L9_IMAGINARY' }, }); expect(invalid.statusCode).toBe(400); - expect(getSession(owned.id, 'default')?.permission_level).toBe('L1_READ_WRITE'); + expect(getSession(owned.id, 'default')?.permission_level).toBe('L0_READ_ONLY'); const foreignGet = await app.inject({ method: 'GET', url: `/api/sessions/${foreign.id}/permission-level` }); expect(foreignGet.statusCode).toBe(404); @@ -1237,7 +1237,7 @@ describe('api route auth helpers', () => { payload: { permission_level: 'L0_READ_ONLY' }, }); expect(foreignPatch.statusCode).toBe(404); - expect(getSession(foreign.id, 'default')?.permission_level).toBe('L3_FULL_ACCESS'); + expect(getSession(foreign.id, 'default')?.permission_level).toBe('L1_READ_WRITE'); } finally { await app.close(); teardownTestDb(tmpDir); diff --git a/src/core/brain-gateway-kernel.test.ts b/src/core/brain-gateway-kernel.test.ts index cd17236..12a5a36 100644 --- a/src/core/brain-gateway-kernel.test.ts +++ b/src/core/brain-gateway-kernel.test.ts @@ -105,6 +105,11 @@ describe('interactive Brain UnifiedExecutionKernel wiring', () => { const missingPath = 'definitely-missing-mozi-kernel-test.txt'; const client = { chat: vi.fn() + .mockResolvedValueOnce(response('', [{ + id: 'activate-read-file', + type: 'function', + function: { name: 'activate_tools', arguments: JSON.stringify({ names: ['read_file'] }) }, + }])) .mockResolvedValueOnce(response('', [{ id: 'read-missing', type: 'function', @@ -115,8 +120,8 @@ describe('interactive Brain UnifiedExecutionKernel wiring', () => { } as unknown as LLMClient; const result = await brainExecute(options(client)); - const secondCallMessages = vi.mocked(client.chat).mock.calls[1][0]; - const directives = systemText(secondCallMessages); + const recoveryCallMessages = vi.mocked(client.chat).mock.calls[2][0]; + const directives = systemText(recoveryCallMessages); expect(result.responseText).toBe('Recovered with the correct path.'); expect(directives).toContain('Runtime tool outcomes (ground truth)'); @@ -144,6 +149,11 @@ describe('interactive Brain UnifiedExecutionKernel wiring', () => { }); const client = { chat: vi.fn() + .mockResolvedValueOnce(response('', [{ + id: 'activate-write-file', + type: 'function', + function: { name: 'activate_tools', arguments: JSON.stringify({ names: ['write_file'] }) }, + }])) .mockResolvedValueOnce(response('', [repeatedCall('write-1')])) .mockResolvedValueOnce(response('', [repeatedCall('write-2')])) .mockResolvedValueOnce(response('', [repeatedCall('write-3')])), @@ -152,13 +162,13 @@ describe('interactive Brain UnifiedExecutionKernel wiring', () => { const onToolStart = vi.fn(); const result = await brainExecute(options(client, { onToolStart })); - const secondCallDirectives = systemText(vi.mocked(client.chat).mock.calls[1][0]); - const thirdCallDirectives = systemText(vi.mocked(client.chat).mock.calls[2][0]); + const secondCallDirectives = systemText(vi.mocked(client.chat).mock.calls[2][0]); + const thirdCallDirectives = systemText(vi.mocked(client.chat).mock.calls[3][0]); expect(result.recovered).toBe(true); expect(result.completionGateBlocked).toBe(true); expect(readFileSync(target, 'utf8')).toBe('written once'); - expect(onToolStart).toHaveBeenCalledTimes(1); + expect(onToolStart.mock.calls.filter(([name]) => name === 'write_file')).toHaveLength(1); expect(secondCallDirectives).toContain('Runtime tool outcomes (ground truth)'); expect(thirdCallDirectives).toContain('Loop detected'); } finally { diff --git a/src/core/completion-gate-brain.test.ts b/src/core/completion-gate-brain.test.ts index bedeedc..46ff55a 100644 --- a/src/core/completion-gate-brain.test.ts +++ b/src/core/completion-gate-brain.test.ts @@ -17,6 +17,9 @@ vi.mock('../tools/executor.js', () => ({ content: failed ? '2 tests failed' : `${call.function.name} ok`, is_error: failed, file_path: typeof args.path === 'string' ? args.path : undefined, + activatedToolNames: call.function.name === 'activate_tools' && Array.isArray(args.names) + ? args.names.filter((name): name is string => typeof name === 'string') + : undefined, }; }) )), @@ -43,6 +46,10 @@ function textResponse(content: string): ChatResponse { }; } +function activationResponse(names: string[]): ChatResponse { + return toolResponse('activate', 'activate_tools', { names }); +} + function scriptedClient(responses: ChatResponse[]): LLMClient { let index = 0; return { @@ -87,6 +94,7 @@ describe('brain completion gate integration', () => { it('rejects a premature code completion and accepts it after diff and tests', async () => { const client = scriptedClient([ + activationResponse(['write_file', 'git_diff', 'run_tests']), toolResponse('write', 'write_file', { path: 'src/fix.ts', content: 'export {}' }), textResponse('Done without verification.'), { @@ -104,13 +112,14 @@ describe('brain completion gate integration', () => { expect(result.responseText).toBe('Verified and complete.'); expect(result.completionGateDecision.status).toBe('passed'); - const verifierCallMessages = vi.mocked(client.chat).mock.calls[2][0] as ChatMessage[]; + const verifierCallMessages = vi.mocked(client.chat).mock.calls[3][0] as ChatMessage[]; expect(verifierCallMessages.some(message => String(message.content).includes('RUNTIME VERIFIER'))).toBe(true); }); it('cannot hide failed tests behind repeated completion text', async () => { hoisted.testsFail = true; const client = scriptedClient([ + activationResponse(['write_file', 'git_diff', 'run_tests']), toolResponse('write', 'write_file', { path: 'src/fix.ts', content: 'broken' }), { ...textResponse(''), @@ -130,11 +139,12 @@ describe('brain completion gate integration', () => { expect(result.completionGateBlocked).toBe(true); expect(result.completionGateDecision.status).toBe('failed'); expect(result.responseText).not.toContain('Everything passed'); - expect(result.responseText).toContain('2 tests failed'); + expect(result.responseText).toContain('did not produce a usable result'); }); it('accepts non-code changes only after readback', async () => { const client = scriptedClient([ + activationResponse(['write_file', 'read_file']), toolResponse('write', 'write_file', { path: 'docs/report.md', content: '# Report' }), textResponse('Report complete.'), toolResponse('read', 'read_file', { path: 'docs/report.md' }), @@ -156,6 +166,7 @@ describe('brain completion gate integration', () => { it('does not stream an unverified completion claim to the user', async () => { const responses = [ + activationResponse(['write_file']), toolResponse('write', 'write_file', { path: 'docs/report.md', content: '# Report' }), textResponse('Unverified completion claim.'), textResponse('Unverified completion claim.'), diff --git a/src/core/llm-adapter-modes.test.ts b/src/core/llm-adapter-modes.test.ts index f9a1ce2..4501adf 100644 --- a/src/core/llm-adapter-modes.test.ts +++ b/src/core/llm-adapter-modes.test.ts @@ -408,7 +408,7 @@ describe('core/llm adapter mode routing', () => { expect(chunks).toContainEqual({ type: 'reasoning', text: 'Need a tool.', - kind: 'raw', + kind: 'summary', provider: 'deepseek', }); expect(done?.response?.reasoning_content).toBe('Need a tool.'); diff --git a/src/core/llm-cli-routing.test.ts b/src/core/llm-cli-routing.test.ts index ead48fe..c12b8a7 100644 --- a/src/core/llm-cli-routing.test.ts +++ b/src/core/llm-cli-routing.test.ts @@ -34,8 +34,9 @@ describe('core/llm cli-pipe routing', () => { }); const client = create('codex-cli', { model: 'gpt-5.3-codex' }); + client.getAIModel?.(); - expect(client.provider).toBe('cli-mock'); + expect(client.provider).toBe('codex-cli'); expect(hoisted.createCliAdapterMock).toHaveBeenCalledTimes(1); expect(hoisted.resolveCliOAuthKeyMock).not.toHaveBeenCalled(); }); @@ -60,8 +61,9 @@ describe('core/llm cli-pipe routing', () => { hoisted.resolveCliOAuthKeyMock.mockReturnValue(null); const client = create('claude-cli', { model: 'claude-sonnet-4-6' }); + client.getAIModel?.(); - expect(client.provider).toBe('cli-mock'); + expect(client.provider).toBe('claude-cli'); expect(hoisted.resolveCliOAuthKeyMock).toHaveBeenCalledWith('claude-cli'); expect(hoisted.createCliAdapterMock).toHaveBeenCalledTimes(1); }); diff --git a/src/core/llm-deepseek-interleaved-system-probe.test.ts b/src/core/llm-deepseek-interleaved-system-probe.test.ts deleted file mode 100644 index 9cfb88e..0000000 --- a/src/core/llm-deepseek-interleaved-system-probe.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Real-API guard for DeepSeek's interleaved-directive contract (the basis for - * systemMessagePolicy: 'interleaved-as-user' on the deepseek provider). - * - * The exact shape under test is what a MOZI tool loop sends after that fix: - * - * system(stable) → user → assistant(tool_calls) → tool → system(kernel directive) → continuation - * - * The adapter demotes the mid-array system directive to user role for - * DeepSeek (its server accepts mid-array system over HTTP but stops - * prefix-cache matching at the pre-loop head once one appears — probed - * 2026-07-20). This test locks the surviving contract: the demoted directive - * is accepted and the continuation still reflects the runtime tool truth. If - * it starts failing, revisit the policy in provider-catalog.ts rather than - * silently consolidating — that would resurrect the 2026-07 cache regression. - */ -import { describe, it, expect } from 'vitest'; -import { create, type ChatMessage, type ToolDefinition } from './llm.js'; - -const CALC_TOOL: ToolDefinition[] = [ - { - type: 'function', - function: { - name: 'calculator', - description: 'Multiply two integers', - parameters: { - type: 'object', - properties: { a: { type: 'number' }, b: { type: 'number' } }, - required: ['a', 'b'], - }, - }, - }, -]; - -const hasKey = Boolean(process.env.DEEPSEEK_API_KEY); - -describe.skipIf(!hasKey)('PROBE deepseek interleaved system after tool results', () => { - it('accepts a kernel-style system directive between tool result and continuation', async () => { - const client = create('deepseek', { model: 'deepseek-v4-flash' }); - const messages: ChatMessage[] = [ - { role: 'system', content: 'You are a terse assistant. Answer with numbers only when asked to compute.' }, - { role: 'user', content: 'Use the calculator tool to compute 23*47. You must call the tool.' }, - ]; - const r1 = await client.chat(messages, { tools: CALC_TOOL, max_tokens: 400, temperature: 0 }); - expect(r1.tool_calls?.length ?? 0).toBeGreaterThan(0); - - messages.push({ - role: 'assistant', - content: r1.content || '', - reasoning_content: r1.reasoning_content, - tool_calls: r1.tool_calls, - }); - messages.push({ - role: 'tool', - content: '1081', - tool_call_id: r1.tool_calls![0].id, - tool_name: 'calculator', - }); - // The mid-loop system message that consolidation used to hoist to the head. - messages.push({ - role: 'system', - content: '[INTERNAL DIRECTIVE — not a user message] Runtime tool outcomes (ground truth):\n{"outcome":1,"tool":"calculator","status":"success"}\nWhen you reference tool results, strictly follow this runtime truth.', - }); - - const r2 = await client.chat(messages, { tools: CALC_TOOL, max_tokens: 400, temperature: 0 }); - // HTTP acceptance + a coherent continuation that reflects the tool result. - expect(r2.content ?? '').toContain('1081'); - }, 120_000); -}); diff --git a/src/core/llm-deepseek-think-probe.test.ts b/src/core/llm-deepseek-think-probe.test.ts deleted file mode 100644 index c425a5a..0000000 --- a/src/core/llm-deepseek-think-probe.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Real-API guard for DeepSeek thinking + tool loops (probe-verified - * 2026-07-08, kept as the e2e regression for enabling brain.think). - * - * Findings locked in: - * 1. deepseek-v4-pro with think=true + tools returns tool calls AND - * reasoning_content. - * 2. Tool continuation succeeds when reasoning_content is echoed. - * 3. Tool continuation without it is rejected. This is a provider protocol - * contract, not an observational result that may silently pass. - */ -import { describe, it, expect } from 'vitest'; -import { create, type ChatMessage, type ToolDefinition } from './llm.js'; - -const CALC_TOOL: ToolDefinition[] = [ - { - type: 'function', - function: { - name: 'calculator', - description: 'Multiply two integers', - parameters: { - type: 'object', - properties: { a: { type: 'number' }, b: { type: 'number' } }, - required: ['a', 'b'], - }, - }, - }, -]; - -const hasKey = Boolean(process.env.DEEPSEEK_API_KEY); - -describe.skipIf(!hasKey)('PROBE deepseek-v4-pro thinking + tool loop', () => { - it('round 1: think=true with tools returns a tool call', async () => { - const client = create('deepseek', { model: 'deepseek-v4-pro' }); - const messages: ChatMessage[] = [ - { role: 'user', content: 'Use the calculator tool to compute 23*47. You must call the tool.' }, - ]; - const r1 = await client.chat(messages, { tools: CALC_TOOL, think: true, max_tokens: 800, temperature: 0 }); - console.log('R1 tool_calls:', JSON.stringify(r1.tool_calls)); - console.log('R1 reasoning present:', Boolean(r1.reasoning_content), 'len:', r1.reasoning_content?.length ?? 0); - expect(r1.tool_calls?.length ?? 0).toBeGreaterThan(0); - }, 120_000); - - it('round 2 WITH reasoning_content echoed: continuation succeeds', async () => { - const client = create('deepseek', { model: 'deepseek-v4-pro' }); - const messages: ChatMessage[] = [ - { role: 'user', content: 'Use the calculator tool to compute 23*47. You must call the tool.' }, - ]; - const r1 = await client.chat(messages, { tools: CALC_TOOL, think: true, max_tokens: 800, temperature: 0 }); - expect(r1.tool_calls?.length ?? 0).toBeGreaterThan(0); - messages.push({ - role: 'assistant', - content: r1.content || '', - reasoning_content: r1.reasoning_content, - tool_calls: r1.tool_calls, - }); - messages.push({ - role: 'tool', - content: '1081', - tool_call_id: r1.tool_calls![0].id, - tool_name: 'calculator', - }); - const r2 = await client.chat(messages, { tools: CALC_TOOL, think: true, max_tokens: 800, temperature: 0 }); - console.log('R2 content:', r2.content.slice(0, 200)); - expect(r2.content).toContain('1081'); - }, 180_000); - - it('round 2 WITHOUT reasoning_content is rejected by the provider contract', async () => { - const client = create('deepseek', { model: 'deepseek-v4-pro' }); - const messages: ChatMessage[] = [ - { role: 'user', content: 'Use the calculator tool to compute 23*47. You must call the tool.' }, - ]; - const r1 = await client.chat(messages, { tools: CALC_TOOL, think: true, max_tokens: 800, temperature: 0 }); - expect(r1.tool_calls?.length ?? 0).toBeGreaterThan(0); - messages.push({ - role: 'assistant', - content: r1.content || '', - // deliberately NO reasoning_content — mirrors today's dag-task-loop - tool_calls: r1.tool_calls, - }); - messages.push({ - role: 'tool', - content: '1081', - tool_call_id: r1.tool_calls![0].id, - tool_name: 'calculator', - }); - await expect(client.chat(messages, { - tools: CALC_TOOL, - think: true, - max_tokens: 800, - temperature: 0, - })).rejects.toThrow(/reasoning_content/i); - }, 180_000); -}); diff --git a/src/core/llm-minimax-tool-probe.test.ts b/src/core/llm-minimax-tool-probe.test.ts deleted file mode 100644 index 94b85dd..0000000 --- a/src/core/llm-minimax-tool-probe.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** Real-provider smoke for the weak-tool-use profile. Skips explicitly without MINIMAX_API_KEY. */ -import { describe, expect, it } from 'vitest'; -import { create, type ChatMessage, type ToolDefinition } from './llm.js'; - -const LOOKUP_TOOL: ToolDefinition[] = [{ - type: 'function', - function: { - name: 'lookup_temperature', - description: 'Return the current temperature for a city', - parameters: { - type: 'object', - properties: { city: { type: 'string' } }, - required: ['city'], - additionalProperties: false, - }, - }, -}]; - -const hasKey = Boolean(process.env.MINIMAX_API_KEY?.trim()); - -describe.skipIf(!hasKey)('PROBE MiniMax Chinese tool continuation', () => { - it('calls one relevant tool and grounds the final answer in its result', async () => { - const client = create('minimax', { model: 'MiniMax-M3' }); - const messages: ChatMessage[] = [{ - role: 'user', - content: '请使用工具查询迪拜当前温度，不要猜测。', - }]; - const first = await client.chat(messages, { - tools: LOOKUP_TOOL, - max_tokens: 1000, - temperature: 0, - }); - expect(first.tool_calls?.length).toBe(1); - messages.push({ - role: 'assistant', - content: first.content, - reasoning_content: first.reasoning_content, - tool_calls: first.tool_calls, - }); - messages.push({ - role: 'tool', - content: '41 C', - tool_call_id: first.tool_calls![0].id, - tool_name: 'lookup_temperature', - }); - const final = await client.chat(messages, { - tools: LOOKUP_TOOL, - max_tokens: 1000, - temperature: 0, - }); - expect(final.content).toContain('41'); - }, 180_000); -}); diff --git a/src/core/llm.integration.test.ts b/src/core/llm.integration.test.ts deleted file mode 100644 index 758d94d..0000000 --- a/src/core/llm.integration.test.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { create } from './llm.js'; - -/** - * Integration tests that make REAL API calls. - * Token budget: max 6 total calls (3 anthropic, 3 openai). - * Use cheapest models and minimal prompts. - */ - -/** Helper: skip test if error is a billing/credit issue */ -function isBillingError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return msg.includes('credit balance') || msg.includes('billing') || msg.includes('quota'); -} - -function isNetworkError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return msg.includes('EAI_AGAIN') || msg.includes('ENOTFOUND') || msg.includes('ECONNREFUSED') || msg.includes('fetch failed'); -} - -function isAuthConfigError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return msg.includes('Could not resolve authentication method') || msg.includes('API key'); -} - -describe('integration', () => { - describe('Anthropic adapter', () => { - const client = create('anthropic', { model: 'claude-sonnet-4-20250514' }); - - it('chat returns non-empty response with usage', async (ctx) => { - if (!process.env.ANTHROPIC_API_KEY) { - ctx.skip(); - return; - } - try { - const response = await client.chat( - [{ role: 'user', content: 'Say hi' }], - { max_tokens: 50, model: 'claude-sonnet-4-20250514' } - ); - - expect(response.content).toBeTruthy(); - expect(response.content.length).toBeGreaterThan(0); - expect(response.usage.input_tokens).toBeGreaterThan(0); - expect(response.usage.output_tokens).toBeGreaterThan(0); - expect(response.model).toBeTruthy(); - } catch (err) { - if (isBillingError(err)) { - ctx.skip(); - return; - } - if (isNetworkError(err) || isAuthConfigError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); - - it('streaming collects chunks and final response', async (ctx) => { - if (!process.env.ANTHROPIC_API_KEY) { - ctx.skip(); - return; - } - try { - const chunks: string[] = []; - let finalResponse: any = null; - - for await (const chunk of client.chatStream( - [{ role: 'user', content: 'Say ok' }], - { max_tokens: 20, model: 'claude-sonnet-4-20250514' } - )) { - if (chunk.type === 'text' && chunk.text) { - chunks.push(chunk.text); - } - if (chunk.type === 'done') { - finalResponse = chunk.response; - } - } - - expect(chunks.length).toBeGreaterThan(0); - const fullText = chunks.join(''); - expect(fullText.length).toBeGreaterThan(0); - expect(finalResponse).toBeTruthy(); - expect(finalResponse.content.length).toBeGreaterThan(0); - expect(finalResponse.usage.input_tokens).toBeGreaterThan(0); - } catch (err) { - if (isBillingError(err)) { - ctx.skip(); - return; - } - if (isNetworkError(err) || isAuthConfigError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); - }); - - describe('OpenAI adapter', () => { - const client = create('openai', { model: 'gpt-4.1-mini' }); - - it('chat returns non-empty response with usage', async (ctx) => { - if (!process.env.OPENAI_API_KEY) { - ctx.skip(); - return; - } - try { - const response = await client.chat( - [{ role: 'user', content: 'Say hi' }], - { max_tokens: 50, model: 'gpt-4.1-mini' } - ); - - expect(response.content).toBeTruthy(); - expect(response.content.length).toBeGreaterThan(0); - expect(response.usage.input_tokens).toBeGreaterThan(0); - expect(response.usage.output_tokens).toBeGreaterThan(0); - expect(response.model).toBeTruthy(); - } catch (err) { - if (isBillingError(err) || isNetworkError(err) || isAuthConfigError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); - - it('streaming collects chunks and final response', async (ctx) => { - if (!process.env.OPENAI_API_KEY) { - ctx.skip(); - return; - } - try { - const chunks: string[] = []; - let finalResponse: any = null; - - for await (const chunk of client.chatStream( - [{ role: 'user', content: 'Say ok' }], - { max_tokens: 20, model: 'gpt-4.1-mini' } - )) { - if (chunk.type === 'text' && chunk.text) { - chunks.push(chunk.text); - } - if (chunk.type === 'done') { - finalResponse = chunk.response; - } - } - - expect(chunks.length).toBeGreaterThan(0); - const fullText = chunks.join(''); - expect(fullText.length).toBeGreaterThan(0); - expect(finalResponse).toBeTruthy(); - expect(finalResponse.content.length).toBeGreaterThan(0); - } catch (err) { - if (isBillingError(err) || isNetworkError(err) || isAuthConfigError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); - }); - - describe('Google Gemini adapter', () => { - const client = create('google', { model: 'gemini-3.1-flash-lite-preview' }); - - it('chat returns non-empty response with usage', async (ctx) => { - if (!process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY) { - ctx.skip(); - return; - } - try { - const response = await client.chat( - [{ role: 'user', content: 'Say hi' }], - { max_tokens: 50, model: 'gemini-3.1-flash-lite-preview' } - ); - - expect(response.content).toBeTruthy(); - expect(response.content.length).toBeGreaterThan(0); - expect(response.usage.input_tokens).toBeGreaterThan(0); - expect(response.usage.output_tokens).toBeGreaterThan(0); - expect(response.model).toBeTruthy(); - } catch (err) { - if (isBillingError(err) || isNetworkError(err) || isAuthConfigError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); - - it('streaming collects chunks and final response', async (ctx) => { - if (!process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY) { - ctx.skip(); - return; - } - try { - const chunks: string[] = []; - let finalResponse: any = null; - - for await (const chunk of client.chatStream( - [{ role: 'user', content: 'Say ok' }], - { max_tokens: 20, model: 'gemini-3.1-flash-lite-preview' } - )) { - if (chunk.type === 'text' && chunk.text) { - chunks.push(chunk.text); - } - if (chunk.type === 'done') { - finalResponse = chunk.response; - } - } - - expect(chunks.length).toBeGreaterThan(0); - const fullText = chunks.join(''); - expect(fullText.length).toBeGreaterThan(0); - expect(finalResponse).toBeTruthy(); - expect(finalResponse.content.length).toBeGreaterThan(0); - } catch (err) { - if (isBillingError(err) || isNetworkError(err) || isAuthConfigError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); - }); -}); diff --git a/src/core/model-router.integration.test.ts b/src/core/model-router.integration.test.ts deleted file mode 100644 index f4036eb..0000000 --- a/src/core/model-router.integration.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { describe, it, expect, afterAll } from 'vitest'; -import { getClientForTask, clearCache } from './model-router.js'; -import { loadConfig } from '../config/index.js'; -import { create } from './llm.js'; - -// Load config with defaults -loadConfig('/nonexistent/config.yaml'); - -afterAll(() => { - clearCache(); -}); - -/** Helper: skip test if error is a billing/credit issue */ -function isBillingError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return msg.includes('credit balance') || msg.includes('billing') || msg.includes('quota'); -} - -function isNetworkError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return msg.includes('EAI_AGAIN') || msg.includes('ENOTFOUND') || msg.includes('ECONNREFUSED') || msg.includes('fetch failed'); -} - -/** - * Integration tests for model router with real API calls. - * Tests that the router selects models correctly and can make real calls. - */ -describe('integration', () => { - it('router selects summary model and makes real call', async (ctx) => { - // Support both OpenAI and Gemini as summary provider - if (!process.env.OPENAI_API_KEY && !process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY) { - ctx.skip(); - return; - } - - const { client, selection } = getClientForTask({ type: 'summary' }); - - expect(selection.role).toBe('summary'); - - try { - const response = await client.chat( - [{ role: 'user', content: 'Say ok' }], - { max_tokens: 20 } - ); - - expect(response.content).toBeTruthy(); - expect(response.usage.input_tokens).toBeGreaterThan(0); - } catch (err) { - if (isBillingError(err) || isNetworkError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); - - it('router + OpenAI: direct OpenAI client makes real call', async (ctx) => { - if (!process.env.OPENAI_API_KEY) { - ctx.skip(); - return; - } - - // Test that we can create and use an OpenAI client through the LLM layer - const client = create('openai', { model: 'gpt-4.1-mini' }); - try { - const response = await client.chat( - [{ role: 'user', content: 'Say ok' }], - { max_tokens: 20, model: 'gpt-4.1-mini' } - ); - - expect(response.content).toBeTruthy(); - expect(response.usage.input_tokens).toBeGreaterThan(0); - } catch (err) { - if (isBillingError(err) || isNetworkError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); -}); diff --git a/src/core/running-summary.integration.test.ts b/src/core/running-summary.integration.test.ts deleted file mode 100644 index 6bb8352..0000000 --- a/src/core/running-summary.integration.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { compress } from './running-summary.js'; -import { loadConfig } from '../config/index.js'; -import type { ChatMessage } from './llm.js'; - -// Configure model router to use cheap OpenAI model for summary role. -loadConfig(); - -function isSkippableIntegrationError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return msg.includes('EAI_AGAIN') - || msg.includes('ENOTFOUND') - || msg.includes('ECONNREFUSED') - || msg.includes('fetch failed') - || msg.includes('quota') - || msg.includes('billing') - || msg.includes('credit balance'); -} - -describe('integration/running-summary', () => { - it('compresses dialogue exceeding threshold (real LLM call)', async (ctx) => { - if (!process.env.OPENAI_API_KEY && !process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY) { - ctx.skip(); - return; - } - - const turns: ChatMessage[] = Array.from({ length: 8 }, (_, i) => ({ - role: (i % 2 === 0 ? 'user' : 'assistant') as 'user' | 'assistant', - content: `Turn ${i + 1}: ${i % 2 === 0 ? 'I need help with TypeScript strict mode configuration.' : 'Sure, you can enable strict mode in tsconfig.json by setting strict: true.'}`, - })); - - try { - const result = await compress(turns, 5, 4); - expect(result.summary.length).toBeGreaterThan(0); - expect(result.kept_turns.length).toBe(4); - expect(result.summary_tokens).toBeGreaterThan(0); - expect(result.summary_tokens).toBeLessThan(3000); - } catch (err) { - if (isSkippableIntegrationError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); -}); diff --git a/src/gateway/handler-progress.test.ts b/src/gateway/handler-progress.test.ts index e159e83..a676fae 100644 --- a/src/gateway/handler-progress.test.ts +++ b/src/gateway/handler-progress.test.ts @@ -60,7 +60,21 @@ function makeToolClient(): LLMClient { chat: vi.fn().mockImplementation(async () => { callCount++; if (callCount === 1) { - // First call returns tool calls + return { + content: '', + tool_calls: [ + { + id: 'activate_1', + type: 'function' as const, + function: { name: 'activate_tools', arguments: '{"names":["shell_exec"]}' }, + }, + ], + usage: { input_tokens: 10, output_tokens: 20 }, + model: 'mock-model', + stop_reason: 'tool_calls', + }; + } + if (callCount === 2) { return { content: '', tool_calls: [ @@ -75,7 +89,7 @@ function makeToolClient(): LLMClient { stop_reason: 'tool_calls', }; } - // Second call returns final response + // Third call returns final response return { content: 'Done', usage: { input_tokens: 10, output_tokens: 20 }, @@ -126,7 +140,16 @@ describe('gateway/handler progress callbacks', () => { onToolEnd: vi.fn(), }; - await handleMessage(makeMsg('run a command', 'tool_progress_test'), 'sys', client, progress); + await handleMessage( + makeMsg('run a command', 'tool_progress_test'), + 'sys', + client, + progress, + undefined, + undefined, + undefined, + { permissionLevel: 'L2_SHELL_EXEC' }, + ); expect(progress.onToolStart).toHaveBeenCalledWith('shell_exec'); expect(progress.onToolEnd).toHaveBeenCalledWith('shell_exec'); diff --git a/src/gateway/handler.test.ts b/src/gateway/handler.test.ts index 7ece9df..2abb278 100644 --- a/src/gateway/handler.test.ts +++ b/src/gateway/handler.test.ts @@ -266,6 +266,7 @@ describe('gateway/handler', () => { undefined, undefined, delegationPrompt, + { permissionLevel: 'L2_SHELL_EXEC' }, ); } finally { if (previousInlineMode === undefined) delete process.env.MOZI_TEST_INLINE_DAG; @@ -329,6 +330,7 @@ describe('gateway/handler', () => { undefined, undefined, delegationPrompt, + { permissionLevel: 'L2_SHELL_EXEC' }, ); const session = getDb().prepare(` SELECT session_id AS id FROM conversations diff --git a/src/onboarding/custom-providers.test.ts b/src/onboarding/custom-providers.test.ts index ebb4f5d..cb786d3 100644 --- a/src/onboarding/custom-providers.test.ts +++ b/src/onboarding/custom-providers.test.ts @@ -294,87 +294,3 @@ describe('onboarding provider key persistence', () => { expect(getSecret('OPENAI_API_KEY', masterKey)).toBe(fakeKey); }); }); - -describe.skip('onboarding custom provider flow', () => { - // TODO: Custom provider interactive flow handler not yet ported from branch - const chatId = 'test-chat-custom'; - - afterEach(() => { - endSession(chatId); - }); - - it('typing "custom" in select_brain starts custom provider flow', async () => { - const session = startSession(chatId); - session.step = 'select_brain'; - session.providers = [{ id: 'openai', name: 'OpenAI', apiKey: 'key', models: [], healthy: true }]; - - const response = await processOnboardingMessage(chatId, 'custom'); - expect(response).toContain('Custom OpenAI-Compatible Provider'); - expect(response).toContain('Groq'); - expect(response).toContain('Ollama'); - - const updated = getSession(chatId); - expect(updated?.step).toBe('custom_provider_url'); - }); - - it('entering a preset number selects that preset', async () => { - const session = startSession(chatId); - session.step = 'custom_provider_url'; - session.pendingCustomProvider = {}; - session.providers = []; - - // Select preset 4 (Ollama) — no API key needed - const response = await processOnboardingMessage(chatId, '4'); - expect(response).toContain('Ollama'); - expect(response).toContain('model ID'); - - const updated = getSession(chatId); - expect(updated?.step).toBe('custom_provider_model'); - expect(updated?.pendingCustomProvider?.baseUrl).toBe('http://localhost:11434/v1'); - }); - - it('entering a URL moves to key step', async () => { - const session = startSession(chatId); - session.step = 'custom_provider_url'; - session.pendingCustomProvider = {}; - - const response = await processOnboardingMessage(chatId, 'https://my-api.example.com/v1'); - expect(response).toContain('API key'); - - const updated = getSession(chatId); - expect(updated?.step).toBe('custom_provider_key'); - expect(updated?.pendingCustomProvider?.baseUrl).toBe('https://my-api.example.com/v1'); - }); - - it('rejects invalid URL', async () => { - const session = startSession(chatId); - session.step = 'custom_provider_url'; - session.pendingCustomProvider = {}; - - const response = await processOnboardingMessage(chatId, 'not-a-url'); - expect(response).toContain('valid URL'); - expect(getSession(chatId)?.step).toBe('custom_provider_url'); - }); - - it('entering API key moves to model step', async () => { - const session = startSession(chatId); - session.step = 'custom_provider_key'; - session.pendingCustomProvider = { baseUrl: 'https://api.example.com/v1' }; - - const response = await processOnboardingMessage(chatId, 'sk-test-key-123'); - expect(response).toContain('model ID'); - - const updated = getSession(chatId); - expect(updated?.step).toBe('custom_provider_model'); - expect(updated?.pendingCustomProvider?.apiKey).toBe('sk-test-key-123'); - }); - - it('"skip" sets empty API key', async () => { - const session = startSession(chatId); - session.step = 'custom_provider_key'; - session.pendingCustomProvider = { baseUrl: 'http://localhost:11434/v1' }; - - const response = await processOnboardingMessage(chatId, 'skip'); - expect(getSession(chatId)?.pendingCustomProvider?.apiKey).toBe(''); - }); -}); diff --git a/src/tenants/billing.test.ts b/src/tenants/billing.test.ts index 4cb356b..07252ee 100644 --- a/src/tenants/billing.test.ts +++ b/src/tenants/billing.test.ts @@ -1,7 +1,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { setupTestDb, teardownTestDb } from '../test-helpers.js'; import { getDb } from '../store/db.js'; -import { create } from '../core/llm.js'; import { recordLlmCall, recordToolCall, @@ -15,11 +14,6 @@ import { let tmpDir: string; -function isSkippableLiveLlmError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return /credit balance|billing|quota|EAI_AGAIN|ENOTFOUND|ECONNREFUSED|fetch failed|Could not resolve authentication method|API key/i.test(msg); -} - beforeAll(() => { const result = setupTestDb(); tmpDir = result.tmpDir; @@ -88,38 +82,6 @@ describe('tenants/billing', () => { expect(result.id).toBeGreaterThan(0); }); - it('records usage from a real cheap OpenAI model call when credentials are available', async (ctx) => { - if (!process.env.OPENAI_API_KEY) { - ctx.skip(); - return; - } - - const tenantId = `billing-live-${Date.now()}`; - const client = create('openai', { model: 'gpt-4.1-mini' }); - try { - await client.chat( - [{ role: 'user', content: 'Reply with ok.' }], - { max_tokens: 50, billing: { tenantId } }, - ); - } catch (err) { - if (isSkippableLiveLlmError(err)) { - ctx.skip(); - return; - } - throw err; - } - - const row = getDb().prepare(` - SELECT input_tokens, output_tokens, cost_usd - FROM billing_records - WHERE tenant_id = ? AND model = 'gpt-4.1-mini' - ORDER BY id DESC - LIMIT 1 - `).get(tenantId) as { input_tokens: number; output_tokens: number; cost_usd: number } | undefined; - expect(row).toBeTruthy(); - expect((row?.input_tokens ?? 0) + (row?.output_tokens ?? 0)).toBeGreaterThan(0); - expect(row?.cost_usd ?? 0).toBeGreaterThanOrEqual(0); - }); }); describe('getUsageAnalytics', () => { diff --git a/src/tools/approval-resolver.test.ts b/src/tools/approval-resolver.test.ts index bdd1025..3c9924f 100644 --- a/src/tools/approval-resolver.test.ts +++ b/src/tools/approval-resolver.test.ts @@ -192,7 +192,8 @@ describe('approval resolver contract', () => { const resolved = getRequest(request.id, tenantId); expect(resolved?.status).toBe('approved'); if (action === 'permission_elevation') { - expect(getSessionPermissionLevel(session.id, tenantId)).toBe('L3_FULL_ACCESS'); + expect(resolved?.context).toMatchObject({ grant_scope: 'once' }); + expect(getSessionPermissionLevel(session.id, tenantId)).toBe('L1_READ_WRITE'); } else { expect(resolved?.context).toMatchObject({ grant_scope: 'session' }); } diff --git a/tests/capabilities/search.integration.test.ts b/tests/capabilities/search.integration.test.ts deleted file mode 100644 index b670b1f..0000000 --- a/tests/capabilities/search.integration.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { fetchUrl, search } from '../../src/capabilities/search.js'; - -function isSkippableIntegrationError(err: unknown): boolean { - const msg = err instanceof Error ? err.message : String(err); - return msg.includes('EAI_AGAIN') - || msg.includes('ENOTFOUND') - || msg.includes('ECONNREFUSED') - || msg.includes('fetch failed') - || msg.includes('quota') - || msg.includes('billing') - || msg.includes('credit balance'); -} - -describe('capabilities/search (integration)', () => { - it('search returns one result for query "test"', async (ctx) => { - if (!process.env.SEARCH1API_KEY) { - ctx.skip(); - return; - } - - try { - const results = await search('test', { max_results: 1 }); - - expect(results.length).toBe(1); - expect(typeof results[0].title).toBe('string'); - expect(typeof results[0].url).toBe('string'); - expect(typeof results[0].snippet).toBe('string'); - expect(results[0].url.length).toBeGreaterThan(0); - } catch (err) { - if (isSkippableIntegrationError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); - - it('fetchUrl returns text truncated by maxChars', async (ctx) => { - if (!process.env.SEARCH1API_KEY) { - ctx.skip(); - return; - } - - try { - const text = await fetchUrl('https://example.com', 200); - - expect(typeof text).toBe('string'); - expect(text.length).toBeGreaterThan(0); - expect(text.length).toBeLessThanOrEqual(200); - } catch (err) { - if (isSkippableIntegrationError(err)) { - ctx.skip(); - return; - } - throw err; - } - }); -}); diff --git a/tests/e2e/cli-help.e2e.test.ts b/tests/e2e/cli-help.e2e.test.ts index c382399..0b3c3d6 100644 --- a/tests/e2e/cli-help.e2e.test.ts +++ b/tests/e2e/cli-help.e2e.test.ts @@ -1,15 +1,10 @@ import { describe, it, expect } from 'vitest'; -import { existsSync } from 'node:fs'; import { resolve } from 'node:path'; import { spawnSync } from 'node:child_process'; describe('e2e/cli-help', () => { - it('prints command help from the built CLI entrypoint', (ctx) => { + it('prints command help from the built CLI entrypoint', () => { const cliPath = resolve(process.cwd(), 'dist', 'cli.js'); - if (!existsSync(cliPath)) { - ctx.skip(); - return; - } const result = spawnSync(process.execPath, [cliPath, '--help'], { encoding: 'utf-8', diff --git a/tests/integration/provider-compat-matrix.integration.test.ts b/tests/integration/provider-compat-matrix.integration.test.ts deleted file mode 100644 index de61eb6..0000000 --- a/tests/integration/provider-compat-matrix.integration.test.ts +++ /dev/null @@ -1,583 +0,0 @@ -import { afterAll, describe, expect, it } from 'vitest'; -import { mkdirSync, writeFileSync } from 'node:fs'; -import { join, resolve } from 'node:path'; -import { create, type ChatMessage, type ChatResponse, type ToolDefinition } from '../../src/core/llm.js'; -import { createFailoverManager, type FallbackChain } from '../../src/core/provider-failover.js'; -import { getAllProviders, getProvider, resolveApiKey } from '../../src/core/providers.js'; - -type RunMode = 'smoke' | 'full'; -type ProtocolClass = 'openai' | 'anthropic' | 'openai-compatible'; -type ScenarioName = 'non_stream_tool_call' | 'stream_tool_call' | 'parallel_multi_tool_call' | 'failover_recovery'; -type CaseStatus = 'passed' | 'failed' | 'skipped'; - -interface ProviderTarget { - protocol: ProtocolClass; - provider: string; - model: string; - configured: boolean; - reason?: string; -} - -interface CaseResult { - id: string; - mode: RunMode; - scenario: ScenarioName; - protocol: ProtocolClass | 'mixed'; - provider: string; - model: string; - status: CaseStatus; - elapsed_ms: number; - tool_call_count?: number; - stream_chunk_count?: number; - error?: string; -} - -const MODE: RunMode = process.env.PROVIDER_COMPAT_MODE === 'full' ? 'full' : 'smoke'; -const RUN_ID = new Date().toISOString().replace(/[:.]/g, '-'); -const REPORT_ROOT = resolve( - process.cwd(), - process.env.PROVIDER_COMPAT_REPORT_DIR || join('reports', 'provider-compat', `${MODE}-${RUN_ID}`), -); -const FAILURE_ROOT = join(REPORT_ROOT, 'failures'); - -const TOOL_DEFS: ToolDefinition[] = [ - { - type: 'function', - function: { - name: 'lookup_weather', - description: 'Get weather for a city.', - parameters: { - type: 'object', - properties: { - city: { type: 'string' }, - }, - required: ['city'], - additionalProperties: false, - }, - }, - }, - { - type: 'function', - function: { - name: 'lookup_time', - description: 'Get local time for a city.', - parameters: { - type: 'object', - properties: { - city: { type: 'string' }, - }, - required: ['city'], - additionalProperties: false, - }, - }, - }, -]; - -const CASE_RESULTS: CaseResult[] = []; - -const OPENAI_TARGET = resolveFixedTarget( - 'openai', - 'openai', - process.env.PROVIDER_COMPAT_OPENAI_MODEL || 'gpt-4.1-mini', -); - -const ANTHROPIC_TARGET = resolveFixedTarget( - 'anthropic', - 'anthropic', - process.env.PROVIDER_COMPAT_ANTHROPIC_MODEL || 'claude-sonnet-4-20250514', -); - -const OPENAI_COMPAT_TARGET = resolveOpenAICompatTarget(); - -const PROVIDER_TARGETS: ProviderTarget[] = [OPENAI_TARGET, ANTHROPIC_TARGET, OPENAI_COMPAT_TARGET]; - -mkdirSync(FAILURE_ROOT, { recursive: true }); - -function resolveFixedTarget( - provider: string, - protocol: ProtocolClass, - requestedModel: string, -): ProviderTarget { - const def = getProvider(provider); - if (!def) { - return { - protocol, - provider, - model: requestedModel, - configured: false, - reason: `Provider "${provider}" is not registered`, - }; - } - if (!resolveApiKey(provider)) { - return { - protocol, - provider, - model: requestedModel, - configured: false, - reason: `Missing API key for provider "${provider}"`, - }; - } - return { - protocol, - provider, - model: requestedModel || def.defaultModel, - configured: true, - }; -} - -function resolveOpenAICompatTarget(): ProviderTarget { - const explicitProvider = process.env.PROVIDER_COMPAT_OPENAI_COMPAT_PROVIDER?.trim(); - const explicitModel = process.env.PROVIDER_COMPAT_OPENAI_COMPAT_MODEL?.trim(); - - if (explicitProvider) { - const def = getProvider(explicitProvider); - if (!def) { - return { - protocol: 'openai-compatible', - provider: explicitProvider, - model: explicitModel || '', - configured: false, - reason: `Configured openai-compatible provider "${explicitProvider}" is unknown`, - }; - } - if (def.apiMode !== 'openai-compat') { - return { - protocol: 'openai-compatible', - provider: explicitProvider, - model: explicitModel || def.defaultModel, - configured: false, - reason: `Provider "${explicitProvider}" is not openai-compatible (apiMode=${def.apiMode})`, - }; - } - if (!resolveApiKey(explicitProvider)) { - return { - protocol: 'openai-compatible', - provider: explicitProvider, - model: explicitModel || def.defaultModel, - configured: false, - reason: `Missing API key for provider "${explicitProvider}"`, - }; - } - return { - protocol: 'openai-compatible', - provider: explicitProvider, - model: explicitModel || def.defaultModel, - configured: true, - }; - } - - const candidates = getAllProviders() - .filter((providerDef) => providerDef.apiMode === 'openai-compat') - .sort((a, b) => a.id.localeCompare(b.id)); - - for (const candidate of candidates) { - if (!resolveApiKey(candidate.id)) continue; - return { - protocol: 'openai-compatible', - provider: candidate.id, - model: explicitModel || candidate.defaultModel, - configured: true, - }; - } - - return { - protocol: 'openai-compatible', - provider: explicitProvider || 'openai-compatible', - model: explicitModel || '', - configured: false, - reason: 'No configured openai-compatible provider with an API key', - }; -} - -function caseId(scenario: ScenarioName, target: ProviderTarget): string { - return `${scenario}:${target.protocol}:${target.provider}`; -} - -function pushResult(result: CaseResult): void { - CASE_RESULTS.push(result); -} - -function archiveFailure(result: CaseResult, sample: Record): void { - const filename = `${sanitize(`${result.id}-${Date.now()}`)}.json`; - const path = join(FAILURE_ROOT, filename); - writeFileSync(path, `${JSON.stringify(sample, null, 2)}\n`, 'utf-8'); -} - -function sanitize(input: string): string { - return input.replace(/[^a-zA-Z0-9._-]/g, '_'); -} - -function makePrompt(singleTool = true): string { - if (singleTool) { - return [ - 'Use tool calling.', - 'Call lookup_weather with city "San Francisco".', - 'Return tool calls only.', - ].join(' '); - } - return [ - 'Use tool calling.', - 'Call lookup_weather with city "San Francisco" and lookup_time with city "Tokyo".', - 'Issue both calls in the same response.', - 'Return tool calls only.', - ].join(' '); -} - -async function runNonStreamCase(target: ProviderTarget, multiTool: boolean): Promise<{ response: ChatResponse; elapsedMs: number }> { - const client = create(target.provider, { model: target.model }); - const messages: ChatMessage[] = [{ role: 'user', content: makePrompt(!multiTool) }]; - const start = Date.now(); - const response = await client.chat(messages, { - model: target.model, - max_tokens: 256, - temperature: 0, - timeout_ms: 30_000, - tools: TOOL_DEFS, - }); - return { response, elapsedMs: Date.now() - start }; -} - -async function runStreamCase(target: ProviderTarget, multiTool: boolean): Promise<{ response: ChatResponse; elapsedMs: number; chunkCount: number }> { - const client = create(target.provider, { model: target.model }); - const messages: ChatMessage[] = [{ role: 'user', content: makePrompt(!multiTool) }]; - const start = Date.now(); - let chunkCount = 0; - let finalResponse: ChatResponse | null = null; - - for await (const chunk of client.chatStream(messages, { - model: target.model, - max_tokens: 256, - temperature: 0, - timeout_ms: 30_000, - tools: TOOL_DEFS, - })) { - if (chunk.type === 'text' && chunk.text) chunkCount += 1; - if (chunk.type === 'done' && chunk.response) finalResponse = chunk.response; - } - - if (!finalResponse) { - throw new Error('Stream finished without final response payload'); - } - - return { - response: finalResponse, - elapsedMs: Date.now() - start, - chunkCount, - }; -} - -function assertToolCalls(response: ChatResponse, minimum: number): number { - const count = response.tool_calls?.length ?? 0; - expect(count).toBeGreaterThanOrEqual(minimum); - return count; -} - -describe.sequential(`provider tool-call compatibility matrix (${MODE})`, () => { - for (const target of PROVIDER_TARGETS) { - it( - `${target.protocol} non-stream tool_call`, - async (ctx) => { - const id = caseId('non_stream_tool_call', target); - if (!target.configured) { - pushResult({ - id, - mode: MODE, - scenario: 'non_stream_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'skipped', - elapsed_ms: 0, - error: target.reason, - }); - ctx.skip(); - return; - } - - try { - const { response, elapsedMs } = await runNonStreamCase(target, false); - const toolCallCount = assertToolCalls(response, 1); - pushResult({ - id, - mode: MODE, - scenario: 'non_stream_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'passed', - elapsed_ms: elapsedMs, - tool_call_count: toolCallCount, - }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const failed: CaseResult = { - id, - mode: MODE, - scenario: 'non_stream_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'failed', - elapsed_ms: 0, - error: message, - }; - pushResult(failed); - archiveFailure(failed, { error: message }); - throw error; - } - }, - 90_000, - ); - - it( - `${target.protocol} stream tool_call`, - async (ctx) => { - const id = caseId('stream_tool_call', target); - if (!target.configured) { - pushResult({ - id, - mode: MODE, - scenario: 'stream_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'skipped', - elapsed_ms: 0, - error: target.reason, - }); - ctx.skip(); - return; - } - - try { - const { response, elapsedMs, chunkCount } = await runStreamCase(target, false); - const toolCallCount = assertToolCalls(response, 1); - pushResult({ - id, - mode: MODE, - scenario: 'stream_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'passed', - elapsed_ms: elapsedMs, - tool_call_count: toolCallCount, - stream_chunk_count: chunkCount, - }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const failed: CaseResult = { - id, - mode: MODE, - scenario: 'stream_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'failed', - elapsed_ms: 0, - error: message, - }; - pushResult(failed); - archiveFailure(failed, { error: message }); - throw error; - } - }, - 90_000, - ); - } - - const multiTargets = MODE === 'full' - ? PROVIDER_TARGETS - : [OPENAI_TARGET.configured ? OPENAI_TARGET : OPENAI_COMPAT_TARGET]; - - for (const target of multiTargets) { - it( - `${target.protocol} parallel multi tool_call`, - async (ctx) => { - const id = caseId('parallel_multi_tool_call', target); - if (!target.configured) { - pushResult({ - id, - mode: MODE, - scenario: 'parallel_multi_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'skipped', - elapsed_ms: 0, - error: target.reason, - }); - ctx.skip(); - return; - } - - try { - const { response, elapsedMs } = await runNonStreamCase(target, true); - const toolCallCount = assertToolCalls(response, 2); - pushResult({ - id, - mode: MODE, - scenario: 'parallel_multi_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'passed', - elapsed_ms: elapsedMs, - tool_call_count: toolCallCount, - }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const failed: CaseResult = { - id, - mode: MODE, - scenario: 'parallel_multi_tool_call', - protocol: target.protocol, - provider: target.provider, - model: target.model, - status: 'failed', - elapsed_ms: 0, - error: message, - }; - pushResult(failed); - archiveFailure(failed, { error: message }); - throw error; - } - }, - 90_000, - ); - } - - it( - 'failover recovery from primary failure to fallback success', - async (ctx) => { - const configuredTargets = PROVIDER_TARGETS.filter((target) => target.configured); - const primary = configuredTargets[0]; - const fallback = configuredTargets[1]; - const id = 'failover_recovery:mixed'; - - if (!primary || !fallback) { - pushResult({ - id, - mode: MODE, - scenario: 'failover_recovery', - protocol: 'mixed', - provider: primary?.provider || 'unavailable', - model: primary?.model || 'unavailable', - status: 'skipped', - elapsed_ms: 0, - error: 'Need at least 2 configured providers for failover recovery scenario', - }); - ctx.skip(); - return; - } - - const chain: FallbackChain = { - primary: { - provider: primary.provider, - model: `nonexistent-model-for-failover-${Date.now()}`, - }, - fallbacks: [{ - provider: fallback.provider, - model: fallback.model, - }], - }; - const manager = createFailoverManager(chain); - - try { - const start = Date.now(); - const response = await manager.chat( - [{ role: 'user', content: 'Reply with exactly: failover-ok' }], - // Reasoning-capable fallbacks may consume a small output allowance - // before emitting visible text. Keep this smoke assertion strict, - // but give the fallback enough room to produce its final answer. - { max_tokens: 256, timeout_ms: 30_000 }, - ); - expect(response.content.trim().length).toBeGreaterThan(0); - expect(['fallback', 'normal']).toContain(manager.getState().mode); - pushResult({ - id, - mode: MODE, - scenario: 'failover_recovery', - protocol: 'mixed', - provider: `${primary.provider}->${fallback.provider}`, - model: `${chain.primary.model}->${fallback.model}`, - status: 'passed', - elapsed_ms: Date.now() - start, - }); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const failed: CaseResult = { - id, - mode: MODE, - scenario: 'failover_recovery', - protocol: 'mixed', - provider: `${primary.provider}->${fallback.provider}`, - model: `${chain.primary.model}->${fallback.model}`, - status: 'failed', - elapsed_ms: 0, - error: message, - }; - pushResult(failed); - archiveFailure(failed, { error: message, chain }); - throw error; - } finally { - manager.destroy(); - } - }, - 120_000, - ); -}); - -afterAll(() => { - const totals = { - passed: CASE_RESULTS.filter((result) => result.status === 'passed').length, - failed: CASE_RESULTS.filter((result) => result.status === 'failed').length, - skipped: CASE_RESULTS.filter((result) => result.status === 'skipped').length, - }; - - const summary = { - run_id: RUN_ID, - generated_at: new Date().toISOString(), - mode: MODE, - totals, - results: CASE_RESULTS, - }; - - mkdirSync(REPORT_ROOT, { recursive: true }); - writeFileSync( - join(REPORT_ROOT, 'compatibility-report.json'), - `${JSON.stringify(summary, null, 2)}\n`, - 'utf-8', - ); - - const md = [ - '# Provider Tool-Calling Compatibility Report', - '', - `- Run ID: ${RUN_ID}`, - `- Mode: ${MODE}`, - `- Generated At: ${summary.generated_at}`, - `- Passed: ${totals.passed}`, - `- Failed: ${totals.failed}`, - `- Skipped: ${totals.skipped}`, - '', - '## Results', - '', - '| Scenario | Protocol | Provider | Model | Status | Tool Calls | Stream Chunks | Elapsed(ms) |', - '| --- | --- | --- | --- | --- | --- | --- | --- |', - ...CASE_RESULTS.map((result) => [ - result.scenario, - result.protocol, - result.provider, - result.model, - result.status, - result.tool_call_count ?? '-', - result.stream_chunk_count ?? '-', - result.elapsed_ms, - ].join(' | ').replace(/^/, '| ').concat(' |')), - '', - `Failure samples: \`${join(REPORT_ROOT, 'failures')}\``, - '', - ].join('\n'); - - writeFileSync(join(REPORT_ROOT, 'compatibility-report.md'), md, 'utf-8'); -});