From 2acc68bee225cec17dca526ae549190436591ddf Mon Sep 17 00:00:00 2001 From: fanxiaotuGod Date: Mon, 20 Jul 2026 13:37:33 -0700 Subject: [PATCH] fix: supprot gpt-5-nano --- docs/help/learning-objectives.md | 2 + .../unit/helpKnowledgeService.test.js | 10 ++ .../unit/openAIStreamingRequest.test.js | 44 +++++- routes/create/services/llmService.js | 148 ++++++++++-------- routes/create/utils/openAIRequestUtils.js | 102 ++++++++++++ .../generation/AIPlanGenerationTrace.test.tsx | 10 ++ .../generation/AIPlanGenerationTrace.tsx | 1 + .../generation/generationTraceLog.ts | 1 + 8 files changed, 253 insertions(+), 65 deletions(-) create mode 100644 routes/create/utils/openAIRequestUtils.js diff --git a/docs/help/learning-objectives.md b/docs/help/learning-objectives.md index 214138a..2b14b76 100644 --- a/docs/help/learning-objectives.md +++ b/docs/help/learning-objectives.md @@ -14,6 +14,8 @@ The **Live generation log** explains what CREATE is doing before model text appe When drafting starts, the same log adds a **Live model draft** section. After the draft returns, CREATE checks whether the required source sections are covered and may run a targeted repair pass before saving the objectives. A long preparation stage does not by itself mean generation is stuck, but an error event in the log identifies the stage that needs attention. +GPT-5 models use one output budget for both internal reasoning and the visible objective JSON. CREATE reserves a larger budget for these models. If OpenAI reports that the first request exhausted that budget before completing the draft, CREATE clears the incomplete draft and retries once with more room. If the retry also stops, check the visible error, try a processed-material subset, or select another configured model rather than repeatedly submitting the same request. + ## Add existing or manual objectives Paste existing objectives into the generation instructions when you want the model to preserve or refine a provided set. Use **Add Manually** when wording must be stored exactly as entered. Manual entry is also useful when a required objective is not stated explicitly in the uploaded material. diff --git a/routes/create/__tests__/unit/helpKnowledgeService.test.js b/routes/create/__tests__/unit/helpKnowledgeService.test.js index 7ec41df..1e9f085 100644 --- a/routes/create/__tests__/unit/helpKnowledgeService.test.js +++ b/routes/create/__tests__/unit/helpKnowledgeService.test.js @@ -63,6 +63,16 @@ describe('CREATE Guide knowledge retrieval', () => { expect(sources.some(source => source.section === 'Understand the generation log')).toBe(true); }); + test('retrieves GPT-5 output-budget retry guidance for learning objectives', async () => { + const sources = await helpKnowledgeService.retrieve( + 'Why did GPT-5 stop before returning learning objective text, and will CREATE retry the output budget?', + { route: '/course/course-1/quiz/quiz-1?tab=objectives', activeTab: 'Learning Objectives' }, + 4 + ); + + expect(sources.some(source => source.section === 'Understand the generation log')).toBe(true); + }); + test('retrieves source-type material preview guidance', async () => { const sources = await helpKnowledgeService.retrieve( 'Does the material eye icon open the PDF and show extracted text for a URL?', diff --git a/routes/create/__tests__/unit/openAIStreamingRequest.test.js b/routes/create/__tests__/unit/openAIStreamingRequest.test.js index 9bbf08a..aeafeba 100644 --- a/routes/create/__tests__/unit/openAIStreamingRequest.test.js +++ b/routes/create/__tests__/unit/openAIStreamingRequest.test.js @@ -1,9 +1,13 @@ import { describe, expect, test } from '@jest/globals'; import { + buildOpenAIIncompleteResponseError, buildOpenAIStreamingRequest, + extractResponsesOutputText, + getLearningObjectiveCompletionOptions, + isOpenAIOutputBudgetError, isGpt5Family, parseCoursePromptReviewResponse -} from '../../services/llmService.js'; +} from '../../utils/openAIRequestUtils.js'; describe('OpenAI streaming request configuration', () => { test('recognizes GPT-5 family model names', () => { @@ -17,18 +21,52 @@ describe('OpenAI streaming request configuration', () => { prompt: 'Generate a question', temperature: 0.7, maxTokens: 4000, - useResponsesApi: true + useResponsesApi: true, + reasoningEffort: 'none' }); expect(request).toEqual({ model: 'gpt-5.4-nano', input: 'Generate a question', max_output_tokens: 4000, - stream: true + stream: true, + reasoning: { effort: 'none' } }); expect(request.temperature).toBeUndefined(); }); + test('allocates a reasoning-safe LO budget for GPT-5.4 nano with one larger retry', () => { + expect(getLearningObjectiveCompletionOptions('gpt-5.4-nano')).toEqual({ + maxTokens: 12000, + reasoningEffort: 'none' + }); + expect(getLearningObjectiveCompletionOptions('gpt-5.4-nano', true)).toEqual({ + maxTokens: 24000, + reasoningEffort: 'none' + }); + expect(getLearningObjectiveCompletionOptions('gpt-4o-mini')).toEqual({ + maxTokens: 2600, + reasoningEffort: null + }); + }); + + test('recovers final Responses API text when no delta event was delivered', () => { + expect(extractResponsesOutputText({ + output: [{ + type: 'message', + content: [{ type: 'output_text', text: '{"objectives":[]}' }] + }] + })).toBe('{"objectives":[]}'); + }); + + test('classifies max-output incomplete responses as retryable budget errors', () => { + const error = buildOpenAIIncompleteResponseError('gpt-5.4-nano', 'max_output_tokens'); + + expect(error.message).toContain('used its output budget'); + expect(error.incompleteReason).toBe('max_output_tokens'); + expect(isOpenAIOutputBudgetError(error)).toBe(true); + }); + test('uses Chat Completions parameters for GPT-4o mini', () => { const request = buildOpenAIStreamingRequest({ model: 'gpt-4o-mini', diff --git a/routes/create/services/llmService.js b/routes/create/services/llmService.js index e6bde45..2e2be50 100644 --- a/routes/create/services/llmService.js +++ b/routes/create/services/llmService.js @@ -12,6 +12,24 @@ import { QUESTION_TYPES } from '../config/constants.js'; import UserApiKey from '../models/UserApiKey.js'; import User from '../models/User.js'; import { normalizeGeneratedQuestionText } from '../utils/questionTextLimits.js'; +import { + buildOpenAIIncompleteResponseError, + buildOpenAIStreamingRequest, + extractResponsesOutputText, + getLearningObjectiveCompletionOptions, + isGpt5Family, + isOpenAIOutputBudgetError, + parseCoursePromptReviewResponse +} from '../utils/openAIRequestUtils.js'; +export { + buildOpenAIIncompleteResponseError, + buildOpenAIStreamingRequest, + extractResponsesOutputText, + getLearningObjectiveCompletionOptions, + isGpt5Family, + isOpenAIOutputBudgetError, + parseCoursePromptReviewResponse +}; const ADMIN_CWLS = (process.env.ADMIN_CWLS || '').split(',').map(s => s.trim()).filter(Boolean); export function normalizeLearningObjectiveText(value) { @@ -24,57 +42,6 @@ function normalizeOptionalText(value) { return typeof value === 'string' ? value.trim() : ''; } -export function isGpt5Family(model = '') { - return model.toLowerCase().startsWith('gpt-5'); -} - -export function buildOpenAIStreamingRequest({ model, prompt, temperature, maxTokens, useResponsesApi }) { - if (useResponsesApi) { - return { - model, - input: prompt, - max_output_tokens: maxTokens, - stream: true - }; - } - - return { - model, - messages: [{ role: 'user', content: prompt }], - max_completion_tokens: maxTokens, - stream: true, - ...(!isGpt5Family(model) ? { temperature } : {}) - }; -} - -export function parseCoursePromptReviewResponse(content = '') { - const withoutFence = String(content) - .replace(/```json\s*/gi, '') - .replace(/```/g, '') - .trim(); - const jsonMatch = withoutFence.match(/\{[\s\S]*\}/); - if (!jsonMatch) { - throw new Error('Prompt review did not return a JSON object'); - } - - const parsed = JSON.parse(jsonMatch[0]); - const normalizeItems = value => ( - Array.isArray(value) - ? value.filter(item => typeof item === 'string' && item.trim()).map(item => item.trim()).slice(0, 5) - : [] - ); - const revisedPrompt = typeof parsed.revisedPrompt === 'string' - ? parsed.revisedPrompt.trim().slice(0, 12000) - : ''; - - return { - warnings: normalizeItems(parsed.warnings), - suggestions: normalizeItems(parsed.suggestions), - revisedPrompt, - changeSummary: normalizeItems(parsed.changeSummary) - }; -} - function extractBalancedJson(value = '') { const text = String(value) .replace(/```json\s*/gi, '') @@ -330,11 +297,14 @@ class QuizLLMService { userId = null, llmConfig: providedLLMConfig = null, temperature = 0.3, - maxTokens = 2400 + maxTokens = 2400, + reasoningEffort = null }, onStreamChunk = null) { const llmConfig = providedLLMConfig || await this.resolveUserLLMConfig(userId); const { provider, model } = llmConfig; let accumulatedContent = ''; + let finalResponseText = ''; + let incompleteReason = ''; if (provider === 'openai') { const OpenAI = (await import('openai')).default; @@ -347,7 +317,8 @@ class QuizLLMService { prompt, temperature, maxTokens, - useResponsesApi + useResponsesApi, + reasoningEffort }); const stream = useResponsesApi ? await openai.responses.create(request) @@ -367,9 +338,21 @@ class QuizLLMService { }); } + if (useResponsesApi && chunk.type === 'response.output_text.done' && typeof chunk.text === 'string') { + finalResponseText = chunk.text; + } + + if (useResponsesApi && ['response.completed', 'response.incomplete'].includes(chunk.type)) { + finalResponseText = extractResponsesOutputText(chunk.response) || finalResponseText; + } + if (useResponsesApi && chunk.type === 'response.failed') { throw new Error(chunk.response?.error?.message || 'OpenAI Responses API reported a failed response'); } + + if (useResponsesApi && chunk.type === 'response.incomplete') { + incompleteReason = chunk.response?.incomplete_details?.reason || 'unknown'; + } } } else { const options = this.getSendMessageOptions(temperature, maxTokens, llmConfig); @@ -402,6 +385,20 @@ class QuizLLMService { } } + if (incompleteReason) { + throw buildOpenAIIncompleteResponseError(model, incompleteReason); + } + + if (!accumulatedContent && finalResponseText) { + accumulatedContent = finalResponseText; + onStreamChunk?.(accumulatedContent, { + partial: true, + totalLength: accumulatedContent.length, + model, + deliveredAsSingleChunk: true + }); + } + if (!accumulatedContent.trim()) { throw new Error(`Model ${model} completed without returning output text`); } @@ -2555,15 +2552,40 @@ Learning Objectives:`; }; const temperature = 0.3; - const maxTokens = 2600; + const completionOptions = getLearningObjectiveCompletionOptions(model); progress('draft-started', `Calling ${model} to draft the learning objectives...`); - const response = await this.streamCompletion({ - prompt, - userId, - llmConfig, - temperature, - maxTokens - }, streamOutputCallback); + let response; + try { + response = await this.streamCompletion({ + prompt, + userId, + llmConfig, + temperature, + ...completionOptions + }, streamOutputCallback); + } catch (error) { + if (!isOpenAIOutputBudgetError(error)) { + throw error; + } + + const retryOptions = getLearningObjectiveCompletionOptions(model, true); + progress( + 'draft-retry', + `${model} used its output budget before returning a complete draft. Retrying once with more room...` + ); + streamOutputCallback?.('', { + reset: true, + reason: 'output-budget-retry', + model + }); + response = await this.streamCompletion({ + prompt, + userId, + llmConfig, + temperature, + ...retryOptions + }, streamOutputCallback); + } progress('draft-complete', 'Learning objective draft returned. Checking source coverage...'); let objectives = parseObjectivesResponse(response.content); @@ -2614,7 +2636,9 @@ ${targetCount ? `- Keep exactly ${targetCount} main objectives.` : ''}`; userId, llmConfig, temperature: 0.2, - maxTokens: 2800 + ...(isGpt5Family(model) + ? getLearningObjectiveCompletionOptions(model) + : { maxTokens: 2800 }) }, streamOutputCallback); const repairedObjectives = parseObjectivesResponse(repairResponse.content); if (repairedObjectives.length > 0) { diff --git a/routes/create/utils/openAIRequestUtils.js b/routes/create/utils/openAIRequestUtils.js new file mode 100644 index 0000000..03ca820 --- /dev/null +++ b/routes/create/utils/openAIRequestUtils.js @@ -0,0 +1,102 @@ +export function isGpt5Family(model = '') { + return model.toLowerCase().startsWith('gpt-5'); +} + +export function buildOpenAIStreamingRequest({ + model, + prompt, + temperature, + maxTokens, + useResponsesApi, + reasoningEffort = null +}) { + if (useResponsesApi) { + return { + model, + input: prompt, + max_output_tokens: maxTokens, + stream: true, + ...(reasoningEffort ? { reasoning: { effort: reasoningEffort } } : {}) + }; + } + + return { + model, + messages: [{ role: 'user', content: prompt }], + max_completion_tokens: maxTokens, + stream: true, + ...(!isGpt5Family(model) ? { temperature } : {}) + }; +} + +export function extractResponsesOutputText(response = {}) { + if (typeof response.output_text === 'string' && response.output_text) { + return response.output_text; + } + + return (Array.isArray(response.output) ? response.output : []) + .flatMap(item => Array.isArray(item?.content) ? item.content : []) + .filter(part => part?.type === 'output_text' && typeof part.text === 'string') + .map(part => part.text) + .join(''); +} + +export function getLearningObjectiveCompletionOptions(model, retry = false) { + if (!isGpt5Family(model)) { + return { maxTokens: 2600, reasoningEffort: null }; + } + + return { + // GPT-5 output budgets include both hidden reasoning and visible JSON. + // Reserve enough room for both, then double it for the single bounded retry. + maxTokens: retry ? 24000 : 12000, + // GPT-5.4 nano supports `none` and is intended for extraction-style work. + // Older GPT-5 aliases do not all support `none`, so keep those on `low`. + reasoningEffort: model.toLowerCase().startsWith('gpt-5.4-nano') ? 'none' : 'low' + }; +} + +export function isOpenAIOutputBudgetError(error) { + return error?.code === 'OPENAI_MAX_OUTPUT_TOKENS'; +} + +export function buildOpenAIIncompleteResponseError(model, reason = 'unknown') { + const error = new Error( + reason === 'max_output_tokens' + ? `Model ${model} used its output budget before completing the visible response` + : `Model ${model} returned an incomplete response (${reason})` + ); + error.code = reason === 'max_output_tokens' + ? 'OPENAI_MAX_OUTPUT_TOKENS' + : 'OPENAI_RESPONSE_INCOMPLETE'; + error.incompleteReason = reason; + return error; +} + +export function parseCoursePromptReviewResponse(content = '') { + const withoutFence = String(content) + .replace(/```json\s*/gi, '') + .replace(/```/g, '') + .trim(); + const jsonMatch = withoutFence.match(/\{[\s\S]*\}/); + if (!jsonMatch) { + throw new Error('Prompt review did not return a JSON object'); + } + + const parsed = JSON.parse(jsonMatch[0]); + const normalizeItems = value => ( + Array.isArray(value) + ? value.filter(item => typeof item === 'string' && item.trim()).map(item => item.trim()).slice(0, 5) + : [] + ); + const revisedPrompt = typeof parsed.revisedPrompt === 'string' + ? parsed.revisedPrompt.trim().slice(0, 12000) + : ''; + + return { + warnings: normalizeItems(parsed.warnings), + suggestions: normalizeItems(parsed.suggestions), + revisedPrompt, + changeSummary: normalizeItems(parsed.changeSummary) + }; +} diff --git a/src/components/generation/AIPlanGenerationTrace.test.tsx b/src/components/generation/AIPlanGenerationTrace.test.tsx index 251621b..a681b1f 100644 --- a/src/components/generation/AIPlanGenerationTrace.test.tsx +++ b/src/components/generation/AIPlanGenerationTrace.test.tsx @@ -110,4 +110,14 @@ describe('public generation log formatting', () => { expect(buildPublicWorkflowLog([], true, 'Preparing source inventory...')) .toBe('[WAIT] Preparing source inventory...'); }); + + it('explains the bounded output-budget retry without exposing request internals', () => { + const log = buildPublicWorkflowLog([{ + status: 'draft-retry', + message: 'The model used its output budget. Retrying once...', + }], true, 'Preparing source inventory...'); + + expect(log).toContain('retrying once with a larger output budget'); + expect(log).not.toContain('24000'); + }); }); diff --git a/src/components/generation/AIPlanGenerationTrace.tsx b/src/components/generation/AIPlanGenerationTrace.tsx index f6c73c1..e5aef9d 100644 --- a/src/components/generation/AIPlanGenerationTrace.tsx +++ b/src/components/generation/AIPlanGenerationTrace.tsx @@ -60,6 +60,7 @@ export default function AIPlanGenerationTrace({ ); const modelDraftStarted = Boolean(streamedText) || steps.some(step => [ 'draft-started', + 'draft-retry', 'llm-started', 'draft-complete', 'llm-complete', diff --git a/src/components/generation/generationTraceLog.ts b/src/components/generation/generationTraceLog.ts index 1032e49..59be9f0 100644 --- a/src/components/generation/generationTraceLog.ts +++ b/src/components/generation/generationTraceLog.ts @@ -17,6 +17,7 @@ const PUBLIC_STEP_DETAILS: Record = { 'budget-complete': 'A deterministic question budget has been allocated across the learning objectives.', 'context-complete': 'Learning objectives, assigned materials, and existing question history are ready for the model.', 'draft-started': 'The model is now drafting the learning-objective JSON from the prepared context.', + 'draft-retry': 'The first request ended before a complete visible draft was returned. CREATE is retrying once with a larger output budget.', 'llm-started': 'The model is now drafting the quiz blueprint from the prepared context.', 'draft-complete': 'The model draft has returned and is being checked against the source inventory.', 'llm-complete': 'The model draft has returned and is being validated.',