From 3e56c26bdaf7b55a9d35398ca2a18e986334edb4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 4 Jul 2026 04:13:24 +0000 Subject: [PATCH 1/4] Initial plan From 3442d688fb91b8844f11704ebf8bf35c12ae7288 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 4 Jul 2026 04:18:03 +0000 Subject: [PATCH 2/4] Add OpenAI Responses API (JSON Schema mode) support --- js/owrap.ai.js | 100 ++++++++++++++++++++++++++ tests/autoTestAll.AI.js | 146 ++++++++++++++++++++++++++++++++++++++ tests/autoTestAll.AI.yaml | 66 ++++++++++++----- 3 files changed, 294 insertions(+), 18 deletions(-) diff --git a/js/owrap.ai.js b/js/owrap.ai.js index e922f509..b5332ac8 100644 --- a/js/owrap.ai.js +++ b/js/owrap.ai.js @@ -698,6 +698,51 @@ OpenWrap.ai.prototype.__gpttypes = { if (isMap(res) && isDef(res.error)) return res return res.content }, + rawResponse: (aPrompt, aModel, aTemperature, aJsonSchemaFlag, aTools, aResponseSchema) => { + aPrompt = _$(aPrompt, "aPrompt").default(__) + aTemperature = _$(aTemperature, "aTemperature").isNumber().default(_temperature) + aModel = _$(aModel, "aModel").isString().default(_model) + aJsonSchemaFlag = _$(aJsonSchemaFlag, "aJsonSchemaFlag").isBoolean().default(false) + if (isUnDef(aTools)) { + aTools = Object.keys(_r.tools) + } else if (isMap(aTools)) { + aTools = Object.keys(aTools) + } + aTools = _$(aTools, "aTools").isArray().default([]) + + _resetStats() + var msgs = [] + if (isString(aPrompt)) aPrompt = [ aPrompt ] + aPrompt = _r.conversation.concat(aPrompt) + msgs = aPrompt.filter(c => isDef(c)).map(c => isMap(c) ? c : { role: "user", content: c }) + + _r.conversation = msgs + if (_noSystem) msgs = msgs.map(m => { if (m.role == "system") m.role = "developer"; return m }) + var body = { + model: aModel, + temperature: aTemperature, + messages: msgs + } + + if (isDef(aResponseSchema) && isMap(aResponseSchema)) { + body.response_format = { + type: "json_schema", + json_schema: { + name: aResponseSchema.name || "Response", + description: aResponseSchema.description || "API Response", + schema: aResponseSchema.schema || {}, + strict: aResponseSchema.strict !== false + } + } + } + + body = merge(body, aOptions.params) + if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'client'}, merge({_t:nowNano(),_f:'client'}, body)) + var _res = _r._request(_route("chat/completions", aModel), body) + if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'llm'}, merge({_t:nowNano(),_f:'llm'}, _res)) + _captureStats(_res, body) + return _res + }, rawImgGen: (aPrompt, aModel) => { aPrompt = _$(aPrompt, "aPrompt").default(__) aModel = _$(aModel, "aModel").isString().default(_model) @@ -3573,6 +3618,39 @@ OpenWrap.ai.prototype.gpt.prototype.rawPromptWithStats = function(aPrompt, aRole return { response: response, stats: this.getLastStats() } } +/** + * + * ow.ai.gpt.jsonSchemaPrompt(aPrompt, aResponseSchema, aModel, aTemperature, tools) : Object + * Executes a prompt using OpenAI's JSON Schema response format, enforcing structured output validated against aResponseSchema.\ + * aResponseSchema should be a map with: name (string), description (string), schema (JSON Schema object), strict (boolean, defaults to true).\ + * Returns the parsed JSON response matching the provided schema.\ + * Only supported by the "openai" provider; throws if the underlying provider does not implement rawResponse. + * + */ +OpenWrap.ai.prototype.gpt.prototype.jsonSchemaPrompt = function(aPrompt, aResponseSchema, aModel, aTemperature, tools) { + if (!isFunction(this.model.rawResponse)) throw "JSON Schema responses not supported by this provider" + var response = this.model.rawResponse(aPrompt, aModel, aTemperature, false, tools, aResponseSchema) + if (isArray(response.choices) && response.choices.length > 0) { + if (response.choices[0].finish_reason == "stop") { + var content = response.choices[0].message.content + return isString(content) ? jsonParse(content, __, __, true) : content + } + } + return response +} + +/** + * + * ow.ai.gpt.jsonSchemaPromptWithStats(aPrompt, aResponseSchema, aModel, aTemperature, tools) : Map + * Executes jsonSchemaPrompt and returns the parsed response together with any reported statistics ({ response, stats }).\ + * aResponseSchema should be a map with: name (string), description (string), schema (JSON Schema object), strict (boolean, defaults to true). + * + */ +OpenWrap.ai.prototype.gpt.prototype.jsonSchemaPromptWithStats = function(aPrompt, aResponseSchema, aModel, aTemperature, tools) { + var response = this.jsonSchemaPrompt(aPrompt, aResponseSchema, aModel, aTemperature, tools) + return { response: response, stats: this.getLastStats() } +} + /** * * ow.ai.gpt.promptImgGen(aPrompt, aModel, anOutputPathPrefix) : Array @@ -4380,6 +4458,28 @@ global.$gpt = function(aModel) { * */ importConversation: (aExport) => { _g.importConversation(aExport); return _r }, + /** + * + * $gpt.jsonSchemaPrompt(aPrompt, aResponseSchema, aModel, aTemperature, tools) : Object + * Executes a prompt using OpenAI's JSON Schema response format, enforcing structured output validated against aResponseSchema.\ + * aResponseSchema should be a map with: name (string), description (string), schema (JSON Schema object), strict (boolean, defaults to true).\ + * Returns the parsed JSON response matching the provided schema.\ + * Only supported by the "openai" provider. + * + */ + jsonSchemaPrompt: (aPrompt, aResponseSchema, aModel, aTemperature, tools) => { + return _g.jsonSchemaPrompt(aPrompt, aResponseSchema, aModel, aTemperature, tools) + }, + /** + * + * $gpt.jsonSchemaPromptWithStats(aPrompt, aResponseSchema, aModel, aTemperature, tools) : Map + * Executes jsonSchemaPrompt and returns the parsed response together with any reported statistics ({ response, stats }).\ + * aResponseSchema should be a map with: name (string), description (string), schema (JSON Schema object), strict (boolean, defaults to true). + * + */ + jsonSchemaPromptWithStats: (aPrompt, aResponseSchema, aModel, aTemperature, tools) => { + return _g.jsonSchemaPromptWithStats(aPrompt, aResponseSchema, aModel, aTemperature, tools) + }, /** * * $gpt.close() diff --git a/tests/autoTestAll.AI.js b/tests/autoTestAll.AI.js index 5e593b20..af8d2b3c 100644 --- a/tests/autoTestAll.AI.js +++ b/tests/autoTestAll.AI.js @@ -815,4 +815,150 @@ ow.test.assert(deltas, [ "part-1", "part-2" ], "Problem preserving Ollama streaming deltas after warmup."); ow.test.assert(res.content, "part-1part-2", "Problem aggregating Ollama streaming content after warmup."); }; + + exports.testAIOpenAIRawResponse = function() { + ow.loadAI(); + + var g = new ow.ai.gpt("openai", { key: "test-key", model: "gpt-4o" }); + var requests = []; + var schema = { + name: "PersonInfo", + description: "Extract person details", + schema: { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" } + }, + required: ["name", "age"], + additionalProperties: false + }, + strict: true + }; + + g.model._request = function(url, body) { + requests.push({ url: url, body: __cloneForTest(body) }); + return { + model: "gpt-4o", + choices: [ + { + finish_reason: "stop", + message: { role: "assistant", content: '{"name":"Alice","age":30}' } + } + ], + usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 } + }; + }; + + var rawRes = g.model.rawResponse("Extract: Alice, 30", "gpt-4o", 0.5, false, [], schema); + ow.test.assert(requests.length, 1, "Problem: rawResponse should make exactly one request."); + ow.test.assert(requests[0].url.indexOf("chat/completions") >= 0, true, "Problem: rawResponse should use chat/completions endpoint."); + ow.test.assert(requests[0].body.response_format.type, "json_schema", "Problem: rawResponse should set response_format.type to json_schema."); + ow.test.assert(requests[0].body.response_format.json_schema.name, "PersonInfo", "Problem: rawResponse should pass schema name."); + ow.test.assert(requests[0].body.response_format.json_schema.strict, true, "Problem: rawResponse should pass strict flag."); + ow.test.assert(isArray(rawRes.choices) && rawRes.choices.length > 0, true, "Problem: rawResponse should return choices array."); + }; + + exports.testAIOpenAIJsonSchemaPrompt = function() { + ow.loadAI(); + + var g = new ow.ai.gpt("openai", { key: "test-key", model: "gpt-4o" }); + var schema = { + name: "PersonInfo", + description: "Extract person details", + schema: { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" } + }, + required: ["name", "age"], + additionalProperties: false + }, + strict: true + }; + + g.model._request = function(url, body) { + return { + model: "gpt-4o", + choices: [ + { + finish_reason: "stop", + message: { role: "assistant", content: '{"name":"Alice","age":30}' } + } + ], + usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 } + }; + }; + + var result = g.jsonSchemaPrompt("Extract: Alice, 30", schema, "gpt-4o", 0.5); + ow.test.assert(isMap(result), true, "Problem: jsonSchemaPrompt should return a parsed map."); + ow.test.assert(result.name, "Alice", "Problem: jsonSchemaPrompt should parse name from JSON."); + ow.test.assert(result.age, 30, "Problem: jsonSchemaPrompt should parse age from JSON."); + }; + + exports.testAIOpenAIJsonSchemaPromptWithStats = function() { + ow.loadAI(); + + var g = new ow.ai.gpt("openai", { key: "test-key", model: "gpt-4o" }); + var schema = { + name: "PersonInfo", + schema: { type: "object", properties: { name: { type: "string" } }, required: ["name"], additionalProperties: false }, + strict: true + }; + + g.model._request = function(url, body) { + return { + model: "gpt-4o", + choices: [ + { + finish_reason: "stop", + message: { role: "assistant", content: '{"name":"Bob"}' } + } + ], + usage: { prompt_tokens: 15, completion_tokens: 8, total_tokens: 23 } + }; + }; + + var result = g.jsonSchemaPromptWithStats("Get name: Bob", schema); + ow.test.assert(isMap(result), true, "Problem: jsonSchemaPromptWithStats should return a map."); + ow.test.assert(isDef(result.response), true, "Problem: jsonSchemaPromptWithStats should have response field."); + ow.test.assert(isDef(result.stats), true, "Problem: jsonSchemaPromptWithStats should have stats field."); + ow.test.assert(result.response.name, "Bob", "Problem: jsonSchemaPromptWithStats response should contain parsed name."); + ow.test.assert(result.stats.tokens.total, 23, "Problem: jsonSchemaPromptWithStats stats should contain token usage."); + }; + + exports.testAIOpenAIJsonSchemaPromptSchemaDefault = function() { + ow.loadAI(); + + var g = new ow.ai.gpt("openai", { key: "test-key", model: "gpt-4o" }); + var requests = []; + + g.model._request = function(url, body) { + requests.push(__cloneForTest(body)); + return { + model: "gpt-4o", + choices: [ { finish_reason: "stop", message: { role: "assistant", content: '{"val":1}' } } ], + usage: { prompt_tokens: 5, completion_tokens: 5, total_tokens: 10 } + }; + }; + + // Schema without description or strict – should use defaults + g.model.rawResponse("test", "gpt-4o", 0.5, false, [], { name: "MySchema", schema: { type: "object", additionalProperties: false } }); + ow.test.assert(requests[0].response_format.json_schema.description, "API Response", "Problem: rawResponse should default description to 'API Response'."); + ow.test.assert(requests[0].response_format.json_schema.strict, true, "Problem: rawResponse should default strict to true."); + }; + + exports.testAIOpenAIJsonSchemaPromptUnsupportedProvider = function() { + ow.loadAI(); + + var g = new ow.ai.gpt("gemini", { key: "test-key" }); + var threw = false; + try { + g.jsonSchemaPrompt("test", { name: "T", schema: {} }); + } catch(e) { + threw = true; + } + ow.test.assert(threw, true, "Problem: jsonSchemaPrompt should throw for providers that don't support rawResponse."); + }; })(); diff --git a/tests/autoTestAll.AI.yaml b/tests/autoTestAll.AI.yaml index f7e2ec2b..1421e0e1 100644 --- a/tests/autoTestAll.AI.yaml +++ b/tests/autoTestAll.AI.yaml @@ -120,10 +120,35 @@ jobs: to : oJob Test exec: args.func = args.tests.testAIOllamaStreamingToolExecutionAndConversationIds; - - name: AI::Ollama streaming warmup on fresh conversation - from: AI::Init - to : oJob Test - exec: args.func = args.tests.testAIOllamaStreamingWarmsModelOnFreshConversation; + - name: AI::Ollama streaming warmup on fresh conversation + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOllamaStreamingWarmsModelOnFreshConversation; + + - name: AI::OpenAI rawResponse method + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIRawResponse; + + - name: AI::OpenAI jsonSchemaPrompt + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIJsonSchemaPrompt; + + - name: AI::OpenAI jsonSchemaPromptWithStats + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIJsonSchemaPromptWithStats; + + - name: AI::OpenAI jsonSchemaPrompt schema defaults + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIJsonSchemaPromptSchemaDefault; + + - name: AI::OpenAI jsonSchemaPrompt unsupported provider + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIJsonSchemaPromptUnsupportedProvider; todo: # ow.ai tests @@ -137,17 +162,22 @@ todo: - AI::LSTM network save and load - AI::Normalize with schema - AI::Normalize features array - - AI::Perceptron XOR save and load - - AI::Test ID3 - - AI::Test C45 - - AI::Test KMeans - - AI::GPT prompt argument routing - - AI::OpenAI transport modes - - AI::OpenAI stats capture includes cached and reasoning - - AI::Anthropic prompt caching headers - - AI::Anthropic prompt caching body and stats - - AI::OpenAI tool recursion without duplication - - AI::OpenAI streaming tool recursion without duplication - - AI::OpenAI export conversation falls back to tool name - - AI::Ollama streaming tool execution and ids - - AI::Ollama streaming warmup on fresh conversation + - AI::Perceptron XOR save and load + - AI::Test ID3 + - AI::Test C45 + - AI::Test KMeans + - AI::GPT prompt argument routing + - AI::OpenAI transport modes + - AI::OpenAI stats capture includes cached and reasoning + - AI::Anthropic prompt caching headers + - AI::Anthropic prompt caching body and stats + - AI::OpenAI tool recursion without duplication + - AI::OpenAI streaming tool recursion without duplication + - AI::OpenAI export conversation falls back to tool name + - AI::Ollama streaming tool execution and ids + - AI::Ollama streaming warmup on fresh conversation + - AI::OpenAI rawResponse method + - AI::OpenAI jsonSchemaPrompt + - AI::OpenAI jsonSchemaPromptWithStats + - AI::OpenAI jsonSchemaPrompt schema defaults + - AI::OpenAI jsonSchemaPrompt unsupported provider From bb3d97e2b8298ea8ea36031eb05485fc2743c95d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 4 Jul 2026 04:20:00 +0000 Subject: [PATCH 3/4] Fix test formatting for consistency with existing test style --- tests/autoTestAll.AI.js | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/autoTestAll.AI.js b/tests/autoTestAll.AI.js index af8d2b3c..d86fc725 100644 --- a/tests/autoTestAll.AI.js +++ b/tests/autoTestAll.AI.js @@ -903,7 +903,14 @@ var g = new ow.ai.gpt("openai", { key: "test-key", model: "gpt-4o" }); var schema = { name: "PersonInfo", - schema: { type: "object", properties: { name: { type: "string" } }, required: ["name"], additionalProperties: false }, + schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"], + additionalProperties: false + }, strict: true }; @@ -938,13 +945,21 @@ requests.push(__cloneForTest(body)); return { model: "gpt-4o", - choices: [ { finish_reason: "stop", message: { role: "assistant", content: '{"val":1}' } } ], + choices: [ + { + finish_reason: "stop", + message: { role: "assistant", content: '{"val":1}' } + } + ], usage: { prompt_tokens: 5, completion_tokens: 5, total_tokens: 10 } }; }; // Schema without description or strict – should use defaults - g.model.rawResponse("test", "gpt-4o", 0.5, false, [], { name: "MySchema", schema: { type: "object", additionalProperties: false } }); + g.model.rawResponse("test", "gpt-4o", 0.5, false, [], { + name: "MySchema", + schema: { type: "object", additionalProperties: false } + }); ow.test.assert(requests[0].response_format.json_schema.description, "API Response", "Problem: rawResponse should default description to 'API Response'."); ow.test.assert(requests[0].response_format.json_schema.strict, true, "Problem: rawResponse should default strict to true."); }; From 46e07dd897fb91065e48fd171ac6987671e59623 Mon Sep 17 00:00:00 2001 From: nmaguiar Date: Mon, 6 Jul 2026 14:23:12 +0100 Subject: [PATCH 4/4] feat(ai): add support for OpenAI responses API Implement the /responses endpoint integration in owrap.ai.js, including response schema normalization, input text conversion, and error handling. Update AI tests to verify the new responses endpoint usage and JSON schema formatting. --- js/owrap.ai.js | 159 ++++++++++++++++++++++++++---- tests/autoTestAll.AI.js | 200 ++++++++++++++++++++++++++++++++------ tests/autoTestAll.AI.yaml | 52 ++++++---- 3 files changed, 346 insertions(+), 65 deletions(-) diff --git a/js/owrap.ai.js b/js/owrap.ai.js index b5332ac8..584d03ab 100644 --- a/js/owrap.ai.js +++ b/js/owrap.ai.js @@ -189,6 +189,74 @@ OpenWrap.ai.prototype.__gpttypes = { return merge(_h, aOptions.headers) } var _buildURL = aURI => _joinURL(aOptions.url, aURI) + var _responsesAvailable = __ + var _normalizeResponseSchema = aResponseSchema => { + if (!isMap(aResponseSchema)) return __ + return { + name : aResponseSchema.name || "Response", + description: aResponseSchema.description || "API Response", + schema : aResponseSchema.schema || {}, + strict : aResponseSchema.strict !== false + } + } + var _toResponseInputText = aContent => { + if (isMap(aContent) || isArray(aContent)) return stringify(aContent, __, "") + return isDef(aContent) ? String(aContent) : "" + } + var _toResponsesInput = aMsgs => { + return _$(aMsgs, "aMsgs").isArray().default([]).map(m => { + if (!isMap(m)) return { role: "user", content: [ { type: "input_text", text: _toResponseInputText(m) } ] } + if (isArray(m.content)) { + return { + role : isString(m.role) ? m.role : "user", + content: clone(m.content) + } + } + return { + role : isString(m.role) ? m.role : "user", + content: [ { type: "input_text", text: _toResponseInputText(m.content) } ] + } + }) + } + var _extractResponseErrorMessage = aResponse => { + if (!isMap(aResponse)) return "" + if (isMap(aResponse.error)) { + if (isString(aResponse.error.message)) return aResponse.error.message + if (isString(aResponse.error.error)) return aResponse.error.error + return stringify(aResponse.error, __, "") + } + if (isString(aResponse.message)) return aResponse.message + return stringify(aResponse, __, "") + } + var _isResponsesUnavailable = aResponse => { + if (!isMap(aResponse)) return false + var _status = isDef(aResponse.status) ? Number(aResponse.status) : (isMap(aResponse.error) && isDef(aResponse.error.status) ? Number(aResponse.error.status) : __) + if ([ 404, 405, 410, 501 ].indexOf(_status) >= 0) return true + + var _code = (isMap(aResponse.error) && isDef(aResponse.error.code) ? aResponse.error.code : aResponse.code) + if (isDef(_code)) { + _code = String(_code).toLowerCase() + if (_code.indexOf("not_found") >= 0 || _code.indexOf("unsupported") >= 0 || _code.indexOf("unknown_url") >= 0 || _code.indexOf("unknown_endpoint") >= 0) return true + } + + var _type = (isMap(aResponse.error) && isDef(aResponse.error.type) ? aResponse.error.type : aResponse.type) + if (isDef(_type)) { + _type = String(_type).toLowerCase() + if (_type.indexOf("not_found") >= 0 || _type.indexOf("unsupported") >= 0) return true + } + + var _msg = _extractResponseErrorMessage(aResponse).toLowerCase() + return ( + _msg.indexOf("/responses") >= 0 && ( + _msg.indexOf("not found") >= 0 || + _msg.indexOf("unsupported") >= 0 || + _msg.indexOf("unknown") >= 0 || + _msg.indexOf("not available") >= 0 || + _msg.indexOf("unavailable") >= 0 || + _msg.indexOf("does not exist") >= 0 + ) + ) || _msg.indexOf("unknown endpoint") >= 0 || _msg.indexOf("unknown url") >= 0 + } var _captureStats = (aResponse, aRequestBody) => { if (!isMap(aResponse)) { _lastStats = __ @@ -207,13 +275,23 @@ OpenWrap.ai.prototype.__gpttypes = { if (isDef(aResponse.usage.prompt_tokens)) tokens.prompt = aResponse.usage.prompt_tokens if (isDef(aResponse.usage.completion_tokens)) tokens.completion = aResponse.usage.completion_tokens if (isDef(aResponse.usage.total_tokens)) tokens.total = aResponse.usage.total_tokens + if (isDef(aResponse.usage.input_tokens)) tokens.prompt = aResponse.usage.input_tokens + if (isDef(aResponse.usage.output_tokens)) tokens.completion = aResponse.usage.output_tokens + if (isDef(aResponse.usage.total_tokens)) tokens.total = aResponse.usage.total_tokens if (isMap(aResponse.usage.prompt_tokens_details)) { if (isDef(aResponse.usage.prompt_tokens_details.cached_tokens)) tokens.cached = aResponse.usage.prompt_tokens_details.cached_tokens if (isDef(aResponse.usage.prompt_tokens_details.audio_tokens)) tokens.audio = aResponse.usage.prompt_tokens_details.audio_tokens } + if (isMap(aResponse.usage.input_tokens_details)) { + if (isDef(aResponse.usage.input_tokens_details.cached_tokens)) tokens.cached = aResponse.usage.input_tokens_details.cached_tokens + if (isDef(aResponse.usage.input_tokens_details.audio_tokens)) tokens.audio = aResponse.usage.input_tokens_details.audio_tokens + } if (isMap(aResponse.usage.completion_tokens_details)) { if (isDef(aResponse.usage.completion_tokens_details.reasoning_tokens)) tokens.reasoning = aResponse.usage.completion_tokens_details.reasoning_tokens } + if (isMap(aResponse.usage.output_tokens_details)) { + if (isDef(aResponse.usage.output_tokens_details.reasoning_tokens)) tokens.reasoning = aResponse.usage.output_tokens_details.reasoning_tokens + } if (Object.keys(tokens).length > 0) stats.tokens = tokens stats.usage = aResponse.usage } @@ -718,29 +796,57 @@ OpenWrap.ai.prototype.__gpttypes = { _r.conversation = msgs if (_noSystem) msgs = msgs.map(m => { if (m.role == "system") m.role = "developer"; return m }) - var body = { - model: aModel, + var _schema = _normalizeResponseSchema(aResponseSchema) + var _responsesBody = { + model : aModel, temperature: aTemperature, - messages: msgs + input : _toResponsesInput(msgs) + } + if (isDef(_schema)) { + _responsesBody.text = { + format: merge({ type: "json_schema" }, _schema) + } } - if (isDef(aResponseSchema) && isMap(aResponseSchema)) { - body.response_format = { - type: "json_schema", - json_schema: { - name: aResponseSchema.name || "Response", - description: aResponseSchema.description || "API Response", - schema: aResponseSchema.schema || {}, - strict: aResponseSchema.strict !== false - } + var _chatBody = { + model : aModel, + temperature: aTemperature, + messages : msgs + } + if (isDef(_schema)) { + _chatBody.response_format = { + type : "json_schema", + json_schema: _schema } } - body = merge(body, aOptions.params) - if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'client'}, merge({_t:nowNano(),_f:'client'}, body)) - var _res = _r._request(_route("chat/completions", aModel), body) - if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'llm'}, merge({_t:nowNano(),_f:'llm'}, _res)) - _captureStats(_res, body) + _responsesBody = merge(_responsesBody, aOptions.params) + _chatBody = merge(_chatBody, aOptions.params) + + var _res = __ + var _requestBody = __ + if (_responsesAvailable !== false) { + _requestBody = _responsesBody + if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'client'}, merge({_t:nowNano(),_f:'client'}, _requestBody)) + _res = _r._request(_route("responses", aModel), _requestBody) + if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'llm'}, merge({_t:nowNano(),_f:'llm'}, _res)) + if (isMap(_res) && isMap(_res.error) && _isResponsesUnavailable(_res)) { + _responsesAvailable = false + _requestBody = _chatBody + if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'client'}, merge({_t:nowNano(),_f:'client'}, _requestBody)) + _res = _r._request(_route("chat/completions", aModel), _requestBody) + if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'llm'}, merge({_t:nowNano(),_f:'llm'}, _res)) + } else if (!isMap(_res) || isUnDef(_res.error)) { + _responsesAvailable = true + } + } else { + _requestBody = _chatBody + if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'client'}, merge({_t:nowNano(),_f:'client'}, _requestBody)) + _res = _r._request(_route("chat/completions", aModel), _requestBody) + if (isDef(_debugCh)) $ch(_debugCh).set({_t:nowNano(),_f:'llm'}, merge({_t:nowNano(),_f:'llm'}, _res)) + } + + _captureStats(_res, _requestBody) return _res }, rawImgGen: (aPrompt, aModel) => { @@ -3630,6 +3736,25 @@ OpenWrap.ai.prototype.gpt.prototype.rawPromptWithStats = function(aPrompt, aRole OpenWrap.ai.prototype.gpt.prototype.jsonSchemaPrompt = function(aPrompt, aResponseSchema, aModel, aTemperature, tools) { if (!isFunction(this.model.rawResponse)) throw "JSON Schema responses not supported by this provider" var response = this.model.rawResponse(aPrompt, aModel, aTemperature, false, tools, aResponseSchema) + if (isString(response.output_text) && response.output_text.length > 0) { + return jsonParse(response.output_text, __, __, true) + } + if (isArray(response.output)) { + var _parts = [] + response.output.forEach(o => { + if (isArray(o.content)) { + o.content.forEach(c => { + if (isString(c.text)) _parts.push(c.text) + else if (isDef(c.json)) _parts.push(c.json) + }) + } + }) + if (_parts.length > 0) { + var _out = _parts[0] + if (_parts.length > 1 && _parts.every(isString)) _out = _parts.join("") + return isString(_out) ? jsonParse(_out, __, __, true) : _out + } + } if (isArray(response.choices) && response.choices.length > 0) { if (response.choices[0].finish_reason == "stop") { var content = response.choices[0].message.content diff --git a/tests/autoTestAll.AI.js b/tests/autoTestAll.AI.js index d86fc725..5b60013e 100644 --- a/tests/autoTestAll.AI.js +++ b/tests/autoTestAll.AI.js @@ -404,11 +404,13 @@ var openai = new ow.ai.gpt("openai", { key: "test-key", url: "https://api.openai.com" }); var openaiTransport = openai.model.__debugTransport("chat/completions", "gpt-test"); ow.test.assert(openaiTransport.url, "https://api.openai.com/v1/chat/completions", "Problem building default OpenAI chat URL."); + ow.test.assert(openai.model.__debugTransport("responses", "gpt-test").url, "https://api.openai.com/v1/responses", "Problem building default OpenAI responses URL."); ow.test.assert(isDef(openaiTransport.headers.Authorization), true, "Problem setting default OpenAI bearer authorization."); ow.test.assert(isUnDef(openaiTransport.headers["api-key"]), true, "Problem avoiding api-key on default OpenAI transport."); var openaiBaseUrl = new ow.ai.gpt("openai", { key: "test-key", url: "https://api.openai.com/v1" }); ow.test.assert(openaiBaseUrl.model.__debugTransport("chat/completions", "gpt-test").url, "https://api.openai.com/v1/chat/completions", "Problem preserving an OpenAI v1 base URL."); + ow.test.assert(openaiBaseUrl.model.__debugTransport("responses", "gpt-test").url, "https://api.openai.com/v1/responses", "Problem preserving an OpenAI v1 responses URL."); var azureV1 = new ow.ai.gpt("openai", { key : "test-key", @@ -417,6 +419,7 @@ }); var azureV1Transport = azureV1.model.__debugTransport("chat/completions", "deployment-a"); ow.test.assert(azureV1Transport.url, "https://example.openai.azure.com/openai/v1/chat/completions", "Problem building Azure OpenAI v1 chat URL."); + ow.test.assert(azureV1.model.__debugTransport("responses", "deployment-a").url, "https://example.openai.azure.com/openai/v1/responses", "Problem building Azure OpenAI v1 responses URL."); ow.test.assert("" + azureV1Transport.headers["api-key"], "test-key", "Problem setting Azure OpenAI v1 api-key header."); ow.test.assert(isUnDef(azureV1Transport.headers.Authorization), true, "Problem suppressing Azure OpenAI v1 bearer authorization."); @@ -426,6 +429,7 @@ mode: "azure-openai-v1" }); ow.test.assert(azureV1BaseUrl.model.__debugTransport("chat/completions", "deployment-a").url, "https://example.openai.azure.com/openai/v1/chat/completions", "Problem preserving an Azure OpenAI v1 base URL."); + ow.test.assert(azureV1BaseUrl.model.__debugTransport("responses", "deployment-a").url, "https://example.openai.azure.com/openai/v1/responses", "Problem preserving an Azure OpenAI v1 responses URL."); var azureLegacy = new ow.ai.gpt("openai", { key : "test-key", @@ -436,6 +440,7 @@ }); var azureLegacyTransport = azureLegacy.model.__debugTransport("chat/completions", "ignored"); ow.test.assert(azureLegacyTransport.url, "https://example.openai.azure.com/openai/deployments/deployment-a/chat/completions?api-version=2024-10-21", "Problem building Azure OpenAI legacy deployment chat URL."); + ow.test.assert(azureLegacy.model.__debugTransport("responses", "ignored").url, "https://example.openai.azure.com/openai/deployments/deployment-a/responses?api-version=2024-10-21", "Problem building Azure OpenAI legacy deployment responses URL."); ow.test.assert("" + azureLegacyTransport.headers["api-key"], "test-key", "Problem setting Azure OpenAI legacy api-key header."); var foundryV1 = new ow.ai.gpt("openai", { @@ -445,6 +450,7 @@ }); var foundryV1Transport = foundryV1.model.__debugTransport("chat/completions", "deployment-a"); ow.test.assert(foundryV1Transport.url, "https://example.services.ai.azure.com/openai/v1/chat/completions", "Problem building Foundry v1 chat URL."); + ow.test.assert(foundryV1.model.__debugTransport("responses", "deployment-a").url, "https://example.services.ai.azure.com/openai/v1/responses", "Problem building Foundry v1 responses URL."); ow.test.assert("" + foundryV1Transport.headers["api-key"], "test-key", "Problem setting Foundry v1 api-key header."); var foundryPreview = new ow.ai.gpt("openai", { @@ -455,6 +461,7 @@ }); var foundryPreviewTransport = foundryPreview.model.__debugTransport("chat/completions", "deployment-a"); ow.test.assert(foundryPreviewTransport.url, "https://example.services.ai.azure.com/models/chat/completions?api-version=2024-05-01-preview", "Problem building Foundry model inference preview chat URL."); + ow.test.assert(foundryPreview.model.__debugTransport("responses", "deployment-a").url, "https://example.services.ai.azure.com/models/responses?api-version=2024-05-01-preview", "Problem building Foundry model inference preview responses URL."); ow.test.assert("" + foundryPreviewTransport.headers["api-key"], "test-key", "Problem setting Foundry preview api-key header."); }; @@ -840,23 +847,26 @@ requests.push({ url: url, body: __cloneForTest(body) }); return { model: "gpt-4o", - choices: [ + output_text: '{"name":"Alice","age":30}', + output: [ { - finish_reason: "stop", - message: { role: "assistant", content: '{"name":"Alice","age":30}' } + content: [ + { type: "output_text", text: '{"name":"Alice","age":30}' } + ] } ], - usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 } + usage: { input_tokens: 20, output_tokens: 10, total_tokens: 30 } }; }; var rawRes = g.model.rawResponse("Extract: Alice, 30", "gpt-4o", 0.5, false, [], schema); ow.test.assert(requests.length, 1, "Problem: rawResponse should make exactly one request."); - ow.test.assert(requests[0].url.indexOf("chat/completions") >= 0, true, "Problem: rawResponse should use chat/completions endpoint."); - ow.test.assert(requests[0].body.response_format.type, "json_schema", "Problem: rawResponse should set response_format.type to json_schema."); - ow.test.assert(requests[0].body.response_format.json_schema.name, "PersonInfo", "Problem: rawResponse should pass schema name."); - ow.test.assert(requests[0].body.response_format.json_schema.strict, true, "Problem: rawResponse should pass strict flag."); - ow.test.assert(isArray(rawRes.choices) && rawRes.choices.length > 0, true, "Problem: rawResponse should return choices array."); + ow.test.assert(requests[0].url.indexOf("responses") >= 0, true, "Problem: rawResponse should prefer the responses endpoint."); + ow.test.assert(requests[0].body.text.format.type, "json_schema", "Problem: rawResponse should set text.format.type to json_schema."); + ow.test.assert(requests[0].body.text.format.name, "PersonInfo", "Problem: rawResponse should pass schema name."); + ow.test.assert(requests[0].body.text.format.strict, true, "Problem: rawResponse should pass strict flag."); + ow.test.assert(requests[0].body.input[0].content[0].text, "Extract: Alice, 30", "Problem: rawResponse should convert prompt messages to responses input."); + ow.test.assert(rawRes.output_text, '{"name":"Alice","age":30}', "Problem: rawResponse should return the responses payload."); }; exports.testAIOpenAIJsonSchemaPrompt = function() { @@ -881,13 +891,14 @@ g.model._request = function(url, body) { return { model: "gpt-4o", - choices: [ + output: [ { - finish_reason: "stop", - message: { role: "assistant", content: '{"name":"Alice","age":30}' } + content: [ + { type: "output_text", text: '{"name":"Alice","age":30}' } + ] } ], - usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 } + usage: { input_tokens: 20, output_tokens: 10, total_tokens: 30 } }; }; @@ -917,13 +928,15 @@ g.model._request = function(url, body) { return { model: "gpt-4o", - choices: [ + output_text: '{"name":"Bob"}', + output: [ { - finish_reason: "stop", - message: { role: "assistant", content: '{"name":"Bob"}' } + content: [ + { type: "output_text", text: '{"name":"Bob"}' } + ] } ], - usage: { prompt_tokens: 15, completion_tokens: 8, total_tokens: 23 } + usage: { input_tokens: 15, output_tokens: 8, total_tokens: 23 } }; }; @@ -945,13 +958,15 @@ requests.push(__cloneForTest(body)); return { model: "gpt-4o", - choices: [ + output_text: '{"val":1}', + output: [ { - finish_reason: "stop", - message: { role: "assistant", content: '{"val":1}' } + content: [ + { type: "output_text", text: '{"val":1}' } + ] } ], - usage: { prompt_tokens: 5, completion_tokens: 5, total_tokens: 10 } + usage: { input_tokens: 5, output_tokens: 5, total_tokens: 10 } }; }; @@ -960,20 +975,143 @@ name: "MySchema", schema: { type: "object", additionalProperties: false } }); - ow.test.assert(requests[0].response_format.json_schema.description, "API Response", "Problem: rawResponse should default description to 'API Response'."); - ow.test.assert(requests[0].response_format.json_schema.strict, true, "Problem: rawResponse should default strict to true."); + ow.test.assert(requests[0].text.format.description, "API Response", "Problem: rawResponse should default description to 'API Response'."); + ow.test.assert(requests[0].text.format.strict, true, "Problem: rawResponse should default strict to true."); + }; + + exports.testAIOpenAIRawResponseFallsBackToChatCompletions = function() { + ow.loadAI(); + + var g = new ow.ai.gpt("openai", { key: "test-key", model: "gpt-4o" }); + var requests = []; + var schema = { + name: "PersonInfo", + schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"], + additionalProperties: false + } + }; + + g.model._request = function(url, body) { + requests.push({ url: url, body: __cloneForTest(body) }); + if (url.indexOf("responses") >= 0) { + return { + error: { + message: "The /responses endpoint is not available for this deployment.", + code: "not_found" + }, + status: 404 + }; + } + return { + model: "gpt-4o", + choices: [ + { + finish_reason: "stop", + message: { role: "assistant", content: '{"name":"Alice"}' } + } + ], + usage: { prompt_tokens: 8, completion_tokens: 4, total_tokens: 12 } + }; + }; + + var rawRes = g.model.rawResponse("Extract: Alice", "gpt-4o", 0.5, false, [], schema); + ow.test.assert(requests.length, 2, "Problem: rawResponse should retry with chat/completions when responses is unavailable."); + ow.test.assert(requests[0].url.indexOf("responses") >= 0, true, "Problem: rawResponse should try responses first."); + ow.test.assert(requests[1].url.indexOf("chat/completions") >= 0, true, "Problem: rawResponse should fall back to chat/completions."); + ow.test.assert(requests[1].body.response_format.type, "json_schema", "Problem: chat fallback should keep json_schema response_format."); + ow.test.assert(isArray(rawRes.choices) && rawRes.choices.length > 0, true, "Problem: chat fallback should return the chat response payload."); + }; + + exports.testAIOpenAIRawResponseCachesChatFallback = function() { + ow.loadAI(); + + var g = new ow.ai.gpt("openai", { key: "test-key", model: "gpt-4o" }); + var requests = []; + var schema = { + name: "PersonInfo", + schema: { + type: "object", + properties: { + name: { type: "string" } + }, + required: ["name"], + additionalProperties: false + } + }; + + g.model._request = function(url, body) { + requests.push(url); + if (url.indexOf("responses") >= 0) { + return { + error: { + message: "The /responses endpoint is not available for this deployment.", + code: "not_found" + }, + status: 404 + }; + } + return { + model: "gpt-4o", + choices: [ + { + finish_reason: "stop", + message: { role: "assistant", content: '{"name":"Alice"}' } + } + ], + usage: { prompt_tokens: 8, completion_tokens: 4, total_tokens: 12 } + }; + }; + + g.model.rawResponse("Extract: Alice", "gpt-4o", 0.5, false, [], schema); + g.model.rawResponse("Extract: Alice again", "gpt-4o", 0.5, false, [], schema); + ow.test.assert(requests.filter(r => r.indexOf("responses") >= 0).length, 1, "Problem: rawResponse should stop retrying responses after endpoint unavailability."); + ow.test.assert(requests.filter(r => r.indexOf("chat/completions") >= 0).length, 2, "Problem: rawResponse should reuse chat/completions fallback after responses becomes unavailable."); + }; + + exports.testAIOpenAIRawResponseDoesNotFallbackOnOtherErrors = function() { + ow.loadAI(); + + var g = new ow.ai.gpt("openai", { key: "test-key", model: "gpt-4o" }); + var requests = []; + + g.model._request = function(url, body) { + requests.push(url); + return { + error: { + message: "Invalid schema for text.format", + code: "invalid_request_error" + }, + status: 400 + }; + }; + + var res = g.model.rawResponse("Extract: Alice", "gpt-4o", 0.5, false, [], { + name: "PersonInfo", + schema: { type: "object" } + }); + ow.test.assert(requests.length, 1, "Problem: rawResponse should not fall back for non-availability errors."); + ow.test.assert(requests[0].indexOf("responses") >= 0, true, "Problem: rawResponse should return the original responses error."); + ow.test.assert(isMap(res.error), true, "Problem: rawResponse should return the original error response."); }; exports.testAIOpenAIJsonSchemaPromptUnsupportedProvider = function() { ow.loadAI(); - var g = new ow.ai.gpt("gemini", { key: "test-key" }); - var threw = false; - try { - g.jsonSchemaPrompt("test", { name: "T", schema: {} }); - } catch(e) { - threw = true; - } - ow.test.assert(threw, true, "Problem: jsonSchemaPrompt should throw for providers that don't support rawResponse."); + [ "gemini", "ollama", "anthropic" ].forEach(provider => { + var g = provider == "ollama" ? new ow.ai.gpt(provider, { url: "http://127.0.0.1:11434", model: "llama-test" }) : new ow.ai.gpt(provider, { key: "test-key" }); + var threw = false; + try { + g.jsonSchemaPrompt("test", { name: "T", schema: {} }); + } catch(e) { + threw = true; + ow.test.assert(String(e), "JSON Schema responses not supported by this provider", "Problem: unsupported providers should throw a consistent error."); + } + ow.test.assert(threw, true, "Problem: jsonSchemaPrompt should throw for providers that don't support rawResponse."); + }); }; })(); diff --git a/tests/autoTestAll.AI.yaml b/tests/autoTestAll.AI.yaml index 1421e0e1..bf5199a4 100644 --- a/tests/autoTestAll.AI.yaml +++ b/tests/autoTestAll.AI.yaml @@ -125,10 +125,10 @@ jobs: to : oJob Test exec: args.func = args.tests.testAIOllamaStreamingWarmsModelOnFreshConversation; - - name: AI::OpenAI rawResponse method - from: AI::Init - to : oJob Test - exec: args.func = args.tests.testAIOpenAIRawResponse; + - name: AI::OpenAI rawResponse prefers responses + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIRawResponse; - name: AI::OpenAI jsonSchemaPrompt from: AI::Init @@ -140,15 +140,30 @@ jobs: to : oJob Test exec: args.func = args.tests.testAIOpenAIJsonSchemaPromptWithStats; - - name: AI::OpenAI jsonSchemaPrompt schema defaults - from: AI::Init - to : oJob Test - exec: args.func = args.tests.testAIOpenAIJsonSchemaPromptSchemaDefault; - - - name: AI::OpenAI jsonSchemaPrompt unsupported provider - from: AI::Init - to : oJob Test - exec: args.func = args.tests.testAIOpenAIJsonSchemaPromptUnsupportedProvider; + - name: AI::OpenAI jsonSchemaPrompt schema defaults + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIJsonSchemaPromptSchemaDefault; + + - name: AI::OpenAI rawResponse fallback to chat completions + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIRawResponseFallsBackToChatCompletions; + + - name: AI::OpenAI rawResponse caches chat fallback + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIRawResponseCachesChatFallback; + + - name: AI::OpenAI rawResponse keeps non availability errors + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIRawResponseDoesNotFallbackOnOtherErrors; + + - name: AI::OpenAI jsonSchemaPrompt unsupported provider + from: AI::Init + to : oJob Test + exec: args.func = args.tests.testAIOpenAIJsonSchemaPromptUnsupportedProvider; todo: # ow.ai tests @@ -176,8 +191,11 @@ todo: - AI::OpenAI export conversation falls back to tool name - AI::Ollama streaming tool execution and ids - AI::Ollama streaming warmup on fresh conversation - - AI::OpenAI rawResponse method - - AI::OpenAI jsonSchemaPrompt - - AI::OpenAI jsonSchemaPromptWithStats - - AI::OpenAI jsonSchemaPrompt schema defaults + - AI::OpenAI rawResponse prefers responses + - AI::OpenAI jsonSchemaPrompt + - AI::OpenAI jsonSchemaPromptWithStats + - AI::OpenAI jsonSchemaPrompt schema defaults + - AI::OpenAI rawResponse fallback to chat completions + - AI::OpenAI rawResponse caches chat fallback + - AI::OpenAI rawResponse keeps non availability errors - AI::OpenAI jsonSchemaPrompt unsupported provider