From 8b47bf5f64850989fba8ecf4723c56661871a62c Mon Sep 17 00:00:00 2001 From: adas0x Date: Tue, 6 Oct 2026 07:58:14 -0300 Subject: [PATCH] chat: clamp generation to OpenAI response_format via StructuredOutputConfig --- src/engine/ov_genai/llm.py | 5 ++ src/engine/ov_genai/utils.py | 17 ++++- src/engine/ov_genai/vlm.py | 5 ++ src/server/routes/openai.py | 1 + .../modeling/contract_ovgenai_llm_and_vlm.py | 8 +++ src/server/schemas/requests_openai.py | 1 + src/server/utils/structured_output.py | 67 +++++++++++++++++ tests/unit/test_ov_genai_llm_unit.py | 57 ++++++++++++++- tests/unit/test_ov_genai_vlm_unit.py | 30 ++++++++ tests/unit/test_structured_output_unit.py | 72 +++++++++++++++++++ 10 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 src/server/utils/structured_output.py create mode 100644 tests/unit/test_structured_output_unit.py diff --git a/src/engine/ov_genai/llm.py b/src/engine/ov_genai/llm.py index 673fdd8f..4f3e0baf 100755 --- a/src/engine/ov_genai/llm.py +++ b/src/engine/ov_genai/llm.py @@ -1,5 +1,6 @@ from src.engine.ov_genai.utils import ( apply_temperature, + build_structured_output_config, extract_scheduler_config_from_loader, ) import asyncio @@ -350,4 +351,8 @@ def create_generation_config(self, config: OVGenAI_GenConfig) -> GenerationConfi else: default_tokens = int(os.getenv('OPENARC_DEFAULT_NUM_ASSISTANT_TOKENS', '3')) generation_kwargs.num_assistant_tokens = default_tokens + + structured_output = build_structured_output_config(config.response_format) + if structured_output is not None: + generation_kwargs.structured_output_config = structured_output return generation_kwargs diff --git a/src/engine/ov_genai/utils.py b/src/engine/ov_genai/utils.py index 746a6440..ce2d925d 100644 --- a/src/engine/ov_genai/utils.py +++ b/src/engine/ov_genai/utils.py @@ -1,11 +1,12 @@ """OV GenAI engine utilities.""" import logging -from typing import Literal +from typing import Any, Literal, Mapping -from openvino_genai import GenerationConfig, SchedulerConfig +from openvino_genai import GenerationConfig, SchedulerConfig, StructuredOutputConfig from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import SchedulerConfigSchema from src.server.schemas.registration import ModelLoadConfig +from src.server.utils.structured_output import structured_output_kwargs logger = logging.getLogger(__name__) @@ -22,6 +23,18 @@ def apply_temperature(generation_config: GenerationConfig, temperature: float) - generation_config.do_sample = False +def build_structured_output_config( + response_format: Mapping[str, Any] | None, +) -> StructuredOutputConfig | None: + """Build the OpenVINO GenAI structured-output config for a request, if any. + + Returns ``None`` when the request asks for no constraint, so callers leave + the generation config's own default untouched. + """ + kwargs = structured_output_kwargs(response_format) + return StructuredOutputConfig(**kwargs) if kwargs else None + + def generate_ov_scheduler_config(scheduler_config: SchedulerConfigSchema) -> dict: """Generates a SchedulerConfig object from the scheduler config model. diff --git a/src/engine/ov_genai/vlm.py b/src/engine/ov_genai/vlm.py index 392ac199..445c0bdf 100644 --- a/src/engine/ov_genai/vlm.py +++ b/src/engine/ov_genai/vlm.py @@ -1,5 +1,6 @@ from src.engine.ov_genai.utils import ( apply_temperature, + build_structured_output_config, extract_scheduler_config_from_loader, ) import asyncio @@ -430,4 +431,8 @@ def create_generation_config(self, config: OVGenAI_GenConfig) -> GenerationConfi generation_kwargs.frequency_penalty = config.frequency_penalty if config.presence_penalty: generation_kwargs.presence_penalty = config.presence_penalty + + structured_output = build_structured_output_config(config.response_format) + if structured_output is not None: + generation_kwargs.structured_output_config = structured_output return generation_kwargs diff --git a/src/server/routes/openai.py b/src/server/routes/openai.py index 01d5df36..8ef56789 100644 --- a/src/server/routes/openai.py +++ b/src/server/routes/openai.py @@ -205,6 +205,7 @@ async def openai_chat_completions( "seed": request.seed, "frequency_penalty": request.frequency_penalty, "presence_penalty": request.presence_penalty, + "response_format": request.response_format, } if parser_module is not None: config_kwargs["tool_call_parser"] = tool_parser_name diff --git a/src/server/schemas/modeling/contract_ovgenai_llm_and_vlm.py b/src/server/schemas/modeling/contract_ovgenai_llm_and_vlm.py index 700d8675..2240b46a 100644 --- a/src/server/schemas/modeling/contract_ovgenai_llm_and_vlm.py +++ b/src/server/schemas/modeling/contract_ovgenai_llm_and_vlm.py @@ -97,6 +97,14 @@ class OVGenAI_GenConfig(BaseModel): default={}, description="Additional arguments to apply to the chat template." ) + response_format: Optional[Dict[str, Any]] = Field( + default=None, + description=( + "OpenAI response_format. 'json_schema' or 'json_object' clamps " + "generation to that grammar (OpenVINO GenAI StructuredOutputConfig / " + "xgrammar); 'text'/None leaves it unconstrained." + ), + ) @property def text_messages(self) -> List[Dict[str, Any]]: diff --git a/src/server/schemas/requests_openai.py b/src/server/schemas/requests_openai.py index 4a5318af..d8e6c7fe 100644 --- a/src/server/schemas/requests_openai.py +++ b/src/server/schemas/requests_openai.py @@ -46,6 +46,7 @@ class OpenAIChatCompletionRequest(BaseModel): frequency_penalty: Optional[float] = None presence_penalty: Optional[float] = None chat_template_kwargs: Optional[dict] = {} + response_format: Optional[Dict[str, Any]] = None class OpenAICompletionRequest(BaseModel): diff --git a/src/server/utils/structured_output.py b/src/server/utils/structured_output.py new file mode 100644 index 00000000..dfc57b1f --- /dev/null +++ b/src/server/utils/structured_output.py @@ -0,0 +1,67 @@ +"""Translate an OpenAI ``response_format`` into OpenVINO GenAI structured output. + +The OV GenAI engines clamp decoding with ``StructuredOutputConfig`` (xgrammar +backend); this module is the pure half of that wiring, mapping the request field +to the config's kwargs. Building the OpenVINO object is the engine's job. +""" +import json +from typing import Any, Dict, Mapping + +UNCONSTRAINED_TYPES = frozenset({"text", "text/plain"}) + +NOT_AN_OBJECT = "response_format must be an object with a 'type' key, got {kind}" +JSON_SCHEMA_FIELD_REQUIRED = ( + "response_format type 'json_schema' requires a 'json_schema' field" +) +JSON_SCHEMA_INVALID = "response_format.json_schema must be an object or a JSON string" +UNSUPPORTED_TYPE = ( + "unsupported response_format type {fmt_type!r}; " + "supported types: json_schema, json_object, text" +) + + +class ResponseFormatError(ValueError): + """A response_format that cannot be mapped to a grammar constraint.""" + + +def structured_output_kwargs( + response_format: Mapping[str, Any] | None, +) -> Dict[str, str] | None: + """Return ``StructuredOutputConfig`` kwargs for an OpenAI ``response_format``. + + ``json_schema`` (OpenAI's ``{name, strict, schema}`` wrapper or a bare + schema) and ``json_object`` produce a grammar constraint; ``text``/None + produce none. Anything else raises ``ResponseFormatError`` so a caller that + asked for clamped JSON never silently receives free-form text. + """ + if not response_format: + return None + if not isinstance(response_format, Mapping): + raise ResponseFormatError(NOT_AN_OBJECT.format(kind=type(response_format).__name__)) + + fmt_type = response_format.get("type") + if fmt_type is None or fmt_type in UNCONSTRAINED_TYPES: + return None + if fmt_type == "json_object": + return {"json_schema": json.dumps({"type": "object"})} + if fmt_type == "json_schema": + return {"json_schema": _json_schema_string(response_format)} + + raise ResponseFormatError(UNSUPPORTED_TYPE.format(fmt_type=fmt_type)) + + +def _json_schema_string(response_format: Mapping[str, Any]) -> str: + """Extract the JSON schema string from a ``json_schema`` response_format.""" + json_schema = response_format.get("json_schema") + if json_schema is None: + raise ResponseFormatError(JSON_SCHEMA_FIELD_REQUIRED) + if isinstance(json_schema, Mapping): + # OpenAI nests the schema under "schema"; llama.cpp and others pass the + # schema itself. "schema" is not a JSON Schema keyword, so a top-level + # "schema" key is unambiguously the wrapper. + json_schema = json_schema.get("schema", json_schema) + if isinstance(json_schema, str): + return json_schema + if isinstance(json_schema, Mapping): + return json.dumps(json_schema) + raise ResponseFormatError(JSON_SCHEMA_INVALID) diff --git a/tests/unit/test_ov_genai_llm_unit.py b/tests/unit/test_ov_genai_llm_unit.py index b136e5f9..6da3f232 100644 --- a/tests/unit/test_ov_genai_llm_unit.py +++ b/tests/unit/test_ov_genai_llm_unit.py @@ -1,11 +1,14 @@ +import json from unittest.mock import AsyncMock, MagicMock import pytest # type: ignore[import] import src.engine.ov_genai.llm as llm_module +import src.engine.ov_genai.utils as utils_module from src.engine.ov_genai.llm import OVGenAI_LLM from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig +from src.server.utils.structured_output import ResponseFormatError MODEL_PATH ="some_fake_url/Qwen3-Reranker-0.6B-fp16-ov" @@ -270,11 +273,20 @@ async def test_unload_model_resets_state(monkeypatch: pytest.MonkeyPatch, load_c gc_mock.assert_called_once() - class DummyGenerationConfig: pass +class CapturingStructuredOutputConfig: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + +def _patch_generation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(llm_module, "GenerationConfig", DummyGenerationConfig) + monkeypatch.setattr(utils_module, "StructuredOutputConfig", CapturingStructuredOutputConfig) + + def test_create_generation_config_zero_temperature_disables_sampling( monkeypatch: pytest.MonkeyPatch, load_config: ModelLoadConfig ) -> None: @@ -299,3 +311,46 @@ def test_create_generation_config_positive_temperature_keeps_sampling( assert getattr(config, "do_sample", None) is None assert config.temperature == 0.7 + + +def test_create_generation_config_applies_json_schema_clamp( + monkeypatch: pytest.MonkeyPatch, load_config: ModelLoadConfig +) -> None: + _patch_generation(monkeypatch) + llm = OVGenAI_LLM(load_config) + llm.model = None + + gen_config = OVGenAI_GenConfig( + response_format={ + "type": "json_schema", + "json_schema": {"name": "x", "strict": True, "schema": {"type": "object"}}, + } + ) + generation = llm.create_generation_config(gen_config) + + clamped = generation.structured_output_config + assert isinstance(clamped, CapturingStructuredOutputConfig) + assert json.loads(clamped.kwargs["json_schema"]) == {"type": "object"} + + +def test_create_generation_config_without_response_format_leaves_it_unset( + monkeypatch: pytest.MonkeyPatch, load_config: ModelLoadConfig +) -> None: + _patch_generation(monkeypatch) + llm = OVGenAI_LLM(load_config) + llm.model = None + + generation = llm.create_generation_config(OVGenAI_GenConfig()) + + assert getattr(generation, "structured_output_config", None) is None + + +def test_create_generation_config_rejects_unsupported_response_format( + monkeypatch: pytest.MonkeyPatch, load_config: ModelLoadConfig +) -> None: + _patch_generation(monkeypatch) + llm = OVGenAI_LLM(load_config) + llm.model = None + + with pytest.raises(ResponseFormatError): + llm.create_generation_config(OVGenAI_GenConfig(response_format={"type": "regex"})) diff --git a/tests/unit/test_ov_genai_vlm_unit.py b/tests/unit/test_ov_genai_vlm_unit.py index c29b28dc..0759553e 100644 --- a/tests/unit/test_ov_genai_vlm_unit.py +++ b/tests/unit/test_ov_genai_vlm_unit.py @@ -6,6 +6,7 @@ import pytest # type: ignore[import] import src.engine.ov_genai.vlm as vlm_module +import src.engine.ov_genai.utils as utils_module from src.engine.ov_genai.vlm import OVGenAI_VLM from src.server.schemas.registration import EngineType, ModelLoadConfig, ModelType from src.server.schemas.modeling.contract_ovgenai_llm_and_vlm import OVGenAI_GenConfig @@ -270,3 +271,32 @@ def test_vlm_create_generation_config_zero_temperature_disables_sampling( assert config.do_sample is False assert config.temperature == 0.0 + + +class CapturingVlmStructuredOutputConfig: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + +def test_vlm_create_generation_config_applies_json_schema_clamp( + monkeypatch: pytest.MonkeyPatch, load_config: ModelLoadConfig +) -> None: + monkeypatch.setattr(vlm_module, "GenerationConfig", DummyVlmGenerationConfig) + monkeypatch.setattr( + utils_module, "StructuredOutputConfig", CapturingVlmStructuredOutputConfig + ) + + vlm = OVGenAI_VLM(load_config) + vlm.model_path = None + + gen_config = OVGenAI_GenConfig( + response_format={ + "type": "json_schema", + "json_schema": {"name": "x", "schema": {"type": "object", "properties": {}}}, + } + ) + generation = vlm.create_generation_config(gen_config) + + clamped = generation.structured_output_config + assert isinstance(clamped, CapturingVlmStructuredOutputConfig) + assert json.loads(clamped.kwargs["json_schema"]) == {"type": "object", "properties": {}} diff --git a/tests/unit/test_structured_output_unit.py b/tests/unit/test_structured_output_unit.py new file mode 100644 index 00000000..bfc71bf2 --- /dev/null +++ b/tests/unit/test_structured_output_unit.py @@ -0,0 +1,72 @@ +import json +from typing import Any, Mapping + +import pytest # type: ignore[import] + +from src.server.utils.structured_output import ( + JSON_SCHEMA_FIELD_REQUIRED, + JSON_SCHEMA_INVALID, + NOT_AN_OBJECT, + ResponseFormatError, + UNSUPPORTED_TYPE, + structured_output_kwargs, +) + +OBJECT_SCHEMA = {"type": "object", "properties": {"contacts": {"type": "array"}}} + + +@pytest.mark.parametrize( + "response_format", + [None, {}, {"type": "text"}, {"type": "text/plain"}], +) +def test_unconstrained_formats_return_none( + response_format: Mapping[str, Any] | None, +) -> None: + assert structured_output_kwargs(response_format) is None + + +def test_json_object_clamps_to_any_object() -> None: + assert structured_output_kwargs({"type": "json_object"}) == { + "json_schema": json.dumps({"type": "object"}) + } + + +@pytest.mark.parametrize( + "json_schema", + [ + {"name": "contact", "strict": True, "schema": OBJECT_SCHEMA}, + OBJECT_SCHEMA, + json.dumps(OBJECT_SCHEMA), + {"schema": json.dumps(OBJECT_SCHEMA)}, + ], + ids=["openai-wrapper", "bare-schema", "pre-encoded", "encoded-in-wrapper"], +) +def test_json_schema_is_clamped(json_schema: Any) -> None: + result = structured_output_kwargs( + {"type": "json_schema", "json_schema": json_schema} + ) + assert json.loads(result["json_schema"]) == OBJECT_SCHEMA + + +def test_missing_json_schema_field_raises() -> None: + with pytest.raises(ResponseFormatError) as exc: + structured_output_kwargs({"type": "json_schema"}) + assert str(exc.value) == JSON_SCHEMA_FIELD_REQUIRED + + +def test_invalid_json_schema_raises() -> None: + with pytest.raises(ResponseFormatError) as exc: + structured_output_kwargs({"type": "json_schema", "json_schema": 42}) + assert str(exc.value) == JSON_SCHEMA_INVALID + + +def test_unsupported_type_raises() -> None: + with pytest.raises(ResponseFormatError) as exc: + structured_output_kwargs({"type": "regex"}) + assert str(exc.value) == UNSUPPORTED_TYPE.format(fmt_type="regex") + + +def test_non_mapping_raises() -> None: + with pytest.raises(ResponseFormatError) as exc: + structured_output_kwargs("json_object") + assert str(exc.value) == NOT_AN_OBJECT.format(kind="str")