From 4679e5062425c4ecc131ee5481af427e15198080 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Fri, 14 Aug 2026 05:01:56 +0800 Subject: [PATCH] feat: support vit-gpt2 image captioning --- .../image-to-text_fp16_config_decoder.json | 507 ++++++++++++++++++ .../image-to-text_fp16_config_encoder.json | 73 +++ .../image-to-text_fp32_config_decoder.json | 485 +++++++++++++++++ .../image-to-text_fp32_config_encoder.json | 51 ++ .../modelkit/eval/image_to_text_evaluator.py | 7 +- src/winml/modelkit/inference/pipeline.py | 93 +++- src/winml/modelkit/loader/resolution.py | 5 + .../unit/eval/test_image_to_text_evaluator.py | 32 +- tests/unit/inference/test_pipeline.py | 129 +++++ .../unit/loader/test_detect_task_and_class.py | 50 +- 10 files changed, 1422 insertions(+), 10 deletions(-) create mode 100644 examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp16_config_decoder.json create mode 100644 examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp16_config_encoder.json create mode 100644 examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp32_config_decoder.json create mode 100644 examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp32_config_encoder.json diff --git a/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp16_config_decoder.json b/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp16_config_decoder.json new file mode 100644 index 000000000..7d8344530 --- /dev/null +++ b/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp16_config_decoder.json @@ -0,0 +1,507 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "decoder_input_ids", + "dtype": "int32", + "shape": [ + 1, + 1 + ], + "value_range": [ + 0, + 50257 + ] + }, + { + "name": "encoder_hidden_states", + "dtype": "float32", + "shape": [ + 1, + 197, + 768 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "decoder_attention_mask", + "dtype": "int64", + "shape": [ + 1, + 1024 + ] + }, + { + "name": "cache_position", + "dtype": "int64", + "shape": [ + 1 + ] + }, + { + "name": "past_0_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_0_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_1_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_1_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_2_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_2_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_3_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_3_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_4_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_4_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_5_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_5_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_6_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_6_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_7_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_7_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_8_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_8_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_9_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_9_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_10_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_10_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_11_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_11_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "logits" + }, + { + "name": "present_0_key" + }, + { + "name": "present_0_value" + }, + { + "name": "present_1_key" + }, + { + "name": "present_1_value" + }, + { + "name": "present_2_key" + }, + { + "name": "present_2_value" + }, + { + "name": "present_3_key" + }, + { + "name": "present_3_value" + }, + { + "name": "present_4_key" + }, + { + "name": "present_4_value" + }, + { + "name": "present_5_key" + }, + { + "name": "present_5_value" + }, + { + "name": "present_6_key" + }, + { + "name": "present_6_value" + }, + { + "name": "present_7_key" + }, + { + "name": "present_7_value" + }, + { + "name": "present_8_key" + }, + { + "name": "present_8_value" + }, + { + "name": "present_9_key" + }, + { + "name": "present_9_value" + }, + { + "name": "present_10_key" + }, + { + "name": "present_10_value" + }, + { + "name": "present_11_key" + }, + { + "name": "present_11_value" + } + ], + "compatibility": { + "transformers_attention": "eager" + } + }, + "optim": { + "gelu_fusion": true, + "layer_norm_fusion": true, + "matmul_add_fusion": true, + "remove_isnan_in_attention_mask": true, + "reshape_mergedreshape": true + }, + "quant": { + "mode": "fp16", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint8", + "per_channel": false, + "symmetric": false, + "weight_symmetric": null, + "activation_symmetric": null, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "task": "text2text-generation", + "model_id": "nlpconnect/vit-gpt2-image-captioning", + "model_type": "vision-encoder-decoder", + "fp16_keep_io_types": true, + "fp16_op_block_list": null + }, + "compile": null, + "loader": { + "task": "text2text-generation", + "model_class": "VisionDecoderWrapper", + "model_type": "vision-encoder-decoder" + } +} \ No newline at end of file diff --git a/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp16_config_encoder.json b/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp16_config_encoder.json new file mode 100644 index 000000000..b317d5dd4 --- /dev/null +++ b/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp16_config_encoder.json @@ -0,0 +1,73 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "pixel_values", + "dtype": "float32", + "shape": [ + 1, + 3, + 224, + 224 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "encoder_hidden_states" + } + ], + "compatibility": { + "transformers_attention": "eager" + } + }, + "optim": { + "gelu_fusion": true, + "layer_norm_fusion": true, + "matmul_add_fusion": true, + "remove_isnan_in_attention_mask": true, + "reshape_mergedreshape": true + }, + "quant": { + "mode": "fp16", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint8", + "per_channel": false, + "symmetric": false, + "weight_symmetric": null, + "activation_symmetric": null, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "task": "image-feature-extraction", + "model_id": "nlpconnect/vit-gpt2-image-captioning", + "model_type": "vision-encoder-decoder", + "fp16_keep_io_types": true, + "fp16_op_block_list": null + }, + "compile": null, + "loader": { + "task": "image-feature-extraction", + "model_class": "VisionEncoderWrapper", + "model_type": "vision-encoder-decoder" + } +} \ No newline at end of file diff --git a/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp32_config_decoder.json b/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp32_config_decoder.json new file mode 100644 index 000000000..10b042903 --- /dev/null +++ b/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp32_config_decoder.json @@ -0,0 +1,485 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "decoder_input_ids", + "dtype": "int32", + "shape": [ + 1, + 1 + ], + "value_range": [ + 0, + 50257 + ] + }, + { + "name": "encoder_hidden_states", + "dtype": "float32", + "shape": [ + 1, + 197, + 768 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "decoder_attention_mask", + "dtype": "int64", + "shape": [ + 1, + 1024 + ] + }, + { + "name": "cache_position", + "dtype": "int64", + "shape": [ + 1 + ] + }, + { + "name": "past_0_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_0_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_1_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_1_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_2_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_2_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_3_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_3_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_4_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_4_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_5_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_5_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_6_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_6_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_7_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_7_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_8_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_8_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_9_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_9_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_10_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_10_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_11_key", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + }, + { + "name": "past_11_value", + "dtype": "float32", + "shape": [ + 1, + 12, + 1024, + 64 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "logits" + }, + { + "name": "present_0_key" + }, + { + "name": "present_0_value" + }, + { + "name": "present_1_key" + }, + { + "name": "present_1_value" + }, + { + "name": "present_2_key" + }, + { + "name": "present_2_value" + }, + { + "name": "present_3_key" + }, + { + "name": "present_3_value" + }, + { + "name": "present_4_key" + }, + { + "name": "present_4_value" + }, + { + "name": "present_5_key" + }, + { + "name": "present_5_value" + }, + { + "name": "present_6_key" + }, + { + "name": "present_6_value" + }, + { + "name": "present_7_key" + }, + { + "name": "present_7_value" + }, + { + "name": "present_8_key" + }, + { + "name": "present_8_value" + }, + { + "name": "present_9_key" + }, + { + "name": "present_9_value" + }, + { + "name": "present_10_key" + }, + { + "name": "present_10_value" + }, + { + "name": "present_11_key" + }, + { + "name": "present_11_value" + } + ], + "compatibility": { + "transformers_attention": "eager" + } + }, + "optim": { + "gelu_fusion": true, + "layer_norm_fusion": true, + "matmul_add_fusion": true, + "remove_isnan_in_attention_mask": true, + "reshape_mergedreshape": true + }, + "quant": null, + "compile": null, + "loader": { + "task": "text2text-generation", + "model_class": "VisionDecoderWrapper", + "model_type": "vision-encoder-decoder" + } +} \ No newline at end of file diff --git a/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp32_config_encoder.json b/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp32_config_encoder.json new file mode 100644 index 000000000..3898cf8fd --- /dev/null +++ b/examples/recipes/nlpconnect_vit-gpt2-image-captioning/cpu/cpu/image-to-text_fp32_config_encoder.json @@ -0,0 +1,51 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "pixel_values", + "dtype": "float32", + "shape": [ + 1, + 3, + 224, + 224 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "encoder_hidden_states" + } + ], + "compatibility": { + "transformers_attention": "eager" + } + }, + "optim": { + "gelu_fusion": true, + "layer_norm_fusion": true, + "matmul_add_fusion": true, + "remove_isnan_in_attention_mask": true, + "reshape_mergedreshape": true + }, + "quant": null, + "compile": null, + "loader": { + "task": "image-feature-extraction", + "model_class": "VisionEncoderWrapper", + "model_type": "vision-encoder-decoder" + } +} \ No newline at end of file diff --git a/src/winml/modelkit/eval/image_to_text_evaluator.py b/src/winml/modelkit/eval/image_to_text_evaluator.py index db63404ed..78ec4bf2c 100644 --- a/src/winml/modelkit/eval/image_to_text_evaluator.py +++ b/src/winml/modelkit/eval/image_to_text_evaluator.py @@ -73,7 +73,12 @@ def compute(self) -> dict[str, Any]: continue try: - out = self.pipe(image, text="") + pipeline_kwargs = ( + {"max_new_tokens": 32, "generate_kwargs": {"num_beams": 1}} + if isinstance(references, (list, tuple)) + else {} + ) + out = self.pipe(image, text="", **pipeline_kwargs) except Exception as e: logger.warning("Pipeline call failed (skipping): %s", e) skipped += 1 diff --git a/src/winml/modelkit/inference/pipeline.py b/src/winml/modelkit/inference/pipeline.py index 36b56ee58..cd2079bce 100644 --- a/src/winml/modelkit/inference/pipeline.py +++ b/src/winml/modelkit/inference/pipeline.py @@ -510,11 +510,95 @@ def _create_extractive_question_answering_pipeline( } -def _pipeline_component_kwargs(task: str, model_id: str | None) -> dict[str, str]: +class _ImageToTextProcessor: + """Compose separate image and text processors for encoder-decoder captioning.""" + + def __init__( + self, + image_processor: Any, + tokenizer: Any, + decoder_start_token_id: int, + ) -> None: + if not callable(image_processor): + raise TypeError("Image-to-text requires a callable image processor.") + if not callable(tokenizer) or not callable(getattr(tokenizer, "batch_decode", None)): + raise TypeError("Image-to-text requires a callable tokenizer with batch decoding.") + self.image_processor = image_processor + self.tokenizer = tokenizer + self.decoder_start_token_id = decoder_start_token_id + + def __call__( + self, + images: Any = None, + text: Any = None, + **kwargs: Any, + ) -> Any: + if images is None: + return self.tokenizer(text, **kwargs) + import torch + + inputs = self.image_processor(images, **kwargs) + if text: + inputs.update(self.tokenizer(text, **kwargs)) + else: + inputs["input_ids"] = torch.tensor([[self.decoder_start_token_id]], dtype=torch.int64) + return inputs + + def post_process_image_text_to_text(self, generated_outputs: Any, **kwargs: Any) -> Any: + return self.tokenizer.batch_decode(generated_outputs, **kwargs) + + +def _load_image_to_text_processor( + model_id: str, + *, + model: Any, + trust_remote_code: bool = False, +) -> Any: + """Load or compose the image and text capabilities required by image-to-text.""" + from transformers import AutoImageProcessor, AutoProcessor, ProcessorMixin + + load_kwargs = {"trust_remote_code": True} if trust_remote_code else {} + processor = AutoProcessor.from_pretrained(model_id, **load_kwargs) + if isinstance(processor, ProcessorMixin): + if not callable(getattr(processor, "image_processor", None)): + raise TypeError("Image-to-text processor is missing a callable image processor.") + if not callable(getattr(processor, "tokenizer", None)): + raise TypeError("Image-to-text processor is missing a callable tokenizer.") + if not callable(getattr(processor, "post_process_image_text_to_text", None)): + raise TypeError("Image-to-text processor is missing generation decoding support.") + return processor + + decoder_start_token_id = getattr( + getattr(model, "config", None), + "decoder_start_token_id", + None, + ) + if not isinstance(decoder_start_token_id, int): + raise TypeError("Image-to-text model config is missing decoder_start_token_id.") + image_processor = AutoImageProcessor.from_pretrained(model_id, **load_kwargs) + return _ImageToTextProcessor(image_processor, processor, decoder_start_token_id) + + +def _pipeline_component_kwargs( + task: str, + model_id: str | None, + *, + model: Any = None, + trust_remote_code: bool = False, +) -> dict[str, Any]: """Select model components from the resolved pipeline's capabilities.""" if model_id is None: return {} + if task == "image-text-to-text": + return { + "processor": _load_image_to_text_processor( + model_id, + model=model, + trust_remote_code=trust_remote_code, + ) + } + from transformers.pipelines import check_task _, targeted_task, _ = check_task(task) @@ -566,7 +650,12 @@ def create_pipeline( # "device" is for HF pipeline tensor placement, not ORT EP. # WinMLSession handles device delegation internally. "device": device, - **_pipeline_component_kwargs(hf_task, model_id), + **_pipeline_component_kwargs( + hf_task, + model_id, + model=model, + trust_remote_code=trust_remote_code, + ), } if trust_remote_code: kwargs["trust_remote_code"] = True diff --git a/src/winml/modelkit/loader/resolution.py b/src/winml/modelkit/loader/resolution.py index a1a372353..828cd12cd 100644 --- a/src/winml/modelkit/loader/resolution.py +++ b/src/winml/modelkit/loader/resolution.py @@ -668,6 +668,11 @@ def resolve_task( # --- Stage 4: composite tag (detection path) -------------------------- composite = _composite_components_for_task(model_type, opt_task) if model_type else None + if surfaced == "image-text-to-text" and model_type: + image_to_text_composite = _composite_components_for_task(model_type, "image-to-text") + if image_to_text_composite is not None: + surfaced = "image-to-text" + composite = image_to_text_composite if source is None: # structural invariant: Stage 1d always sets a source raise RuntimeError("resolve_task: internal invariant violated — source was not set") diff --git a/tests/unit/eval/test_image_to_text_evaluator.py b/tests/unit/eval/test_image_to_text_evaluator.py index a1f0b5d03..f2ac72568 100644 --- a/tests/unit/eval/test_image_to_text_evaluator.py +++ b/tests/unit/eval/test_image_to_text_evaluator.py @@ -15,8 +15,6 @@ def make_evaluator(columns_mapping=None): """Instantiate evaluator with mocked dataset + pipeline.""" - import transformers - from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig mapping = columns_mapping or {} @@ -40,12 +38,12 @@ def make_evaluator(columns_mapping=None): dataset=DatasetConfig(path="Teklia/IAM-line", columns_mapping=mapping), ) - # Resolve the lazy Transformers export before patching it. - assert hasattr(transformers, "pipeline") with ( patch("datasets.load_dataset", return_value=mock_ds), - patch("transformers.pipelines.pipeline", return_value=mock_pipe), - patch.object(transformers, "pipeline", return_value=mock_pipe), + patch( + "winml.modelkit.inference.pipeline.create_pipeline", + return_value=mock_pipe, + ), ): return WinMLImageToTextEvaluator(config, model) @@ -125,6 +123,28 @@ def test_dict_output_shape(self): assert result["cer"] == 0.0 assert result["n_samples"] == 1 + def test_caps_caption_generation_without_changing_ocr_calls(self): + ev = make_evaluator() + ev.data = [ + {"image": "caption-image", "text": ["caption one", "caption two"]}, + {"image": "ocr-image", "text": "OCR TEXT"}, + ] + ev.pipe = MagicMock( + side_effect=[ + [{"generated_text": "caption one"}], + [{"generated_text": "OCR TEXT"}], + ] + ) + + ev.compute() + + assert ev.pipe.call_args_list[0].kwargs == { + "text": "", + "max_new_tokens": 32, + "generate_kwargs": {"num_beams": 1}, + } + assert ev.pipe.call_args_list[1].kwargs == {"text": ""} + def test_skips_samples_with_missing_data(self): """None image or None text → skipped, n_samples reflects actual count.""" ev = make_evaluator() diff --git a/tests/unit/inference/test_pipeline.py b/tests/unit/inference/test_pipeline.py index 2482e3218..d33d5b1ed 100644 --- a/tests/unit/inference/test_pipeline.py +++ b/tests/unit/inference/test_pipeline.py @@ -29,6 +29,8 @@ _adapt_tokenizer_padding, _detect_tokenizer_dict_param, _ExtractiveQuestionAnsweringPipeline, + _ImageToTextProcessor, + _load_image_to_text_processor, _pipeline_component_kwargs, create_pipeline, ) @@ -187,6 +189,133 @@ class ProcessorOnlyPipeline: assert result == {"processor": "model-id"} + def test_image_to_text_composes_separate_image_processor_and_tokenizer(self) -> None: + tokenizer = MagicMock() + image_processor = MagicMock() + model = MagicMock() + model.config.decoder_start_token_id = 42 + + with ( + patch("transformers.AutoProcessor.from_pretrained", return_value=tokenizer), + patch( + "transformers.AutoImageProcessor.from_pretrained", + return_value=image_processor, + ), + ): + result = _pipeline_component_kwargs( + "image-text-to-text", "model-id", model=model + ) + + processor = result["processor"] + assert isinstance(processor, _ImageToTextProcessor) + assert processor.tokenizer is tokenizer + assert processor.image_processor is image_processor + assert processor.decoder_start_token_id == 42 + + def test_image_to_text_preserves_existing_combined_processor(self) -> None: + from transformers import ProcessorMixin, ViTImageProcessor + + class CombinedProcessor(ProcessorMixin): + image_processor_class = "AutoImageProcessor" + tokenizer_class = "AutoTokenizer" + + processor = CombinedProcessor(ViTImageProcessor(), _make_fast_qa_tokenizer()) + with ( + patch("transformers.AutoProcessor.from_pretrained", return_value=processor), + patch("transformers.AutoImageProcessor.from_pretrained") as load_image_processor, + ): + result = _load_image_to_text_processor("model-id", model=MagicMock()) + + assert result is processor + load_image_processor.assert_not_called() + + @pytest.mark.parametrize("missing_component", ["image_processor", "tokenizer"]) + def test_image_to_text_rejects_incomplete_combined_processor( + self, missing_component: str + ) -> None: + from transformers import ProcessorMixin, ViTImageProcessor + + class CombinedProcessor(ProcessorMixin): + image_processor_class = "AutoImageProcessor" + tokenizer_class = "AutoTokenizer" + + processor = CombinedProcessor(ViTImageProcessor(), _make_fast_qa_tokenizer()) + setattr(processor, missing_component, None) + + with ( + patch("transformers.AutoProcessor.from_pretrained", return_value=processor), + pytest.raises(TypeError, match=f"callable {missing_component.replace('_', ' ')}"), + ): + _load_image_to_text_processor("model-id", model=MagicMock()) + + @pytest.mark.parametrize( + ("image_processor", "tokenizer", "message"), + [ + (object(), MagicMock(), "callable image processor"), + (MagicMock(), object(), "callable tokenizer with batch decoding"), + ], + ) + def test_image_to_text_rejects_missing_component_capability( + self, + image_processor: Any, + tokenizer: Any, + message: str, + ) -> None: + with pytest.raises(TypeError, match=message): + _ImageToTextProcessor(image_processor, tokenizer, 42) + + def test_image_to_text_rejects_missing_decoder_start_token(self) -> None: + tokenizer = MagicMock() + image_processor = MagicMock() + model = MagicMock() + model.config.decoder_start_token_id = None + + with ( + patch("transformers.AutoProcessor.from_pretrained", return_value=tokenizer), + patch( + "transformers.AutoImageProcessor.from_pretrained", + return_value=image_processor, + ), + pytest.raises(TypeError, match="missing decoder_start_token_id"), + ): + _load_image_to_text_processor("model-id", model=model) + + def test_composed_processor_preprocesses_images_and_decodes_generation(self) -> None: + tokenizer = MagicMock() + tokenizer.batch_decode.return_value = ["a caption"] + image_processor = MagicMock(return_value={"pixel_values": torch.ones(1, 3, 4, 4)}) + processor = _ImageToTextProcessor(image_processor, tokenizer, 42) + + result = processor(images="image", text="", return_tensors="pt") + decoded = processor.post_process_image_text_to_text( + torch.tensor([[1, 2]]), skip_special_tokens=True + ) + + image_processor.assert_called_once_with("image", return_tensors="pt") + assert set(result) == {"pixel_values", "input_ids"} + assert result["input_ids"].tolist() == [[42]] + tokenizer.batch_decode.assert_called_once() + assert decoded == ["a caption"] + + def test_composed_processor_preserves_nonempty_text_prompt(self) -> None: + tokenizer = MagicMock(return_value={"input_ids": torch.tensor([[7, 8]])}) + image_processor = MagicMock(return_value={"pixel_values": torch.ones(1, 3, 4, 4)}) + processor = _ImageToTextProcessor(image_processor, tokenizer, 42) + + result = processor(images="image", text="describe", return_tensors="pt") + + tokenizer.assert_called_once_with("describe", return_tensors="pt") + assert result["input_ids"].tolist() == [[7, 8]] + + def test_composed_processor_public_signatures_match_pipeline_calls(self) -> None: + call_parameters = inspect.signature(_ImageToTextProcessor.__call__).parameters + decode_parameters = inspect.signature( + _ImageToTextProcessor.post_process_image_text_to_text + ).parameters + + assert {"images", "text", "kwargs"} <= set(call_parameters) + assert {"generated_outputs", "kwargs"} <= set(decode_parameters) + class TestCreatePipeline: def test_threads_trust_remote_code_to_pipeline(self) -> None: diff --git a/tests/unit/loader/test_detect_task_and_class.py b/tests/unit/loader/test_detect_task_and_class.py index b6492578b..61d1e486c 100644 --- a/tests/unit/loader/test_detect_task_and_class.py +++ b/tests/unit/loader/test_detect_task_and_class.py @@ -94,10 +94,58 @@ def test_fallback_to_arch_class_when_tasksmanager_fails(self): r = resolve_task(config) - assert r.task == "image-text-to-text" + assert r.task == "image-to-text" # Should fallback to architecture class assert r.model_class == BlipForConditionalGeneration + def test_auto_detected_alias_uses_registered_image_to_text_composite(self): + import winml.modelkit.models.hf # noqa: F401 + + config = MagicMock() + config.architectures = ["VisionEncoderDecoderModel"] + config.model_type = "vision-encoder-decoder" + config._name_or_path = "" + + with patch( + "winml.modelkit.loader.resolution._infer_task_from_architecture", + return_value="image-text-to-text", + ): + r = resolve_task(config) + + assert r.task == "image-to-text" + assert r.optimum_task == "image-to-text" + assert r.composite is not None + + def test_auto_detected_alias_is_preserved_without_image_to_text_composite(self): + config = MagicMock() + config.architectures = ["BlipForConditionalGeneration"] + config.model_type = "unregistered-multimodal" + config._name_or_path = "" + + with patch( + "winml.modelkit.loader.resolution._infer_task_from_architecture", + return_value="image-text-to-text", + ): + r = resolve_task(config) + + assert r.task == "image-text-to-text" + assert r.optimum_task == "image-text-to-text" + assert r.composite is None + + def test_explicit_image_to_text_is_unchanged(self): + import winml.modelkit.models.hf # noqa: F401 + + config = MagicMock() + config.architectures = ["VisionEncoderDecoderModel"] + config.model_type = "vision-encoder-decoder" + config._name_or_path = "" + + r = resolve_task(config, task="image-to-text") + + assert r.task == "image-to-text" + assert r.optimum_task == "image-to-text" + assert r.composite is not None + class TestModelTaskDefaultsOverride: """Tests for per-model-type default-task auto-detection override.