From 12bcecb93607aea8cf41203367dd5d3377879a37 Mon Sep 17 00:00:00 2001 From: Devam0311 Date: Tue, 1 Sep 2026 16:33:46 +0530 Subject: [PATCH] Use negative crop coordinates when building SDXL negative time ids `_get_add_time_ids` accepts `negative_crops_coords_top_left` and the aesthetic-score branch uses it, but the other branch built the negative time ids from `crops_coords_top_left`. Callers asking for different positive and negative crop conditioning silently got the positive coordinates in both, so negative micro-conditioning was wrong. Fixed in the SDXL img2img and inpaint pipelines, the modular SDXL img2img conditioning step, and the five pipelines that carry this method via `# Copied from`. Add a regression test asserting the negative time ids carry the negative crop coordinates. Ref #13610 (Issue 2) Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01XBYeq5vB4DNZDEaqUsroKR --- .../stable_diffusion_xl/before_denoise.py | 2 +- .../pipeline_controlnet_sd_xl_img2img.py | 2 +- ...pipeline_controlnet_union_sd_xl_img2img.py | 2 +- .../pipeline_pag_controlnet_sd_xl_img2img.py | 2 +- .../pag/pipeline_pag_sd_xl_img2img.py | 2 +- .../pag/pipeline_pag_sd_xl_inpaint.py | 2 +- .../pipeline_stable_diffusion_xl_img2img.py | 2 +- .../pipeline_stable_diffusion_xl_inpaint.py | 2 +- .../test_stable_diffusion_xl_img2img.py | 35 +++++++++++++++++++ 9 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/diffusers/modular_pipelines/stable_diffusion_xl/before_denoise.py b/src/diffusers/modular_pipelines/stable_diffusion_xl/before_denoise.py index 92c74219bd06..c11dd545f9fb 100644 --- a/src/diffusers/modular_pipelines/stable_diffusion_xl/before_denoise.py +++ b/src/diffusers/modular_pipelines/stable_diffusion_xl/before_denoise.py @@ -1154,7 +1154,7 @@ def _get_add_time_ids( ) else: add_time_ids = list(original_size + crops_coords_top_left + target_size) - add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + add_neg_time_ids = list(negative_original_size + negative_crops_coords_top_left + negative_target_size) passed_add_embed_dim = ( components.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py index 1de2f5d66da0..fbc6a05e4eda 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py @@ -1006,7 +1006,7 @@ def _get_add_time_ids( ) else: add_time_ids = list(original_size + crops_coords_top_left + target_size) - add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + add_neg_time_ids = list(negative_original_size + negative_crops_coords_top_left + negative_target_size) passed_add_embed_dim = ( self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim diff --git a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py index dee22942eb8f..d2ef1b86b4e3 100644 --- a/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py +++ b/src/diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py @@ -996,7 +996,7 @@ def _get_add_time_ids( ) else: add_time_ids = list(original_size + crops_coords_top_left + target_size) - add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + add_neg_time_ids = list(negative_original_size + negative_crops_coords_top_left + negative_target_size) passed_add_embed_dim = ( self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim diff --git a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py index bfe45761d5a0..ef840293778c 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py @@ -1012,7 +1012,7 @@ def _get_add_time_ids( ) else: add_time_ids = list(original_size + crops_coords_top_left + target_size) - add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + add_neg_time_ids = list(negative_original_size + negative_crops_coords_top_left + negative_target_size) passed_add_embed_dim = ( self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py index 201d16a86f8a..41bf5decab8e 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py @@ -874,7 +874,7 @@ def _get_add_time_ids( ) else: add_time_ids = list(original_size + crops_coords_top_left + target_size) - add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + add_neg_time_ids = list(negative_original_size + negative_crops_coords_top_left + negative_target_size) passed_add_embed_dim = ( self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim diff --git a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py index 9e70a7779f1e..a08228ef8261 100644 --- a/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py +++ b/src/diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py @@ -965,7 +965,7 @@ def _get_add_time_ids( ) else: add_time_ids = list(original_size + crops_coords_top_left + target_size) - add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + add_neg_time_ids = list(negative_original_size + negative_crops_coords_top_left + negative_target_size) passed_add_embed_dim = ( self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py index c7a13ca02524..ee2327408c3e 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -860,7 +860,7 @@ def _get_add_time_ids( ) else: add_time_ids = list(original_size + crops_coords_top_left + target_size) - add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + add_neg_time_ids = list(negative_original_size + negative_crops_coords_top_left + negative_target_size) passed_add_embed_dim = ( self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py index 3f18cbe21d0f..5f863d4bf04a 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -965,7 +965,7 @@ def _get_add_time_ids( ) else: add_time_ids = list(original_size + crops_coords_top_left + target_size) - add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + add_neg_time_ids = list(negative_original_size + negative_crops_coords_top_left + negative_target_size) passed_add_embed_dim = ( self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim diff --git a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_img2img.py b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_img2img.py index 15c271950764..6e44c5600701 100644 --- a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_img2img.py +++ b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_img2img.py @@ -15,6 +15,7 @@ import gc import random +from types import SimpleNamespace import numpy as np import pytest @@ -357,6 +358,40 @@ def test_stable_diffusion_xl_img2img_negative_conditions(self): assert (image_slice_with_no_neg_conditions - image_slice_with_neg_conditions).abs().max() > 1e-4 + def test_get_add_time_ids_uses_negative_crops_coords_top_left(self): + # Regression: the non-aesthetic-score branch built the negative time ids from + # `crops_coords_top_left` rather than `negative_crops_coords_top_left`, so asking for + # different positive and negative crop conditioning silently applied the positive + # coordinates to both. The aesthetic-score branch above it was already correct. + # + # Driven through a stand-in rather than the dummy pipeline: this branch emits six time ids + # instead of five, which the dummy UNet's `add_embedding` is not sized for. + stand_in = SimpleNamespace( + config=SimpleNamespace(requires_aesthetics_score=False), + unet=SimpleNamespace( + config=SimpleNamespace(addition_time_embed_dim=1), + add_embedding=SimpleNamespace(linear_1=SimpleNamespace(in_features=7)), + ), + ) + + _, add_neg_time_ids = StableDiffusionXLImg2ImgPipeline._get_add_time_ids( + stand_in, + original_size=(64, 64), + crops_coords_top_left=(1, 2), + target_size=(64, 64), + aesthetic_score=6.0, + negative_aesthetic_score=2.5, + negative_original_size=(32, 32), + negative_crops_coords_top_left=(9, 10), + negative_target_size=(32, 32), + dtype=torch.float32, + text_encoder_projection_dim=1, + ) + + assert add_neg_time_ids.flatten().tolist() == [32.0, 32.0, 9.0, 10.0, 32.0, 32.0], ( + "Negative time ids should use `negative_crops_coords_top_left`." + ) + def test_pipeline_interrupt(self): sd_pipe = self.get_pipeline().to(torch_device)