From 0bb5845a5335e29ae88231bb4937f151244baab6 Mon Sep 17 00:00:00 2001 From: ErenAta16 Date: Fri, 17 Jul 2026 19:20:25 +0300 Subject: [PATCH 1/5] Fix _merged_adapters bookkeeping desync on partial unfuse_lora(components=...) _merged_adapters is a single set shared across the whole pipeline, while actual merge state is tracked per component by PEFT. When the same adapter is fused into multiple components (the default when fuse_lora() is called with no adapter_names) and later only some of those components are unfused via unfuse_lora(components=[...]), the old code removed the adapter name from _merged_adapters unconditionally, even when it was still physically merged into the base weights of the untouched component(s). num_fused_loras/fused_loras would then report the pipeline as having nothing (or less) fused than it actually does. Track unmerge candidates during the per-component unmerge loop, then only drop an adapter from the pipeline-wide set once confirmed unmerged in every _lora_loadable_modules component via the new _is_adapter_merged_in_any_component helper. Also adds a regression test (test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping in tests/lora/utils.py) covering the partial-components case, since the existing fuse/unfuse tests only ever pass every loadable component at once. Fixes #14214. --- src/diffusers/loaders/lora_base.py | 30 ++++++++++++++++++++--- tests/lora/utils.py | 39 ++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index dc01475ac8de..402f7beda285 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -664,6 +664,13 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): if len(components) == 0: raise ValueError("`components` cannot be an empty list.") + # self._merged_adapters is shared across the whole pipeline (all _lora_loadable_modules), while + # merge state is tracked per component by PEFT. The same adapter name can be fused into multiple + # components independently, so unmerging it in only some of them doesn't mean it's fully unfused. + # Track candidates here and only drop them from the pipeline-wide set once confirmed unmerged + # everywhere, below. + unmerge_candidates: set[str] = set() + for fuse_component in components: if fuse_component not in self._lora_loadable_modules: raise ValueError(f"{fuse_component} is not found in {self._lora_loadable_modules=}.") @@ -673,11 +680,28 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): if issubclass(model.__class__, (ModelMixin, PreTrainedModel)): for module in model.modules(): if isinstance(module, BaseTunerLayer): - for adapter in set(module.merged_adapters): - if adapter and adapter in self._merged_adapters: - self._merged_adapters = self._merged_adapters - {adapter} + unmerge_candidates.update(module.merged_adapters) module.unmerge() + for adapter in unmerge_candidates: + if adapter not in self._merged_adapters: + continue + if not self._is_adapter_merged_in_any_component(adapter): + self._merged_adapters = self._merged_adapters - {adapter} + + def _is_adapter_merged_in_any_component(self, adapter_name: str) -> bool: + """Whether `adapter_name` is still merged into the base weights of any `_lora_loadable_modules` + component, used to keep the pipeline-wide `_merged_adapters` bookkeeping in sync with the real, + per-component PEFT merge state after a partial `unfuse_lora(components=...)` call.""" + for component in self._lora_loadable_modules: + model = getattr(self, component, None) + if model is None or not issubclass(model.__class__, (ModelMixin, PreTrainedModel)): + continue + for module in model.modules(): + if isinstance(module, BaseTunerLayer) and adapter_name in module.merged_adapters: + return True + return False + def set_adapters( self, adapter_names: list[str] | str, diff --git a/tests/lora/utils.py b/tests/lora/utils.py index 38aec8ce4807..ad5aaf1ab868 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -1711,6 +1711,45 @@ def test_simple_inference_with_text_lora_denoiser_fused_multi( pipe.unfuse_lora(components=self.pipeline_class._lora_loadable_modules) self.assertTrue(pipe.num_fused_loras == 0, f"{pipe.num_fused_loras=}, {pipe.fused_loras=}") + def test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping(self): + """ + `_merged_adapters` (backing `num_fused_loras`/`fused_loras`) is shared across the whole pipeline, + while actual merge state is tracked per component by PEFT. Fusing an adapter into every loadable + component and then unfusing only *some* of them should not report the adapter as fully unfused, + it's still merged into the untouched component(s). + """ + if "text_encoder" not in self.pipeline_class._lora_loadable_modules: + return + + components, text_lora_config, denoiser_lora_config = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + self.assertTrue(check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder") + + denoiser = pipe.transformer if self.unet_kwargs is None else pipe.unet + denoiser_component_name = "unet" if self.unet_kwargs is not None else "transformer" + denoiser.add_adapter(denoiser_lora_config, "adapter-1") + self.assertTrue(check_if_lora_correctly_set(denoiser), "Lora not correctly set in denoiser.") + + pipe.fuse_lora(components=["text_encoder", denoiser_component_name], adapter_names=["adapter-1"]) + self.assertTrue(pipe.num_fused_loras == 1, f"{pipe.num_fused_loras=}, {pipe.fused_loras=}") + + # Only unfuse the text encoder; the denoiser still has "adapter-1" merged into its base weights. + pipe.unfuse_lora(components=["text_encoder"]) + self.assertTrue( + pipe.num_fused_loras == 1, + f"adapter-1 is still merged into {denoiser_component_name}, but num_fused_loras dropped to " + f"{pipe.num_fused_loras=}, {pipe.fused_loras=}", + ) + self.assertIn("adapter-1", pipe.fused_loras) + + # Now unfuse the remaining component; bookkeeping should correctly drop to 0. + pipe.unfuse_lora(components=[denoiser_component_name]) + self.assertTrue(pipe.num_fused_loras == 0, f"{pipe.num_fused_loras=}, {pipe.fused_loras=}") + def test_lora_scale_kwargs_match_fusion(self, expected_atol: float = 1e-3, expected_rtol: float = 1e-3): attention_kwargs_name = determine_attention_kwargs_name(self.pipeline_class) From cacdeee5fc3c4e412e49ed59238171014e2e104a Mon Sep 17 00:00:00 2001 From: ErenAta16 Date: Sat, 18 Jul 2026 11:53:17 +0300 Subject: [PATCH 2/5] Use pytest.skip convention for unsupported text-encoder LoRA case Matches the established skip pattern used elsewhere in this file instead of a silent early return, per review feedback. --- tests/lora/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lora/utils.py b/tests/lora/utils.py index ad5aaf1ab868..a7385602a4cf 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -1718,8 +1718,8 @@ def test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping(self): component and then unfusing only *some* of them should not report the adapter as fully unfused, it's still merged into the untouched component(s). """ - if "text_encoder" not in self.pipeline_class._lora_loadable_modules: - return + if not self.supports_text_encoder_loras: + pytest.skip("Skipping test as text encoder LoRAs are not currently supported.") components, text_lora_config, denoiser_lora_config = self.get_dummy_components() pipe = self.pipeline_class(**components) From f6c54bc19a6d33e322bb4be273e8d0cc1cb2b74d Mon Sep 17 00:00:00 2001 From: ErenAta16 Date: Sat, 18 Jul 2026 12:27:56 +0300 Subject: [PATCH 3/5] Document pipeline-wide fused-adapter tracking; strengthen bookkeeping test - unfuse_lora docstring now notes num_fused_loras/fused_loras are tracked pipeline-wide, not per component. - Regression test now also asserts the adapter is absent from fused_loras after it's been unfused from every component. --- src/diffusers/loaders/lora_base.py | 5 +++++ tests/lora/utils.py | 1 + 2 files changed, 6 insertions(+) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index 402f7beda285..166755f6ec63 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -635,6 +635,11 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): unfuse_text_encoder (`bool`, defaults to `True`): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the LoRA parameters then it won't have any effect. + + Note that `num_fused_loras`/`fused_loras` (backed by `self._merged_adapters`) are tracked pipeline-wide, + not per component. An adapter fused into multiple `_lora_loadable_modules` components only drops out of + `fused_loras` once it has been unfused from all of them; unfusing it from only some components leaves it + listed as fused. """ if components is None: components = [] diff --git a/tests/lora/utils.py b/tests/lora/utils.py index a7385602a4cf..370f01c0e414 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -1749,6 +1749,7 @@ def test_fuse_unfuse_partial_components_keeps_merged_adapter_bookkeeping(self): # Now unfuse the remaining component; bookkeeping should correctly drop to 0. pipe.unfuse_lora(components=[denoiser_component_name]) self.assertTrue(pipe.num_fused_loras == 0, f"{pipe.num_fused_loras=}, {pipe.fused_loras=}") + self.assertNotIn("adapter-1", pipe.fused_loras) def test_lora_scale_kwargs_match_fusion(self, expected_atol: float = 1e-3, expected_rtol: float = 1e-3): attention_kwargs_name = determine_attention_kwargs_name(self.pipeline_class) From e11c893ed647d0bd4aade10983a247c4e1318d30 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 18 Jul 2026 09:59:35 +0000 Subject: [PATCH 4/5] Apply style fixes --- src/diffusers/loaders/lora_base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index 166755f6ec63..f31a30b33916 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -636,10 +636,10 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the LoRA parameters then it won't have any effect. - Note that `num_fused_loras`/`fused_loras` (backed by `self._merged_adapters`) are tracked pipeline-wide, - not per component. An adapter fused into multiple `_lora_loadable_modules` components only drops out of - `fused_loras` once it has been unfused from all of them; unfusing it from only some components leaves it - listed as fused. + Note that `num_fused_loras`/`fused_loras` (backed by `self._merged_adapters`) are tracked pipeline-wide, not + per component. An adapter fused into multiple `_lora_loadable_modules` components only drops out of + `fused_loras` once it has been unfused from all of them; unfusing it from only some components leaves it listed + as fused. """ if components is None: components = [] @@ -696,8 +696,8 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): def _is_adapter_merged_in_any_component(self, adapter_name: str) -> bool: """Whether `adapter_name` is still merged into the base weights of any `_lora_loadable_modules` - component, used to keep the pipeline-wide `_merged_adapters` bookkeeping in sync with the real, - per-component PEFT merge state after a partial `unfuse_lora(components=...)` call.""" + component, used to keep the pipeline-wide `_merged_adapters` bookkeeping in sync with the real, per-component + PEFT merge state after a partial `unfuse_lora(components=...)` call.""" for component in self._lora_loadable_modules: model = getattr(self, component, None) if model is None or not issubclass(model.__class__, (ModelMixin, PreTrainedModel)): From b52aa289ad1c710f615a258019954541f8b2aa44 Mon Sep 17 00:00:00 2001 From: ErenAta16 Date: Sat, 18 Jul 2026 15:24:27 +0300 Subject: [PATCH 5/5] Trim unfuse_lora docstring to user-facing behavior only Drop the _merged_adapters implementation detail per review feedback; fused_loras is what's actually exposed to callers. --- src/diffusers/loaders/lora_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/diffusers/loaders/lora_base.py b/src/diffusers/loaders/lora_base.py index f31a30b33916..51b30698118e 100644 --- a/src/diffusers/loaders/lora_base.py +++ b/src/diffusers/loaders/lora_base.py @@ -636,10 +636,9 @@ def unfuse_lora(self, components: list[str] | None = None, **kwargs): Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the LoRA parameters then it won't have any effect. - Note that `num_fused_loras`/`fused_loras` (backed by `self._merged_adapters`) are tracked pipeline-wide, not - per component. An adapter fused into multiple `_lora_loadable_modules` components only drops out of - `fused_loras` once it has been unfused from all of them; unfusing it from only some components leaves it listed - as fused. + Note that `fused_loras` is tracked pipeline-wide, not per component. An adapter fused into multiple + components only drops out of `fused_loras` once it has been unfused from all of them; unfusing it from + only some components leaves it listed as fused. """ if components is None: components = []