diff --git a/tests/integration/model_bridge/test_vit_adapter.py b/tests/integration/model_bridge/test_vit_adapter.py new file mode 100644 index 000000000..8329284f8 --- /dev/null +++ b/tests/integration/model_bridge/test_vit_adapter.py @@ -0,0 +1,386 @@ +"""Integration tests for the ViT / DeiT architecture adapter. + +Modeled on tests/integration/model_bridge/test_mamba_adapter.py — verifies +wrap-don't-reimplement behavior against real HF checkpoints: +- Forward pass matches HF (bridge substitutes hooked submodules into the real + HF model and calls its real forward — see bridge.py's `is_visual_model` + branch, which does `output = self.original_model(**kwargs)`) +- Submodule hooks fire with expected shapes (embed, attn q/k/v/o, mlp in/out, + ln_final, unembed/classifier) +- DeiT's distillation token is invisible to the adapter (sequence length + differs from ViT's by one extra token, entirely inside VisionEmbeddingsBridge) +- Bare ViTModel (no classifier) return-value shape via return_type="logits" +- DeiTForImageClassificationWithTeacher is rejected, not silently mishandled +- prepare_model() doesn't assume a `.encoder` wrapper exists on the HF model + (transformers now puts blocks directly on `.layers`) + +Four checkpoints, matching the ones already sanity-checked manually (added +facebook/deit-small-patch16-224 as a real DeiTForImageClassification load — +previously declared as MODEL_DEIT_CLASSIFIER but never actually used): +- google/vit-base-patch16-224 (ViTForImageClassification, prefix "vit.") +- google/vit-base-patch16-224-in21k (bare ViTModel, no prefix, no classifier) +- facebook/deit-small-patch16-224 (DeiTForImageClassification, prefix "deit.") +- facebook/deit-small-distilled-patch16-224 (bare DeiTModel, distillation token) + +NOTE ON RUNNING THESE: they download real weights from HuggingFace (a few +hundred MB total) — same cost class as the existing Mamba/SmolLM3 integration +tests, no special marker needed, but expect the first run to be slow. +""" + +from types import SimpleNamespace + +import pytest +import torch +from transformers import ( + DeiTForImageClassification, + DeiTModel, + ViTForImageClassification, + ViTModel, +) + +from transformer_lens.model_bridge.bridge import TransformerBridge +from transformer_lens.model_bridge.generalized_components.vision_classifier_head import ( + VisionClassifierHeadBridge, +) + +MODEL_VIT_CLASSIFIER = "google/vit-base-patch16-224" +MODEL_VIT_BARE = "google/vit-base-patch16-224-in21k" +MODEL_DEIT_CLASSIFIER = "facebook/deit-small-patch16-224" + + +def _pixel_values(batch: int = 1, image_size: int = 224) -> torch.Tensor: + return torch.randn(batch, 3, image_size, image_size) + + +# --------------------------------------------------------------------------- +# ViTForImageClassification (the "normal" case) +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module") +def vit_bridge(): + # Explicitly load the classification model to bypass boot_transformers auto-loader logic + hf_model = ViTForImageClassification.from_pretrained(MODEL_VIT_CLASSIFIER) + return TransformerBridge.boot_transformers( + MODEL_VIT_CLASSIFIER, hf_model=hf_model, device="cpu" + ) + + +class TestViTClassifierBridgeCreation: + def test_block_count(self, vit_bridge): + assert len(vit_bridge.blocks) == vit_bridge.original_model.config.num_hidden_layers + + def test_config_flags(self, vit_bridge): + assert vit_bridge.cfg.normalization_type == "LN" + assert vit_bridge.cfg.positional_embedding_type == "standard" + assert vit_bridge.cfg.is_visual_model is True + assert vit_bridge.cfg.gated_mlp is False + assert vit_bridge.cfg.attn_only is False + + def test_has_core_components(self, vit_bridge): + assert hasattr(vit_bridge, "embed") + assert hasattr(vit_bridge, "blocks") + assert hasattr(vit_bridge, "ln_final") + assert hasattr(vit_bridge, "unembed") + + def test_unembed_is_vision_classifier_head(self, vit_bridge): + assert isinstance(vit_bridge.unembed, VisionClassifierHeadBridge) + + def test_d_model_matches_hf_config(self, vit_bridge): + assert vit_bridge.cfg.d_model == vit_bridge.original_model.config.hidden_size + + def test_n_heads_matches_hf_config(self, vit_bridge): + assert vit_bridge.cfg.n_heads == vit_bridge.original_model.config.num_attention_heads + + def test_blocks_point_at_flat_layers_attribute(self, vit_bridge): + """Current transformers has no ViTEncoder wrapper any more — blocks + live directly on `vit.layers`, not `vit.encoder.layer`.""" + assert vit_bridge.adapter.component_mapping["blocks"].name == "vit.layers" + + def test_n_ctx_matches_real_patch_grid(self, vit_bridge): + """Regression check for the previously-broken n_ctx propagation: for + google/vit-base-patch16-224 (224px image, patch16 -> 14x14 patches), + cfg.n_ctx should be patches + CLS token = 197, not the generic + HF-config-fallback default of 2048.""" + assert vit_bridge.cfg.n_ctx == 197 + + +class TestViTClassifierForwardPass: + def test_forward_returns_correct_shape(self, vit_bridge): + pixel_values = _pixel_values() + with torch.no_grad(): + output = vit_bridge(pixel_values) + num_labels = vit_bridge.original_model.config.num_labels + assert output.shape == (1, num_labels) + assert not torch.isnan(output).any() + + def test_forward_matches_hf(self, vit_bridge): + """Wrap-don't-reimplement: bridge substitutes hooked submodules into the + real HF model and calls its real forward, so this should match HF very + tightly (mamba's equivalent test asserts bit-exact; using a tight atol + here to be safe against BLAS/backend-dependent float noise across + machines rather than assuming bit-exactness untested).""" + pixel_values = _pixel_values() + hf_model = vit_bridge.original_model + with torch.no_grad(): + bridge_out = vit_bridge(pixel_values) + hf_out = hf_model(pixel_values).logits + max_diff = (bridge_out - hf_out).abs().max().item() + assert max_diff < 1e-4, f"Bridge vs HF max diff = {max_diff}" + + def test_batch_of_images(self, vit_bridge): + pixel_values = _pixel_values(batch=3) + with torch.no_grad(): + output = vit_bridge(pixel_values) + assert output.shape[0] == 3 + + +class TestViTClassifierHookCoverage: + @pytest.fixture(scope="class") + def cache(self, vit_bridge): + pixel_values = _pixel_values() + with torch.no_grad(): + _, cache = vit_bridge.run_with_cache(pixel_values) + return cache + + def test_embed_hooks_fire(self, cache, vit_bridge): + assert "embed.hook_in" in cache + assert "embed.hook_out" in cache + assert cache["embed.hook_in"].shape == (1, 3, 224, 224) + seq_len = cache["embed.hook_out"].shape[1] + d_model = vit_bridge.cfg.d_model + assert cache["embed.hook_out"].shape == (1, seq_len, d_model) + # ViT: CLS + patches, no distillation token (contrast with DeiT below). + assert seq_len == 1 + (224 // 16) ** 2 + + def test_block_submodule_hooks_fire(self, cache, vit_bridge): + n_layers = vit_bridge.cfg.n_layers + for i in [0, n_layers - 1]: + for submod in ("q", "k", "v", "o"): + assert f"blocks.{i}.attn.{submod}.hook_in" in cache + assert f"blocks.{i}.attn.{submod}.hook_out" in cache + assert f"blocks.{i}.mlp.in.hook_out" in cache + assert f"blocks.{i}.mlp.out.hook_out" in cache + + def test_mlp_hook_aliases_resolve(self, cache): + """hook_mlp_in/out are redirected via hook_alias_overrides in vit.py.""" + assert torch.equal(cache["blocks.0.hook_mlp_in"], cache["blocks.0.mlp.in.hook_in"]) + assert torch.equal(cache["blocks.0.hook_mlp_out"], cache["blocks.0.mlp.out.hook_out"]) + + def test_ln_final_hook_fires(self, cache): + assert "ln_final.hook_normalized" in cache + + def test_unembed_hooks_fire_with_pooled_shape(self, cache, vit_bridge): + """VisionClassifierHeadBridge only ever sees the pooled (batch, hidden) + CLS-token tensor — HF's ViTForImageClassification.forward() does the + `sequence_output[:, 0, :]` slicing itself before calling self.classifier, + and that real forward path is what's running here.""" + d_model = vit_bridge.cfg.d_model + num_labels = vit_bridge.original_model.config.num_labels + assert cache["unembed.hook_in"].shape == (1, d_model) + assert cache["unembed.hook_out"].shape == (1, num_labels) + + +# --------------------------------------------------------------------------- +# Bare ViTModel (no classifier) — google/vit-base-patch16-224-in21k +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module") +def vit_bare_bridge(): + # Explicitly load the bare model + hf_model = ViTModel.from_pretrained(MODEL_VIT_BARE) + return TransformerBridge.boot_transformers(MODEL_VIT_BARE, hf_model=hf_model, device="cpu") + + +class TestViTBareModel: + def test_no_unembed_component(self, vit_bare_bridge): + assert "unembed" not in vit_bare_bridge.adapter.component_mapping + + def test_blocks_point_at_flat_layers_attribute_no_prefix(self, vit_bare_bridge): + assert vit_bare_bridge.adapter.component_mapping["blocks"].name == "layers" + + def test_forward_does_not_crash(self, vit_bare_bridge): + pixel_values = _pixel_values() + with torch.no_grad(): + vit_bare_bridge(pixel_values) # must not raise + + def test_forward_returns_last_hidden_state_tensor(self, vit_bare_bridge): + """ + For a bare ViTModel, HF's own forward returns BaseModelOutputWithPooling, + which has no `.logits` and is not a tuple subclass. bridge.py's forward() + now falls back to `output.last_hidden_state` in that case, so + return_type="logits" (the default) returns a plain tensor instead of the + raw HF output object. This mirrors the choice made for bare text encoders + that hit the same code path. + """ + pixel_values = _pixel_values() + hf_model = vit_bare_bridge.original_model + with torch.no_grad(): + output = vit_bare_bridge(pixel_values) + hf_out = hf_model(pixel_values).last_hidden_state + assert isinstance(output, torch.Tensor) + assert output.shape == hf_out.shape + assert torch.allclose(output, hf_out, atol=1e-4) + + +# --------------------------------------------------------------------------- +# DeiTForImageClassification — distillation token must stay invisible +# Previously MODEL_DEIT_CLASSIFIER was declared but never used, and the only +# DeiT fixture (deit_bridge, below) loaded a bare DeiTModel — so the +# "deit."-prefix + real classifier-head path had no coverage. This fixture +# and test class close that gap with a real DeiTForImageClassification load. +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module") +def deit_classifier_bridge(): + hf_model = DeiTForImageClassification.from_pretrained(MODEL_DEIT_CLASSIFIER) + return TransformerBridge.boot_transformers( + MODEL_DEIT_CLASSIFIER, hf_model=hf_model, device="cpu" + ) + + +class TestDeiTClassifierBridge: + def test_prefix_is_deit_for_classifier_model(self, deit_classifier_bridge): + assert deit_classifier_bridge.adapter.component_mapping["blocks"].name == "deit.layers" + + def test_unembed_is_vision_classifier_head(self, deit_classifier_bridge): + assert isinstance(deit_classifier_bridge.unembed, VisionClassifierHeadBridge) + + def test_n_ctx_matches_real_patch_grid(self, deit_classifier_bridge): + assert deit_classifier_bridge.cfg.n_ctx == 197 + + def test_forward_matches_hf(self, deit_classifier_bridge): + pixel_values = _pixel_values() + hf_model = deit_classifier_bridge.original_model + with torch.no_grad(): + bridge_out = deit_classifier_bridge(pixel_values) + hf_out = hf_model(pixel_values).logits + max_diff = (bridge_out - hf_out).abs().max().item() + assert max_diff < 1e-4, f"Bridge vs HF max diff = {max_diff}" + + def test_forward_returns_correct_shape(self, deit_classifier_bridge): + pixel_values = _pixel_values() + with torch.no_grad(): + output = deit_classifier_bridge(pixel_values) + num_labels = deit_classifier_bridge.original_model.config.num_labels + assert output.shape == (1, num_labels) + + +# --------------------------------------------------------------------------- +# DeiT (bare model) — distillation token must stay invisible +# --------------------------------------------------------------------------- +# 1. Use the distilled checkpoint to actually test distillation token logic +MODEL_DEIT_DISTILLED = "facebook/deit-small-distilled-patch16-224" + + +@pytest.fixture(scope="module") +def deit_bridge(): + # 2. Load the bare Hugging Face model manually to bypass the + # DeiTForImageClassificationWithTeacher dual-head that your adapter rejects. + hf_model = DeiTModel.from_pretrained(MODEL_DEIT_DISTILLED) + hf_model.config.architectures = ["DeiTModel"] + + # 3. Pass the instantiated HF model to boot_transformers. + # (Assuming your boot_transformers accepts an hf_model kwarg like HookedTransformer) + return TransformerBridge.boot_transformers( + MODEL_DEIT_DISTILLED, hf_model=hf_model, device="cpu" + ) + + +class TestDeiTBridge: + def test_prefix_is_empty_for_bare_model(self, deit_bridge): + # Because we loaded a bare DeiTModel, the components are at the root. + # Your prepare_model should assign prefix="" (not "deit.") + assert deit_bridge.adapter.component_mapping["embed"].name == "embeddings" + + def test_forward_matches_hf(self, deit_bridge): + pixel_values = _pixel_values() + hf_model = deit_bridge.original_model + + with torch.no_grad(): + bridge_out = deit_bridge(pixel_values) + # Bare models return a BaseModelOutput object with last_hidden_state + hf_out = hf_model(pixel_values).last_hidden_state + + # Account for your adapter returning the raw HF object for bare models + # (as you documented in TestViTBareModel) + if not isinstance(bridge_out, torch.Tensor): + bridge_out = bridge_out.last_hidden_state + + max_diff = (bridge_out - hf_out).abs().max().item() + assert max_diff < 1e-4, f"Bridge vs HF max diff = {max_diff}" + + def test_sequence_length_includes_distillation_token(self, deit_bridge): + """DeiT embeddings prepend CLS *and* distillation tokens (vs. ViT's CLS + only) — invisible to the adapter per vision_embeddings.py's docstring, + but it should still show up in the actual hidden-state shape, one token + longer than the equivalent ViT sequence length.""" + pixel_values = _pixel_values() + with torch.no_grad(): + _, cache = deit_bridge.run_with_cache(pixel_values) + + seq_len = cache["embed.hook_out"].shape[1] + num_patches = (224 // 16) ** 2 + + assert seq_len == num_patches + 2 # CLS + distillation + patches + + +# --------------------------------------------------------------------------- +# DeiTForImageClassificationWithTeacher — must raise, not silently mishandle +# --------------------------------------------------------------------------- + + +class TestDeiTWithTeacherRejected: + def test_loading_raises_not_implemented(self): + """No small public DeiTForImageClassificationWithTeacher checkpoint is + assumed here — this drives the same code path prepare_model() exercises + via a HF class instance check, without a full weights download. + """ + from transformers import DeiTConfig, DeiTForImageClassificationWithTeacher + + from transformer_lens.model_bridge.supported_architectures.vit import ( + ViTArchitectureAdapter, + ) + + config = DeiTConfig( + hidden_size=32, + num_hidden_layers=2, + num_attention_heads=2, + intermediate_size=64, + image_size=32, + patch_size=16, + num_labels=10, + ) + hf_model = DeiTForImageClassificationWithTeacher(config) + + adapter = ViTArchitectureAdapter.__new__(ViTArchitectureAdapter) + adapter.cfg = None # prepare_model() doesn't touch cfg before the raise + with pytest.raises(NotImplementedError): + ViTArchitectureAdapter.prepare_model(adapter, hf_model) + + +# --------------------------------------------------------------------------- +# Regression test: prepare_model() must not assume `.encoder` exists. +# --------------------------------------------------------------------------- + + +class TestPrepareModelDoesNotAssumeEncoderWrapper: + def test_minimal_bare_model_stub_does_not_crash(self): + """Reproduces the AttributeError seen after upgrading transformers: + 'types.SimpleNamespace' object has no attribute 'encoder'. + + A stub with none of vit/deit/classifier/cls_classifier/ + distillation_classifier set should be treated like a bare, no-prefix + model — prepare_model() must not reach for `hf_model.encoder.layer` + (that wrapper module no longer exists in current transformers; blocks + live directly on `.layers`). + """ + from transformer_lens.model_bridge.supported_architectures.vit import ( + ViTArchitectureAdapter, + ) + + hf_model = SimpleNamespace() + + adapter = ViTArchitectureAdapter.__new__(ViTArchitectureAdapter) + adapter.cfg = None + + ViTArchitectureAdapter.prepare_model(adapter, hf_model) # must not raise + + assert adapter.component_mapping["blocks"].name == "layers" + assert "unembed" not in adapter.component_mapping diff --git a/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py b/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py new file mode 100644 index 000000000..cdf99ae1a --- /dev/null +++ b/tests/unit/model_bridge/supported_architectures/test_vit_adapter.py @@ -0,0 +1,359 @@ +"""Unit tests for ViTArchitectureAdapter. + +Modeled on tests/unit/model_bridge/supported_architectures/test_hubert_adapter.py +(the closest existing analog: HubertArchitectureAdapter also sets a modality flag +— is_audio_model there, is_visual_model here — and rebinds a prefix + injects a +head in prepare_model()). + +Tests cover: +- Component mapping structure for a bare ViTModel/DeiTModel (default, before + prepare_model() sees the real HF model) +- Block/attn/mlp submodule mapping and hook-alias overrides +- Weight conversion key set and rearrange patterns (weights + biases) +- Anti-drift config flags (is_visual_model, positional_embedding_type, + normalization_type, supports_fold_ln, supports_generation) +- prepare_model(): bare model / ViTForImageClassification / DeiTForImageClassification + prefix rebinding + classifier injection, and the DeiTForImageClassificationWithTeacher + guard rail + +NOTE (transformers ViT/DeiT flattening refactor): current transformers removed +the separate `ViTEncoder`/`ViTSelfAttention`/`ViTSelfOutput`/`ViTIntermediate`/ +`ViTOutput` wrapper modules. Blocks now live directly at `.layers` +(not `.encoder.layer`); `ViTAttention`/`DeiTAttention` own `q_proj`/ +`k_proj`/`v_proj`/`o_proj` directly (not nested `attention.query`/`attention.key`/ +`attention.value`/`output.dense`); and `ViTLayer.mlp` is already a flat `ViTMLP` +with `fc1`/`fc2` (not `intermediate.dense`/`output.dense`). All the HF-path +assertions below reflect that. + +NOT covered here (needs a real HF model + real forward pass — see the +integration test at tests/integration/model_bridge/test_vit_adapter.py instead): +- Numeric parity with HF +- Hook firing / cache shapes +- n_ctx propagation: This is cross-checked against a real checkpoint + (google/vit-base-patch16-224 -> n_ctx == 197) in the integration test. +""" + +from types import SimpleNamespace + +import pytest + +from transformer_lens.config.transformer_bridge_config import TransformerBridgeConfig +from transformer_lens.conversion_utils.conversion_steps import RearrangeTensorConversion +from transformer_lens.conversion_utils.param_processing_conversion import ( + ParamProcessingConversion, +) +from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + LinearBridge, + MLPBridge, + NormalizationBridge, +) +from transformer_lens.model_bridge.generalized_components.vision_classifier_head import ( + VisionClassifierHeadBridge, +) +from transformer_lens.model_bridge.generalized_components.vision_embeddings import ( + VisionEmbeddingsBridge, +) +from transformer_lens.model_bridge.supported_architectures.vit import ( + ViTArchitectureAdapter, +) + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- + + +def _make_cfg( + n_heads: int = 12, + d_model: int = 768, + n_layers: int = 12, + d_vocab: int = 1000, + n_ctx: int = 197, + **overrides, +) -> TransformerBridgeConfig: + """Return a minimal TransformerBridgeConfig for ViT/DeiT adapter tests. + + n_ctx=197 is the real value for a 224x224 image / patch16 model + (14*14 patches + 1 CLS token) — used as the *input* cfg here so tests can + tell apart "the adapter left this alone" from "the adapter zeroed it". + """ + cfg = TransformerBridgeConfig( + d_model=d_model, + d_head=d_model // n_heads, + n_heads=n_heads, + n_layers=n_layers, + n_ctx=n_ctx, + d_vocab=d_vocab, + architecture="ViTForImageClassification", + ) + for k, v in overrides.items(): + setattr(cfg, k, v) + return cfg + + +@pytest.fixture() +def adapter() -> ViTArchitectureAdapter: + """Fresh adapter per test — prepare_model() mutates component_mapping in place.""" + return ViTArchitectureAdapter(_make_cfg()) + + +# --------------------------------------------------------------------------- +# Component mapping — bare ViTModel/DeiTModel (default, pre prepare_model()) +# --------------------------------------------------------------------------- + + +class TestViTComponentMappingBare: + """Default mapping assumes no prefix and no classifier (matches __init__).""" + + def test_top_level_keys(self, adapter: ViTArchitectureAdapter) -> None: + assert set(adapter.component_mapping.keys()) == {"embed", "blocks", "ln_final"} + + def test_no_unembed_by_default(self, adapter: ViTArchitectureAdapter) -> None: + assert "unembed" not in adapter.component_mapping + + def test_bridge_types(self, adapter: ViTArchitectureAdapter) -> None: + mapping = adapter.component_mapping + assert isinstance(mapping["embed"], VisionEmbeddingsBridge) + assert isinstance(mapping["blocks"], BlockBridge) + assert isinstance(mapping["ln_final"], NormalizationBridge) + + def test_top_level_hf_paths(self, adapter: ViTArchitectureAdapter) -> None: + mapping = adapter.component_mapping + assert mapping["embed"].name == "embeddings" + # No more ViTEncoder wrapper — blocks sit directly on `.layers`. + assert mapping["blocks"].name == "layers" + assert mapping["ln_final"].name == "layernorm" + + def test_block_submodule_keys(self, adapter: ViTArchitectureAdapter) -> None: + assert set(adapter.component_mapping["blocks"].submodules.keys()) == { + "ln1", + "ln2", + "attn", + "mlp", + } + + def test_block_submodule_types(self, adapter: ViTArchitectureAdapter) -> None: + blocks = adapter.component_mapping["blocks"] + assert isinstance(blocks.submodules["ln1"], NormalizationBridge) + assert isinstance(blocks.submodules["ln2"], NormalizationBridge) + assert isinstance(blocks.submodules["attn"], AttentionBridge) + assert isinstance(blocks.submodules["mlp"], MLPBridge) + + def test_block_submodule_hf_paths(self, adapter: ViTArchitectureAdapter) -> None: + blocks = adapter.component_mapping["blocks"] + assert blocks.submodules["ln1"].name == "layernorm_before" + assert blocks.submodules["ln2"].name == "layernorm_after" + assert blocks.submodules["attn"].name == "attention" + assert blocks.submodules["mlp"].name == "mlp" + + def test_attn_submodule_keys(self, adapter: ViTArchitectureAdapter) -> None: + attn = adapter.component_mapping["blocks"].submodules["attn"] + assert set(attn.submodules.keys()) == {"q", "k", "v", "o"} + + def test_attn_qkvo_hf_paths(self, adapter: ViTArchitectureAdapter) -> None: + """Paths here are relative to the already-resolved "attn" bridge module + (block.attention). ViTAttention/DeiTAttention now owns q_proj/k_proj/ + v_proj/o_proj directly (flattened, no nested self-attention + separate + output.dense module), so no "attention." prefix.""" + attn = adapter.component_mapping["blocks"].submodules["attn"] + assert attn.submodules["q"].name == "q_proj" + assert attn.submodules["k"].name == "k_proj" + assert attn.submodules["v"].name == "v_proj" + assert attn.submodules["o"].name == "o_proj" + + def test_attn_submodules_are_linear_bridges(self, adapter: ViTArchitectureAdapter) -> None: + attn = adapter.component_mapping["blocks"].submodules["attn"] + for sub in attn.submodules.values(): + assert isinstance(sub, LinearBridge) + + def test_mlp_submodule_hf_paths(self, adapter: ViTArchitectureAdapter) -> None: + """Paths here are relative to the already-resolved "mlp" bridge module + (block.mlp). ViTLayer.mlp is a real ViTMLP now (fc1/fc2 directly) — no + more intermediate/output split, so no wrapper shim is needed.""" + mlp = adapter.component_mapping["blocks"].submodules["mlp"] + assert mlp.submodules["in"].name == "fc1" + assert mlp.submodules["out"].name == "fc2" + + def test_mlp_hook_alias_overrides(self, adapter: ViTArchitectureAdapter) -> None: + aliases = adapter.component_mapping["blocks"].hook_aliases + assert aliases.get("hook_mlp_out") == "mlp.out.hook_out" + assert aliases.get("hook_mlp_in") == "mlp.in.hook_in" + + +# --------------------------------------------------------------------------- +# Anti-drift config flags +# --------------------------------------------------------------------------- + + +class TestViTAdapterConfig: + """Flags that must not silently regress.""" + + def test_normalization_type_is_ln(self, adapter: ViTArchitectureAdapter) -> None: + assert adapter.cfg.normalization_type == "LN" + + def test_positional_embedding_type_is_standard(self, adapter: ViTArchitectureAdapter) -> None: + assert adapter.cfg.positional_embedding_type == "standard" + + def test_is_visual_model_true(self, adapter: ViTArchitectureAdapter) -> None: + """Only vision adapters set this flag; text/audio adapters must not.""" + assert adapter.cfg.is_visual_model is True + + def test_is_audio_model_not_set_true(self, adapter: ViTArchitectureAdapter) -> None: + assert getattr(adapter.cfg, "is_audio_model", False) is False + + def test_final_rms_is_false(self, adapter: ViTArchitectureAdapter) -> None: + assert adapter.cfg.final_rms is False + + def test_gated_mlp_is_false(self, adapter: ViTArchitectureAdapter) -> None: + assert adapter.cfg.gated_mlp is False + + def test_attn_only_is_false(self, adapter: ViTArchitectureAdapter) -> None: + assert adapter.cfg.attn_only is False + + def test_supports_generation_is_false(self) -> None: + assert ViTArchitectureAdapter.supports_generation is False + + def test_supports_fold_ln_true_pre_ln(self, adapter: ViTArchitectureAdapter) -> None: + """ViT/DeiT are pre-LN — unlike BertArchitectureAdapter's post-LN False.""" + assert adapter.supports_fold_ln is True + + +# --------------------------------------------------------------------------- +# Weight processing conversions +# --------------------------------------------------------------------------- + + +class TestViTWeightConversions: + def test_exact_conversion_key_set(self, adapter: ViTArchitectureAdapter) -> None: + assert set(adapter.weight_processing_conversions.keys()) == { + "blocks.{i}.attn.q.weight", + "blocks.{i}.attn.k.weight", + "blocks.{i}.attn.v.weight", + "blocks.{i}.attn.q.bias", + "blocks.{i}.attn.k.bias", + "blocks.{i}.attn.v.bias", + "blocks.{i}.attn.o.weight", + } + + def test_qkv_weight_pattern(self, adapter: ViTArchitectureAdapter) -> None: + for slot in ("q", "k", "v"): + conv = adapter.weight_processing_conversions[f"blocks.{{i}}.attn.{slot}.weight"] + assert isinstance(conv, ParamProcessingConversion) + assert isinstance(conv.tensor_conversion, RearrangeTensorConversion) + assert conv.tensor_conversion.pattern == "(h d_head) d_model -> h d_model d_head" + + def test_qkv_bias_pattern(self, adapter: ViTArchitectureAdapter) -> None: + for slot in ("q", "k", "v"): + conv = adapter.weight_processing_conversions[f"blocks.{{i}}.attn.{slot}.bias"] + assert isinstance(conv, ParamProcessingConversion) + assert isinstance(conv.tensor_conversion, RearrangeTensorConversion) + assert conv.tensor_conversion.pattern == "(h d_head) -> h d_head" + + def test_o_weight_pattern(self, adapter: ViTArchitectureAdapter) -> None: + conv = adapter.weight_processing_conversions["blocks.{i}.attn.o.weight"] + assert isinstance(conv, ParamProcessingConversion) + assert isinstance(conv.tensor_conversion, RearrangeTensorConversion) + assert conv.tensor_conversion.pattern == "d_model (h d_head) -> h d_head d_model" + + def test_qkv_weight_head_axis(self, adapter: ViTArchitectureAdapter) -> None: + for slot in ("q", "k", "v"): + conv = adapter.weight_processing_conversions[f"blocks.{{i}}.attn.{slot}.weight"] + assert conv.tensor_conversion.axes_lengths["h"] == 12 + + def test_qkv_bias_head_axis(self, adapter: ViTArchitectureAdapter) -> None: + for slot in ("q", "k", "v"): + conv = adapter.weight_processing_conversions[f"blocks.{{i}}.attn.{slot}.bias"] + assert conv.tensor_conversion.axes_lengths["h"] == 12 + + +# --------------------------------------------------------------------------- +# prepare_model() — prefix rebinding + classifier injection +# --------------------------------------------------------------------------- + + +class TestViTPrepareModel: + """prepare_model() detects bare/ViT/DeiT wrappers and rebuilds the mapping.""" + + def _bare_model(self) -> object: + """No 'vit'/'deit'/'classifier' attribute — mimics bare ViTModel/DeiTModel.""" + return SimpleNamespace() + + def _vit_for_classification(self) -> object: + return SimpleNamespace(vit=SimpleNamespace(), classifier=SimpleNamespace()) + + def _deit_for_classification(self) -> object: + return SimpleNamespace(deit=SimpleNamespace(), classifier=SimpleNamespace()) + + def _deit_with_teacher(self) -> object: + """Dual cls_classifier + distillation_classifier — explicitly unsupported.""" + return SimpleNamespace( + deit=SimpleNamespace(), + cls_classifier=SimpleNamespace(), + distillation_classifier=SimpleNamespace(), + ) + + # -- bare model -------------------------------------------------------- + + def test_bare_model_keeps_no_prefix(self, adapter: ViTArchitectureAdapter) -> None: + adapter.prepare_model(self._bare_model()) + assert adapter.component_mapping["embed"].name == "embeddings" + assert adapter.component_mapping["blocks"].name == "layers" + assert adapter.component_mapping["ln_final"].name == "layernorm" + + def test_bare_model_has_no_unembed(self, adapter: ViTArchitectureAdapter) -> None: + adapter.prepare_model(self._bare_model()) + assert "unembed" not in adapter.component_mapping + + def test_bare_model_does_not_require_encoder_attribute( + self, adapter: ViTArchitectureAdapter + ) -> None: + """Regression test: prepare_model() must not assume a `.encoder` + wrapper module exists on the HF model. Current transformers removed + it — blocks live directly on `.layers` — so a minimal stub + with no `.encoder` attribute at all must still work.""" + hf_model = SimpleNamespace() + assert not hasattr(hf_model, "encoder") + adapter.prepare_model(hf_model) # must not raise AttributeError + + # -- ViTForImageClassification ----------------------------------------- + + def test_vit_classification_adds_vit_prefix(self, adapter: ViTArchitectureAdapter) -> None: + adapter.prepare_model(self._vit_for_classification()) + assert adapter.component_mapping["embed"].name == "vit.embeddings" + assert adapter.component_mapping["blocks"].name == "vit.layers" + assert adapter.component_mapping["ln_final"].name == "vit.layernorm" + + def test_vit_classification_adds_unembed(self, adapter: ViTArchitectureAdapter) -> None: + adapter.prepare_model(self._vit_for_classification()) + assert "unembed" in adapter.component_mapping + assert isinstance(adapter.component_mapping["unembed"], VisionClassifierHeadBridge) + + def test_classifier_is_never_prefixed(self, adapter: ViTArchitectureAdapter) -> None: + """classifier is always a top-level attr on the *ForImageClassification wrapper.""" + adapter.prepare_model(self._vit_for_classification()) + assert adapter.component_mapping["unembed"].name == "classifier" + + # -- DeiTForImageClassification ------------------------------------------ + + def test_deit_classification_adds_deit_prefix(self, adapter: ViTArchitectureAdapter) -> None: + adapter.prepare_model(self._deit_for_classification()) + assert adapter.component_mapping["embed"].name == "deit.embeddings" + assert adapter.component_mapping["blocks"].name == "deit.layers" + assert adapter.component_mapping["ln_final"].name == "deit.layernorm" + + def test_deit_classification_adds_unembed(self, adapter: ViTArchitectureAdapter) -> None: + adapter.prepare_model(self._deit_for_classification()) + assert adapter.component_mapping["unembed"].name == "classifier" + + # -- DeiTForImageClassificationWithTeacher guard rail -------------------- + + def test_deit_with_teacher_raises(self, adapter: ViTArchitectureAdapter) -> None: + with pytest.raises(NotImplementedError): + adapter.prepare_model(self._deit_with_teacher()) + + def test_deit_with_teacher_message_mentions_reason( + self, adapter: ViTArchitectureAdapter + ) -> None: + with pytest.raises(NotImplementedError, match="distillation_classifier"): + adapter.prepare_model(self._deit_with_teacher()) diff --git a/transformer_lens/config/transformer_bridge_config.py b/transformer_lens/config/transformer_bridge_config.py index 7b183f097..ddfa446de 100644 --- a/transformer_lens/config/transformer_bridge_config.py +++ b/transformer_lens/config/transformer_bridge_config.py @@ -90,6 +90,8 @@ def __init__( attn_implementation: Optional[str] = None, # Audio model configuration is_audio_model: bool = False, + # Vision model (ViT, DeiT) configuration + is_visual_model: bool = False, # Stateful model configuration (e.g., Mamba SSMs use cache_params, # not past_key_values, so generation delegates to hf_generate) is_stateful: bool = False, @@ -185,6 +187,8 @@ def __init__( self.attn_implementation = attn_implementation # Audio model configuration self.is_audio_model = is_audio_model + # Vision model (ViT, DeiT) configuration + self.is_visual_model = is_visual_model # Stateful model configuration self.is_stateful = is_stateful # Multimodal configuration diff --git a/transformer_lens/factories/architecture_adapter_factory.py b/transformer_lens/factories/architecture_adapter_factory.py index 5def69852..63d8c119e 100644 --- a/transformer_lens/factories/architecture_adapter_factory.py +++ b/transformer_lens/factories/architecture_adapter_factory.py @@ -147,6 +147,7 @@ T5Gemma2ArchitectureAdapter, T5GemmaArchitectureAdapter, VaultGemmaArchitectureAdapter, + ViTArchitectureAdapter, XGLMArchitectureAdapter, YoutuArchitectureAdapter, Zamba2ArchitectureAdapter, @@ -310,6 +311,10 @@ "MinGPTForCausalLM": MingptArchitectureAdapter, "GPTNeoForCausalLM": NeoArchitectureAdapter, "GPTNeoXForCausalLM": NeoxArchitectureAdapter, + "ViTModel": ViTArchitectureAdapter, + "ViTForImageClassification": ViTArchitectureAdapter, + "DeiTModel": ViTArchitectureAdapter, + "DeiTForImageClassification": ViTArchitectureAdapter, # "DeiTForImageClassificationWithTeacher" unsupported for now } diff --git a/transformer_lens/model_bridge/bridge.py b/transformer_lens/model_bridge/bridge.py index f48fbae53..3cec4a623 100644 --- a/transformer_lens/model_bridge/bridge.py +++ b/transformer_lens/model_bridge/bridge.py @@ -844,7 +844,7 @@ def enable_compatibility_mode( if not getattr(self.adapter, "supports_compatibility_mode", True): raise RuntimeError( f"{type(self.adapter).__name__} does not support compatibility mode: " - "its stored-processed-weights forward is known to diverge from the " + "its stored-processed-weights is known to diverge from the " "reference model. Use the default bridge forward instead." ) @@ -1760,9 +1760,10 @@ def forward( (fragile across HF versions) or exception-based layer skipping (corrupts model state). Raises NotImplementedError if a non-None value is passed. stop_at_layer: Layer to stop forward pass at - pixel_values: Optional image tensor for multimodal models (e.g., LLaVA, Gemma3). + pixel_values: Optional image tensor for multimodal models (e.g., LLaVA, Gemma3) + and vision models (eg. ViT, DeiT). The tensor is passed directly to the underlying HuggingFace model. - Only valid when cfg.is_multimodal is True. + Only valid when cfg.is_multimodal is True or cfg.is_visual_model is True. input_values: Optional audio waveform tensor for audio models (e.g., HuBERT). The tensor is passed directly to the underlying HuggingFace model. Only valid when cfg.is_audio_model is True. @@ -1822,6 +1823,7 @@ def forward( isinstance(input, list) and len(input) > 1 and not getattr(self.cfg, "is_audio_model", False) + and not getattr(self.cfg, "is_visual_model", False) ) try: @@ -1831,6 +1833,11 @@ def forward( "Audio models require tensor input (raw waveform), not text. " "Pass a torch.Tensor or use the input_values parameter." ) + if getattr(self.cfg, "is_visual_model", False): + raise ValueError( + "Visual models require tensor input (pixel values), not text. " + "Pass a torch.Tensor or use the pixel_values parameter." + ) if _is_batched_list and padding_side is None: # Force left-padding so real tokens are flush-right. _orig_padding_side = self.tokenizer.padding_side @@ -1918,10 +1925,13 @@ def forward( # Handle pixel_values for multimodal models if pixel_values is not None: - if not getattr(self.cfg, "is_multimodal", False): + if not ( + getattr(self.cfg, "is_multimodal", False) + or getattr(self.cfg, "is_visual_model", False) + ): raise ValueError( - "pixel_values can only be passed to multimodal models " - "(cfg.is_multimodal must be True)" + "pixel_values can only be passed to multimodal or vision models " + "(cfg.is_multimodal or cfg.is_visual_model must be True)" ) kwargs["pixel_values"] = pixel_values @@ -1946,6 +1956,19 @@ def forward( "Audio models require tensor input (raw waveform). " "Pass a torch.Tensor or use input_values parameter." ) + elif getattr(self.cfg, "is_visual_model", False): + # "pixel_values" may already be in kwargs from the "if pixel_values is not None:" + # gate above (explicit pixel_values=... call); otherwise treat `input` itself as + # the image tensor, matching how the audio branch treats `input` as the waveform. + if "pixel_values" not in kwargs: + if isinstance(input, torch.Tensor): + kwargs["pixel_values"] = input + else: + raise ValueError( + "Visual models require tensor input (pixel values). " + "Pass a torch.Tensor as `input` or use the pixel_values parameter." + ) + output = self.original_model(**kwargs) elif _is_inputs_embeds: output = self.original_model(inputs_embeds=input_ids, **kwargs) else: @@ -1957,6 +1980,13 @@ def forward( logits = output.logits elif isinstance(output, tuple) and len(output) > 0: logits = output[0] + elif hasattr(output, "last_hidden_state"): + # Bare encoder models (ViTModel, DeiTModel, BertModel, etc. without + # a task head) return e.g. BaseModelOutput/BaseModelOutputWithPooling, + # which has neither `.logits` nor tuple semantics. Fall back to + # `last_hidden_state` so return_type="logits" still yields a plain + # tensor rather than silently handing back the raw HF output object. + logits = output.last_hidden_state else: logits = output if return_type == "logits": @@ -1970,6 +2000,14 @@ def forward( "Audio models do not support return_type='loss'. " "CTC loss requires aligned frame-level labels." ) + if getattr(self.cfg, "is_visual_model", False): + raise ValueError( + "Vision classification models do not support return_type='loss' " + "via this path (no next-token LM target exists for image " + "classification). Compute cross-entropy against `labels` " + "yourself from the returned logits, or use hf_generate()-style " + "direct access to self.original_model for HF's own loss." + ) if _is_inputs_embeds: raise ValueError( "Cannot compute loss with inputs_embeds — token IDs required for labels." diff --git a/transformer_lens/model_bridge/generalized_components/__init__.py b/transformer_lens/model_bridge/generalized_components/__init__.py index d8e0cc828..00225510b 100644 --- a/transformer_lens/model_bridge/generalized_components/__init__.py +++ b/transformer_lens/model_bridge/generalized_components/__init__.py @@ -137,6 +137,12 @@ from transformer_lens.model_bridge.generalized_components.vision_projection import ( VisionProjectionBridge, ) +from transformer_lens.model_bridge.generalized_components.vision_embeddings import ( + VisionEmbeddingsBridge, +) +from transformer_lens.model_bridge.generalized_components.vision_classifier_head import ( + VisionClassifierHeadBridge, +) __all__ = [ "AttentionBridge", @@ -193,4 +199,6 @@ "SSMBlockBridge", "SSMMixerBridge", "VisionProjectionBridge", + "VisionEmbeddingsBridge", + "VisionClassifierHeadBridge", ] diff --git a/transformer_lens/model_bridge/generalized_components/vision_classifier_head.py b/transformer_lens/model_bridge/generalized_components/vision_classifier_head.py new file mode 100644 index 000000000..8e9dfbbad --- /dev/null +++ b/transformer_lens/model_bridge/generalized_components/vision_classifier_head.py @@ -0,0 +1,37 @@ +"""CLS-token classifier head bridge component. + +HF's ViTForImageClassification.forward() / DeiTForImageClassification.forward() +already pool the CLS token before calling self.classifier: + + sequence_output = outputs.last_hidden_state + pooled_output = sequence_output[:, 0, :] + logits = self.classifier(pooled_output) + +So this component only ever receives an already-pooled (batch, hidden) tensor. +No slicing needed here — this is a thin, hook-named pass-through. + +Deliberately NOT covering DeiTForImageClassificationWithTeacher (dual +cls_classifier + distillation_classifier head) — see vit.py's docstring. +""" + +from typing import Any + +from torch import Tensor + +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) + + +class VisionClassifierHeadBridge(GeneralizedComponent): + """Wraps the classifier nn.Linear that HF calls with an already-pooled CLS token.""" + + def forward(self, pooled_output: Tensor, **kwargs: Any) -> Tensor: + original_component = self.original_component + if original_component is None: + raise RuntimeError( + f"Original component not set for {self.name}. Call set_original_component() first." + ) + pooled_output = self.hook_in(pooled_output) + logits = original_component(pooled_output) + return self.hook_out(logits) diff --git a/transformer_lens/model_bridge/generalized_components/vision_embeddings.py b/transformer_lens/model_bridge/generalized_components/vision_embeddings.py new file mode 100644 index 000000000..2bcbc50f9 --- /dev/null +++ b/transformer_lens/model_bridge/generalized_components/vision_embeddings.py @@ -0,0 +1,64 @@ +"""Vision (ViT-style) embeddings bridge component. + +Wraps a HF `ViTEmbeddings` / `DeiTEmbeddings` module directly and forwards through +it unmodified. The patch-conv projection, CLS-token (and, for DeiT, distillation- +token) concatenation, and position-embedding addition are HF's math — we don't +reimplement any of it, we just point at the real module and hook its input/output. + +Why one class covers both ViT and DeiT: `ViTEmbeddings.forward` and +`DeiTEmbeddings.forward` have an *identical* signature + (pixel_values, bool_masked_pos=None, interpolate_pos_encoding=False) +The only structural difference between them (DeiT prepends a distillation token +in addition to CLS, and sizes `position_embeddings` for +2 slots instead of +1) +lives entirely inside whichever module `set_original_component` resolves to — it's +invisible to this wrapper, so no DeiT-specific branching is needed here. + +Verified against the real generalized_components/base.py (not extrapolated): +`GeneralizedComponent.forward(*args, **kwargs)` already does exactly what we need +— it hooks the input via `self.hook_in`, casts it to match the wrapped module's +own parameter dtype (equivalent to HF's own `pixel_values.to(expected_dtype)` +"kept for BC" cast in ViTModel.forward — both land on the same dtype in the +overwhelming common case of a non-mixed-precision checkpoint), calls +`self.original_component(*args, **kwargs)`, and hooks the output via +`self.hook_out`. So this subclass only needs to guarantee `pixel_values` reaches +`super().forward()` *positionally* — the base class's own kwarg-name sniffing +list (`input`, `hidden_states`, `input_ids`, `query_input`, `x`, `inputs_embeds`) +doesn't include `pixel_values`, so if this component were ever called +all-keyword (`self.embeddings(pixel_values=x, ...)`, unlike the two HF call +sites we've confirmed) the base class's hook_in would silently never fire. +Re-emitting positionally here closes that gap regardless of how *we* were +called. +""" + +from typing import Any, Optional + +import torch +from torch import Tensor + +from transformer_lens.model_bridge.generalized_components.base import ( + GeneralizedComponent, +) + + +class VisionEmbeddingsBridge(GeneralizedComponent): + """Bridge for ViTEmbeddings / DeiTEmbeddings. + + Point `name=` at the embeddings module directly, e.g.: + VisionEmbeddingsBridge(name="embeddings") # bare ViTModel/DeiTModel + VisionEmbeddingsBridge(name="vit.embeddings") # ViTForImageClassification + VisionEmbeddingsBridge(name="deit.embeddings") # DeiTForImageClassification + """ + + def forward( + self, + pixel_values: Tensor, + bool_masked_pos: Optional[torch.BoolTensor] = None, + interpolate_pos_encoding: Optional[bool] = None, + **kwargs: Any, + ) -> Tensor: + return super().forward( + pixel_values, + bool_masked_pos=bool_masked_pos, + interpolate_pos_encoding=interpolate_pos_encoding, + **kwargs, + ) diff --git a/transformer_lens/model_bridge/sources/transformers.py b/transformer_lens/model_bridge/sources/transformers.py index 34ef4226b..d13fbcf2b 100644 --- a/transformer_lens/model_bridge/sources/transformers.py +++ b/transformer_lens/model_bridge/sources/transformers.py @@ -157,6 +157,19 @@ def map_default_transformer_lens_config(hf_config): tl_config.n_ctx = source_config.max_length elif hasattr(source_config, "seq_length"): tl_config.n_ctx = source_config.seq_length + elif hasattr(source_config, "image_size") and hasattr(source_config, "patch_size"): + # Vision Transformers calculate sequence length dynamically: + # (image_size / patch_size)^2 + 1 (for the CLS token) + image_size = source_config.image_size + patch_size = source_config.patch_size + + # HF configs allow these to be integers or tuples/lists + img_h = image_size[0] if isinstance(image_size, (list, tuple)) else image_size + img_w = image_size[1] if isinstance(image_size, (list, tuple)) else image_size + patch_h = patch_size[0] if isinstance(patch_size, (list, tuple)) else patch_size + patch_w = patch_size[1] if isinstance(patch_size, (list, tuple)) else patch_size + + tl_config.n_ctx = (img_h // patch_h) * (img_w // patch_w) + 1 elif hasattr(source_config, "max_sequence_length"): tl_config.n_ctx = source_config.max_sequence_length else: @@ -367,6 +380,8 @@ def get_hf_model_class_for_architecture(architecture: str): MASKED_LM_ARCHITECTURES, MULTIMODAL_ARCHITECTURES, SEQ2SEQ_ARCHITECTURES, + VISION_ARCHITECTURES, + VISION_CLASSIFICATION_ARCHITECTURES, ) if architecture in SEQ2SEQ_ARCHITECTURES or architecture in AUDIO_TEXT_ARCHITECTURES: @@ -388,6 +403,14 @@ def get_hf_model_class_for_architecture(architecture: str): return AutoModelForCTC from transformers import AutoModel + return AutoModel + elif architecture in VISION_ARCHITECTURES: + if architecture in VISION_CLASSIFICATION_ARCHITECTURES: + from transformers import AutoModelForImageClassification + + return AutoModelForImageClassification + from transformers import AutoModel + return AutoModel else: return AutoModelForCausalLM @@ -813,7 +836,8 @@ def boot( use_fast = getattr(adapter.cfg, "use_fast", True) # Audio models use feature extractors, not text tokenizers _is_audio = getattr(adapter.cfg, "is_audio_model", False) - if _is_audio and tokenizer is None: + _is_visual = getattr(adapter.cfg, "is_visual_model", False) + if (_is_audio or _is_visual) and tokenizer is None: tokenizer = None # Skip tokenizer loading for audio models elif tokenizer is not None: tokenizer = setup_tokenizer(tokenizer, default_padding_side=default_padding_side) diff --git a/transformer_lens/model_bridge/supported_architectures/__init__.py b/transformer_lens/model_bridge/supported_architectures/__init__.py index e5d60f958..ba60f5309 100644 --- a/transformer_lens/model_bridge/supported_architectures/__init__.py +++ b/transformer_lens/model_bridge/supported_architectures/__init__.py @@ -252,6 +252,9 @@ from transformer_lens.model_bridge.supported_architectures.switch_transformers import ( SwitchTransformersArchitectureAdapter, ) +from transformer_lens.model_bridge.supported_architectures.vit import ( + ViTArchitectureAdapter, +) from transformer_lens.model_bridge.supported_architectures.t5 import T5ArchitectureAdapter from transformer_lens.model_bridge.supported_architectures.t5gemma import T5GemmaArchitectureAdapter from transformer_lens.model_bridge.supported_architectures.t5gemma2 import ( @@ -396,6 +399,8 @@ "Starcoder2ArchitectureAdapter", "T5ArchitectureAdapter", "T5GemmaArchitectureAdapter", + "T5Gemma2ArchitectureAdapter", + "ViTArchitectureAdapter", "XGLMArchitectureAdapter", "Zamba2ArchitectureAdapter", "HrmTextArchitectureAdapter", diff --git a/transformer_lens/model_bridge/supported_architectures/vit.py b/transformer_lens/model_bridge/supported_architectures/vit.py new file mode 100644 index 000000000..d0bad6e0f --- /dev/null +++ b/transformer_lens/model_bridge/supported_architectures/vit.py @@ -0,0 +1,204 @@ +"""ViT / DeiT architecture adapter. + +Supports HF `ViTModel`, `ViTForImageClassification`, `DeiTModel`, +`DeiTForImageClassification` (single CLS-token classifier head). Encoder blocks +are structurally near-identical between ViT and DeiT — same field names +(layernorm_before/after, attention.{q,k,v,o}_proj, mlp.{fc1,fc2}) — differing +only in the embeddings module used (DeiT's carries an extra distillation token, +which is invisible to this adapter — see vision_embeddings.py). + +NOT covered: `DeiTForImageClassificationWithTeacher` (dual cls+distillation head, +averaged). See vision_classifier_head.py's docstring for why, and prepare_model() +below raises loudly if you load one anyway rather than silently producing wrong +logits. + +ViT/DeiT blocks are pre-LN (LayerNorm applied *before* attention/MLP, residual +added after — same shape as Llama/GPT2), unlike BERT's post-LN. That's why +`supports_fold_ln = True` here where BertArchitectureAdapter sets it False. + +NOTE (transformers >= the ViT/DeiT flattening refactor): as of this version of +`modeling_vit.py`, HF removed the separate `ViTEncoder` wrapper — the blocks +now live directly at `.layers` on the model, not `.encoder.layer`. +Attention was flattened too: `ViTAttention` now owns `q_proj`/`k_proj`/`v_proj`/ +`o_proj` directly (no more nested `attention.attention.{query,key,value}` + +`output.dense`). And `ViTLayer` already exposes a flat `.mlp` submodule +(`ViTMLP` with `.fc1`/`.fc2`) instead of the old `intermediate`/`output` split. +Because of this, the old `ViTMLPWrapper` shim, the block-forward tuple-unwrapping +monkey-patch (`ViTLayer.forward` returns a plain tensor now, not a tuple), and +the `hf_model.encoder_layer = hf_model.encoder.layer` aliasing hack are all +gone — the component mapping below points straight at the real attributes. +""" + +from typing import Any, Dict + +from transformer_lens.conversion_utils.conversion_steps import RearrangeTensorConversion +from transformer_lens.conversion_utils.param_processing_conversion import ( + ParamProcessingConversion, +) +from transformer_lens.model_bridge.architecture_adapter import ArchitectureAdapter +from transformer_lens.model_bridge.generalized_components import ( + AttentionBridge, + BlockBridge, + LinearBridge, + MLPBridge, + NormalizationBridge, +) +from transformer_lens.model_bridge.generalized_components.vision_classifier_head import ( + VisionClassifierHeadBridge, +) +from transformer_lens.model_bridge.generalized_components.vision_embeddings import ( + VisionEmbeddingsBridge, +) + + +class ViTArchitectureAdapter(ArchitectureAdapter): + """Architecture adapter for ViT and (non-distilled-head) DeiT vision models.""" + + supports_generation: bool = False + + # Vision models have no tokenizer, so the default text-shaped verify_models + # phases can't run against them. Mirrors falcon_h1.py's opt-out until + # verify_models grows a vision-model-aware phase set (tracked separately). + applicable_phases: list[int] = [] + + def __init__(self, cfg: Any) -> None: + super().__init__(cfg) + + self.cfg.is_visual_model = True + self.cfg.normalization_type = "LN" + self.cfg.positional_embedding_type = "standard" + self.cfg.final_rms = False + self.cfg.gated_mlp = False + self.cfg.attn_only = False + self.supports_fold_ln = True + + n_heads = self.cfg.n_heads + + self.weight_processing_conversions = { + "blocks.{i}.attn.q.weight": ParamProcessingConversion( + tensor_conversion=RearrangeTensorConversion( + "(h d_head) d_model -> h d_model d_head", h=n_heads + ), + ), + "blocks.{i}.attn.k.weight": ParamProcessingConversion( + tensor_conversion=RearrangeTensorConversion( + "(h d_head) d_model -> h d_model d_head", h=n_heads + ), + ), + "blocks.{i}.attn.v.weight": ParamProcessingConversion( + tensor_conversion=RearrangeTensorConversion( + "(h d_head) d_model -> h d_model d_head", h=n_heads + ), + ), + "blocks.{i}.attn.q.bias": ParamProcessingConversion( + tensor_conversion=RearrangeTensorConversion("(h d_head) -> h d_head", h=n_heads), + ), + "blocks.{i}.attn.k.bias": ParamProcessingConversion( + tensor_conversion=RearrangeTensorConversion("(h d_head) -> h d_head", h=n_heads), + ), + "blocks.{i}.attn.v.bias": ParamProcessingConversion( + tensor_conversion=RearrangeTensorConversion("(h d_head) -> h d_head", h=n_heads), + ), + "blocks.{i}.attn.o.weight": ParamProcessingConversion( + tensor_conversion=RearrangeTensorConversion( + "d_model (h d_head) -> h d_head d_model", h=n_heads + ), + ), + } + + self.component_mapping = self._build_component_mapping(prefix="", with_classifier=False) + + def _build_component_mapping(self, prefix: str, with_classifier: bool) -> Dict[str, Any]: + p = prefix + mapping: Dict[str, Any] = { + "embed": VisionEmbeddingsBridge(name=f"{p}embeddings"), + "blocks": BlockBridge( + # HF's ViTModel/DeiTModel no longer wrap blocks in a `.encoder` + # module — they sit directly on `.layers`. + name=f"{p}layers", + hook_alias_overrides={ + "hook_mlp_out": "mlp.out.hook_out", + "hook_mlp_in": "mlp.in.hook_in", + }, + submodules={ + "ln1": NormalizationBridge( + name="layernorm_before", + config=self.cfg, + use_native_layernorm_autograd=True, + ), + "ln2": NormalizationBridge( + name="layernorm_after", + config=self.cfg, + use_native_layernorm_autograd=True, + ), + "attn": AttentionBridge( + name="attention", + config=self.cfg, + submodules={ + # Submodule paths here are resolved relative to the + # already-resolved "attn" bridge module itself + # (block.attention). ViTAttention/DeiTAttention now + # owns q_proj/k_proj/v_proj/o_proj directly + # (flattened, no nested self-attention + separate + # output.dense module), so no "attention." prefix. + "q": LinearBridge(name="q_proj"), + "k": LinearBridge(name="k_proj"), + "v": LinearBridge(name="v_proj"), + "o": LinearBridge(name="o_proj"), + }, + ), + "mlp": MLPBridge( + name="mlp", + config=self.cfg, + submodules={ + # ViTLayer.mlp is a real ViTMLP module now + # (fc1/fc2) — no more intermediate/output split, + # so no wrapper shim is needed. + "in": LinearBridge(name="fc1"), + "out": LinearBridge(name="fc2"), + }, + ), + }, + ), + "ln_final": NormalizationBridge( + name=f"{p}layernorm", + config=self.cfg, + use_native_layernorm_autograd=True, + ), + } + if with_classifier: + mapping["unembed"] = VisionClassifierHeadBridge(name="classifier") + return mapping + + def prepare_model(self, hf_model: Any) -> None: + """Detect ViTForImageClassification vs DeiTForImageClassification vs a bare + *Model, and add/omit the classifier head + prefix accordingly. + + No structural patching of the HF model is needed any more: current + transformers ViT/DeiT blocks already expose a flat `.mlp` (fc1/fc2) and + return plain tensors from `forward`, and the blocks live directly on + `.layers` rather than behind a now-removed `.encoder` wrapper. + This method only has to figure out the right prefix/classifier and + build the component mapping to point at those real attributes. + """ + if hasattr(hf_model, "cls_classifier") and hasattr(hf_model, "distillation_classifier"): + raise NotImplementedError( + "DeiTForImageClassificationWithTeacher (dual cls_classifier + " + "distillation_classifier head, averaged) isn't supported by this " + "adapter yet — see vision_classifier_head.py's docstring for why, " + "and what to check before adding it." + ) + + # Check for the wrapper module created by HF's ForImageClassification classes. + # Bare ViTModel / DeiTModel have components at the root, so prefix must be "". + if hasattr(hf_model, "deit") and not hasattr(hf_model, "vit"): + prefix = "deit." + elif hasattr(hf_model, "vit"): + prefix = "vit." + else: + prefix = "" + + with_classifier = hasattr(hf_model, "classifier") + self.component_mapping = self._build_component_mapping( + prefix=prefix, with_classifier=with_classifier + ) diff --git a/transformer_lens/tools/model_registry/__init__.py b/transformer_lens/tools/model_registry/__init__.py index 07931038b..aa31a3b0b 100644 --- a/transformer_lens/tools/model_registry/__init__.py +++ b/transformer_lens/tools/model_registry/__init__.py @@ -187,6 +187,10 @@ "T5GemmaForConditionalGeneration", "T5Gemma2ForConditionalGeneration", "XGLMForCausalLM", + "ViTModel", + "ViTForImageClassification", + "DeiTModel", + "DeiTForImageClassification", "Zamba2ForCausalLM", } @@ -335,6 +339,10 @@ "T5GemmaForConditionalGeneration": ["google"], "T5Gemma2ForConditionalGeneration": ["google"], "XGLMForCausalLM": ["facebook"], + "ViTModel": ["google"], + "ViTForImageClassification": ["google"], + "DeiTModel": ["facebook"], + "DeiTForImageClassification": ["facebook"], "Zamba2ForCausalLM": ["Zyphra"], } diff --git a/transformer_lens/tools/model_registry/generate_report.py b/transformer_lens/tools/model_registry/generate_report.py index 559425cf2..451c35470 100644 --- a/transformer_lens/tools/model_registry/generate_report.py +++ b/transformer_lens/tools/model_registry/generate_report.py @@ -128,6 +128,12 @@ "RwkvForCausalLM": "BlinkDL RWKV-4 WKV linear-attention RNN (Pythia-parallel Pile suite)", "NanoChatForCausalLM": "Karpathy's nanochat education-stack decoder (weightless norms, relu^2 MLP, capped logits)", "HunYuanDenseV1ForCausalLM": "Tencent's open source decoder models", + "ViTModel": "Vision Transformer (bare encoder, no classification head)", + "ViTForImageClassification": "Vision Transformer with an image classification head", + "DeiTModel": "Data-efficient Image Transformer (bare encoder)", + "DeiTForImageClassification": "DeiT with a single CLS-token classification head", + "Wav2Vec2Model": "Facebook's Wav2Vec 2.0 for speech", + "HubertModel": "Facebook's HuBERT for speech", "Cohere2ForCausalLM": "Cohere's Command-A architecture with interleaved sliding-window RoPE and full-attention NoPE layers", "OuroForCausalLM": "ByteDance's Ouro looped language model (LoopLM) with weight-shared iterated depth", "RavenForCausalLM": "tomg-group-umd's Huginn depth-recurrent decoder (prelude / weight-tied recurrent core / coda) with runtime recurrence count", @@ -153,14 +159,10 @@ "MT5ForConditionalGeneration": "Multilingual T5", "WhisperForConditionalGeneration": "OpenAI's Whisper speech recognition", "CLIPModel": "OpenAI's CLIP vision-language model", - "ViTModel": "Google's Vision Transformer", "SwinModel": "Microsoft's Swin Transformer for vision", - "DeiTModel": "Facebook's Data-efficient Image Transformer", "BeitModel": "Microsoft's BERT pre-training for images", "ConvNextModel": "Facebook's ConvNeXt modernized ConvNet", "SegformerModel": "NVIDIA's SegFormer for segmentation", - "Wav2Vec2Model": "Facebook's Wav2Vec 2.0 for speech", - "HubertModel": "Facebook's HuBERT for speech", "SpeechT5Model": "Microsoft's SpeechT5 for speech tasks", "BlipModel": "Salesforce's BLIP vision-language model", "Blip2Model": "Salesforce's BLIP-2 with frozen LLM", diff --git a/transformer_lens/utilities/architectures.py b/transformer_lens/utilities/architectures.py index 5dda51702..3312fd9f8 100644 --- a/transformer_lens/utilities/architectures.py +++ b/transformer_lens/utilities/architectures.py @@ -71,6 +71,19 @@ "HubertForSequenceClassification", } +# Vision-only (non-multimodal, no text tower) encoder models. Split into the +# two HF AutoModel classes they load under: bare encoders load via AutoModel, +# classification heads load via AutoModelForImageClassification. +VISION_MODEL_ARCHITECTURES: set[str] = { + "ViTModel", + "DeiTModel", +} +VISION_CLASSIFICATION_ARCHITECTURES: set[str] = { + "ViTForImageClassification", + "DeiTForImageClassification", +} +VISION_ARCHITECTURES: set[str] = VISION_MODEL_ARCHITECTURES | VISION_CLASSIFICATION_ARCHITECTURES + # Text models whose remote code registers only under plain AutoModel # (the class itself carries the LM head). BASE_AUTOMODEL_ARCHITECTURES: set[str] = {