diff --git a/families/llama/checkpoint_mapper.py b/families/llama/checkpoint_mapper.py index c931921d75..00f5a10bc2 100644 --- a/families/llama/checkpoint_mapper.py +++ b/families/llama/checkpoint_mapper.py @@ -116,6 +116,8 @@ def load_standard_weights( f"Embedding shape {embedding.shape} != ({vocab}, {hidden})" ) weights["embedding"] = embedding.astype(target_dtype) + # Jiaxin Deng: keep only the stored embedding while loading decoder layers. + del embedding def _load_layer(layer_idx: int) -> tuple[int, WeightDict, int, int]: prefix = f"layer.{layer_idx}" @@ -226,7 +228,11 @@ def _projection(stem: str, hf_name: str) -> np.ndarray: ) else: # Tied embeddings - weights["w_out"] = _transpose_2d(embedding.copy(), "embedding_tied", precision=precision) + weights["w_out"] = _transpose_2d( + weights["embedding"], "embedding_tied", precision=precision + ) + if np.shares_memory(weights["w_out"], weights["embedding"]): + weights["w_out"] = weights["w_out"].copy() weights["_attention_size"] = attention_size # type: ignore[assignment] weights["_kv_attention_size"] = kv_attention_size # type: ignore[assignment] diff --git a/families/llama/model.py b/families/llama/model.py index 9f19dc95cb..ef700f50cf 100644 --- a/families/llama/model.py +++ b/families/llama/model.py @@ -283,6 +283,9 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: verbose=bool(request.verbose), quant_ctx=quant_ctx, ) + # Note (Jiaxin Deng): avoid retaining both plans during decode compilation. + writer.add_bytes("prefill.plan", prefill) + del prefill config.raw["_decoder_engine_role"] = "decode" decode = _build_engine( config, @@ -293,7 +296,6 @@ def _build_native(request: "BuildRequest", writer: "BundleWriter") -> None: quant_ctx=quant_ctx, ) writer.add_bytes("engine.plan", decode) - writer.add_bytes("prefill.plan", prefill) layout = "split" config.raw.pop("_decoder_engine_role", None) diff --git a/families/llama/tests/test_checkpoint_cache.py b/families/llama/tests/test_checkpoint_cache.py new file mode 100644 index 0000000000..9f765a0ca5 --- /dev/null +++ b/families/llama/tests/test_checkpoint_cache.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise staged checkpoint lookup without Hub metadata or network access.""" + +import pytest +from huggingface_hub import constants +from huggingface_hub.errors import LocalEntryNotFoundError + +from .test_e2e import _checkpoint + + +@pytest.fixture +def offline_cache(tmp_path, monkeypatch): + monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True) + return tmp_path + + +def test_pinned_checkpoint_uses_snapshot_without_cached_tree(offline_cache): + revision = "a" * 40 + snapshot = offline_cache / "models--trtmc-test--llama" / "snapshots" / revision + snapshot.mkdir(parents=True) + (snapshot / "config.json").write_text("{}", encoding="utf-8") + + assert _checkpoint({"hf_id": "trtmc-test/llama", "hf_revision": revision}) == snapshot + + +def test_missing_checkpoint_is_not_replaced_by_another_revision(offline_cache): + cached = offline_cache / "models--trtmc-test--llama" / "snapshots" / ("a" * 40) + cached.mkdir(parents=True) + (cached / "config.json").write_text("{}", encoding="utf-8") + + with pytest.raises(LocalEntryNotFoundError): + _checkpoint({"hf_id": "trtmc-test/llama", "hf_revision": "b" * 40}) + + +def test_staged_snapshot_still_requires_config(offline_cache): + revision = "a" * 40 + snapshot = offline_cache / "models--trtmc-test--llama" / "snapshots" / revision + snapshot.mkdir(parents=True) + + with pytest.raises(AssertionError): + _checkpoint({"hf_id": "trtmc-test/llama", "hf_revision": revision}) diff --git a/families/llama/tests/test_support.py b/families/llama/tests/test_support.py new file mode 100644 index 0000000000..0bacb43246 --- /dev/null +++ b/families/llama/tests/test_support.py @@ -0,0 +1,350 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU coverage of build memory lifetime and bundle publication.""" + +import gc +import importlib.util +import json +from pathlib import Path +import struct +import sys +import tracemalloc +import types +from types import SimpleNamespace +import weakref + +import numpy as np +import pytest +from safetensors.numpy import save_file + +from tensorrt_model_connect import BuildRequest +from tensorrt_model_connect.bundle_writer import BundleWriter + +from .. import checkpoint_mapper +from ..config import ModelConfig + + +@pytest.fixture +def request_and_writer(tmp_path, monkeypatch): + # Jiaxin Deng: stub only GPU compilation; exercise the real build and writer on CPU. + with monkeypatch.context() as imports: + for module_name, function_name in ( + ("dual_profile_decoder_builder", "build_dual_profile_decoder_engine"), + ("standard_decoder_builder", "build_standard_decoder_engine"), + ): + module = types.ModuleType(f"families.llama.{module_name}") + setattr(module, function_name, None) + imports.setitem(sys.modules, module.__name__, module) + spec = importlib.util.spec_from_file_location( + "families.llama._plan_lifetime_model", Path(__file__).parents[1] / "model.py" + ) + model = importlib.util.module_from_spec(spec) + spec.loader.exec_module(model) + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + (checkpoint / "config.json").write_text( + json.dumps( + { + "model_type": "llama", + "hidden_size": 8, + "intermediate_size": 16, + "num_hidden_layers": 1, + "num_attention_heads": 2, + "num_key_value_heads": 1, + "vocab_size": 32, + "max_position_embeddings": 32, + "eos_token_id": 2, + } + ) + ) + monkeypatch.setattr(model, "load_standard_weights", lambda *args, **kwargs: {}) + destination = tmp_path / "model.bundle" + request = BuildRequest( + model_dir=checkpoint, + output_path=destination, + family="llama", + task="text_generation", + precision="fp16", + max_sequence_length=16, + ) + writer = BundleWriter(destination) + yield model, request, writer + writer.abort() + + +def test_prefill_plan_is_released_before_decode_build(request_and_writer, monkeypatch): + model, request, writer = request_and_writer + released = [] + + class Plan(bytes): + def __del__(self): + released.append(True) + + def build_engine(config, *args, **kwargs): + if config.raw["_decoder_engine_role"] == "prefill": + return Plan(b"prefill bytes") + gc.collect() + assert released, "prefill plan is still resident during decode build" + return b"decode bytes" + + monkeypatch.setattr(model, "_build_engine", build_engine) + model.build(request, writer) + writer.finish() + data = request.output_path.read_bytes() + header_size = struct.unpack("