diff --git a/tests/shared/granite4_equivalence.py b/tests/shared/granite4_equivalence.py index e5206e1..01e0beb 100644 --- a/tests/shared/granite4_equivalence.py +++ b/tests/shared/granite4_equivalence.py @@ -31,11 +31,22 @@ def assert_close(actual, expected, *, atol, rtol, msg=""): This avoids the joint formula (atol + rtol * |b|) where the two tolerances inflate each other's budget. + + ``atol=rtol=0.0`` is a valid bit-exact gate. Passing it here rather than + to ``torch.testing.assert_close`` is deliberate: that function's ``msg=`` + string *replaces* its whole diagnostic, so a failure reports only the + message and none of the numbers needed to tell a last-bit drift from a + real regression. + + Matching non-finite entries count as equal. Logprob tensors are padded + with ``-inf`` for positions vLLM did not return, and ``-inf - -inf`` is + ``nan``, which fails every comparison — including ``diff <= 0.0`` — and + would surface as a bogus ``worst=nan``. """ diff = (actual - expected).abs() abs_ok = diff <= atol rel_ok = diff <= rtol * expected.abs() - ok = abs_ok | rel_ok + ok = abs_ok | rel_ok | (actual == expected) if not ok.all(): num_bad = (~ok).sum().item() @@ -256,6 +267,21 @@ def get_tolerances(layer_types, long_sequence=False, has_kv_hidden=False): position while upstream embeds the original control id. Visible positions attending to the control position pick up that delta. + Case 1's bit-exactness holds for the *eager* graph only. Under + torch.compile the switch graph and the upstream graph are different + graphs, so inductor is free to fuse and tile them differently; at + production dimensions that is enough to round one bf16 logit's last bit + apart. It surfaced as ``test_logits_match[4.0-micro]`` failing on vLLM + 0.20 while passing on 0.19 — with 4.0-1b and 4.0-350m (same 40 layers, + same dense stack) passing on both, and the same test bit-exact on the HF + backend, which is why the divergence is attributed to compilation rather + than to the weight transfer. vLLM's per-engine kernel choice is visible + in its own logs: the failing engine reported + ``IrOpPriorityConfig(rms_norm=['native'])`` where its sibling reported + ``['vllm_c', 'native']``. The vLLM callers that want a bit-exact gate + therefore pass ``enforce_eager=True``; see + ``tests/vllm/_granite4_fullsize_tests.py``. + Args: layer_types: list of "attention" strings long_sequence: unused (kept for API compatibility) diff --git a/tests/vllm/_granite4_fullsize_tests.py b/tests/vllm/_granite4_fullsize_tests.py index 57a131c..67604a7 100644 --- a/tests/vllm/_granite4_fullsize_tests.py +++ b/tests/vllm/_granite4_fullsize_tests.py @@ -3,6 +3,10 @@ Full-size Granite 4 equivalence tests via vllm.LLM. Requires CUDA GPU and vLLM installed. + +Runs eager. These are the bit-exact tests, and bit-exactness is a property of +the module tree, not of whatever inductor decided to fuse in each of the two +graphs — see the comment on ``enforce_eager`` below. """ import pytest @@ -53,22 +57,39 @@ def test_logits_match(self, model_name, tmp_path): tmpdir=tmp_path, max_model_len=64, gpu_memory_utilization=0.4, + # Compare the two module trees, not two compilations of them. + # torch.compile/inductor pick fusions and kernels per graph, and + # the switch graph is not the upstream graph, so at production + # dimensions the compiled paths can land a bf16 last bit + # differently while the eager math is identical. That is what + # broke 4.0-micro under vLLM 0.20 and not 0.19 (see the note in + # get_tolerances). enforce_eager removes the compiler, the + # CUDA-graph capture and vLLM's on-disk compile cache from the + # comparison, which restores a meaningful bit-exact gate. + # The compiled path is still covered, by the tests that are + # tolerant of a flipped last bit: + # tests/composer/test_skinning_equivalence.py (real weights) and + # tests/vllm/test_generation_equivalence.py (distribution). + enforce_eager=True, ) tol = get_tolerances(layer_types) - if tol is None: - torch.testing.assert_close( - switch, - upstream, - atol=0.0, - rtol=0.0, - msg=f"{model_name}: logprobs should be bit-exact", - ) - else: - assert_close( - switch, - upstream, - atol=tol[0], - rtol=tol[1], - msg=f"{model_name}: full-size logprobs diverge", - ) + atol, rtol = (0.0, 0.0) if tol is None else tol + assert_close( + switch, + upstream, + atol=atol, + rtol=rtol, + msg=f"{model_name}: full-size logprobs diverge", + ) + + # Print the margin on a pass, not only into the failure message. A + # bit-exact gate that has quietly started riding at one ULP is a + # regression in progress, and this is the only place it shows. + delta = (switch - upstream).abs() + finite = delta[delta.isfinite()] + print( + f"{model_name}: {finite.numel()} finite logprob entries, " + f"max |delta| = {finite.max().item():.3e} " + f"(atol={atol:.1e} rtol={rtol:.1e})" + )