From 12e1540cb8f840885dde379e5d41b59f5eb920be Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Tue, 15 Sep 2026 00:38:39 +0000 Subject: [PATCH 1/5] Reuse a whole recipe via $import, deprecate recipe_type, and backfill NVIDIA checkpoint recipes Squashes the shengliangx/batch-backfill-recipe work (PR #2376) into one commit for a clean rebase onto the model_type/ rename (PR #2328). - Backfill modelopt_recipes/models/// entries for the quantized checkpoints NVIDIA publishes, plus a models/ tier README. - Let a recipe's kind be declared once and deprecate metadata.recipe_type; reject a recipe that delegates via $import to a different kind of recipe. - Support and document reusing a whole recipe via $import (aliasing), and drop the one-off recipe_backfill tool and its generated index. - Stop emitting the deprecated recipe_type in the shipped recipes. Signed-off-by: Shengliang Xu --- CHANGELOG.rst | 4 + docs/source/guides/10_recipes.rst | 55 ++- modelopt/recipe/config.py | 66 ++- modelopt/recipe/loader.py | 172 ++++++- modelopt/torch/opt/config_loader.py | 14 +- .../kv_fp8_nvfp4_cast_kl_div_at_5p4bits.yaml | 1 - .../auto_quantize/nvfp4_fp8_at_5p4bits.yaml | 1 - .../nvfp4_fp8_kl_div_at_5p4bits.yaml | 1 - .../nvfp4_mse_fp8_at_6p0bits.yaml | 1 - ...w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml | 1 - .../w4a8_awq_beta_fp8_at_6p0bits.yaml | 1 - .../general/ptq/fp8_default-kv_fp8.yaml | 2 +- .../general/ptq/fp8_default-kv_fp8_cast.yaml | 2 +- .../ptq/int4_blockwise_weight_only.yaml | 2 +- .../general/ptq/mxfp4_mlp_weight_only.yaml | 2 +- .../ptq/nvfp4_act_headroom-kv_fp8_cast.yaml | 2 +- .../general/ptq/nvfp4_default-kv_fp8.yaml | 2 +- .../ptq/nvfp4_default-kv_fp8_cast.yaml | 2 +- .../ptq/nvfp4_default-kv_none-gptq.yaml | 2 +- .../ptq/nvfp4_default-kv_nvfp4_cast.yaml | 2 +- .../ptq/nvfp4_experts_only-kv_fp8.yaml | 2 +- .../ptq/nvfp4_experts_only-kv_fp8_cast.yaml | 2 +- .../nvfp4_experts_only-kv_fp8_layerwise.yaml | 2 +- ..._experts_only-kv_fp8_layerwise_export.yaml | 2 +- ...experts_only-kv_fp8_layerwise_offload.yaml | 2 +- ...experts_only_input_scale1-kv_fp8_cast.yaml | 2 +- .../nvfp4_experts_only_mse-kv_fp8_cast.yaml | 2 +- .../general/ptq/nvfp4_mlp_only-kv_fp8.yaml | 2 +- .../ptq/nvfp4_mlp_only-kv_fp8_cast.yaml | 2 +- .../ptq/nvfp4_mlp_only-novit-kv_fp8.yaml | 2 +- .../ptq/nvfp4_mlp_only_mse-kv_fp8_cast.yaml | 2 +- .../general/ptq/nvfp4_mlp_weight_only.yaml | 2 +- .../general/ptq/nvfp4_omlp_only-kv_fp8.yaml | 2 +- .../ptq/nvfp4_omlp_only-kv_fp8_cast.yaml | 2 +- .../ptq/nvfp4_weight_only-kv_fp16.yaml | 2 +- .../ptq/nvfp4_weight_only-kv_fp8_cast.yaml | 2 +- .../qad/nvfp4_dual_lsq-mse_init-fp8_kv.yaml | 2 +- .../qad/nvfp4_lsq-mse_init-fp8_kv.yaml | 2 +- .../general/speculative_decoding/dflash.yaml | 2 +- .../general/speculative_decoding/domino.yaml | 2 +- .../general/speculative_decoding/dspark.yaml | 2 +- .../general/speculative_decoding/eagle3.yaml | 2 +- .../speculative_decoding/lilicorr.yaml | 2 +- .../speculative_decoding/lilicorr_conv.yaml | 2 +- .../ptq/nvfp4_experts_only.yaml | 2 +- .../gemma/ptq/int8_sq-kv_fp8_cast.yaml | 2 +- .../gemma/ptq/w4a8_awq-kv_fp8_cast.yaml | 2 +- .../gemma4/ptq/w4a8_awq-kv_fp8_cast.yaml | 2 +- .../ptq/mxfp8_nvfp4_experts.yaml | 2 +- .../minimax_m3_vl/ptq/nvfp4_experts_only.yaml | 2 +- .../mpt/ptq/w4a8_awq-kv_fp8_cast.yaml | 2 +- .../ptq/fp8_output_quant_proj.yaml | 2 +- .../ptq/nvfp4_output_quant_proj.yaml | 2 +- .../nemotron_vl/ptq/nvfp4-kv_fp8_cast.yaml | 2 +- .../qwen3_5/ptq/fp8_vision-kv_none.yaml | 2 +- .../ptq/fp8_vision_lm-kv_fp8_cast.yaml | 2 +- .../ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml | 2 +- .../w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml | 2 +- .../nvfp4_experts_mse-fp8_rest-kv_fp8.yaml | 2 +- .../ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml | 2 +- .../w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml | 2 +- ...w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml | 1 - ...8_module_spaces_at_6p0bits-active_moe.yaml | 1 - .../qwen3_vl/ptq/fp8_vision-kv_none.yaml | 2 +- .../ptq/fp8_vision_lm-kv_fp8_cast.yaml | 2 +- .../ptq/nvfp4_experts_only-kv_fp8_cast.yaml | 2 +- .../step3p7/ptq/nvfp4_mlp_only-kv_fp8.yaml | 2 +- modelopt_recipes/model_type/vit/ptq/fp8.yaml | 6 +- ...self_attn-fp8_linear_attn-kv_fp8_cast.yaml | 2 +- modelopt_recipes/models/README.md | 71 ++- .../ptq/nvfp4_experts_only.yaml | 2 +- .../auto_quantize/w4a16_nvfp4_4o6_mixed.yaml | 1 - .../ptq/nvfp4-max-calib.yaml | 2 +- .../ptq/nvfp4_experts-fp8_pb_attention.yaml | 2 +- .../ptq/fp8_moe_mamba-kv_fp8_cast.yaml | 83 ++++ .../ptq/nvfp4_w4a16.yaml | 2 +- .../ptq/fp8_moe_mamba-kv_fp8_cast.yaml | 74 +++ .../ptq/nvfp4-max-calib.yaml | 11 +- .../ptq/nvfp4-mse.yaml | 11 +- .../ptq/nvfp4-4o6.yaml | 11 +- .../ptq/w4a16_nvfp4_4o6.yaml | 2 +- .../dspark_warmstart.yaml | 2 +- .../ptq/fp8-kv_fp8_cast.yaml | 72 +++ .../nvfp4_experts-fp8_rest-kv_fp8_cast.yaml | 99 ++++ .../Step-3.5-Flash/ptq/nvfp4-mlp-only.yaml | 2 +- .../nvfp4_experts_dense_mlp-kv_fp8_cast.yaml | 2 +- modelopt_recipes/ptq.md | 36 +- modelopt_recipes/timm/resnet/ptq/fp8.yaml | 1 - modelopt_recipes/timm/resnet/ptq/int8.yaml | 1 - tests/unit/recipe/test_loader.py | 453 +++++++++++++++++- tools/precommit/check_modelopt_recipes.py | 21 +- 91 files changed, 1284 insertions(+), 112 deletions(-) create mode 100644 modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml create mode 100644 modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml create mode 100644 modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/fp8-kv_fp8_cast.yaml create mode 100644 modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/nvfp4_experts-fp8_rest-kv_fp8_cast.yaml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ee4648f101b..53588aec0b2 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,8 @@ Changelog *Quantization* +- A recipe can now **delegate its whole body to another recipe** with a top-level ``$import``, overriding only ``metadata``. ``metadata.recipe_type`` became optional along with it: a recipe states its kind with a ``# modelopt-schema:`` comment, with ``metadata.recipe_type``, or by delegating to a recipe that does, and only a recipe that another file imports has to carry the schema comment. Whatever a recipe does state must be true: a schema comment and a ``recipe_type`` must agree, and so must a recipe and the recipe it delegates to. ``modelopt_recipes/models/`` uses this for checkpoint entries that a portable recipe already reproduces: the entry aliases that recipe instead of copying it. +- Backfill the recipes behind NVIDIA's already-published checkpoints under ``modelopt_recipes/models/``, so a released checkpoint's quantization scheme is reachable from its own model-hub path rather than only from the general tier. This batch covers the NVIDIA-source releases: NVIDIA-Nemotron-3-Nano-30B-A3B FP8, NVIDIA-Nemotron-3-Super-120B-A12B FP8, and Nemotron-3-Nano-Omni-30B-A3B-Reasoning (FP8 and NVFP4); the remaining source-model orgs follow in separate changes. A release is only backfilled when its model card supports a post-training quantization recipe -- one whose card documents quantization-aware distillation after PTQ is deliberately left out, since no PTQ recipe reproduces it. - Add ``layerwise.export_dir``: layerwise calibration writes each decoder layer to its own quantized checkpoint shard as it finishes, so no separate ``export_hf_checkpoint()`` pass is needed and, with ``layerwise.checkpoint_dir``, an interrupted run resumes without redoing finished layers. Calibration writes the layer shards; ``finalize()`` on the exporter left on the model adds the tail shard, the index and the config artifacts, and the checkpoint does not load until it runs. ``examples/hf_ptq`` does this for you. Supports FP8 and NVFP4 on single-process models, resident or offloaded, including multimodal models and models with MTP layers; other formats and placements raise ``NotImplementedError`` before calibration starts. - Add an end-to-end BEVFormer ONNX PTQ example with temporal calibration data generation, INT8 and FP8 quantization, TensorRT engine building, and nuScenes accuracy evaluation. See `examples/onnx_ptq/bevformer/README.md `_ for details. @@ -25,12 +27,14 @@ Changelog - Rename the architecture-specific recipe tier from ``modelopt_recipes/huggingface/`` to ``modelopt_recipes/model_type/`` to clarify that it holds recipes shared across every checkpoint of a Hugging Face ``model_type``. Saved ``--recipe huggingface//...`` paths still resolve via a backward-compatibility alias but now emit a ``FutureWarning``, so update them to ``model_type//...`` as the ``huggingface/`` prefix is deprecated. - The single-format quantization CLI flags are deprecated in favour of ``--recipe`` and will be removed in a future release; passing one now emits a ``FutureWarning``. ``examples/hf_ptq``: ``--qformat`` and ``--kv_cache_qformat``. ``examples/megatron_bridge/quantize.py``: ``--quant_cfg``, ``--kv_cache_quant`` and ``--weight_only``. ``examples/torch_onnx/torch_quant_to_onnx.py``: ``--qformat``. A recipe carries the quantization config, the calibration algorithm and the KV-cache setting in one file, so they cannot drift apart the way separate flags can -- and ``--recipe`` already took precedence over all six, silently on ``hf_ptq`` and with a warning on ``megatron_bridge`` -- with one gap the recipe closes rather than inherits: a weight AutoQuantize recipe that omits ``kv_cache`` still falls back to ``--kv_cache_qformat``, so set ``kv_cache`` in the recipe when migrating. Use a recipe from ``modelopt_recipes/general/ptq/``, an architecture-specific one under ``modelopt_recipes/model_type//``, or a checkpoint-specific one under ``modelopt_recipes/models/``. The warning fires only when a flag is passed explicitly: ``--qformat`` defaults to ``fp8`` and ``--kv_cache_qformat`` to ``fp8_cast``, so warning on the defaults would fire on every run, including runs that correctly use ``--recipe``. ``examples/speculative_decoding/scripts/quantize_drafter.py`` keeps ``--qformat`` undeprecated: it has no ``--recipe`` alternative yet. - The TensorRT-LLM checkpoint export format is deprecated and will be removed in 0.49.0: ``export_tensorrt_llm_checkpoint`` and ``torch_to_tensorrt_llm_checkpoint`` now emit a ``DeprecationWarning`` on use. Use ``export_hf_checkpoint``, which exports a unified Hugging Face checkpoint deployable on TensorRT-LLM, vLLM and SGLang. Its implementation moved to ``modelopt.torch.export.trtllm``, so import those two functions from there and the ``ModelConfig`` dataclasses from ``modelopt.torch.export.trtllm.model_config``; both functions remain importable from ``modelopt.torch.export`` for this release only. +- Deprecate ``metadata.recipe_type`` in recipe YAML. A recipe now says what kind it is with a ``# modelopt-schema:`` comment naming its schema class, or by delegating to a recipe that does; ``recipe_type`` is still read and still honoured, so a recipe outside this repo keeps working unchanged, but new recipes should leave it out and every recipe shipped here has been converted. Where both are present they must agree, and so must a recipe and the recipe it delegates to -- a disagreement is an error rather than a silent preference. **Bug Fixes** - Fix ONNX INT8 entropy calibration failing or producing invalid quantization parameters for FP16 activations. - Fix ``--use_fsdp2`` HuggingFace checkpoint export gathering the whole model onto rank 0, which made export the dominant phase of a PTQ run and could exhaust host memory on large models. The model is now split into per-decoder-layer units dealt round-robin across ranks; each rank gathers every unit but keeps, packs, and writes only the ones it owns, so a rank buffers roughly ``model / world_size`` instead of the whole checkpoint, and rank 0 writes the combined index. Export configurations that cannot be split this way now raise instead of producing a mismatched checkpoint: FSDP2 combined with another DTensor parallelism (for example FSDP2 + tensor parallel on a 2-D mesh; HSDP is supported), models whose decoder layers cannot be discovered, a decoder layer object reused across layers, and a module that holds the decoder layers while owning parameters of its own. - Speed up ``mtq.quantize`` on FSDP2-sharded fused-MoE models. Promoting static-block weight quantizers gathered each expert's slice of the fused weight across ranks even though only quantizer state is read, adding a collective per expert to calibration. +- Fix the NVIDIA-Nemotron-3 Super and Ultra NVFP4 recipes quantizing the MTP block on the Megatron-Core path, where it is a live ``model.mtp`` submodule the recipes' broad ``*mixer.*`` patterns matched into, contrary to their own descriptions. Hugging Face runs were unaffected: the ``nemotron_h`` class discards the MTP tensors on load. - Add FP8 and INT8 recipes that quantize timm ResNet shortcut inputs immediately before residual adds. The torch ONNX example now accepts PTQ and AutoQuantize recipes through ``--recipe`` and uses ``--qformat`` when no recipe is provided. ResNet supports only FP8 and INT8 because TensorRT has limited convolution kernel support; AutoQuantize and other quantization formats are no longer supported for ResNet. - Fix a DDP hang in DFlash training at scale where a rank whose batch contained no valid anchor skipped the draft forward, leaving its rotary buffer list shorter than other ranks' and causing ``broadcast_buffers`` to hang. The buffer is now created during ``modify()`` before training begins. - Fix ``megatron_generate`` dropping the VLM vision inputs (``pixel_values`` / ``image_grid_thw`` / ``image_sizes``) after the first generated token when KV-cache decoding is off, including the automatic fallback under sequence parallelism, which made generation silently ignore the image. No other ModelOpt feature is affected. diff --git a/docs/source/guides/10_recipes.rst b/docs/source/guides/10_recipes.rst index 4c6c25cb23c..bf1131e2157 100644 --- a/docs/source/guides/10_recipes.rst +++ b/docs/source/guides/10_recipes.rst @@ -515,19 +515,37 @@ General PTQ recipes are model-agnostic and apply to any supported architecture: - NVFP4 for output projection + MLP layers, FP8 KV cache * - ``general/ptq/nvfp4_weight_only-kv_fp8_cast`` - NVFP4 W4A16 weight-only, FP8 KV cache with constant amax + * - ``general/ptq/fp8_default-kv_fp16`` + - FP8 per-tensor W8A8, KV cache left unquantized + * - ``general/ptq/nvfp4_default-kv_fp16`` + - NVFP4 W4A4, KV cache left unquantized + * - ``general/ptq/nvfp4_mlp_only-kv_fp16`` + - NVFP4 for MLP layers only, KV cache left unquantized + * - ``general/ptq/nvfp4_experts_only-kv_fp16`` + - NVFP4 for MoE expert layers only, KV cache left unquantized + +See `modelopt_recipes/ptq.md `_ +for the full list and for guidance on choosing between them. Model-specific recipes ---------------------- Model-specific recipes come in two tiers: architecture recipes keyed by a Hugging Face ``model_type`` under ``model_type///``, and -checkpoint mirrors keyed by a model-hub path under +checkpoint entries keyed by a model-hub path under ``models////``. See `modelopt_recipes/model_type/README.md `_ and `modelopt_recipes/models/README.md `_ for the layout conventions and recipe-lookup order. +A checkpoint entry comes in two forms. A **mirror** carries its own body, because the +release uses a per-layer scheme no portable recipe expresses. An **alias** has no body of +its own: a general or architecture recipe already produces that scheme, so the entry +imports that recipe wholesale and exists to make it findable from the checkpoint's hub +path. See `modelopt_recipes/ptq.md `_ +for what each checkpoint entry does. + .. note:: ``model_type/`` was previously named ``huggingface/``. Old @@ -547,6 +565,41 @@ for the layout conventions and recipe-lookup order. - MXFP8 language-model base with MSE-calibrated NVFP4 routed experts for MiniMax-M3 +Delegating to another recipe +---------------------------- + +A recipe can hand its whole body to another recipe with a top-level ``$import`` and keep +only its own ``metadata``. Keys given alongside the ``$import`` override the imported +ones, so the body -- ``quantize``, its algorithm and every ``quant_cfg`` entry -- is +inherited unchanged: + +.. code-block:: yaml + + imports: + base: general/ptq/nvfp4_default-kv_fp8_cast + + $import: base + metadata: + description: What this checkpoint uses the base recipe for. + +Note the missing ``recipe_type``. A recipe states its kind in whichever of these it +likes, and the loader takes the first that answers: a ``# modelopt-schema:`` comment +naming its schema class, ``metadata.recipe_type``, or -- as here -- the recipe it +delegates to. ``metadata.recipe_type`` is **deprecated**: it is still read and still +honoured, so no existing recipe needs changing, but new recipes should declare their +schema instead. It remains the only option for a directory-format recipe's +``metadata.yml``, which has no comment preamble to read. + +Stating more than one is allowed, but they must agree, and that extends across the +import: a recipe and the recipe it delegates to must be the same kind, since the import +takes over the whole body. A PTQ recipe importing an EAGLE one is rejected as a kind +mismatch rather than left to fail on whatever the spliced sections do to the schema. + +What *is* required: a recipe that another file imports must carry the schema comment, +since ``$import`` resolution needs it to validate the imported payload. A recipe nothing +imports needs no comment at all. This is how the checkpoint aliases under ``models/`` +name the recipe behind a release without duplicating its body. + Loading recipes =============== diff --git a/modelopt/recipe/config.py b/modelopt/recipe/config.py index 28de372adad..4767a58b8d2 100644 --- a/modelopt/recipe/config.py +++ b/modelopt/recipe/config.py @@ -19,7 +19,7 @@ import warnings from enum import Enum -from typing import Literal +from typing import ClassVar, Literal from pydantic import Field, field_validator, model_validator @@ -67,9 +67,16 @@ class RecipeType(str, Enum): class RecipeMetadataConfig(ModeloptBaseConfig): """YAML shape of the recipe metadata section.""" - recipe_type: RecipeType = Field( + recipe_type: RecipeType | None = ModeloptField( + default=None, title="Recipe type", - description="The type of the recipe (e.g. PTQ).", + description="The type of the recipe (e.g. PTQ). **Deprecated** in recipe YAML: " + "the ``# modelopt-schema:`` comment naming the recipe's schema class already says " + "which kind it is -- and it is the same declaration that makes the file " + "``$import``-able -- so the class fills this in. Still read and still honoured, so " + "no existing recipe needs changing, but new recipes should leave it out. It remains " + "the only option where there is no schema comment to read, as in a directory-format " + "recipe's ``metadata.yml``. When both are present they must agree.", ) description: str = ModeloptField( default=_DEFAULT_RECIPE_DESCRIPTION, @@ -78,10 +85,10 @@ class RecipeMetadataConfig(ModeloptBaseConfig): ) -def _metadata_field(recipe_type: RecipeType): - """Build the metadata Pydantic field with the recipe_type baked into the default.""" +def _metadata_field(): + """Build a metadata Pydantic field that defaults to the owning class's recipe type.""" return ModeloptField( - default={"recipe_type": recipe_type, "description": _DEFAULT_RECIPE_DESCRIPTION}, + default={"description": _DEFAULT_RECIPE_DESCRIPTION}, title="Metadata", description="Recipe metadata containing the recipe type and description.", validate_default=True, @@ -94,16 +101,43 @@ class ModelOptRecipeBase(ModeloptBaseConfig): If a layer name matches ``"*output_layer*"``, the attributes will be replaced with ``{"enable": False}``. """ + #: The kind of recipe this class *is*. Set on every concrete subclass; it is the + #: single source of truth for ``metadata.recipe_type``, which the validator below + #: fills in so a recipe file never has to repeat what its schema already states. + RECIPE_TYPE: ClassVar[RecipeType | None] = None + metadata: RecipeMetadataConfig = Field( title="Metadata", description="Recipe metadata containing the recipe type and description. " "Required: a recipe without a ``metadata`` section is rejected so that a " - "missing section can't silently fall back to a default recipe type.", + "recipe always says what it is for.", ) + @model_validator(mode="after") + def _resolve_recipe_type(self): + """Fill ``metadata.recipe_type`` from the schema class, or reject a mismatch. + + The schema class already determines the kind, so a recipe file that declares its + schema needs no ``recipe_type``. One that states it anyway must state the truth -- + a silent disagreement between the two would make the file mean different things + to the loader and to a reader. + """ + if self.RECIPE_TYPE is None: + return self + if self.metadata.recipe_type is None: + self.metadata.recipe_type = self.RECIPE_TYPE + elif self.metadata.recipe_type != self.RECIPE_TYPE: + raise ValueError( + f"metadata.recipe_type is {self.metadata.recipe_type.value!r} but this recipe " + f"is a {type(self).__name__}, which is {self.RECIPE_TYPE.value!r}. Drop the " + "recipe_type (the schema declares it) or correct it." + ) + return self + @property def recipe_type(self) -> RecipeType: """Return the recipe type from metadata.""" + assert self.metadata.recipe_type is not None, "recipe_type was not resolved" return self.metadata.recipe_type @property @@ -115,6 +149,8 @@ def description(self) -> str: class ModelOptPTQRecipe(ModelOptRecipeBase): """Our config class for PTQ recipes.""" + RECIPE_TYPE: ClassVar[RecipeType] = RecipeType.PTQ + quantize: QuantizeConfig = Field( title="PTQ config", description="PTQ config containing quant_cfg and algorithm. Required: a PTQ " @@ -308,7 +344,9 @@ def _has_search_space(self): class ModelOptAutoQuantizeRecipe(ModelOptRecipeBase): """Our config class for AutoQuantize recipes.""" - metadata: RecipeMetadataConfig = _metadata_field(RecipeType.AUTO_QUANTIZE) + RECIPE_TYPE: ClassVar[RecipeType] = RecipeType.AUTO_QUANTIZE + + metadata: RecipeMetadataConfig = _metadata_field() quantize: QuantizeConfig | None = ModeloptField( default=None, @@ -380,7 +418,9 @@ class ModelOptSpeculativeRecipeBase(ModelOptRecipeBase): class ModelOptEagleRecipe(ModelOptSpeculativeRecipeBase): """Our config class for EAGLE speculative decoding recipes.""" - metadata: RecipeMetadataConfig = _metadata_field(RecipeType.SPECULATIVE_EAGLE) + RECIPE_TYPE: ClassVar[RecipeType] = RecipeType.SPECULATIVE_EAGLE + + metadata: RecipeMetadataConfig = _metadata_field() eagle: EagleConfig = ModeloptField( default=EagleConfig(), @@ -409,7 +449,9 @@ def _warn_rope_vs_training_seq_len(self) -> ModelOptEagleRecipe: class ModelOptDFlashRecipe(ModelOptSpeculativeRecipeBase): """Our config class for DFlash speculative decoding recipes.""" - metadata: RecipeMetadataConfig = _metadata_field(RecipeType.SPECULATIVE_DFLASH) + RECIPE_TYPE: ClassVar[RecipeType] = RecipeType.SPECULATIVE_DFLASH + + metadata: RecipeMetadataConfig = _metadata_field() dflash: DFlashConfig = ModeloptField( default=DFlashConfig(), @@ -431,7 +473,9 @@ def _derive_dflash_offline(self) -> ModelOptDFlashRecipe: class ModelOptMedusaRecipe(ModelOptSpeculativeRecipeBase): """Our config class for Medusa speculative decoding recipes.""" - metadata: RecipeMetadataConfig = _metadata_field(RecipeType.SPECULATIVE_MEDUSA) + RECIPE_TYPE: ClassVar[RecipeType] = RecipeType.SPECULATIVE_MEDUSA + + metadata: RecipeMetadataConfig = _metadata_field() medusa: MedusaConfig = ModeloptField( default=MedusaConfig(), diff --git a/modelopt/recipe/loader.py b/modelopt/recipe/loader.py index 91e2bac75de..691c71e3d54 100644 --- a/modelopt/recipe/loader.py +++ b/modelopt/recipe/loader.py @@ -26,7 +26,11 @@ from omegaconf import OmegaConf from modelopt.torch.opt.config_loader import BUILTIN_CONFIG_ROOT as BUILTIN_RECIPES_LIB -from modelopt.torch.opt.config_loader import _alias_builtin_recipe_prefix, load_config +from modelopt.torch.opt.config_loader import ( + _alias_builtin_recipe_prefix, + load_config, + peek_declared_schema, +) from modelopt.torch.quantization.config import QuantizeConfig from .config import ( @@ -114,6 +118,42 @@ def load_recipe( ``eagle`` (EAGLE speculative decoding), ``dflash`` (DFlash speculative decoding) or ``medusa`` (Medusa speculative decoding) sections. The suffix may be omitted and will be probed automatically. + + .. _recipe-alias: + + A recipe can instead **alias an existing recipe**. A top-level ``$import`` + brings the imported recipe in whole, so the file needs nothing else:: + + imports: + base: general/ptq/nvfp4_default-kv_fp8_cast + + $import: base + + That resolves to exactly the imported recipe -- its body, its algorithm and + its metadata alike -- under a second name. + + Keys given alongside the ``$import`` override the imported ones, so an alias + can say what it is for while inheriting everything else:: + + imports: + base: general/ptq/nvfp4_default-kv_fp8_cast + + $import: base + metadata: + description: What this checkpoint uses the base recipe for. + + An override replaces a top-level key outright rather than merging into it, so + a partial ``metadata`` supplies the whole section. Nothing needs restating in + either form: the recipe's kind comes from the imported recipe (see + :func:`_peek_recipe_type`), and the two must agree if the alias states one. + + The one requirement is on the other side -- the imported recipe must carry a + ``# modelopt-schema:`` comment naming its schema class, as every + ``$import``-able file must. + + This is how the checkpoint entries under ``models///`` record + that a released checkpoint is reproduced by an existing recipe without + duplicating it. * A directory containing ``metadata.yml`` and ``quantize.yml`` — **PTQ recipes only**. Speculative-decoding recipes are always single YAML files. @@ -162,19 +202,94 @@ def _apply_dotlist(data: dict, overrides: list[str]) -> dict: return OmegaConf.to_container(merged, resolve=False) -def _peek_recipe_type(recipe_file: Path | Traversable) -> RecipeType | None: - """Extract ``metadata.recipe_type`` from a recipe YAML without resolving $imports. +#: Recipe schema classes by their fully-qualified path, for resolving a +#: ``# modelopt-schema:`` comment to the recipe kind it names. +_RECIPE_SCHEMA_PATHS: dict[str, RecipeType] = { + f"{cls.__module__}.{cls.__qualname__}": rtype for rtype, cls in RECIPE_TYPE_TO_CLASS.items() +} + + +def _peek_recipe_type( + recipe_file: Path | Traversable, _seen: frozenset[str] | None = None +) -> RecipeType | None: + """Determine a recipe's kind without resolving its ``$import`` references. Needed so :func:`load_config` can be called with the correct ``schema_type`` for - typed-list ``$import`` resolution before the full recipe is constructed. + typed-list ``$import`` resolution before the full recipe is constructed -- which is + why this cannot simply wait for the imports to resolve. + + Neither way of saying it is mandatory; a recipe just has to say it *somehow*, and + the three sources are checked in this order: + + 1. a ``# modelopt-schema:`` comment naming the recipe's schema class. Only files + that are **imported** by another one need this -- it is what + ``$import`` resolution requires of any snippet -- so a leaf recipe never has to + carry it; + 2. ``metadata.recipe_type`` in the YAML body. **Deprecated** -- still read and still + honoured, so no existing recipe has to change, but a new one should declare its + schema instead. It remains the only option for a directory recipe's + ``metadata.yml``, which has no comment preamble to read; + 3. the recipe this one **delegates to** via a top-level ``$import``. A checkpoint + alias states neither of the above: its kind is whatever its base is, and the base + must declare a schema to be importable at all, so the walk terminates. + + When more than one source is present they must agree -- + :class:`~modelopt.recipe.config.ModelOptRecipeBase` rejects a recipe whose + ``metadata.recipe_type`` contradicts its schema class. """ import yaml + key = str(recipe_file) + _seen = (_seen or frozenset()) | {key} + + try: + declared = peek_declared_schema(recipe_file) + except ValueError: # multiple modelopt-schema comments; load_config reports it + declared = None + if declared in _RECIPE_SCHEMA_PATHS: + return _RECIPE_SCHEMA_PATHS[declared] + try: raw = yaml.safe_load(recipe_file.read_text()) + except yaml.YAMLError: + return None + if not isinstance(raw, dict): + return None + + try: return RecipeType(raw["metadata"]["recipe_type"]) except (TypeError, KeyError, ValueError): - return None + pass + + for base in delegated_recipe_paths(raw): + if str(base) in _seen: + continue + rtype = _peek_recipe_type(base, _seen) + if rtype is not None: + return rtype + return None + + +def delegated_recipe_paths(raw: dict) -> list[Path | Traversable]: + """Resolve the recipe files a raw recipe body delegates to via top-level ``$import``. + + Returns an empty list for an ordinary recipe. Names that do not appear in the + ``imports`` section, or that do not resolve to a file, are skipped here and reported + by ``$import`` resolution itself with a better message. + """ + ref = raw.get("$import") + imports = raw.get("imports") or {} + if ref is None or not isinstance(imports, dict): + return [] + paths = [] + for name in ref if isinstance(ref, list) else [ref]: + target = imports.get(name) + if not target: + continue + resolved = _resolve_recipe_path(target) + if resolved.is_file(): + paths.append(resolved) + return paths def _load_recipe_from_file( @@ -183,26 +298,53 @@ def _load_recipe_from_file( ) -> ModelOptRecipeBase: """Load a recipe from a YAML file, optionally applying dotlist overrides. - The file must contain a ``metadata`` section with at least ``recipe_type``, - plus the algorithm-specific section (``quantize`` / ``eagle`` / ``dflash`` / ``medusa``). + The file has to say what kind of recipe it is -- see :func:`_peek_recipe_type` for + the three ways to do that -- and to supply the matching body section (``quantize`` / + ``eagle`` / ``dflash`` / ``medusa``), either directly or through a top-level + ``$import`` of a recipe that has one. """ rtype = _peek_recipe_type(recipe_file) if rtype is None: - raise ValueError(f"Recipe file {recipe_file} must contain a 'metadata.recipe_type' field.") + raise ValueError( + f"Recipe file {recipe_file} does not say what kind of recipe it is. Set " + "'metadata.recipe_type', or declare a '# modelopt-schema: " + "modelopt.recipe.config.ModelOptRecipe' comment, or delegate to a recipe " + "that does with a top-level '$import'." + ) schema_class = RECIPE_TYPE_TO_CLASS.get(rtype) if schema_class is None: raise ValueError(f"Unsupported recipe type: {rtype!r}") + import yaml + + raw = yaml.safe_load(recipe_file.read_text()) or {} + raw = raw if isinstance(raw, dict) else {} + + # A recipe that delegates inherits the imported recipe's body wholesale, so the two + # have to be the same kind. Checked here, and against the *declared* kind on both + # sides, so a mismatch reads as a mismatch -- otherwise it surfaces as whatever + # pydantic makes of, say, an ``eagle`` section spliced into a PTQ schema. + for base in delegated_recipe_paths(raw): + base_type = _peek_recipe_type(base) + if base_type is not None and base_type != rtype: + raise ValueError( + f"Recipe file {recipe_file} is a {rtype.value!r} recipe but imports " + f"{base}, which is a {base_type.value!r} recipe. A top-level '$import' " + "takes over the whole body, so both must be the same kind." + ) + # Pre-flight check on the *raw* YAML so the user sees a clear loader-level error # rather than a generic pydantic missing-field error. Speculative recipes' body # sections have field-level defaults, so this check is what keeps their loader # semantics consistent with PTQ. required_section = _REQUIRED_SECTION_PER_RECIPE_TYPE.get(rtype) if required_section is not None: - import yaml - - raw = yaml.safe_load(recipe_file.read_text()) or {} - if not isinstance(raw, dict) or required_section not in raw: + # A recipe may delegate its whole body to another recipe with a top-level + # ``$import`` and override only ``metadata`` -- see :ref:`recipe-alias`. The + # body section then arrives during import resolution, so it cannot be required + # in the raw YAML; pydantic still rejects the result if the import does not + # supply one. + if "$import" not in raw and required_section not in raw: # Strip only the ``speculative_`` prefix so multi-word non-speculative types # (e.g. ``auto_quantize``) keep their full name: AUTO_QUANTIZE, not QUANTIZE. kind = rtype.value.removeprefix("speculative_").upper() @@ -255,6 +397,12 @@ def _load_recipe_from_dir(recipe_dir: Path | Traversable) -> ModelOptRecipeBase: metadata_file = _find_recipe_section_file(recipe_dir, "metadata") metadata = load_config(metadata_file, schema_type=RecipeMetadataConfig) + if metadata.recipe_type is None: + raise ValueError( + f"Recipe directory {recipe_dir}: {metadata_file} must set 'recipe_type'. A " + "directory recipe has no schema comment naming the recipe class, so its " + "metadata is the only place the kind can come from." + ) if metadata.recipe_type == RecipeType.PTQ: quantize_file = _find_recipe_section_file(recipe_dir, "quantize") quantize_cfg = load_config(quantize_file, schema_type=QuantizeConfig) diff --git a/modelopt/torch/opt/config_loader.py b/modelopt/torch/opt/config_loader.py index cb3278dcedb..82cb541fa9e 100644 --- a/modelopt/torch/opt/config_loader.py +++ b/modelopt/torch/opt/config_loader.py @@ -110,7 +110,7 @@ def _alias_builtin_recipe_prefix(config_path: str) -> str: _EXMY_RE = re.compile(r"^[Ee](\d+)[Mm](\d+)$") _EXMY_KEYS = frozenset({"num_bits", "scale_bits"}) -_MODELOPT_SCHEMA_RE = re.compile(r"^\s*#\s*modelopt-schema:\s*(\S+)\s*$") +_MODELOPT_SCHEMA_RE = re.compile(r"^\s*#\s*modelopt-schema:\s*(\S+)\s*$", re.MULTILINE) def _parse_exmy_num_bits(obj: Any) -> Any: @@ -219,6 +219,18 @@ def _parse_modelopt_schema(text: str, config_path: Path | Traversable) -> str | return schema +def peek_declared_schema(config_file: str | Path | Traversable) -> str | None: + """Return the ``# modelopt-schema:`` path a config file declares, if any. + + Reads only the comment preamble -- no YAML parsing and no ``$import`` resolution -- + so a caller can find out what kind of config a file is *before* it can be loaded. + :func:`modelopt.recipe.load_recipe` uses this to pick a recipe's schema class, which + is why a recipe can inherit its body from another one via a top-level ``$import``. + """ + config_path = _resolve_config_path(config_file) + return _parse_modelopt_schema(config_path.read_text(encoding="utf-8"), config_path) + + def _load_raw_config_with_schema(config_file: str | Path | Traversable) -> _RawConfig: """Load a config YAML without resolving ``$import`` references.""" config_path = _resolve_config_path(config_file) diff --git a/modelopt_recipes/general/auto_quantize/kv_fp8_nvfp4_cast_kl_div_at_5p4bits.yaml b/modelopt_recipes/general/auto_quantize/kv_fp8_nvfp4_cast_kl_div_at_5p4bits.yaml index 5f2f15c237f..94bff1e2a55 100644 --- a/modelopt_recipes/general/auto_quantize/kv_fp8_nvfp4_cast_kl_div_at_5p4bits.yaml +++ b/modelopt_recipes/general/auto_quantize/kv_fp8_nvfp4_cast_kl_div_at_5p4bits.yaml @@ -24,7 +24,6 @@ imports: nvfp4: configs/numerics/nvfp4 metadata: - recipe_type: auto_quantize description: Layer-wise FP8-cast/NVFP4-cast KV-cache AutoQuantize at 5.4 bits using forward KL. auto_quantize: diff --git a/modelopt_recipes/general/auto_quantize/nvfp4_fp8_at_5p4bits.yaml b/modelopt_recipes/general/auto_quantize/nvfp4_fp8_at_5p4bits.yaml index 6d62a31f450..e42f04a1bc9 100644 --- a/modelopt_recipes/general/auto_quantize/nvfp4_fp8_at_5p4bits.yaml +++ b/modelopt_recipes/general/auto_quantize/nvfp4_fp8_at_5p4bits.yaml @@ -23,7 +23,6 @@ imports: fp8: configs/ptq/presets/model/fp8 metadata: - recipe_type: auto_quantize description: Mixed NVFP4 + FP8 per-layer search at 5.4 effective bits. auto_quantize: diff --git a/modelopt_recipes/general/auto_quantize/nvfp4_fp8_kl_div_at_5p4bits.yaml b/modelopt_recipes/general/auto_quantize/nvfp4_fp8_kl_div_at_5p4bits.yaml index 5f243f2219d..feba4ba8b73 100644 --- a/modelopt_recipes/general/auto_quantize/nvfp4_fp8_kl_div_at_5p4bits.yaml +++ b/modelopt_recipes/general/auto_quantize/nvfp4_fp8_kl_div_at_5p4bits.yaml @@ -25,7 +25,6 @@ imports: fp8: configs/ptq/presets/model/fp8 metadata: - recipe_type: auto_quantize description: Mixed NVFP4 + FP8 per-layer search at 5.4 effective bits, kl_div scoring (no backprop). auto_quantize: diff --git a/modelopt_recipes/general/auto_quantize/nvfp4_mse_fp8_at_6p0bits.yaml b/modelopt_recipes/general/auto_quantize/nvfp4_mse_fp8_at_6p0bits.yaml index f45c492cdf8..35f750b3ae0 100644 --- a/modelopt_recipes/general/auto_quantize/nvfp4_mse_fp8_at_6p0bits.yaml +++ b/modelopt_recipes/general/auto_quantize/nvfp4_mse_fp8_at_6p0bits.yaml @@ -23,7 +23,6 @@ imports: fp8: configs/ptq/presets/model/fp8 metadata: - recipe_type: auto_quantize description: Mixed NVFP4 (weight-MSE + FP8 sweep) + FP8 per-layer search at 6.0 effective bits. auto_quantize: diff --git a/modelopt_recipes/general/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml b/modelopt_recipes/general/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml index 608a469694f..cb30fd5d599 100644 --- a/modelopt_recipes/general/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml +++ b/modelopt_recipes/general/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml @@ -24,7 +24,6 @@ imports: w4a16_nvfp4: configs/ptq/presets/model/w4a16_nvfp4 metadata: - recipe_type: auto_quantize description: >- Mixed FP8 + NVFP4-weight-only per-layer search at 6.0 effective bits with the active-MoE cost model (expert ratio 0.03125). diff --git a/modelopt_recipes/general/auto_quantize/w4a8_awq_beta_fp8_at_6p0bits.yaml b/modelopt_recipes/general/auto_quantize/w4a8_awq_beta_fp8_at_6p0bits.yaml index 7ebc23adfee..01b3ec1d482 100644 --- a/modelopt_recipes/general/auto_quantize/w4a8_awq_beta_fp8_at_6p0bits.yaml +++ b/modelopt_recipes/general/auto_quantize/w4a8_awq_beta_fp8_at_6p0bits.yaml @@ -23,7 +23,6 @@ imports: fp8: configs/ptq/presets/model/fp8 metadata: - recipe_type: auto_quantize description: Mixed W4A8 AWQ-beta + FP8 per-layer search at 6.0 effective bits. auto_quantize: diff --git a/modelopt_recipes/general/ptq/fp8_default-kv_fp8.yaml b/modelopt_recipes/general/ptq/fp8_default-kv_fp8.yaml index ea2ac567290..6bd20d8ee3a 100644 --- a/modelopt_recipes/general/ptq/fp8_default-kv_fp8.yaml +++ b/modelopt_recipes/general/ptq/fp8_default-kv_fp8.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for W8A8 FP8 E4M3 model quantization with FP8 KV-cache quantization. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- Composes W8A8 FP8 E4M3 model quantization with FP8 KV-cache quantization; uses max calibration. quantize: diff --git a/modelopt_recipes/general/ptq/fp8_default-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/fp8_default-kv_fp8_cast.yaml index 4e24bf53274..006f23ebe3d 100644 --- a/modelopt_recipes/general/ptq/fp8_default-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/fp8_default-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for W8A8 FP8 E4M3 model quantization with FP8 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Composes W8A8 FP8 E4M3 model quantization with FP8 KV-cache cast mode using constant amax; uses max calibration. diff --git a/modelopt_recipes/general/ptq/int4_blockwise_weight_only.yaml b/modelopt_recipes/general/ptq/int4_blockwise_weight_only.yaml index 91bf57971de..ff688a4589f 100644 --- a/modelopt_recipes/general/ptq/int4_blockwise_weight_only.yaml +++ b/modelopt_recipes/general/ptq/int4_blockwise_weight_only.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe metadata: - recipe_type: ptq description: INT4 blockwise weight-only (W4A16, block size 128), max calibration. imports: diff --git a/modelopt_recipes/general/ptq/mxfp4_mlp_weight_only.yaml b/modelopt_recipes/general/ptq/mxfp4_mlp_weight_only.yaml index c78ab4aa14f..da942516053 100644 --- a/modelopt_recipes/general/ptq/mxfp4_mlp_weight_only.yaml +++ b/modelopt_recipes/general/ptq/mxfp4_mlp_weight_only.yaml @@ -16,11 +16,11 @@ # PTQ recipe wrapping the shipped mxfp4_mlp_weight_only preset (the config behind # mtq.MXFP4_MLP_WEIGHT_ONLY_CFG), so the two cannot drift apart. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: preset: configs/ptq/presets/model/mxfp4_mlp_weight_only metadata: - recipe_type: ptq description: >- Applies dynamic MXFP4 to MLP/MoE weight quantizers only (weight-only, activations untouched); no calibration forward pass is required. diff --git a/modelopt_recipes/general/ptq/nvfp4_act_headroom-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_act_headroom-kv_fp8_cast.yaml index a0e79316eb4..29aef844543 100644 --- a/modelopt_recipes/general/ptq/nvfp4_act_headroom-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_act_headroom-kv_fp8_cast.yaml @@ -16,6 +16,7 @@ # using nvfp4_act_headroom calibration: NVFP4 weights on plain max, NVFP4 activation global # scales anchored with headroom. Module coverage matches nvfp4_default-kv_fp8_cast. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Composes dynamic NVFP4 W4A4 model quantization with FP8 KV-cache cast mode using constant amax; uses nvfp4_act_headroom calibration. Weight quantizers use plain max, while the NVFP4 diff --git a/modelopt_recipes/general/ptq/nvfp4_default-kv_fp8.yaml b/modelopt_recipes/general/ptq/nvfp4_default-kv_fp8.yaml index 9be27b7bae9..4120cbfb812 100644 --- a/modelopt_recipes/general/ptq/nvfp4_default-kv_fp8.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_default-kv_fp8.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for dynamic NVFP4 W4A4 model quantization with FP8 KV-cache quantization. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- Composes dynamic NVFP4 W4A4 model quantization with FP8 KV-cache quantization for PTQ, QAT, and QAD; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_default-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_default-kv_fp8_cast.yaml index 312cdd16c8d..05d7cedfc3b 100644 --- a/modelopt_recipes/general/ptq/nvfp4_default-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_default-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for dynamic NVFP4 W4A4 model quantization with FP8 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Composes dynamic NVFP4 W4A4 model quantization with FP8 KV-cache cast mode using constant amax; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_default-kv_none-gptq.yaml b/modelopt_recipes/general/ptq/nvfp4_default-kv_none-gptq.yaml index 5dc5786f0ff..4a927e599b7 100644 --- a/modelopt_recipes/general/ptq/nvfp4_default-kv_none-gptq.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_default-kv_none-gptq.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for NVFP4 W4A4 model quantization with KV quantizers disabled and GPTQ calibration. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: nvfp4: configs/numerics/nvfp4 metadata: - recipe_type: ptq description: >- Applies NVFP4 W4A4 with static weight scales, dynamic inputs, KV quantizers disabled, and GPTQ layerwise calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_default-kv_nvfp4_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_default-kv_nvfp4_cast.yaml index 0acdf6050db..3379887ea84 100644 --- a/modelopt_recipes/general/ptq/nvfp4_default-kv_nvfp4_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_default-kv_nvfp4_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for dynamic NVFP4 W4A4 model quantization with NVFP4 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: kv_nvfp4_cast: configs/ptq/units/kv_nvfp4_cast metadata: - recipe_type: ptq description: >- Composes dynamic NVFP4 W4A4 model quantization with NVFP4 KV-cache cast mode using constant amax; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8.yaml index 85b73e4462f..c1fcfdab966 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for expert-only dynamic NVFP4 quantization with FP8 KV-cache quantization. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 only to expert-layer weight and input quantizers, plus FP8 KV-cache quantization; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_cast.yaml index ea9fa1f60d9..f50084401d7 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for expert-only dynamic NVFP4 quantization with FP8 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 only to expert-layer weight and input quantizers, plus FP8 KV-cache cast mode using constant amax; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise.yaml index f86afb2d2ee..3ef7fd2de28 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise.yaml @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -20,7 +21,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: NVFP4 static weight and dynamic activation for expert layers only (W4A4), FP8 KV cache, max layerwise calibration. quantize: algorithm: diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise_export.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise_export.yaml index 049922cc4fb..556a988e3cf 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise_export.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise_export.yaml @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -20,7 +21,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: > NVFP4 static weight and dynamic activation for expert layers only (W4A4), FP8 KV cache, max layerwise calibration, exporting each decoder layer to a quantized checkpoint shard diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise_offload.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise_offload.yaml index aa525cb4188..0cc500fda9f 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise_offload.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only-kv_fp8_layerwise_offload.yaml @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -20,7 +21,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: > NVFP4 static weight and dynamic activation for expert layers only (W4A4), FP8 KV cache, max layerwise calibration with calib_mutates_weights=False for disk-offloaded single-GPU diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only_input_scale1-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only_input_scale1-kv_fp8_cast.yaml index 1e0e527b50a..5dcd4a45d85 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only_input_scale1-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only_input_scale1-kv_fp8_cast.yaml @@ -22,6 +22,7 @@ # ``amax / (E2M1_MAX * E4M3_MAX) = 1.0``; the per-block E4M3 activation scales remain dynamic. # No activation statistics are collected for these quantizers. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -29,7 +30,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Applies NVFP4 only to expert-layer weight and input quantizers, pinning the expert activation input_scale to 1.0 (constant amax = 2688, no activation calibration), plus FP8 KV-cache cast diff --git a/modelopt_recipes/general/ptq/nvfp4_experts_only_mse-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_experts_only_mse-kv_fp8_cast.yaml index c1ccf3d342d..ebaeab2779c 100644 --- a/modelopt_recipes/general/ptq/nvfp4_experts_only_mse-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_experts_only_mse-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for expert-only NVFP4 quantization with MSE weight calibration and FP8 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Applies static NVFP4 weight scales from MSE FP8-scale sweep and dynamic NVFP4 inputs to expert layers only, plus FP8 KV-cache cast mode. diff --git a/modelopt_recipes/general/ptq/nvfp4_mlp_only-kv_fp8.yaml b/modelopt_recipes/general/ptq/nvfp4_mlp_only-kv_fp8.yaml index 4fd2e0a7558..f19386f0742 100644 --- a/modelopt_recipes/general/ptq/nvfp4_mlp_only-kv_fp8.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_mlp_only-kv_fp8.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for MLP/MoE-only dynamic NVFP4 quantization with FP8 KV-cache quantization. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 only to MLP/MoE weight and input quantizers, plus FP8 KV-cache quantization; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_mlp_only-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_mlp_only-kv_fp8_cast.yaml index a12951bda65..239e2b69b66 100644 --- a/modelopt_recipes/general/ptq/nvfp4_mlp_only-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_mlp_only-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for MLP/MoE-only dynamic NVFP4 quantization with FP8 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 only to MLP/MoE weight and input quantizers, plus FP8 KV-cache cast mode using constant amax; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_mlp_only-novit-kv_fp8.yaml b/modelopt_recipes/general/ptq/nvfp4_mlp_only-novit-kv_fp8.yaml index 2d80d7a5701..65dd8c0a79c 100644 --- a/modelopt_recipes/general/ptq/nvfp4_mlp_only-novit-kv_fp8.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_mlp_only-novit-kv_fp8.yaml @@ -30,6 +30,7 @@ # `nvidia/Kimi-K2.5-NVFP4` checkpoint (which excludes the vision tower and # multimodal projector). +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -38,7 +39,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 only to MLP/MoE weight and input quantizers, plus FP8 KV-cache quantization; uses max calibration. Vision tower (model.visual.* / vision_tower.*) diff --git a/modelopt_recipes/general/ptq/nvfp4_mlp_only_mse-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_mlp_only_mse-kv_fp8_cast.yaml index 18fed45d266..7ba7e96d831 100644 --- a/modelopt_recipes/general/ptq/nvfp4_mlp_only_mse-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_mlp_only_mse-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for MLP/MoE-only NVFP4 quantization with MSE weight calibration and FP8 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Applies static NVFP4 weight scales from MSE FP8-scale sweep and dynamic NVFP4 inputs to MLP/MoE layers, plus FP8 KV-cache cast mode. diff --git a/modelopt_recipes/general/ptq/nvfp4_mlp_weight_only.yaml b/modelopt_recipes/general/ptq/nvfp4_mlp_weight_only.yaml index bfeddd7f163..657be0e9e80 100644 --- a/modelopt_recipes/general/ptq/nvfp4_mlp_weight_only.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_mlp_weight_only.yaml @@ -16,11 +16,11 @@ # PTQ recipe wrapping the shipped nvfp4_mlp_weight_only preset (the config behind # mtq.NVFP4_MLP_WEIGHT_ONLY_CFG), so the two cannot drift apart. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: preset: configs/ptq/presets/model/nvfp4_mlp_weight_only metadata: - recipe_type: ptq description: >- Applies NVFP4 (block size 32) to MLP/MoE weight quantizers only (weight-only, activations untouched); uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_omlp_only-kv_fp8.yaml b/modelopt_recipes/general/ptq/nvfp4_omlp_only-kv_fp8.yaml index 41541e4b2e4..edb2b9ea4e2 100644 --- a/modelopt_recipes/general/ptq/nvfp4_omlp_only-kv_fp8.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_omlp_only-kv_fp8.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for output-projection and MLP/MoE dynamic NVFP4 quantization with FP8 KV-cache quantization. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 to output-projection and MLP/MoE weight and input quantizers, plus FP8 KV-cache quantization; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_omlp_only-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_omlp_only-kv_fp8_cast.yaml index 14da2d92d4a..566699a9de1 100644 --- a/modelopt_recipes/general/ptq/nvfp4_omlp_only-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_omlp_only-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for output-projection and MLP/MoE dynamic NVFP4 quantization with FP8 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 to output-projection and MLP/MoE weight and input quantizers, plus FP8 KV-cache cast mode using constant amax; uses max calibration. diff --git a/modelopt_recipes/general/ptq/nvfp4_weight_only-kv_fp16.yaml b/modelopt_recipes/general/ptq/nvfp4_weight_only-kv_fp16.yaml index 416572e0f80..f0c9deb185b 100644 --- a/modelopt_recipes/general/ptq/nvfp4_weight_only-kv_fp16.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_weight_only-kv_fp16.yaml @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers w4a16_nvfp4: configs/ptq/units/w4_nvfp4 metadata: - recipe_type: ptq description: NVFP4 W4A16 weight-only, BF16 activations, max calibration. No calibration forward pass required. quantize: algorithm: max diff --git a/modelopt_recipes/general/ptq/nvfp4_weight_only-kv_fp8_cast.yaml b/modelopt_recipes/general/ptq/nvfp4_weight_only-kv_fp8_cast.yaml index ff335405f30..3c6e5932fb4 100644 --- a/modelopt_recipes/general/ptq/nvfp4_weight_only-kv_fp8_cast.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_weight_only-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Composed PTQ recipe for NVFP4 W4A16 weight-only quantization with FP8 KV-cache cast mode. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -22,7 +23,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- NVFP4 W4A16 weight-only, BF16 activations, plus FP8 KV-cache cast mode using constant amax; uses max calibration. No calibration forward pass required. diff --git a/modelopt_recipes/general/qad/nvfp4_dual_lsq-mse_init-fp8_kv.yaml b/modelopt_recipes/general/qad/nvfp4_dual_lsq-mse_init-fp8_kv.yaml index 53a9651ed24..1deb76d9c38 100644 --- a/modelopt_recipes/general/qad/nvfp4_dual_lsq-mse_init-fp8_kv.yaml +++ b/modelopt_recipes/general/qad/nvfp4_dual_lsq-mse_init-fp8_kv.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe metadata: - recipe_type: ptq description: >- Learns separate pre-quantization and post-quantization NVFP4 weight scales with MSE initialization and FP8 scale sweep; uses dynamic NVFP4 activations and FP8 KV cache. diff --git a/modelopt_recipes/general/qad/nvfp4_lsq-mse_init-fp8_kv.yaml b/modelopt_recipes/general/qad/nvfp4_lsq-mse_init-fp8_kv.yaml index dba0f7d98fe..ec0f5902123 100644 --- a/modelopt_recipes/general/qad/nvfp4_lsq-mse_init-fp8_kv.yaml +++ b/modelopt_recipes/general/qad/nvfp4_lsq-mse_init-fp8_kv.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe metadata: - recipe_type: ptq description: >- Learns one shared pre-quantization and post-quantization NVFP4 weight scale with MSE initialization and FP8 scale sweep; uses dynamic NVFP4 activations and FP8 KV cache. diff --git a/modelopt_recipes/general/speculative_decoding/dflash.yaml b/modelopt_recipes/general/speculative_decoding/dflash.yaml index 021cccd475d..ffde91a586a 100644 --- a/modelopt_recipes/general/speculative_decoding/dflash.yaml +++ b/modelopt_recipes/general/speculative_decoding/dflash.yaml @@ -1,7 +1,7 @@ # DFlash speculative-decoding training recipe. Override fields via OmegaConf dotlist on the CLI. +# modelopt-schema: modelopt.recipe.config.ModelOptDFlashRecipe metadata: - recipe_type: speculative_dflash description: DFlash training recipe (model/data/training/dflash bundled). # maps to ModelArguments (main.py) diff --git a/modelopt_recipes/general/speculative_decoding/domino.yaml b/modelopt_recipes/general/speculative_decoding/domino.yaml index 6dc21ea66af..e125381ce6e 100644 --- a/modelopt_recipes/general/speculative_decoding/domino.yaml +++ b/modelopt_recipes/general/speculative_decoding/domino.yaml @@ -5,8 +5,8 @@ # Online training is the default path (data.mode=online). Override fields via an # OmegaConf dotlist on the CLI. +# modelopt-schema: modelopt.recipe.config.ModelOptDFlashRecipe metadata: - recipe_type: speculative_dflash description: Domino training recipe (DFlash backbone + causal correction head). # maps to ModelArguments (main.py) diff --git a/modelopt_recipes/general/speculative_decoding/dspark.yaml b/modelopt_recipes/general/speculative_decoding/dspark.yaml index cb7b09f40e9..a071b071372 100644 --- a/modelopt_recipes/general/speculative_decoding/dspark.yaml +++ b/modelopt_recipes/general/speculative_decoding/dspark.yaml @@ -8,8 +8,8 @@ # loss: ce_alpha*CE + l1_alpha*TVD + conf_alpha*confidence_BCE. Online training is # the default path (data.mode=online). Override fields via an OmegaConf dotlist. +# modelopt-schema: modelopt.recipe.config.ModelOptDFlashRecipe metadata: - recipe_type: speculative_dflash description: DSpark training recipe (DFlash backbone + Markov head + confidence head). # maps to ModelArguments (main.py) diff --git a/modelopt_recipes/general/speculative_decoding/eagle3.yaml b/modelopt_recipes/general/speculative_decoding/eagle3.yaml index 1bf42769fca..0a7f6532889 100644 --- a/modelopt_recipes/general/speculative_decoding/eagle3.yaml +++ b/modelopt_recipes/general/speculative_decoding/eagle3.yaml @@ -1,7 +1,7 @@ # EAGLE3 speculative-decoding training recipe. Override fields via OmegaConf dotlist on the CLI. +# modelopt-schema: modelopt.recipe.config.ModelOptEagleRecipe metadata: - recipe_type: speculative_eagle description: EAGLE3 training recipe (model/data/training/eagle bundled). # maps to ModelArguments (main.py) diff --git a/modelopt_recipes/general/speculative_decoding/lilicorr.yaml b/modelopt_recipes/general/speculative_decoding/lilicorr.yaml index 5fcb8d07b56..f96fa44eeef 100644 --- a/modelopt_recipes/general/speculative_decoding/lilicorr.yaml +++ b/modelopt_recipes/general/speculative_decoding/lilicorr.yaml @@ -17,8 +17,8 @@ # the composition of the cross-entropy weight — see the two-line override below. # Override fields via an OmegaConf dotlist. +# modelopt-schema: modelopt.recipe.config.ModelOptDFlashRecipe metadata: - recipe_type: speculative_dflash description: LiLiCorr training recipe (DFlash backbone + candidate-lattice reranker). # maps to ModelArguments (main.py) diff --git a/modelopt_recipes/general/speculative_decoding/lilicorr_conv.yaml b/modelopt_recipes/general/speculative_decoding/lilicorr_conv.yaml index 276f6b5abc2..0fdea64b25c 100644 --- a/modelopt_recipes/general/speculative_decoding/lilicorr_conv.yaml +++ b/modelopt_recipes/general/speculative_decoding/lilicorr_conv.yaml @@ -29,8 +29,8 @@ # does not. # # Override fields via an OmegaConf dotlist. +# modelopt-schema: modelopt.recipe.config.ModelOptDFlashRecipe metadata: - recipe_type: speculative_dflash description: >- LiLiCorr + grouped sublayer convolutions (DFlash backbone + candidate-lattice reranker + DFlash2 sublayer convolutions). diff --git a/modelopt_recipes/model_type/diffusion_gemma/ptq/nvfp4_experts_only.yaml b/modelopt_recipes/model_type/diffusion_gemma/ptq/nvfp4_experts_only.yaml index f8570d136d4..60002032b56 100644 --- a/modelopt_recipes/model_type/diffusion_gemma/ptq/nvfp4_experts_only.yaml +++ b/modelopt_recipes/model_type/diffusion_gemma/ptq/nvfp4_experts_only.yaml @@ -21,6 +21,7 @@ # ``*self_conditioning*``. Matches the ``--qformat nvfp4_experts_only`` # behavior in ``hf_ptq.py``, which defaults ``--kv_cache_qformat=fp8_cast``. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all experts_nvfp4: configs/ptq/units/experts_nvfp4 @@ -28,7 +29,6 @@ imports: disabled_quantizers: model_type/diffusion_gemma/ptq/disabled_quantizers metadata: - recipe_type: ptq description: >- DiffusionGemma PTQ recipe (nvfp4_experts_only): dynamic W4A4 NVFP4 on MoE experts only with FP8 KV-cache cast (constant-amax); attention Q/K/V/O diff --git a/modelopt_recipes/model_type/gemma/ptq/int8_sq-kv_fp8_cast.yaml b/modelopt_recipes/model_type/gemma/ptq/int8_sq-kv_fp8_cast.yaml index 7d3b9bced59..a6a8e7b50ae 100644 --- a/modelopt_recipes/model_type/gemma/ptq/int8_sq-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/gemma/ptq/int8_sq-kv_fp8_cast.yaml @@ -17,6 +17,7 @@ # the SmoothQuant alpha from the default 1.0 to 0.5 to recover accuracy on # Gemma 7B (default alpha causes a regression). +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -25,7 +26,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Gemma INT8 SmoothQuant recipe with FP8 KV-cache cast: alpha=0.5 (instead of the default 1.0) to avoid accuracy regression on Gemma 7B, plus FP8 diff --git a/modelopt_recipes/model_type/gemma/ptq/w4a8_awq-kv_fp8_cast.yaml b/modelopt_recipes/model_type/gemma/ptq/w4a8_awq-kv_fp8_cast.yaml index c649234088e..44728f5392b 100644 --- a/modelopt_recipes/model_type/gemma/ptq/w4a8_awq-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/gemma/ptq/w4a8_awq-kv_fp8_cast.yaml @@ -17,6 +17,7 @@ # optimal-scale search (awq_lite with alpha_step=1) to avoid overflow observed # in TRT-LLM kernels when using the default AWQ search on Gemma. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -25,7 +26,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Gemma W4A8 AWQ recipe with FP8 KV-cache cast: INT4 block weights + FP8 inputs, awq_lite with alpha_step=1 (coarser search) to avoid TRT-LLM diff --git a/modelopt_recipes/model_type/gemma4/ptq/w4a8_awq-kv_fp8_cast.yaml b/modelopt_recipes/model_type/gemma4/ptq/w4a8_awq-kv_fp8_cast.yaml index 28410cc2565..25b5b7267f8 100644 --- a/modelopt_recipes/model_type/gemma4/ptq/w4a8_awq-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/gemma4/ptq/w4a8_awq-kv_fp8_cast.yaml @@ -28,6 +28,7 @@ # branch is kept in BF16 by the shared `default_disabled_quantizers` unit imported # below, which globally disables `*vision_tower*` / `*visual*` / `*embed_vision*`. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -36,7 +37,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Gemma 4 (multimodal) W4A8 AWQ recipe with FP8 KV-cache cast: INT4 block weights + FP8 inputs, awq_lite with alpha_step=1 (coarser search) to avoid diff --git a/modelopt_recipes/model_type/minimax_m3_vl/ptq/mxfp8_nvfp4_experts.yaml b/modelopt_recipes/model_type/minimax_m3_vl/ptq/mxfp8_nvfp4_experts.yaml index a7a14dcdfd1..6c6003e5522 100644 --- a/modelopt_recipes/model_type/minimax_m3_vl/ptq/mxfp8_nvfp4_experts.yaml +++ b/modelopt_recipes/model_type/minimax_m3_vl/ptq/mxfp8_nvfp4_experts.yaml @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -21,7 +22,6 @@ imports: nvfp4_static: configs/numerics/nvfp4_static metadata: - recipe_type: ptq description: >- MiniMax-M3 mixed MXFP8 and NVFP4 PTQ. Routed experts use MSE-calibrated NVFP4 weights and dynamic NVFP4 activations with input_scale fixed to 1.0; diff --git a/modelopt_recipes/model_type/minimax_m3_vl/ptq/nvfp4_experts_only.yaml b/modelopt_recipes/model_type/minimax_m3_vl/ptq/nvfp4_experts_only.yaml index 20d35003307..3e661d16ae4 100644 --- a/modelopt_recipes/model_type/minimax_m3_vl/ptq/nvfp4_experts_only.yaml +++ b/modelopt_recipes/model_type/minimax_m3_vl/ptq/nvfp4_experts_only.yaml @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -20,7 +21,6 @@ imports: nvfp4_static: configs/numerics/nvfp4_static metadata: - recipe_type: ptq description: >- MiniMax-M3 routed-experts-only NVFP4 PTQ with MSE-calibrated static weights and dynamic activations; shared experts and the KV cache stay diff --git a/modelopt_recipes/model_type/mpt/ptq/w4a8_awq-kv_fp8_cast.yaml b/modelopt_recipes/model_type/mpt/ptq/w4a8_awq-kv_fp8_cast.yaml index d559071a99b..07fea8f90f2 100644 --- a/modelopt_recipes/model_type/mpt/ptq/w4a8_awq-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/mpt/ptq/w4a8_awq-kv_fp8_cast.yaml @@ -17,6 +17,7 @@ # optimal-scale search (awq_lite with alpha_step=1) to avoid overflow observed # in TRT-LLM kernels when using the default AWQ search on MPT. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -25,7 +26,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- MPT W4A8 AWQ recipe with FP8 KV-cache cast: INT4 block weights + FP8 inputs, awq_lite with alpha_step=1 (coarser search) to avoid TRT-LLM diff --git a/modelopt_recipes/model_type/nemotron_llama/ptq/fp8_output_quant_proj.yaml b/modelopt_recipes/model_type/nemotron_llama/ptq/fp8_output_quant_proj.yaml index 15665a727e3..47ac2286635 100644 --- a/modelopt_recipes/model_type/nemotron_llama/ptq/fp8_output_quant_proj.yaml +++ b/modelopt_recipes/model_type/nemotron_llama/ptq/fp8_output_quant_proj.yaml @@ -20,6 +20,7 @@ # top of the FP16 activations, so the engine uses MORE activation memory than # an unquantized FP16 engine. [5726458] +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 @@ -27,7 +28,6 @@ imports: fp8: configs/numerics/fp8 metadata: - recipe_type: ptq description: 'Llama-Nemotron embedding/reranking PTQ recipe (fp8 + projection output quantizers): same numerics as the general fp8 preset, plus per-tensor FP8 output quantizers on the projection Linears so TensorRT engines carry FP8 activations between layers.' quantize: diff --git a/modelopt_recipes/model_type/nemotron_llama/ptq/nvfp4_output_quant_proj.yaml b/modelopt_recipes/model_type/nemotron_llama/ptq/nvfp4_output_quant_proj.yaml index 1edf942f2f2..e431d48ab07 100644 --- a/modelopt_recipes/model_type/nemotron_llama/ptq/nvfp4_output_quant_proj.yaml +++ b/modelopt_recipes/model_type/nemotron_llama/ptq/nvfp4_output_quant_proj.yaml @@ -20,6 +20,7 @@ # Output quantizers must stay scoped to GEMM outputs: a DynamicQuantize on # non-GEMM outputs (embeddings, pooling) fails to compile in TensorRT. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all w4a4_nvfp4_nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 @@ -27,7 +28,6 @@ imports: nvfp4: configs/numerics/nvfp4 metadata: - recipe_type: ptq description: 'Llama-Nemotron embedding PTQ recipe (nvfp4 + projection output quantizers): same numerics as the general nvfp4 preset, plus dynamic NVFP4 output quantizers on the projection Linears so TensorRT engines carry FP4 activations between layers.' quantize: diff --git a/modelopt_recipes/model_type/nemotron_vl/ptq/nvfp4-kv_fp8_cast.yaml b/modelopt_recipes/model_type/nemotron_vl/ptq/nvfp4-kv_fp8_cast.yaml index 3ae8e86d6fc..5c4bd6019e5 100644 --- a/modelopt_recipes/model_type/nemotron_vl/ptq/nvfp4-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/nemotron_vl/ptq/nvfp4-kv_fp8_cast.yaml @@ -17,6 +17,7 @@ # Equivalent to the general `nvfp4` preset with quantization disabled # on non-language branches. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all w4a4_nvfp4_nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 @@ -24,7 +25,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: 'Nemotron VL PTQ recipe (nvfp4): same numerics as the general nvfp4 preset, applied to the decoder (text generation) component only (vision/encoder branches are skipped).' quantize: diff --git a/modelopt_recipes/model_type/qwen3_5/ptq/fp8_vision-kv_none.yaml b/modelopt_recipes/model_type/qwen3_5/ptq/fp8_vision-kv_none.yaml index fc2809321f7..d6ed27b32d9 100644 --- a/modelopt_recipes/model_type/qwen3_5/ptq/fp8_vision-kv_none.yaml +++ b/modelopt_recipes/model_type/qwen3_5/ptq/fp8_vision-kv_none.yaml @@ -15,12 +15,12 @@ # Dense Qwen3.5 Vision Encoder FP8 with a high-precision language model and KV cache. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all vision_fp8: model_type/qwen3_vl/ptq/vision_fp8.quant_cfg metadata: - recipe_type: ptq description: >- FP8 quantization of dense Qwen3.5 Vision Encoder Linear layers, including any merger Linears; the language model, KV cache, patch embedding, and vision-attention operands remain in high diff --git a/modelopt_recipes/model_type/qwen3_5/ptq/fp8_vision_lm-kv_fp8_cast.yaml b/modelopt_recipes/model_type/qwen3_5/ptq/fp8_vision_lm-kv_fp8_cast.yaml index e8b42964a4a..173d32b286a 100644 --- a/modelopt_recipes/model_type/qwen3_5/ptq/fp8_vision_lm-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/qwen3_5/ptq/fp8_vision_lm-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Dense Qwen3.5 Vision Encoder and language model FP8 with FP8 KV-cache cast. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 metadata: - recipe_type: ptq description: >- W8A8 FP8 quantization of dense Qwen3.5 Vision Encoder and language-model Linear layers, including any merger Linears, with FP8 KV-cache cast; patch embedding and vision-attention diff --git a/modelopt_recipes/model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml b/modelopt_recipes/model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml index 5d5c5ff9288..2b2f938850a 100644 --- a/modelopt_recipes/model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml @@ -21,11 +21,11 @@ # the snippet lives under # `model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.quant_cfg.yaml`. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: shared_quant_cfg: model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.quant_cfg metadata: - recipe_type: ptq description: >- W4A16 (NVFP4 weights) MLP / FP8 attention / FP8 KV-cast PTQ recipe for HuggingFace `qwen3_5` (dense) models: NVFP4 for MLP projection weights and diff --git a/modelopt_recipes/model_type/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml b/modelopt_recipes/model_type/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml index 774d18c05ae..0134cf28911 100644 --- a/modelopt_recipes/model_type/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml @@ -24,11 +24,11 @@ # the snippet lives under # `model_type/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.quant_cfg.yaml`. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: shared_quant_cfg: model_type/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.quant_cfg metadata: - recipe_type: ptq description: >- W4A16 (NVFP4 weights, MSE calibration) MLP / FP8 attention / FP8 KV-cast PTQ recipe for HuggingFace `qwen3_5` (dense) models: NVFP4 with static scales diff --git a/modelopt_recipes/model_type/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml b/modelopt_recipes/model_type/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml index 068fd9c7796..d9e849a197b 100644 --- a/modelopt_recipes/model_type/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml +++ b/modelopt_recipes/model_type/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml @@ -19,6 +19,7 @@ # Note: fp8_scale_sweep refines ONLY static NVFP4 weights with MSE; all FP8 / dynamic / KV # quantizers stay max-calibrated (modelopt default) -- i.e. MSE applies only to the NVFP4 layers. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -28,7 +29,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- NVFP4 (MSE static weights, dynamic inputs) on LM routed experts; ModelOpt-default FP8 (max-calibrated) on all other Linear layers; FP8 KV cache; MTP block BF16. diff --git a/modelopt_recipes/model_type/qwen3_5_moe/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml b/modelopt_recipes/model_type/qwen3_5_moe/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml index 0dfe8884450..b7543301484 100644 --- a/modelopt_recipes/model_type/qwen3_5_moe/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/qwen3_5_moe/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml @@ -21,11 +21,11 @@ # `model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.yaml`; the snippet lives # under `model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.quant_cfg.yaml`. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: shared_quant_cfg: model_type/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast.quant_cfg metadata: - recipe_type: ptq description: >- W4A16 (NVFP4 weights) MLP / FP8 attention / FP8 KV-cast PTQ recipe for HuggingFace `qwen3_5_moe` models (Qwen3.5-MoE and Qwen3.6-MoE releases): diff --git a/modelopt_recipes/model_type/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml b/modelopt_recipes/model_type/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml index e6903cf47d4..2a10c641a48 100644 --- a/modelopt_recipes/model_type/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/qwen3_5_moe/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.yaml @@ -24,11 +24,11 @@ # snippet lives under # `model_type/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.quant_cfg.yaml`. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: shared_quant_cfg: model_type/qwen3_5/ptq/w4a16_nvfp4_mse-fp8_attn-kv_fp8_cast.quant_cfg metadata: - recipe_type: ptq description: >- W4A16 (NVFP4 weights, MSE calibration) MLP / FP8 attention / FP8 KV-cast PTQ recipe for HuggingFace `qwen3_5_moe` models (Qwen3.5-MoE and Qwen3.6-MoE diff --git a/modelopt_recipes/model_type/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml b/modelopt_recipes/model_type/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml index 02c175a233b..ea1c09961cf 100644 --- a/modelopt_recipes/model_type/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml +++ b/modelopt_recipes/model_type/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_at_6p0bits-active_moe.yaml @@ -25,7 +25,6 @@ imports: w4a16_nvfp4: configs/ptq/presets/model/w4a16_nvfp4 metadata: - recipe_type: auto_quantize description: >- Qwen3.6 MoE: FP8 + NVFP4-weight-only per-layer search at 6.0 effective bits, active-MoE cost model (expert ratio 0.03125), with architecture-specific disabled layers. diff --git a/modelopt_recipes/model_type/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_module_spaces_at_6p0bits-active_moe.yaml b/modelopt_recipes/model_type/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_module_spaces_at_6p0bits-active_moe.yaml index 4286a1214fc..974fb329615 100644 --- a/modelopt_recipes/model_type/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_module_spaces_at_6p0bits-active_moe.yaml +++ b/modelopt_recipes/model_type/qwen3_6_moe/auto_quantize/w4a16_nvfp4_fp8_module_spaces_at_6p0bits-active_moe.yaml @@ -14,7 +14,6 @@ imports: w4a16_nvfp4: configs/ptq/presets/model/w4a16_nvfp4 metadata: - recipe_type: auto_quantize description: >- Qwen3.6 MoE module-specific search based on the Qwen3.5-MoE W4A16/FP8 PTQ recipe: shared experts, attention, and lm_head searched over W4A16 NVFP4 diff --git a/modelopt_recipes/model_type/qwen3_vl/ptq/fp8_vision-kv_none.yaml b/modelopt_recipes/model_type/qwen3_vl/ptq/fp8_vision-kv_none.yaml index f00fd3fcf0f..9fc2e4ee590 100644 --- a/modelopt_recipes/model_type/qwen3_vl/ptq/fp8_vision-kv_none.yaml +++ b/modelopt_recipes/model_type/qwen3_vl/ptq/fp8_vision-kv_none.yaml @@ -15,12 +15,12 @@ # Qwen3-VL Vision Encoder FP8 with a high-precision language model and KV cache. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all vision_fp8: model_type/qwen3_vl/ptq/vision_fp8.quant_cfg metadata: - recipe_type: ptq description: >- FP8 quantization of Qwen3-VL Vision Encoder Linear layers, including primary and deepstack merger Linears; the language model, KV cache, patch embedding, and vision-attention operands diff --git a/modelopt_recipes/model_type/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast.yaml b/modelopt_recipes/model_type/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast.yaml index c0e864f86ee..fbd449d2d42 100644 --- a/modelopt_recipes/model_type/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast.yaml @@ -15,6 +15,7 @@ # Qwen3-VL Vision Encoder and language model FP8 with FP8 KV-cache cast. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -23,7 +24,6 @@ imports: w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 metadata: - recipe_type: ptq description: >- W8A8 FP8 quantization of Qwen3-VL Vision Encoder and language-model Linear layers, including primary and deepstack merger Linears, with FP8 KV-cache cast; patch embedding and diff --git a/modelopt_recipes/model_type/step3p7/ptq/nvfp4_experts_only-kv_fp8_cast.yaml b/modelopt_recipes/model_type/step3p7/ptq/nvfp4_experts_only-kv_fp8_cast.yaml index 086bdf4038c..2623e526319 100644 --- a/modelopt_recipes/model_type/step3p7/ptq/nvfp4_experts_only-kv_fp8_cast.yaml +++ b/modelopt_recipes/model_type/step3p7/ptq/nvfp4_experts_only-kv_fp8_cast.yaml @@ -20,6 +20,7 @@ # `moe.{gate,up,down}_proj`, the router is `moe.gate`, and each MoE layer also has a dense # `share_expert` that stays in BF16. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -27,7 +28,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 to Step-3.7 routed-expert weight and input quantizers, plus FP8 KV-cache cast mode using constant amax; the router, the shared expert and the dense MLPs diff --git a/modelopt_recipes/model_type/step3p7/ptq/nvfp4_mlp_only-kv_fp8.yaml b/modelopt_recipes/model_type/step3p7/ptq/nvfp4_mlp_only-kv_fp8.yaml index b858d895117..e2e485115c2 100644 --- a/modelopt_recipes/model_type/step3p7/ptq/nvfp4_mlp_only-kv_fp8.yaml +++ b/modelopt_recipes/model_type/step3p7/ptq/nvfp4_mlp_only-kv_fp8.yaml @@ -19,6 +19,7 @@ # general `*mlp*` / `*.experts.*` patterns reach only the dense layers' `mlp` submodules and # leave the routed experts — the bulk of the model — in BF16. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -26,7 +27,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- Applies dynamic NVFP4 to Step-3.7 routed-expert and dense-MLP weight and input quantizers, plus calibrated FP8 KV-cache quantization; the router and the shared expert diff --git a/modelopt_recipes/model_type/vit/ptq/fp8.yaml b/modelopt_recipes/model_type/vit/ptq/fp8.yaml index 6eb39351f62..d75b12333e2 100644 --- a/modelopt_recipes/model_type/vit/ptq/fp8.yaml +++ b/modelopt_recipes/model_type/vit/ptq/fp8.yaml @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe metadata: - recipe_type: ptq + description: >- + ViT image classifiers: FP8 W8A8 on every linear plus the attention BMM and + softmax-P quantizers, so the whole attention block runs in FP8 and Torch-TRT can + compile it end to end; output quantizers disabled. Uses max calibration. imports: w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 attention_qkv_fp8: configs/ptq/units/attention_qkv_fp8 diff --git a/modelopt_recipes/models/Qwen/Qwen3.8-2.4T-A95B/ptq/nvfp4_experts_mse-fp8_self_attn-fp8_linear_attn-kv_fp8_cast.yaml b/modelopt_recipes/models/Qwen/Qwen3.8-2.4T-A95B/ptq/nvfp4_experts_mse-fp8_self_attn-fp8_linear_attn-kv_fp8_cast.yaml index f8dcc94dba1..9cc17074f17 100644 --- a/modelopt_recipes/models/Qwen/Qwen3.8-2.4T-A95B/ptq/nvfp4_experts_mse-fp8_self_attn-fp8_linear_attn-kv_fp8_cast.yaml +++ b/modelopt_recipes/models/Qwen/Qwen3.8-2.4T-A95B/ptq/nvfp4_experts_mse-fp8_self_attn-fp8_linear_attn-kv_fp8_cast.yaml @@ -49,6 +49,7 @@ # BF16 before quantizers are inserted, so the scales below are calibrated against BF16 # weights rather than against the shipped FP8. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all nvfp4: configs/numerics/nvfp4 @@ -57,7 +58,6 @@ imports: kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- qwen3_5_moe_text: NVFP4 (MSE static weights, dynamic inputs) on the routed experts + FP8 (W8A8) on all self-attention AND all linear-attention projections; everything else BF16; FP8 KV cache diff --git a/modelopt_recipes/models/README.md b/modelopt_recipes/models/README.md index 3fa0c0fd9bc..0c3f28f08e6 100644 --- a/modelopt_recipes/models/README.md +++ b/modelopt_recipes/models/README.md @@ -5,8 +5,14 @@ This folder holds model-optimization recipes (e.g. PTQ recipes) tuned for a such as the [Hugging Face Hub](https://huggingface.co/), [ModelScope](https://modelscope.cn/), or similar. Unlike [`../model_type/`](../model_type/), which keys recipes by a transformers -`model_type` (an architecture shared by many checkpoints), a recipe here mirrors -**one checkpoint's** quantization scheme verbatim. +`model_type` (an architecture shared by many checkpoints), an entry here is keyed +to **one checkpoint**. + +An entry takes one of two forms: a **mirror**, whose body reproduces a per-layer +scheme no portable recipe can express, or an **alias**, which records that a +`general/` or `model_type//` recipe already produces that +checkpoint's scheme and imports it wholesale. See +[What belongs here](#what-belongs-here) for which to write. ## Folder structure @@ -46,8 +52,9 @@ applies to its mirrors, key it by that base model's id. Prefer the most specific entry that applies to your model: 1. **`models///`** — if there is an entry for your **exact** - checkpoint. It reproduces a validated, often per-component mixed-precision - scheme for that release; use it to match a published quantized checkpoint. + checkpoint. Use it to match a published quantized checkpoint: it either + reproduces a validated, often per-component mixed-precision scheme for that + release, or aliases the portable recipe that does. 2. **[`model_type//`](../model_type/)** — an architecture-level recipe that applies to every checkpoint of that `model_type`. 3. **[`general/`](../general/)** — model-agnostic recipes; a good starting point @@ -71,7 +78,11 @@ recipe = load_recipe("models/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/ptq/nvfp4_w4a ## What belongs here -A recipe earns a place here only when it mirrors **one specific released (or +Two kinds of entry, and the difference matters when you add one. + +### 1. Checkpoint mirrors — the recipe lives here + +A recipe earns a **body** here only when it mirrors **one specific released (or planned) checkpoint** — a hand-mapped, usually per-layer or per-component precision scheme tuned to match that exact release. If the tuning generalizes to every checkpoint of an architecture, it belongs under @@ -80,6 +91,56 @@ model-agnostic, it belongs under [`../general/`](../general/). See [`../ptq.md`](../ptq.md) for what each checkpoint mirror does and how it compares to its general baseline. +### 2. Aliases — the recipe lives elsewhere, the *record* lives here + +Many released checkpoints use a scheme a portable recipe already produces, with no +checkpoint-specific changes at all. Those can still get an entry here, so the recipe +is findable at the checkpoint's own hub path — but the entry is a thin **alias** +that delegates its whole body to that recipe: + +```yaml +imports: + base: general/ptq/nvfp4_default-kv_fp8_cast + +$import: base +metadata: + description: >- + meta-llama/Llama-3.1-8B-Instruct quantized with the general NVFP4 scheme and an + FP8 KV cache in cast mode, as published in nvidia/Llama-3.1-8B-Instruct-NVFP4. +``` + +A top-level `$import` brings in the whole imported recipe; the keys given +alongside it override the imported ones, so the alias supplies its own +`metadata` and inherits `quantize` — algorithm and every `quant_cfg` entry — +unchanged. Nothing is duplicated: editing the base recipe changes every alias +that points at it. + +Note what the alias does *not* say. It has no `recipe_type`, because the kind is +whatever the imported recipe's kind is; a recipe states its kind in whichever of +these it likes, and the loader takes the first that answers: + +1. a `# modelopt-schema:` comment naming its schema class, +2. `metadata.recipe_type` — **deprecated**; still read, but new recipes should + leave it out, +3. the recipe it delegates to via a top-level `$import`. + +Whatever a recipe does state has to be true. Declaring both a schema comment and +a `recipe_type` is fine as long as they agree, and the same holds across a +delegation: a recipe and the recipe it imports must be the same kind, since the +import takes over the whole body. Any disagreement is an error, not a preference. + +The one thing that *is* required: **a recipe another file imports must carry the +schema comment**, because that is what `$import` resolution needs to validate the +imported payload. A recipe nothing imports needs nothing — so aliasing a recipe +for the first time means adding the comment to it in the same change. + +**Which one to write.** Start by assuming an alias, and reach for a body here only +once you have established that no portable recipe expresses the release's scheme — +compare the release's own `hf_quant_config.json` (and, where the distinction matters, +its exported scale tensors) against what the candidate recipe's `quant_cfg` would +produce. A body duplicated from a general recipe is a maintenance liability: it stops +tracking edits to the recipe it was copied from. + ## Sharing content across recipes When several recipes reuse the same body, extract it into a sibling **snippet** diff --git a/modelopt_recipes/models/deepseek-ai/DeepSeek-V4-Pro-0813/ptq/nvfp4_experts_only.yaml b/modelopt_recipes/models/deepseek-ai/DeepSeek-V4-Pro-0813/ptq/nvfp4_experts_only.yaml index 48862491c02..b16f19dfb48 100644 --- a/modelopt_recipes/models/deepseek-ai/DeepSeek-V4-Pro-0813/ptq/nvfp4_experts_only.yaml +++ b/modelopt_recipes/models/deepseek-ai/DeepSeek-V4-Pro-0813/ptq/nvfp4_experts_only.yaml @@ -25,12 +25,12 @@ # are model.layers..ffn.experts..w{1,2,3}. The shared expert is the same # Expert class under a different MoE attribute, so it is disabled explicitly. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all nvfp4: configs/numerics/nvfp4 metadata: - recipe_type: ptq description: >- DeepSeek-V4-Pro-0813 recipe quantizing only the routed-expert linears to NVFP4 (weights and activations). Shared experts, attention, router gates, diff --git a/modelopt_recipes/models/meta-models/Muse-Glimmer-30B/auto_quantize/w4a16_nvfp4_4o6_mixed.yaml b/modelopt_recipes/models/meta-models/Muse-Glimmer-30B/auto_quantize/w4a16_nvfp4_4o6_mixed.yaml index b65ed6cb132..f4cf98771b0 100644 --- a/modelopt_recipes/models/meta-models/Muse-Glimmer-30B/auto_quantize/w4a16_nvfp4_4o6_mixed.yaml +++ b/modelopt_recipes/models/meta-models/Muse-Glimmer-30B/auto_quantize/w4a16_nvfp4_4o6_mixed.yaml @@ -14,7 +14,6 @@ imports: nvfp4_four_over_six: configs/numerics/nvfp4_four_over_six metadata: - recipe_type: auto_quantize description: >- Muse Glimmer language-model W4A16 NVFP4 Four-Over-Six, FP8, and BF16 fallback search at 5.5 weight-effective bits. Use --kv_cache_qformat none. diff --git a/modelopt_recipes/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib.yaml b/modelopt_recipes/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib.yaml index 3474315285b..dcd004f65cb 100644 --- a/modelopt_recipes/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib.yaml +++ b/modelopt_recipes/models/mistralai/Mistral-Medium-3.5-128B/ptq/nvfp4-max-calib.yaml @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -21,7 +22,6 @@ imports: kv_fp8: configs/ptq/units/kv_fp8 metadata: - recipe_type: ptq description: >- NVFP4 W4A4 on interior MLP layers, FP8 W8A8 on edge MLP and attention layers, and an FP8 KV cache; uses max calibration. diff --git a/modelopt_recipes/models/moonshotai/Kimi-K3/ptq/nvfp4_experts-fp8_pb_attention.yaml b/modelopt_recipes/models/moonshotai/Kimi-K3/ptq/nvfp4_experts-fp8_pb_attention.yaml index 30d85bcccf6..2db89f603e2 100644 --- a/modelopt_recipes/models/moonshotai/Kimi-K3/ptq/nvfp4_experts-fp8_pb_attention.yaml +++ b/modelopt_recipes/models/moonshotai/Kimi-K3/ptq/nvfp4_experts-fp8_pb_attention.yaml @@ -18,6 +18,7 @@ # applies this layout as a streaming checkpoint conversion rather than loading # the complete model through examples/hf_ptq/hf_ptq.py. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers @@ -25,7 +26,6 @@ imports: nvfp4: configs/numerics/nvfp4 metadata: - recipe_type: ptq description: >- Kimi-K3 checkpoint-mirror recipe with calibration-free MXFP4-to-NVFP4 routed experts, expert input_scale fixed to 1.0, and 128x128 block-FP8 diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml new file mode 100644 index 00000000000..a442ac3ff82 --- /dev/null +++ b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Recipe for nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16, mirroring +# nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8. +# +# The Nemotron-3 Nano 30B-A3B hybrid has 52 `backbone.layers`: Mamba-2 mixer layers +# (which own `mixer.conv1d` / `mixer.in_proj` / `mixer.out_proj`), MoE layers (which own +# `mixer.experts..{up,down}_proj`, `mixer.shared_experts.*` and the `mixer.gate` +# router), and six attention layers (5, 12, 19, 26, 33, 42). +# +# The Mamba scope is the model-specific part. `mixer.in_proj` / `mixer.out_proj` are +# quantized on every Mamba layer EXCEPT 4, 11, 18, 25, 32 and 41 -- the Mamba layer +# immediately preceding each of the six attention layers, which the release leaves +# BF16. A portable recipe has no way to express that, which is why this checkpoint +# mirror exists. +# +# BF16 everywhere else: all attention (`mixer.{q,k,v,o}_proj`), the `mixer.conv1d` +# depthwise convolutions, the `mixer.gate` routers, and `lm_head`. +# +# `hf_quant_config.json` records `kv_cache_quant_algo: FP8` but the checkpoint ships no +# `k_scale`/`v_scale` tensors, which is the constant-amax (cast) KV cache: a runtime +# that finds no scale uses 1.0, exactly what `kv_fp8_cast` produces. + +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + fp8: configs/numerics/fp8 + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + +metadata: + description: >- + NVIDIA-Nemotron-3-Nano-30B-A3B: FP8 (W8A8) on the MoE routed experts, the shared + experts and the Mamba in/out projections -- except on the six Mamba layers that + precede an attention layer, which stay BF16 -- plus an FP8 KV cache in cast mode; + max calibration. Attention, conv1d, routers and lm_head stay BF16. +quantize: + algorithm: + method: max + layerwise: {enable: false} + quant_cfg: + - $import: base_disable_all + # MoE routed + shared experts. + - quantizer_name: '*mixer.experts.*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.experts.*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.shared_experts.*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.shared_experts.*input_quantizer' + cfg: {$import: fp8} + # Mamba-2 mixer projections. + - quantizer_name: '*mixer.in_proj*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.in_proj*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.out_proj*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.out_proj*input_quantizer' + cfg: {$import: fp8} + - $import: kv_fp8_cast + - $import: default_disabled_quantizers + # The Mamba layer before each attention layer keeps BF16 projections. + # MUST stay last: it undoes the broad in/out_proj enables above. + - {quantizer_name: '*layers.4.mixer.*_proj*', enable: false} + - {quantizer_name: '*layers.11.mixer.*_proj*', enable: false} + - {quantizer_name: '*layers.18.mixer.*_proj*', enable: false} + - {quantizer_name: '*layers.25.mixer.*_proj*', enable: false} + - {quantizer_name: '*layers.32.mixer.*_proj*', enable: false} + - {quantizer_name: '*layers.41.mixer.*_proj*', enable: false} diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/ptq/nvfp4_w4a16.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/ptq/nvfp4_w4a16.yaml index 1f102d52850..b7671d72f24 100644 --- a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/ptq/nvfp4_w4a16.yaml +++ b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16/ptq/nvfp4_w4a16.yaml @@ -26,6 +26,7 @@ # lm_head → FP8 W8A16 (weight-only; overrides its Q8_0 GGUF type) # everything else (F32: conv1d, norms, A_log/D/dt_bias) → bf16 +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all w4a4_nvfp4_nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 @@ -34,7 +35,6 @@ imports: nvfp4: configs/numerics/nvfp4 metadata: - recipe_type: ptq description: >- GGUF Q4_K_M-mirrored Nemotron-H recipe: NVFP4 W4A4 for Q4_K/Q5_0 linears (attn q/k/v/o kept uniform for export fusion), FP8 W8A8 for Q6_K MLP diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml new file mode 100644 index 00000000000..d226050f9cf --- /dev/null +++ b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Recipe for nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16, mirroring +# nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 -- the FP8 sibling of the NVFP4 +# release that `nvfp4-mse.yaml` / `nvfp4-max-calib.yaml` mirror. +# +# The Super hybrid has 88 `backbone.layers`: Mamba-2 mixer layers, MoE layers, and +# eight attention layers (7, 16, 25, 36, 47, 58, 69, 78). +# +# Quantized, all FP8 W8A8 per-tensor: the MoE routed experts +# (`mixer.experts..{up,down}_proj`), the shared experts, and the Mamba +# `mixer.in_proj` / `mixer.out_proj` on every Mamba layer. Unlike the NVFP4 release +# this one is uniform -- there is no per-layer carve-out. +# +# BF16 everywhere else: all attention (`mixer.{q,k,v,o}_proj`), the latent-MoE +# projections (`mixer.fc1_latent_proj` / `fc2_latent_proj`), the `mixer.conv1d` +# depthwise convolutions, the `mixer.gate` routers, `lm_head`, and the whole MTP block. +# +# KV cache: FP8 with a constant amax -- the published checkpoint's `k_scale` is exactly +# 1.0, which is what `use_constant_amax` produces (amax pinned to the E4M3 max, 448). + +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + fp8: configs/numerics/fp8 + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + +metadata: + description: >- + NVIDIA-Nemotron-3-Super-120B-A12B: FP8 W8A8 per-tensor on the MoE routed experts, + the shared experts and the Mamba in/out projections, plus an FP8 KV cache in cast + mode; max calibration. Attention, latent-MoE projections, conv1d, routers, lm_head + and the MTP block stay BF16. +quantize: + algorithm: + method: max + layerwise: {enable: false} + quant_cfg: + - $import: base_disable_all + - quantizer_name: '*mixer.experts.*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.experts.*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.shared_experts.*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.shared_experts.*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.in_proj*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.in_proj*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.out_proj*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.out_proj*input_quantizer' + cfg: {$import: fp8} + - $import: kv_fp8_cast + - $import: default_disabled_quantizers + # Latent-MoE projections stay BF16 -- `*_proj` wildcards above do not reach them, + # but pin it so a widened pattern cannot silently pull them in. + - {quantizer_name: '*latent_proj*', enable: false} diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/nvfp4-max-calib.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/nvfp4-max-calib.yaml index fa0e9ef8f70..b11bb8a6333 100644 --- a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/nvfp4-max-calib.yaml +++ b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/nvfp4-max-calib.yaml @@ -29,8 +29,8 @@ # - SSM cache: FP32 (can be set to FP16 in VLLM) # # Calibration: amax/max calibration comparison variant +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe metadata: - recipe_type: ptq description: Super NVFP4 mixed precision — sparse MoE experts NVFP4 (W4A4, group_size 16); shared experts, mamba in/out_proj FP8 per-tensor; FP8 KV cache; everything else(lm_head/MTP/Latent MOE) stay BF16. Amax calibration comparison variant. quantize: @@ -129,3 +129,12 @@ quantize: enable: true cfg: num_bits: e4m3 + + # MTP block -> BF16. Only reachable on the Megatron-Core path, where the block is a + # live `model.mtp` submodule the broad `*mixer.*` patterns above match into; the HF + # `nemotron_h` class drops the MTP tensors on load + # (`_keys_to_ignore_on_load_unexpected = [r"mtp.*"]`), so this is inert there. Must + # stay last, after the patterns it undoes. Mirrors the same entry in + # `configs/ptq/units/default_disabled_quantizers`, which this recipe does not import. + - quantizer_name: 'mtp.*' + enable: false diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/nvfp4-mse.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/nvfp4-mse.yaml index 05bc9511781..07382992e3b 100644 --- a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/nvfp4-mse.yaml +++ b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/nvfp4-mse.yaml @@ -30,8 +30,8 @@ # Calibration: weight MSE with FP8-scale sweep over the 128 e4m3 scale values # (NVFP4 weights use static block scales selected by MSE; FP8 per-tensor scales # are also chosen via MSE search instead of plain amax). +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe metadata: - recipe_type: ptq description: Super NVFP4 mixed precision — sparse MoE experts NVFP4 (W4A4, group_size 16); shared experts, mamba in/out_proj FP8 per-tensor; FP8 KV cache; everything else(lm_head/MTP/latent MOE) stay BF16. Weight-MSE calibration with FP8 scale sweep. quantize: @@ -132,3 +132,12 @@ quantize: enable: true cfg: num_bits: e4m3 + + # MTP block -> BF16. Only reachable on the Megatron-Core path, where the block is a + # live `model.mtp` submodule the broad `*mixer.*` patterns above match into; the HF + # `nemotron_h` class drops the MTP tensors on load + # (`_keys_to_ignore_on_load_unexpected = [r"mtp.*"]`), so this is inert there. Must + # stay last, after the patterns it undoes. Mirrors the same entry in + # `configs/ptq/units/default_disabled_quantizers`, which this recipe does not import. + - quantizer_name: 'mtp.*' + enable: false diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16/ptq/nvfp4-4o6.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16/ptq/nvfp4-4o6.yaml index 4095e620d08..b08bef2a301 100644 --- a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16/ptq/nvfp4-4o6.yaml +++ b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16/ptq/nvfp4-4o6.yaml @@ -11,13 +11,13 @@ # Weights: 4/6 NVFP4, per-block M=6 vs M=4 selected by MSE. Activations: dynamic NVFP4 (NOT MSE). # Shared experts + mamba in/out_proj + KV cache: FP8. Everything else: BF16. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: nvfp4_four_over_six: configs/numerics/nvfp4_four_over_six nvfp4: configs/numerics/nvfp4 fp8: configs/numerics/fp8 metadata: - recipe_type: ptq description: Ultra NVFP4 with Four-Over-Six (4/6) MSE-selected weights on routed experts (W4A4, block 16); dynamic NVFP4 activations (not MSE); shared experts + mamba in/out_proj + KV cache FP8; everything else BF16. quantize: @@ -79,3 +79,12 @@ quantize: - quantizer_name: '*[kv]_bmm_quantizer' enable: true cfg: {$import: fp8} + + # MTP block -> BF16. Only reachable on the Megatron-Core path, where the block is a + # live `model.mtp` submodule the broad `*mixer.*` patterns above match into; the HF + # `nemotron_h` class drops the MTP tensors on load + # (`_keys_to_ignore_on_load_unexpected = [r"mtp.*"]`), so this is inert there. Must + # stay last, after the patterns it undoes. Mirrors the same entry in + # `configs/ptq/units/default_disabled_quantizers`, which this recipe does not import. + - quantizer_name: 'mtp.*' + enable: false diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/ptq/w4a16_nvfp4_4o6.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/ptq/w4a16_nvfp4_4o6.yaml index ab6933007b8..f393e822eda 100644 --- a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/ptq/w4a16_nvfp4_4o6.yaml +++ b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/ptq/w4a16_nvfp4_4o6.yaml @@ -11,11 +11,11 @@ # - mixer.shared_experts.{up,down}_proj -> mlp.shared_experts.linear_fc{1,2} # - mixer.in_proj / out_proj -> mixer.in_proj / out_proj (same name; W4A16) # - lm_head -> output_layer (W4A16) +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: nvfp4_four_over_six: configs/numerics/nvfp4_four_over_six fp8_default: configs/numerics/fp8 metadata: - recipe_type: ptq description: > Lightning 3.5 W4A16 PTQ, NVFP4 four_over_six (4/6) weight scales. Routed MoE experts, shared experts, and lm_head; FP8 Mamba in_proj/out_proj. diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/speculative_decoding/dspark_warmstart.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/speculative_decoding/dspark_warmstart.yaml index 2736ec2dfbd..eb78e80c466 100644 --- a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/speculative_decoding/dspark_warmstart.yaml +++ b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16/speculative_decoding/dspark_warmstart.yaml @@ -20,8 +20,8 @@ # transformers-5 path (`mamba`->`linear_attention`, `attention`->`full_attention`, plus a # matching `hybrid_override_pattern`); that conversion is not covered here either. +# modelopt-schema: modelopt.recipe.config.ModelOptDFlashRecipe metadata: - recipe_type: speculative_dflash description: Warm-start fine-tune of the released Nemotron-3.5 DSpark drafter (streaming). model: diff --git a/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/fp8-kv_fp8_cast.yaml b/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/fp8-kv_fp8_cast.yaml new file mode 100644 index 00000000000..7ef20299066 --- /dev/null +++ b/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/fp8-kv_fp8_cast.yaml @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Recipe for nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16, mirroring +# nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8. +# +# Nemotron-3-Nano-Omni is the omni-modal (text + vision + audio) sibling of +# NVIDIA-Nemotron-3-Nano-30B-A3B: the same 52-layer Nemotron-H hybrid decoder, nested +# under `language_model.backbone.layers`, plus a RADIO vision tower +# (`vision_model.radio_model.*`), a Conformer sound encoder (`sound_encoder.*`) and the +# `mlp1` / `sound_projection` modality projectors. +# +# Only the language decoder is quantized. The vision tower, the sound encoder and both +# projectors stay BF16 -- and `sound_encoder*`, `sound_projection*` and `mlp1*` match +# none of the shared `default_disabled_quantizers` patterns, so this recipe disables +# them explicitly. That extra exclusion is the model-specific part. +# +# `layerwise.enable=false` is required: the decoder layers nest under +# `language_model.backbone.layers`, where layerwise_calibrate cannot find them. +# +# `hf_quant_config.json` records `kv_cache_quant_algo: FP8` but the checkpoint ships no +# `k_scale`/`v_scale` tensors, which is the constant-amax (cast) KV cache. +# +# Quantized, all FP8 W8A8 per-tensor: every linear in the decoder -- MoE routed and +# shared experts, the Mamba `mixer.in_proj` / `mixer.out_proj` on every Mamba layer, and +# all six attention layers' `mixer.{q,k,v,o}_proj`. BF16: `mixer.conv1d`, the +# `mixer.gate` routers and `lm_head`. + +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + +metadata: + description: >- + Nemotron-3-Nano-Omni-30B-A3B-Reasoning: FP8 W8A8 per-tensor on every language-decoder + linear (MoE experts, shared experts, Mamba in/out projections and attention), plus an + FP8 KV cache in cast mode; max calibration. The RADIO vision tower, the Conformer + sound encoder, both modality projectors, conv1d, routers and lm_head stay BF16. +quantize: + algorithm: + method: max + # Required: the decoder nests under `language_model.backbone.layers`, which + # layerwise_calibrate cannot locate. + layerwise: {enable: false} + quant_cfg: + - $import: base_disable_all + - $import: w8a8_fp8_fp8 + - $import: kv_fp8_cast + - $import: default_disabled_quantizers + # Non-language branches. MUST stay after the broad enables above; none of these + # match a `default_disabled_quantizers` pattern. + - {quantizer_name: '*sound_encoder*', enable: false} + - {quantizer_name: '*sound_projection*', enable: false} + - {quantizer_name: '*radio_model*', enable: false} + - {quantizer_name: 'mlp1.*', enable: false} + - {quantizer_name: '*mixer.conv1d*', enable: false} + - {quantizer_name: '*mixer.gate.*', enable: false} diff --git a/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/nvfp4_experts-fp8_rest-kv_fp8_cast.yaml b/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/nvfp4_experts-fp8_rest-kv_fp8_cast.yaml new file mode 100644 index 00000000000..d72dc4f9cd0 --- /dev/null +++ b/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/nvfp4_experts-fp8_rest-kv_fp8_cast.yaml @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Recipe for nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16, mirroring +# nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4. +# +# Nemotron-3-Nano-Omni is the omni-modal (text + vision + audio) sibling of +# NVIDIA-Nemotron-3-Nano-30B-A3B: the same 52-layer Nemotron-H hybrid decoder, nested +# under `language_model.backbone.layers`, plus a RADIO vision tower +# (`vision_model.radio_model.*`), a Conformer sound encoder (`sound_encoder.*`) and the +# `mlp1` / `sound_projection` modality projectors. +# +# Only the language decoder is quantized. The vision tower, the sound encoder and both +# projectors stay BF16 -- and `sound_encoder*`, `sound_projection*` and `mlp1*` match +# none of the shared `default_disabled_quantizers` patterns, so this recipe disables +# them explicitly. That extra exclusion is the model-specific part. +# +# `layerwise.enable=false` is required: the decoder layers nest under +# `language_model.backbone.layers`, where layerwise_calibrate cannot find them. +# +# `hf_quant_config.json` records `kv_cache_quant_algo: FP8` but the checkpoint ships no +# `k_scale`/`v_scale` tensors, which is the constant-amax (cast) KV cache. +# +# Mixed precision, unlike the FP8 sibling: +# +# MoE routed experts NVFP4 (W4A4) +# MoE shared experts FP8 (W8A8 per-tensor) +# Mamba in/out_proj FP8 (W8A8, every Mamba layer) +# attention o_proj FP8 (W8A8, the six attention layers) +# attention q/k/v_proj BF16 +# conv1d, routers, head BF16 +# KV cache FP8 (cast) +# +# The attention split is the notable part: only the output projection drops to FP8, +# while q/k/v stay BF16 -- the same `omlp`-style scoping the general +# `nvfp4_omlp_only` recipe uses, but with FP8 rather than NVFP4 on that projection. + +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + nvfp4: configs/numerics/nvfp4 + fp8: configs/numerics/fp8 + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + +metadata: + description: >- + Nemotron-3-Nano-Omni-30B-A3B-Reasoning: NVFP4 (W4A4) on the MoE routed experts; FP8 + (W8A8) on the shared experts, the Mamba in/out projections and the attention output + projection; FP8 KV cache in cast mode; max calibration. Attention q/k/v, conv1d, + routers, lm_head, the RADIO vision tower, the Conformer sound encoder and both + modality projectors stay BF16. +quantize: + algorithm: + method: max + layerwise: {enable: false} + quant_cfg: + - $import: base_disable_all + # MoE routed experts -> NVFP4 W4A4. + - quantizer_name: '*mixer.experts.*weight_quantizer' + cfg: {$import: nvfp4} + - quantizer_name: '*mixer.experts.*input_quantizer' + cfg: {$import: nvfp4} + # Shared experts, Mamba projections and the attention output projection -> FP8 W8A8. + - quantizer_name: '*mixer.shared_experts.*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.shared_experts.*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.in_proj*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.in_proj*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.out_proj*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.out_proj*input_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.o_proj*weight_quantizer' + cfg: {$import: fp8} + - quantizer_name: '*mixer.o_proj*input_quantizer' + cfg: {$import: fp8} + - $import: kv_fp8_cast + - $import: default_disabled_quantizers + # Non-language branches; none match a `default_disabled_quantizers` pattern. + - {quantizer_name: '*sound_encoder*', enable: false} + - {quantizer_name: '*sound_projection*', enable: false} + - {quantizer_name: '*radio_model*', enable: false} + - {quantizer_name: 'mlp1.*', enable: false} diff --git a/modelopt_recipes/models/stepfun-ai/Step-3.5-Flash/ptq/nvfp4-mlp-only.yaml b/modelopt_recipes/models/stepfun-ai/Step-3.5-Flash/ptq/nvfp4-mlp-only.yaml index d0adbe00479..f828373358f 100644 --- a/modelopt_recipes/models/stepfun-ai/Step-3.5-Flash/ptq/nvfp4-mlp-only.yaml +++ b/modelopt_recipes/models/stepfun-ai/Step-3.5-Flash/ptq/nvfp4-mlp-only.yaml @@ -15,12 +15,12 @@ # Model-specific PTQ recipe for Step3.5-Flash NVFP4 MLP/MoE quantization with FP8 KV cache. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: fp8: configs/numerics/fp8 nvfp4: configs/numerics/nvfp4 metadata: - recipe_type: ptq description: >- Step3.5-Flash PTQ recipe that enables dynamic NVFP4 on MoE/MLP weight and input quantizers, enables FP8 KV-cache quantizers, and leaves other quantizers disabled. diff --git a/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_dense_mlp-kv_fp8_cast.yaml b/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_dense_mlp-kv_fp8_cast.yaml index edfa05ba9ff..2954aa12abe 100644 --- a/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_dense_mlp-kv_fp8_cast.yaml +++ b/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_dense_mlp-kv_fp8_cast.yaml @@ -53,13 +53,13 @@ # than they are: `*.experts.*` needs a literal `.experts.`, so `mlp.shared_experts.*` does not # match; and `*mlp.gate_proj*` needs `mlp.gate_proj`, which the router `mlp.gate` lacks. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: base_disable_all: configs/ptq/units/base_disable_all nvfp4: configs/numerics/nvfp4 kv_fp8_cast: configs/ptq/units/kv_fp8_cast metadata: - recipe_type: ptq description: >- GLM-5.3-Flash: NVFP4 (W4A4) on the routed experts and on the dense MLP of layers 0-2, plus an FP8 KV cache in cast mode using constant amax; max calibration. Shared experts, diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 963889e98aa..f7aeb31d967 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -152,6 +152,12 @@ of the body scheme. Quantizing the KV cache reduces memory at long context. - **`kv_fp8`** — FP8 E4M3 KV cache with **calibrated** per-tensor amax. The KV scales are measured during the calibration pass. Hopper+. +- **`kv_fp16`** — the KV cache is **not** quantized; it stays at the model's + activation dtype (BF16/FP16). Combine with any body scheme when the deployment + stack does not consume an FP8 KV cache, or when long-context memory is not the + binding constraint. Some published checkpoints ship this way, so a body scheme + gains a `-kv_fp16` variant when one of them needs it. + > **`kv_fp8_cast` vs `kv_fp8`:** both produce an FP8 KV cache. `_cast` uses a > fixed scale and skips the KV calibration step (faster, no extra data > dependence); plain `kv_fp8` calibrates the scale from data. The cast version @@ -237,7 +243,7 @@ The general recipes above are **model-agnostic**: they select layers by wildcard (`*mlp*`, `*self_attn*`, `*[kv]_bmm_quantizer`) and lean on the shared `default_disabled_quantizers` exclusions, so the same file works on any architecture whose module names follow the usual conventions. A recipe only -earns a place under `model_type//` or +earns a **body** under `model_type//` or `models///` when a model has to **deviate** from that baseline. The deviations come in four kinds: @@ -256,6 +262,15 @@ The numerics and standard exclusions are still inherited from `configs/` wherever possible — the model folder captures *only* the delta. Each `/` folder may carry a `README.md` spelling out that delta. +> **Not every `models///` folder holds a deviation.** When a +> general (or `huggingface/`) recipe already produces a released +> checkpoint's scheme, the folder holds a thin **alias** that imports that recipe +> wholesale and overrides only `metadata` — no duplicated `quant_cfg` — so the +> recipe is reachable from the checkpoint's own hub path. [Checkpoint +> mirrors](#checkpoint-mirrors--modelsorgcheckpoint) covers the folders that +> genuinely deviate and aliases the ones that don't; see +> [`models/README.md`](models/README.md) for the alias format and when to write one. + ### Architecture-aware `quant_cfg` — `minimax_m3_vl`, `qwen3_vl`, `qwen3_5`, `qwen3_5_moe`, `vit`, `nemotron_llama` **`minimax_m3_vl/ptq/mxfp8_nvfp4_experts`** applies MXFP8 to the language-model @@ -451,6 +466,25 @@ checkpoint's** quant config verbatim: `general/ptq/nvfp4_experts_only-kv_fp8_cast` — the model-specific delta here is the dense-MLP scope plus the vision-tower exclusion.) +- **`models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast`** + mirrors `nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8`: MoE routed and shared experts plus + the Mamba `mixer.in_proj` / `mixer.out_proj` quantized, **except on layers 4, 11, 18, + 25, 32 and 41** — the Mamba layer immediately preceding each of the six attention + layers, which the release leaves BF16. Attention, `conv1d`, routers and `lm_head` stay + BF16; KV cache FP8 cast. There is deliberately no NVFP4 sibling: that release's model + card says quantization-aware distillation was applied after PTQ, so no PTQ recipe + reproduces it. +- **`models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast`** + mirrors `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8`, the FP8 sibling of the NVFP4 + release above: FP8 W8A8 on the routed experts, shared experts and Mamba in/out + projections on every Mamba layer — uniform, with no per-layer carve-out — and BF16 for + attention, the latent-MoE projections, `conv1d`, routers, `lm_head` and MTP. +- **`models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/{fp8-kv_fp8_cast,nvfp4_experts-fp8_rest-kv_fp8_cast}`** + mirror the omni-modal (text + vision + audio) Nemotron-H releases. The FP8 one is + uniform W8A8 across the decoder; the NVFP4 one is mixed — routed experts NVFP4, shared + experts / Mamba in-out / attention **o_proj only** FP8, q/k/v BF16. Both add explicit + disables for the RADIO vision tower, the Conformer sound encoder and the `mlp1` / + `sound_projection` modality projectors, none of which match a standard exclusion. *Why special:* unlike any general recipe, each is pinned to one checkpoint and captures a model-specific deviation a portable general recipe can't express. Most **mix FP8 and NVFP4 across different component types — or individual layers** — diff --git a/modelopt_recipes/timm/resnet/ptq/fp8.yaml b/modelopt_recipes/timm/resnet/ptq/fp8.yaml index 5040246348c..fdfc3b13fb0 100644 --- a/modelopt_recipes/timm/resnet/ptq/fp8.yaml +++ b/modelopt_recipes/timm/resnet/ptq/fp8.yaml @@ -21,7 +21,6 @@ imports: w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 metadata: - recipe_type: ptq description: FP8 ResNet PTQ with quantized residual connections. quantize: diff --git a/modelopt_recipes/timm/resnet/ptq/int8.yaml b/modelopt_recipes/timm/resnet/ptq/int8.yaml index 8ebb6151a1d..501c063333b 100644 --- a/modelopt_recipes/timm/resnet/ptq/int8.yaml +++ b/modelopt_recipes/timm/resnet/ptq/int8.yaml @@ -21,7 +21,6 @@ imports: int8_per_channel: configs/numerics/int8_per_channel metadata: - recipe_type: ptq description: INT8 ResNet PTQ with quantized residual connections. quantize: diff --git a/tests/unit/recipe/test_loader.py b/tests/unit/recipe/test_loader.py index 006e7d57927..a96fc4f1b82 100644 --- a/tests/unit/recipe/test_loader.py +++ b/tests/unit/recipe/test_loader.py @@ -28,6 +28,7 @@ import modelopt.torch.quantization.config as qcfg from modelopt.recipe.config import ( + RECIPE_TYPE_TO_CLASS, AutoQuantizeConfig, AutoQuantizeConstraints, AutoQuantizeCost, @@ -35,14 +36,17 @@ ModelOptDFlashRecipe, ModelOptEagleRecipe, ModelOptPTQRecipe, + RecipeMetadataConfig, RecipeType, ) from modelopt.recipe.loader import _apply_dotlist, _resolve_recipe_path, load_config, load_recipe from modelopt.torch.opt.config_loader import ( + _MODELOPT_SCHEMA_RE, _alias_builtin_recipe_prefix, _load_raw_config, _resolve_config_path, _schema_type, + peek_declared_schema, ) from modelopt.torch.quantization.config import QuantizerAttributeConfig, normalize_quant_cfg_list from modelopt.torch.quantization.mode import CalibrateModeRegistry, get_modelike_from_algo_cfg @@ -301,8 +305,17 @@ def test_import_resolution_honors_huggingface_alias(): assert str(_resolve_config_path(old)) == str(_resolve_config_path(canonical)) +_PTQ_SCHEMA = "modelopt.recipe.config.ModelOptPTQRecipe" + + def _all_shipped_ptq_recipe_paths(): - """Every shipped PTQ recipe, discovered from disk rather than a hardcoded list.""" + """Every shipped PTQ recipe, discovered from disk rather than a hardcoded list. + + A recipe says it is PTQ with ``metadata.recipe_type``, with a ``# modelopt-schema:`` + comment naming :class:`ModelOptPTQRecipe`, or -- for a checkpoint alias, which states + neither -- by delegating to a recipe that does. All three are picked up, so every + shipped recipe is swept by the tests below. + """ root = files("modelopt_recipes") paths = [] for path in sorted(Path(str(root)).rglob("*.yaml")): @@ -314,7 +327,13 @@ def _all_shipped_ptq_recipe_paths(): # List-shaped fragments (layer-pattern units) are not recipes. if not isinstance(raw, dict): continue - if (raw.get("metadata") or {}).get("recipe_type") == "ptq": + declared = peek_declared_schema(path) + delegates = "$import" in raw # a checkpoint alias inherits its kind from its base + if ( + declared == _PTQ_SCHEMA + or (raw.get("metadata") or {}).get("recipe_type") == "ptq" + or delegates + ): paths.append(str(rel.with_suffix(""))) return paths @@ -476,6 +495,436 @@ def test_load_recipe_unsupported_type_raises(tmp_path): load_recipe(bad) +# --------------------------------------------------------------------------- +# load_recipe — whole-recipe delegation (checkpoint aliases) +# --------------------------------------------------------------------------- + + +_BASE_RECIPE_FOR_ALIAS = """\ +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe +metadata: + recipe_type: ptq + description: the base recipe +quantize: + algorithm: max + quant_cfg: + - quantizer_name: '*' + enable: false + - quantizer_name: '*weight_quantizer' + cfg: {num_bits: 8, axis: 0} +""" + + +def _write_alias_pair(tmp_path, alias_body: str): + """Write a base recipe plus an alias that delegates to it; return the alias path.""" + (tmp_path / "base.yaml").write_text(_BASE_RECIPE_FOR_ALIAS) + alias = tmp_path / "alias.yaml" + alias.write_text(alias_body.format(base=tmp_path / "base.yaml")) + return alias + + +def test_load_recipe_delegates_whole_body_to_import(tmp_path): + """A top-level ``$import`` supplies the body; local keys override the imported ones. + + This is what the checkpoint aliases under ``modelopt_recipes/models/`` rely on: they + name the recipe behind a published checkpoint without copying its body. + """ + alias = _write_alias_pair( + tmp_path, + """\ +imports: + base: {base} + +$import: base +metadata: + recipe_type: ptq + description: the alias +""", + ) + recipe = load_recipe(alias) + assert recipe.description == "the alias" + assert recipe.quantize.model_dump() == load_recipe(tmp_path / "base.yaml").quantize.model_dump() + + +def test_load_recipe_delegating_alias_can_override_the_body(tmp_path): + """A delegating recipe may also replace an imported section outright.""" + alias = _write_alias_pair( + tmp_path, + """\ +imports: + base: {base} + +$import: base +metadata: + recipe_type: ptq + description: overridden body +quantize: + algorithm: max + quant_cfg: + - quantizer_name: '*input_quantizer' + enable: false +""", + ) + quant_cfg = load_recipe(alias).quantize.model_dump()["quant_cfg"] + assert [entry["quantizer_name"] for entry in quant_cfg] == ["*input_quantizer"] + + +def test_load_recipe_infers_kind_from_schema_comment(tmp_path): + """A recipe that declares its schema needs no ``metadata.recipe_type``.""" + recipe = tmp_path / "r.yaml" + recipe.write_text(_BASE_RECIPE_FOR_ALIAS.replace(" recipe_type: ptq\n", "")) + loaded = load_recipe(recipe) + assert loaded.recipe_type == RecipeType.PTQ + assert isinstance(loaded, ModelOptPTQRecipe) + + +def test_load_recipe_infers_kind_from_the_recipe_it_delegates_to(tmp_path): + """A pure alias states neither ``recipe_type`` nor a schema; it inherits both. + + This is the shape the checkpoint aliases under ``modelopt_recipes/models/`` use, so + a released checkpoint's entry carries nothing but a description and the import. + """ + alias = _write_alias_pair( + tmp_path, + """\ +imports: + base: {base} + +$import: base +metadata: + description: nothing but a description +""", + ) + loaded = load_recipe(alias) + assert loaded.recipe_type == RecipeType.PTQ + assert loaded.description == "nothing but a description" + + +def test_load_recipe_rejects_recipe_type_contradicting_its_schema(tmp_path): + """Stating a kind that disagrees with the schema class is an error, not a preference.""" + recipe = tmp_path / "r.yaml" + recipe.write_text( + _BASE_RECIPE_FOR_ALIAS.replace(" recipe_type: ptq\n", " recipe_type: speculative_eagle\n") + ) + with pytest.raises(ValueError, match="recipe_type"): + load_recipe(recipe) + + +def test_load_recipe_without_any_kind_declaration_raises(tmp_path): + """No schema comment, no recipe_type and no delegation: the loader cannot dispatch.""" + recipe = tmp_path / "r.yaml" + recipe.write_text( + _BASE_RECIPE_FOR_ALIAS.replace( + "# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe\n", "" + ).replace(" recipe_type: ptq\n", "") + ) + with pytest.raises(ValueError, match="does not say what kind of recipe it is"): + load_recipe(recipe) + + +_EAGLE_RECIPE_FOR_ALIAS = """\ +# modelopt-schema: modelopt.recipe.config.ModelOptEagleRecipe +metadata: + description: an eagle recipe +eagle: {} +""" + + +@pytest.mark.parametrize( + ("declaration", "label"), + [ + ("# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe\n", "schema comment"), + ("", "metadata.recipe_type"), + ], +) +def test_load_recipe_rejects_delegating_to_a_different_kind(tmp_path, declaration, label): + """Importing a recipe of another kind is an error, however the kinds were declared. + + A top-level ``$import`` takes over the whole body, so a PTQ recipe importing an + EAGLE one would splice an ``eagle`` section into a PTQ schema. Caught as a kind + mismatch rather than left to surface as whatever pydantic makes of the result. + """ + (tmp_path / "base.yaml").write_text(_EAGLE_RECIPE_FOR_ALIAS) + alias = tmp_path / "alias.yaml" + metadata = "metadata:\n description: a ptq recipe\n" + if not declaration: # declare the kind the other way instead + metadata = "metadata:\n recipe_type: ptq\n description: a ptq recipe\n" + alias.write_text( + f"{declaration}imports:\n base: {tmp_path / 'base.yaml'}\n\n$import: base\n{metadata}" + ) + with pytest.raises(ValueError, match=r"is a 'ptq' recipe but imports .*'speculative_eagle'"): + load_recipe(alias) + + +def test_load_recipe_allows_delegating_within_the_same_kind(tmp_path): + """The matching case still loads -- the check rejects mismatches, not delegation.""" + alias = _write_alias_pair( + tmp_path, + """\ +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe +imports: + base: {base} + +$import: base +metadata: + recipe_type: ptq + description: agrees on every axis +""", + ) + assert load_recipe(alias).recipe_type == RecipeType.PTQ + + +def test_load_recipe_reuses_a_whole_recipe_with_no_metadata(tmp_path): + """A recipe can be nothing but an import: two lines, everything inherited. + + With no inline keys to override with, ``metadata`` arrives from the base along with + ``quantize`` -- so this is genuine whole-recipe reuse, not just body reuse. Useful + when a second path should resolve to an existing recipe verbatim; a checkpoint alias + keeps its own description instead, so it can say which release it stands for. + """ + alias = _write_alias_pair(tmp_path, "imports:\n base: {base}\n\n$import: base\n") + aliased, original = load_recipe(alias), load_recipe(tmp_path / "base.yaml") + assert aliased.recipe_type == RecipeType.PTQ + assert aliased.quantize.model_dump() == original.quantize.model_dump() + assert aliased.metadata.model_dump() == original.metadata.model_dump() + assert aliased.description == "the base recipe" + + +def test_load_recipe_delegation_chain_inherits_the_kind(tmp_path): + """Kind resolution follows a chain of delegations, not just one hop.""" + (tmp_path / "base.yaml").write_text(_BASE_RECIPE_FOR_ALIAS) + (tmp_path / "middle.yaml").write_text( + f"# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe\n" + f"imports:\n base: {tmp_path / 'base.yaml'}\n\n$import: base\n" + ) + leaf = tmp_path / "leaf.yaml" + leaf.write_text( + f"imports:\n mid: {tmp_path / 'middle.yaml'}\n\n$import: mid\n" + "metadata:\n description: two hops from the body\n" + ) + loaded = load_recipe(leaf) + assert loaded.recipe_type == RecipeType.PTQ + assert loaded.description == "two hops from the body" + assert loaded.quantize.model_dump() == load_recipe(tmp_path / "base.yaml").quantize.model_dump() + + +def test_load_recipe_delegation_cycle_is_reported_not_hung(tmp_path): + """Two recipes that delegate to each other fail cleanly instead of recursing forever. + + Neither states a kind, so resolution has to walk the import to find one and would + loop without the cycle guard. A ``ValueError`` rather than a ``RecursionError`` is + the assertion that the guard is doing its job. + """ + a, b = tmp_path / "a.yaml", tmp_path / "b.yaml" + a.write_text(f"imports:\n other: {b}\n\n$import: other\n") + b.write_text(f"imports:\n other: {a}\n\n$import: other\n") + with pytest.raises(ValueError, match="does not say what kind of recipe it is"): + load_recipe(a) + + +def test_load_recipe_delegates_via_a_list_of_imports(tmp_path): + """``$import`` accepts a list; the kind comes from the first entry that is a recipe.""" + (tmp_path / "base.yaml").write_text(_BASE_RECIPE_FOR_ALIAS) + (tmp_path / "extra.yaml").write_text( + "# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe\n" + "metadata:\n description: extra\n" + "quantize:\n algorithm: max\n quant_cfg: []\n" + ) + alias = tmp_path / "alias.yaml" + alias.write_text( + f"imports:\n base: {tmp_path / 'base.yaml'}\n extra: {tmp_path / 'extra.yaml'}\n\n" + "$import: [base, extra]\nmetadata:\n description: merged\n" + ) + loaded = load_recipe(alias) + assert loaded.recipe_type == RecipeType.PTQ + # Later imports win, matching the dict-merge semantics of a multi-name $import. + assert loaded.quantize.quant_cfg == [] + + +def test_load_recipe_ignores_a_non_recipe_schema_comment_when_dispatching(tmp_path): + """A schema comment naming something that is not a recipe falls through to metadata. + + Only the recipe schema classes identify a recipe kind; anything else means the file + is a snippet as far as dispatch is concerned, so ``metadata.recipe_type`` still has + to answer. + """ + recipe = tmp_path / "r.yaml" + recipe.write_text( + "# modelopt-schema: modelopt.torch.quantization.config.QuantizeConfig\n" + "metadata:\n recipe_type: ptq\n description: d\n" + "quantize:\n algorithm: max\n quant_cfg: []\n" + ) + assert load_recipe(recipe).recipe_type == RecipeType.PTQ + + +# --------------------------------------------------------------------------- +# peek_declared_schema +# --------------------------------------------------------------------------- + + +def test_peek_declared_schema_reads_the_preamble(tmp_path): + """The declared schema path is returned without parsing or resolving the file.""" + f = tmp_path / "c.yaml" + f.write_text( + "# a comment\n" + "# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe\n" + "metadata:\n description: d\n" + ) + assert peek_declared_schema(f) == "modelopt.recipe.config.ModelOptPTQRecipe" + + +def test_peek_declared_schema_returns_none_without_a_comment(tmp_path): + f = tmp_path / "c.yaml" + f.write_text("metadata:\n recipe_type: ptq\n") + assert peek_declared_schema(f) is None + + +def test_peek_declared_schema_ignores_a_comment_below_the_preamble(tmp_path): + """A comment after the first YAML line is not a declaration -- and must not look like one. + + This is the shape ``test_shipped_modelopt_schema_comments_are_in_the_preamble`` + guards the shipped recipes against. + """ + f = tmp_path / "c.yaml" + f.write_text( + "metadata:\n recipe_type: ptq\n" + "# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe\n" + ) + assert peek_declared_schema(f) is None + + +def test_peek_declared_schema_rejects_two_declarations(tmp_path): + f = tmp_path / "c.yaml" + f.write_text( + "# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe\n" + "# modelopt-schema: modelopt.recipe.config.ModelOptEagleRecipe\n" + "metadata:\n description: d\n" + ) + with pytest.raises(ValueError, match="multiple modelopt-schema"): + peek_declared_schema(f) + + +# --------------------------------------------------------------------------- +# metadata.recipe_type is derived from the schema class +# --------------------------------------------------------------------------- + + +def test_metadata_config_recipe_type_is_optional(tmp_path): + """``RecipeMetadataConfig`` on its own no longer requires a recipe_type.""" + f = tmp_path / "metadata.yaml" + f.write_text("description: no kind stated\n") + metadata = load_config(f, schema_type=RecipeMetadataConfig) + assert metadata.recipe_type is None + assert metadata.description == "no kind stated" + + +#: The body section each recipe class requires, so the metadata behaviour can be checked +#: on every kind rather than only the one with the simplest body. +_MINIMAL_BODIES: dict[RecipeType, dict] = { + RecipeType.PTQ: {"quantize": {"algorithm": "max", "quant_cfg": []}}, + RecipeType.AUTO_QUANTIZE: { + "auto_quantize": { + "constraints": {"effective_bits": 4.8}, + "candidate_formats": [ + {"quant_cfg": [{"quantizer_name": "*", "enable": False}]}, + {"quant_cfg": [{"quantizer_name": "*weight_quantizer", "enable": True}]}, + ], + } + }, + RecipeType.SPECULATIVE_EAGLE: {}, # body sections have field defaults + RecipeType.SPECULATIVE_DFLASH: {}, + RecipeType.SPECULATIVE_MEDUSA: {}, +} + + +@pytest.mark.parametrize( + ("recipe_type", "schema_class"), + sorted(RECIPE_TYPE_TO_CLASS.items(), key=lambda kv: kv[0].value), +) +def test_recipe_class_fills_in_its_own_recipe_type(recipe_type, schema_class): + """Every recipe class knows its kind and fills ``metadata.recipe_type`` from it.""" + assert recipe_type == schema_class.RECIPE_TYPE + recipe = schema_class.model_validate( + {"metadata": {"description": "d"}, **_MINIMAL_BODIES[recipe_type]} + ) + assert recipe.metadata.recipe_type == recipe_type + assert recipe.recipe_type == recipe_type + + +def test_recipe_class_rejects_a_contradicting_recipe_type(): + """Stating the wrong kind is rejected at validation, not silently overwritten.""" + with pytest.raises(ValueError, match="recipe_type"): + ModelOptPTQRecipe.model_validate( + { + "metadata": {"recipe_type": "speculative_eagle", "description": "d"}, + "quantize": {"algorithm": "max", "quant_cfg": []}, + } + ) + + +def test_load_recipe_dir_without_recipe_type_raises(tmp_path): + """A directory recipe has no schema comment, so its metadata must state the kind.""" + (tmp_path / "metadata.yml").write_text("description: no kind stated\n") + (tmp_path / "quantize.yml").write_text("algorithm: max\nquant_cfg: []\n") + with pytest.raises(ValueError, match="recipe_type"): + load_recipe(tmp_path) + + +def test_shipped_modelopt_schema_comments_are_in_the_preamble(): + """A ``modelopt-schema`` comment below the first YAML line is silently ignored. + + :func:`_parse_modelopt_schema` stops at the first non-comment line, so a comment + placed after e.g. ``metadata:`` parses as absent -- the file looks annotated but is + not importable and cannot be dispatched from. Catch that here rather than at the + point some future recipe tries to ``$import`` it. + """ + root = Path(str(files("modelopt_recipes"))) + ignored = [ + str(path.relative_to(root)) + for path in sorted(root.rglob("*.yaml")) + # Use the parser's own pattern so this can't drift from what it accepts, and so + # prose that merely mentions the comment is not mistaken for one. + if not path.is_symlink() + and _MODELOPT_SCHEMA_RE.search(path.read_text(encoding="utf-8")) + and peek_declared_schema(path) is None + ] + assert not ignored, ( + "These files carry a modelopt-schema comment that the parser cannot see; move it " + f"above the first YAML line: {ignored}" + ) + + +def test_load_recipe_delegating_alias_still_needs_a_body(tmp_path): + """Delegation relaxes the raw-YAML check, it does not remove the requirement.""" + (tmp_path / "empty.yaml").write_text( + "# modelopt-schema: modelopt.recipe.config.RecipeMetadataConfig\n" + "recipe_type: ptq\ndescription: not a full recipe\n" + ) + alias = tmp_path / "alias.yaml" + alias.write_text( + f"imports:\n base: {tmp_path / 'empty.yaml'}\n\n" + "$import: base\nmetadata:\n recipe_type: ptq\n description: alias\n" + ) + with pytest.raises(ValueError, match="quantize"): + load_recipe(alias) + + +def test_load_recipe_import_of_recipe_without_schema_raises(tmp_path): + """An imported recipe must declare its ``modelopt-schema``, like any snippet.""" + (tmp_path / "base.yaml").write_text( + _BASE_RECIPE_FOR_ALIAS.replace( + "# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe\n", "" + ) + ) + alias = tmp_path / "alias.yaml" + alias.write_text( + f"imports:\n base: {tmp_path / 'base.yaml'}\n\n" + "$import: base\nmetadata:\n recipe_type: ptq\n description: alias\n" + ) + with pytest.raises(ValueError, match="modelopt-schema"): + load_recipe(alias) + + # --------------------------------------------------------------------------- # load_recipe — directory format # --------------------------------------------------------------------------- diff --git a/tools/precommit/check_modelopt_recipes.py b/tools/precommit/check_modelopt_recipes.py index 88bf49cea24..db0ee31ae5f 100644 --- a/tools/precommit/check_modelopt_recipes.py +++ b/tools/precommit/check_modelopt_recipes.py @@ -32,6 +32,7 @@ from __future__ import annotations +import re import sys from pathlib import Path @@ -46,6 +47,22 @@ {"ptq", "speculative_eagle", "speculative_dflash", "speculative_medusa"} ) +# A recipe usually declares its kind with a ``# modelopt-schema:`` comment naming its +# schema class rather than with ``metadata.recipe_type`` (see modelopt/recipe/loader.py). +# Matched here by name so the hook keeps working without importing modelopt. +_SCHEMA_COMMENT_RE = re.compile( + r"^\s*#\s*modelopt-schema:\s*modelopt\.recipe\.config\.ModelOpt\w+Recipe\s*$", + re.MULTILINE, +) + + +def _declares_recipe_schema(path: Path) -> bool: + """Whether *path* names one of the recipe schema classes in its comment preamble.""" + try: + return bool(_SCHEMA_COMMENT_RE.search(path.read_text(encoding="utf-8"))) + except OSError: + return False + def _check_quant_cfg(quant_cfg, label: str) -> list[str]: """Validate quant_cfg format. *label* is used in error messages.""" @@ -105,7 +122,7 @@ def _check_single_file_recipe(path: Path) -> list[str]: return [] # not a recipe file metadata = data.get("metadata") - if not isinstance(metadata, dict) or "recipe_type" not in metadata: + if not isinstance(metadata, dict) and not _declares_recipe_schema(path): return [] # not a recipe file if "ptq_cfg" in data: @@ -180,6 +197,8 @@ def _is_recipe_file(path: Path) -> bool: return True # let load_recipe report the parse error if not isinstance(data, dict): return False # not a recipe file at all + if _declares_recipe_schema(path): + return True metadata = data.get("metadata") if not isinstance(metadata, dict) or "recipe_type" not in metadata: return False # not a recipe file at all From 80d7d17f02e027a8ebcb2d19f44a839c9e89e109 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Tue, 15 Sep 2026 01:10:06 +0000 Subject: [PATCH 2/5] Swap the NVIDIA Nemotron backfill for Kimi-K2.6 and Qwen3.5-397B-A17B aliases Drop the newly-added NVIDIA Nemotron checkpoint mirrors (Nano-30B-A3B FP8, Super-120B-A12B FP8, Nano-Omni FP8/NVFP4) and instead record two published checkpoints as thin aliases that reuse a portable recipe wholesale: - models/moonshotai/Kimi-K2.6/ptq/nvfp4_experts_only_mse-kv_fp8_cast aliases general/ptq/nvfp4_experts_only_mse-kv_fp8_cast, published as nvidia/Kimi-K2.6-NVFP4. - models/nvidia/Qwen3.5-397B-A17B/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8 aliases model_type/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8, published as nvidia/Qwen3.5-397B-A17B-NVFP4-V2. Update CHANGELOG and ptq.md (drop the Nemotron mirror bullets, add a Checkpoint aliases subsection). The pre-existing nvidia Nemotron-3 mirrors (Super-120B nvfp4, Ultra, 3.5-Lightning, Nano-4B) are unchanged. Signed-off-by: Shengliang Xu --- CHANGELOG.rst | 2 +- .../nvfp4_experts_only_mse-kv_fp8_cast.yaml | 29 ++++++ .../ptq/fp8_moe_mamba-kv_fp8_cast.yaml | 83 ---------------- .../ptq/fp8_moe_mamba-kv_fp8_cast.yaml | 74 -------------- .../ptq/fp8-kv_fp8_cast.yaml | 72 -------------- .../nvfp4_experts-fp8_rest-kv_fp8_cast.yaml | 99 ------------------- .../nvfp4_experts_mse-fp8_rest-kv_fp8.yaml | 29 ++++++ modelopt_recipes/ptq.md | 35 +++---- 8 files changed, 75 insertions(+), 348 deletions(-) create mode 100644 modelopt_recipes/models/moonshotai/Kimi-K2.6/ptq/nvfp4_experts_only_mse-kv_fp8_cast.yaml delete mode 100644 modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml delete mode 100644 modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml delete mode 100644 modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/fp8-kv_fp8_cast.yaml delete mode 100644 modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/nvfp4_experts-fp8_rest-kv_fp8_cast.yaml create mode 100644 modelopt_recipes/models/nvidia/Qwen3.5-397B-A17B/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53588aec0b2..dde933ea451 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,7 +9,7 @@ Changelog *Quantization* - A recipe can now **delegate its whole body to another recipe** with a top-level ``$import``, overriding only ``metadata``. ``metadata.recipe_type`` became optional along with it: a recipe states its kind with a ``# modelopt-schema:`` comment, with ``metadata.recipe_type``, or by delegating to a recipe that does, and only a recipe that another file imports has to carry the schema comment. Whatever a recipe does state must be true: a schema comment and a ``recipe_type`` must agree, and so must a recipe and the recipe it delegates to. ``modelopt_recipes/models/`` uses this for checkpoint entries that a portable recipe already reproduces: the entry aliases that recipe instead of copying it. -- Backfill the recipes behind NVIDIA's already-published checkpoints under ``modelopt_recipes/models/``, so a released checkpoint's quantization scheme is reachable from its own model-hub path rather than only from the general tier. This batch covers the NVIDIA-source releases: NVIDIA-Nemotron-3-Nano-30B-A3B FP8, NVIDIA-Nemotron-3-Super-120B-A12B FP8, and Nemotron-3-Nano-Omni-30B-A3B-Reasoning (FP8 and NVFP4); the remaining source-model orgs follow in separate changes. A release is only backfilled when its model card supports a post-training quantization recipe -- one whose card documents quantization-aware distillation after PTQ is deliberately left out, since no PTQ recipe reproduces it. +- Backfill the recipes behind NVIDIA's already-published checkpoints under ``modelopt_recipes/models/``, so a released checkpoint's quantization scheme is reachable from its own model-hub path rather than only from the general tier. For example, ``moonshotai/Kimi-K2.6`` (published as ``nvidia/Kimi-K2.6-NVFP4``) and ``nvidia/Qwen3.5-397B-A17B`` (``nvidia/Qwen3.5-397B-A17B-NVFP4-V2``) each alias a portable recipe wholesale -- the general expert-only NVFP4 recipe and the ``qwen3_5_moe`` architecture recipe respectively -- rather than copying its body; other checkpoints follow in separate changes. A release is only backfilled when its model card supports a post-training quantization recipe -- one whose card documents quantization-aware distillation after PTQ is deliberately left out, since no PTQ recipe reproduces it. - Add ``layerwise.export_dir``: layerwise calibration writes each decoder layer to its own quantized checkpoint shard as it finishes, so no separate ``export_hf_checkpoint()`` pass is needed and, with ``layerwise.checkpoint_dir``, an interrupted run resumes without redoing finished layers. Calibration writes the layer shards; ``finalize()`` on the exporter left on the model adds the tail shard, the index and the config artifacts, and the checkpoint does not load until it runs. ``examples/hf_ptq`` does this for you. Supports FP8 and NVFP4 on single-process models, resident or offloaded, including multimodal models and models with MTP layers; other formats and placements raise ``NotImplementedError`` before calibration starts. - Add an end-to-end BEVFormer ONNX PTQ example with temporal calibration data generation, INT8 and FP8 quantization, TensorRT engine building, and nuScenes accuracy evaluation. See `examples/onnx_ptq/bevformer/README.md `_ for details. diff --git a/modelopt_recipes/models/moonshotai/Kimi-K2.6/ptq/nvfp4_experts_only_mse-kv_fp8_cast.yaml b/modelopt_recipes/models/moonshotai/Kimi-K2.6/ptq/nvfp4_experts_only_mse-kv_fp8_cast.yaml new file mode 100644 index 00000000000..5385993a2e8 --- /dev/null +++ b/modelopt_recipes/models/moonshotai/Kimi-K2.6/ptq/nvfp4_experts_only_mse-kv_fp8_cast.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Alias recipe for nvidia/Kimi-K2.6-NVFP4 (source moonshotai/Kimi-K2.6). The general +# expert-only NVFP4 scheme -- MSE-swept static weight scales, dynamic NVFP4 inputs, FP8 +# KV cache in cast mode -- reproduces this release with no checkpoint-specific deviation, +# so this entry imports that recipe wholesale instead of copying its body. + +imports: + base: general/ptq/nvfp4_experts_only_mse-kv_fp8_cast + +$import: base +metadata: + description: >- + moonshotai/Kimi-K2.6 quantized with the general expert-only NVFP4 scheme (MSE static + weight scales, dynamic inputs) and an FP8 KV cache in cast mode, as published in + nvidia/Kimi-K2.6-NVFP4. diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml deleted file mode 100644 index a442ac3ff82..00000000000 --- a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Recipe for nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16, mirroring -# nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8. -# -# The Nemotron-3 Nano 30B-A3B hybrid has 52 `backbone.layers`: Mamba-2 mixer layers -# (which own `mixer.conv1d` / `mixer.in_proj` / `mixer.out_proj`), MoE layers (which own -# `mixer.experts..{up,down}_proj`, `mixer.shared_experts.*` and the `mixer.gate` -# router), and six attention layers (5, 12, 19, 26, 33, 42). -# -# The Mamba scope is the model-specific part. `mixer.in_proj` / `mixer.out_proj` are -# quantized on every Mamba layer EXCEPT 4, 11, 18, 25, 32 and 41 -- the Mamba layer -# immediately preceding each of the six attention layers, which the release leaves -# BF16. A portable recipe has no way to express that, which is why this checkpoint -# mirror exists. -# -# BF16 everywhere else: all attention (`mixer.{q,k,v,o}_proj`), the `mixer.conv1d` -# depthwise convolutions, the `mixer.gate` routers, and `lm_head`. -# -# `hf_quant_config.json` records `kv_cache_quant_algo: FP8` but the checkpoint ships no -# `k_scale`/`v_scale` tensors, which is the constant-amax (cast) KV cache: a runtime -# that finds no scale uses 1.0, exactly what `kv_fp8_cast` produces. - -# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe -imports: - base_disable_all: configs/ptq/units/base_disable_all - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers - fp8: configs/numerics/fp8 - kv_fp8_cast: configs/ptq/units/kv_fp8_cast - -metadata: - description: >- - NVIDIA-Nemotron-3-Nano-30B-A3B: FP8 (W8A8) on the MoE routed experts, the shared - experts and the Mamba in/out projections -- except on the six Mamba layers that - precede an attention layer, which stay BF16 -- plus an FP8 KV cache in cast mode; - max calibration. Attention, conv1d, routers and lm_head stay BF16. -quantize: - algorithm: - method: max - layerwise: {enable: false} - quant_cfg: - - $import: base_disable_all - # MoE routed + shared experts. - - quantizer_name: '*mixer.experts.*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.experts.*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.shared_experts.*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.shared_experts.*input_quantizer' - cfg: {$import: fp8} - # Mamba-2 mixer projections. - - quantizer_name: '*mixer.in_proj*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.in_proj*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.out_proj*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.out_proj*input_quantizer' - cfg: {$import: fp8} - - $import: kv_fp8_cast - - $import: default_disabled_quantizers - # The Mamba layer before each attention layer keeps BF16 projections. - # MUST stay last: it undoes the broad in/out_proj enables above. - - {quantizer_name: '*layers.4.mixer.*_proj*', enable: false} - - {quantizer_name: '*layers.11.mixer.*_proj*', enable: false} - - {quantizer_name: '*layers.18.mixer.*_proj*', enable: false} - - {quantizer_name: '*layers.25.mixer.*_proj*', enable: false} - - {quantizer_name: '*layers.32.mixer.*_proj*', enable: false} - - {quantizer_name: '*layers.41.mixer.*_proj*', enable: false} diff --git a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml b/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml deleted file mode 100644 index d226050f9cf..00000000000 --- a/modelopt_recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Recipe for nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16, mirroring -# nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 -- the FP8 sibling of the NVFP4 -# release that `nvfp4-mse.yaml` / `nvfp4-max-calib.yaml` mirror. -# -# The Super hybrid has 88 `backbone.layers`: Mamba-2 mixer layers, MoE layers, and -# eight attention layers (7, 16, 25, 36, 47, 58, 69, 78). -# -# Quantized, all FP8 W8A8 per-tensor: the MoE routed experts -# (`mixer.experts..{up,down}_proj`), the shared experts, and the Mamba -# `mixer.in_proj` / `mixer.out_proj` on every Mamba layer. Unlike the NVFP4 release -# this one is uniform -- there is no per-layer carve-out. -# -# BF16 everywhere else: all attention (`mixer.{q,k,v,o}_proj`), the latent-MoE -# projections (`mixer.fc1_latent_proj` / `fc2_latent_proj`), the `mixer.conv1d` -# depthwise convolutions, the `mixer.gate` routers, `lm_head`, and the whole MTP block. -# -# KV cache: FP8 with a constant amax -- the published checkpoint's `k_scale` is exactly -# 1.0, which is what `use_constant_amax` produces (amax pinned to the E4M3 max, 448). - -# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe -imports: - base_disable_all: configs/ptq/units/base_disable_all - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers - fp8: configs/numerics/fp8 - kv_fp8_cast: configs/ptq/units/kv_fp8_cast - -metadata: - description: >- - NVIDIA-Nemotron-3-Super-120B-A12B: FP8 W8A8 per-tensor on the MoE routed experts, - the shared experts and the Mamba in/out projections, plus an FP8 KV cache in cast - mode; max calibration. Attention, latent-MoE projections, conv1d, routers, lm_head - and the MTP block stay BF16. -quantize: - algorithm: - method: max - layerwise: {enable: false} - quant_cfg: - - $import: base_disable_all - - quantizer_name: '*mixer.experts.*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.experts.*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.shared_experts.*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.shared_experts.*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.in_proj*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.in_proj*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.out_proj*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.out_proj*input_quantizer' - cfg: {$import: fp8} - - $import: kv_fp8_cast - - $import: default_disabled_quantizers - # Latent-MoE projections stay BF16 -- `*_proj` wildcards above do not reach them, - # but pin it so a widened pattern cannot silently pull them in. - - {quantizer_name: '*latent_proj*', enable: false} diff --git a/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/fp8-kv_fp8_cast.yaml b/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/fp8-kv_fp8_cast.yaml deleted file mode 100644 index 7ef20299066..00000000000 --- a/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/fp8-kv_fp8_cast.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Recipe for nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16, mirroring -# nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8. -# -# Nemotron-3-Nano-Omni is the omni-modal (text + vision + audio) sibling of -# NVIDIA-Nemotron-3-Nano-30B-A3B: the same 52-layer Nemotron-H hybrid decoder, nested -# under `language_model.backbone.layers`, plus a RADIO vision tower -# (`vision_model.radio_model.*`), a Conformer sound encoder (`sound_encoder.*`) and the -# `mlp1` / `sound_projection` modality projectors. -# -# Only the language decoder is quantized. The vision tower, the sound encoder and both -# projectors stay BF16 -- and `sound_encoder*`, `sound_projection*` and `mlp1*` match -# none of the shared `default_disabled_quantizers` patterns, so this recipe disables -# them explicitly. That extra exclusion is the model-specific part. -# -# `layerwise.enable=false` is required: the decoder layers nest under -# `language_model.backbone.layers`, where layerwise_calibrate cannot find them. -# -# `hf_quant_config.json` records `kv_cache_quant_algo: FP8` but the checkpoint ships no -# `k_scale`/`v_scale` tensors, which is the constant-amax (cast) KV cache. -# -# Quantized, all FP8 W8A8 per-tensor: every linear in the decoder -- MoE routed and -# shared experts, the Mamba `mixer.in_proj` / `mixer.out_proj` on every Mamba layer, and -# all six attention layers' `mixer.{q,k,v,o}_proj`. BF16: `mixer.conv1d`, the -# `mixer.gate` routers and `lm_head`. - -# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe -imports: - base_disable_all: configs/ptq/units/base_disable_all - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers - w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 - kv_fp8_cast: configs/ptq/units/kv_fp8_cast - -metadata: - description: >- - Nemotron-3-Nano-Omni-30B-A3B-Reasoning: FP8 W8A8 per-tensor on every language-decoder - linear (MoE experts, shared experts, Mamba in/out projections and attention), plus an - FP8 KV cache in cast mode; max calibration. The RADIO vision tower, the Conformer - sound encoder, both modality projectors, conv1d, routers and lm_head stay BF16. -quantize: - algorithm: - method: max - # Required: the decoder nests under `language_model.backbone.layers`, which - # layerwise_calibrate cannot locate. - layerwise: {enable: false} - quant_cfg: - - $import: base_disable_all - - $import: w8a8_fp8_fp8 - - $import: kv_fp8_cast - - $import: default_disabled_quantizers - # Non-language branches. MUST stay after the broad enables above; none of these - # match a `default_disabled_quantizers` pattern. - - {quantizer_name: '*sound_encoder*', enable: false} - - {quantizer_name: '*sound_projection*', enable: false} - - {quantizer_name: '*radio_model*', enable: false} - - {quantizer_name: 'mlp1.*', enable: false} - - {quantizer_name: '*mixer.conv1d*', enable: false} - - {quantizer_name: '*mixer.gate.*', enable: false} diff --git a/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/nvfp4_experts-fp8_rest-kv_fp8_cast.yaml b/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/nvfp4_experts-fp8_rest-kv_fp8_cast.yaml deleted file mode 100644 index d72dc4f9cd0..00000000000 --- a/modelopt_recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/nvfp4_experts-fp8_rest-kv_fp8_cast.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Recipe for nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16, mirroring -# nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4. -# -# Nemotron-3-Nano-Omni is the omni-modal (text + vision + audio) sibling of -# NVIDIA-Nemotron-3-Nano-30B-A3B: the same 52-layer Nemotron-H hybrid decoder, nested -# under `language_model.backbone.layers`, plus a RADIO vision tower -# (`vision_model.radio_model.*`), a Conformer sound encoder (`sound_encoder.*`) and the -# `mlp1` / `sound_projection` modality projectors. -# -# Only the language decoder is quantized. The vision tower, the sound encoder and both -# projectors stay BF16 -- and `sound_encoder*`, `sound_projection*` and `mlp1*` match -# none of the shared `default_disabled_quantizers` patterns, so this recipe disables -# them explicitly. That extra exclusion is the model-specific part. -# -# `layerwise.enable=false` is required: the decoder layers nest under -# `language_model.backbone.layers`, where layerwise_calibrate cannot find them. -# -# `hf_quant_config.json` records `kv_cache_quant_algo: FP8` but the checkpoint ships no -# `k_scale`/`v_scale` tensors, which is the constant-amax (cast) KV cache. -# -# Mixed precision, unlike the FP8 sibling: -# -# MoE routed experts NVFP4 (W4A4) -# MoE shared experts FP8 (W8A8 per-tensor) -# Mamba in/out_proj FP8 (W8A8, every Mamba layer) -# attention o_proj FP8 (W8A8, the six attention layers) -# attention q/k/v_proj BF16 -# conv1d, routers, head BF16 -# KV cache FP8 (cast) -# -# The attention split is the notable part: only the output projection drops to FP8, -# while q/k/v stay BF16 -- the same `omlp`-style scoping the general -# `nvfp4_omlp_only` recipe uses, but with FP8 rather than NVFP4 on that projection. - -# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe -imports: - base_disable_all: configs/ptq/units/base_disable_all - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers - nvfp4: configs/numerics/nvfp4 - fp8: configs/numerics/fp8 - kv_fp8_cast: configs/ptq/units/kv_fp8_cast - -metadata: - description: >- - Nemotron-3-Nano-Omni-30B-A3B-Reasoning: NVFP4 (W4A4) on the MoE routed experts; FP8 - (W8A8) on the shared experts, the Mamba in/out projections and the attention output - projection; FP8 KV cache in cast mode; max calibration. Attention q/k/v, conv1d, - routers, lm_head, the RADIO vision tower, the Conformer sound encoder and both - modality projectors stay BF16. -quantize: - algorithm: - method: max - layerwise: {enable: false} - quant_cfg: - - $import: base_disable_all - # MoE routed experts -> NVFP4 W4A4. - - quantizer_name: '*mixer.experts.*weight_quantizer' - cfg: {$import: nvfp4} - - quantizer_name: '*mixer.experts.*input_quantizer' - cfg: {$import: nvfp4} - # Shared experts, Mamba projections and the attention output projection -> FP8 W8A8. - - quantizer_name: '*mixer.shared_experts.*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.shared_experts.*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.in_proj*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.in_proj*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.out_proj*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.out_proj*input_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.o_proj*weight_quantizer' - cfg: {$import: fp8} - - quantizer_name: '*mixer.o_proj*input_quantizer' - cfg: {$import: fp8} - - $import: kv_fp8_cast - - $import: default_disabled_quantizers - # Non-language branches; none match a `default_disabled_quantizers` pattern. - - {quantizer_name: '*sound_encoder*', enable: false} - - {quantizer_name: '*sound_projection*', enable: false} - - {quantizer_name: '*radio_model*', enable: false} - - {quantizer_name: 'mlp1.*', enable: false} diff --git a/modelopt_recipes/models/nvidia/Qwen3.5-397B-A17B/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml b/modelopt_recipes/models/nvidia/Qwen3.5-397B-A17B/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml new file mode 100644 index 00000000000..220402eabb3 --- /dev/null +++ b/modelopt_recipes/models/nvidia/Qwen3.5-397B-A17B/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Alias recipe for nvidia/Qwen3.5-397B-A17B-NVFP4-V2 (source nvidia/Qwen3.5-397B-A17B). The +# qwen3_5_moe architecture recipe already produces this release's scheme -- NVFP4 (MSE static +# weights, dynamic inputs) on the LM routed experts, ModelOpt-default FP8 elsewhere, and an FP8 +# KV cache -- so this entry imports that recipe wholesale instead of copying its body. + +imports: + base: model_type/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8 + +$import: base +metadata: + description: >- + nvidia/Qwen3.5-397B-A17B quantized with the qwen3_5_moe architecture recipe -- NVFP4 (MSE + static weights, dynamic inputs) on the LM routed experts, ModelOpt-default FP8 on every other + Linear layer, and an FP8 KV cache -- as published in nvidia/Qwen3.5-397B-A17B-NVFP4-V2. diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index f7aeb31d967..ffa2bc27634 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -466,25 +466,6 @@ checkpoint's** quant config verbatim: `general/ptq/nvfp4_experts_only-kv_fp8_cast` — the model-specific delta here is the dense-MLP scope plus the vision-tower exclusion.) -- **`models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast`** - mirrors `nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8`: MoE routed and shared experts plus - the Mamba `mixer.in_proj` / `mixer.out_proj` quantized, **except on layers 4, 11, 18, - 25, 32 and 41** — the Mamba layer immediately preceding each of the six attention - layers, which the release leaves BF16. Attention, `conv1d`, routers and `lm_head` stay - BF16; KV cache FP8 cast. There is deliberately no NVFP4 sibling: that release's model - card says quantization-aware distillation was applied after PTQ, so no PTQ recipe - reproduces it. -- **`models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/ptq/fp8_moe_mamba-kv_fp8_cast`** - mirrors `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8`, the FP8 sibling of the NVFP4 - release above: FP8 W8A8 on the routed experts, shared experts and Mamba in/out - projections on every Mamba layer — uniform, with no per-layer carve-out — and BF16 for - attention, the latent-MoE projections, `conv1d`, routers, `lm_head` and MTP. -- **`models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/ptq/{fp8-kv_fp8_cast,nvfp4_experts-fp8_rest-kv_fp8_cast}`** - mirror the omni-modal (text + vision + audio) Nemotron-H releases. The FP8 one is - uniform W8A8 across the decoder; the NVFP4 one is mixed — routed experts NVFP4, shared - experts / Mamba in-out / attention **o_proj only** FP8, q/k/v BF16. Both add explicit - disables for the RADIO vision tower, the Conformer sound encoder and the `mlp1` / - `sound_projection` modality projectors, none of which match a standard exclusion. *Why special:* unlike any general recipe, each is pinned to one checkpoint and captures a model-specific deviation a portable general recipe can't express. Most **mix FP8 and NVFP4 across different component types — or individual layers** — @@ -494,6 +475,22 @@ is the exception: its deviation is a model-specific *scope* — a wildcard schem plus a load-bearing vision-tower exclusion and the VLM-required `layerwise.enable=false` — rather than a per-component precision map. +### Checkpoint aliases — `models//` + +Some published checkpoints use a scheme a portable recipe already produces, so their +entry is a thin **alias** that imports that recipe wholesale and overrides only +`metadata` (see [`models/README.md`](models/README.md) for the format): + +- **`models/moonshotai/Kimi-K2.6/ptq/nvfp4_experts_only_mse-kv_fp8_cast`** aliases the general + `general/ptq/nvfp4_experts_only_mse-kv_fp8_cast` — expert-only NVFP4 with MSE-swept static + weight scales and dynamic inputs, plus an FP8 KV cache in cast mode — as published in + `nvidia/Kimi-K2.6-NVFP4`. +- **`models/nvidia/Qwen3.5-397B-A17B/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8`** aliases the + `qwen3_5_moe` architecture recipe + `model_type/qwen3_5_moe/ptq/nvfp4_experts_mse-fp8_rest-kv_fp8` — NVFP4 (MSE static weights) + on the routed experts, ModelOpt-default FP8 elsewhere, and an FP8 KV cache — as published in + `nvidia/Qwen3.5-397B-A17B-NVFP4-V2`. + --- For the full catalog and how to pick a starting recipe for a given model, see From aabfe5cd0702d0ccff094dee7dfb602d95f90146 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Tue, 15 Sep 2026 21:59:18 +0000 Subject: [PATCH 3/5] Keep the schema regex line-scoped and the delegation helper private Two cleanups to the recipe-alias change, no behaviour change. _MODELOPT_SCHEMA_RE had re.MULTILINE added to it, but _parse_modelopt_schema iterates text.splitlines() and calls .match() on each line, where ^ and $ already anchor to the single-line string and .match() anchors at position 0 -- so the flag did nothing in production. It was there only so the shipped-recipe guard test could .search() whole-file text. That put a test's requirement into a production regex and implied the parser scans multi-line input, when it deliberately stops at the first non-comment line. The flag is dropped; the test compiles its own multiline variant from the same pattern, so the syntax it accepts still cannot drift from the parser's. delegated_recipe_paths was public-named but is called only from loader.py and is not in __all__, unlike every sibling helper there (_peek_recipe_type, _apply_dotlist, _load_recipe_from_file). Renamed to _delegated_recipe_paths. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Shengliang Xu --- modelopt/recipe/loader.py | 6 +++--- modelopt/torch/opt/config_loader.py | 2 +- tests/unit/recipe/test_loader.py | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modelopt/recipe/loader.py b/modelopt/recipe/loader.py index 691c71e3d54..c24056b7177 100644 --- a/modelopt/recipe/loader.py +++ b/modelopt/recipe/loader.py @@ -261,7 +261,7 @@ def _peek_recipe_type( except (TypeError, KeyError, ValueError): pass - for base in delegated_recipe_paths(raw): + for base in _delegated_recipe_paths(raw): if str(base) in _seen: continue rtype = _peek_recipe_type(base, _seen) @@ -270,7 +270,7 @@ def _peek_recipe_type( return None -def delegated_recipe_paths(raw: dict) -> list[Path | Traversable]: +def _delegated_recipe_paths(raw: dict) -> list[Path | Traversable]: """Resolve the recipe files a raw recipe body delegates to via top-level ``$import``. Returns an empty list for an ordinary recipe. Names that do not appear in the @@ -324,7 +324,7 @@ def _load_recipe_from_file( # have to be the same kind. Checked here, and against the *declared* kind on both # sides, so a mismatch reads as a mismatch -- otherwise it surfaces as whatever # pydantic makes of, say, an ``eagle`` section spliced into a PTQ schema. - for base in delegated_recipe_paths(raw): + for base in _delegated_recipe_paths(raw): base_type = _peek_recipe_type(base) if base_type is not None and base_type != rtype: raise ValueError( diff --git a/modelopt/torch/opt/config_loader.py b/modelopt/torch/opt/config_loader.py index 82cb541fa9e..1f1f4bb2dd7 100644 --- a/modelopt/torch/opt/config_loader.py +++ b/modelopt/torch/opt/config_loader.py @@ -110,7 +110,7 @@ def _alias_builtin_recipe_prefix(config_path: str) -> str: _EXMY_RE = re.compile(r"^[Ee](\d+)[Mm](\d+)$") _EXMY_KEYS = frozenset({"num_bits", "scale_bits"}) -_MODELOPT_SCHEMA_RE = re.compile(r"^\s*#\s*modelopt-schema:\s*(\S+)\s*$", re.MULTILINE) +_MODELOPT_SCHEMA_RE = re.compile(r"^\s*#\s*modelopt-schema:\s*(\S+)\s*$") def _parse_exmy_num_bits(obj: Any) -> Any: diff --git a/tests/unit/recipe/test_loader.py b/tests/unit/recipe/test_loader.py index a96fc4f1b82..87c255d2b27 100644 --- a/tests/unit/recipe/test_loader.py +++ b/tests/unit/recipe/test_loader.py @@ -879,13 +879,16 @@ def test_shipped_modelopt_schema_comments_are_in_the_preamble(): point some future recipe tries to ``$import`` it. """ root = Path(str(files("modelopt_recipes"))) + # The parser matches line by line, so its own pattern is not multiline; recompile it + # here to scan whole files, keeping the accepted syntax identical to the parser's. + _schema_comment_anywhere = re.compile(_MODELOPT_SCHEMA_RE.pattern, re.MULTILINE) ignored = [ str(path.relative_to(root)) for path in sorted(root.rglob("*.yaml")) # Use the parser's own pattern so this can't drift from what it accepts, and so # prose that merely mentions the comment is not mistaken for one. if not path.is_symlink() - and _MODELOPT_SCHEMA_RE.search(path.read_text(encoding="utf-8")) + and _schema_comment_anywhere.search(path.read_text(encoding="utf-8")) and peek_declared_schema(path) is None ] assert not ignored, ( From 1cc02bf6f547881a26666e33c0171aa9f8d09c5c Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Wed, 16 Sep 2026 21:03:27 +0000 Subject: [PATCH 4/5] Address review: phantom doc rows, alias validation, and kind resolution Five findings from CodeRabbit and the Claude reviewer, each verified against the tree before acting. Four documented recipes did not exist (Claude, IMPORTANT). The 10_recipes.rst general-PTQ table listed fp8_default-kv_fp16, nvfp4_default-kv_fp16, nvfp4_mlp_only-kv_fp16 and nvfp4_experts_only-kv_fp16 -- rows this PR added, for recipes that moved to a later backfill batch. A recipe path is the user-facing --recipe interface, so copying any of them yielded "Recipe path ... is not a valid YAML file or directory". Rows removed. The `kv_fp16` KV-mode prose in ptq.md stays: nvfp4_weight_only-kv_fp16 does ship, so the suffix it documents is real. A new test_documented_recipe_paths_resolve closes the direction the existing checks left open: they verify every shipped recipe is documented, not that every documented recipe exists. Confirmed non-vacuous by reinstating one phantom row and watching it fail. Prose that names a recipe family (general/ptq/nvfp4_mlp_only standing for its -kv_* variants) and examples carrying an explicit .yaml suffix are both accepted, since neither is a broken path. The recipe pre-commit hook skipped aliases (Claude). _is_recipe_file required a schema comment or a recipe_type, and an alias carries neither -- so the hook silently ignored exactly the file class this PR introduces, and a typo in an alias's imports: target would not be caught locally. Added the third branch. PTQ discovery assumed $import implies PTQ (CodeRabbit). An alias of a speculative-decoding recipe delegates identically, so the test now asks _peek_recipe_type for the kind instead of re-deriving the rules. Coverage is unchanged at 67 recipes, both aliases included, and it can no longer drift from what load_recipe does. Changelog overstated the restriction (CodeRabbit): any top-level key given alongside a $import overrides the imported one, not just metadata. Docs contradicted themselves (CodeRabbit): three passages still called metadata.recipe_type mandatory or the sole source of a recipe's kind, while the new section below documents three sources. Reworded, with an anchor so they link to it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Shengliang Xu --- CHANGELOG.rst | 2 +- docs/source/guides/10_recipes.rst | 18 +++++------ tests/unit/recipe/test_loader.py | 19 +++++++----- tests/unit/recipe/test_recipe_docs.py | 37 +++++++++++++++++++++++ tools/precommit/check_modelopt_recipes.py | 5 +++ 5 files changed, 61 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b559be03773..1dae21434f1 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,7 @@ Changelog *Quantization* -- A recipe can now **delegate its whole body to another recipe** with a top-level ``$import``, overriding only ``metadata``. ``metadata.recipe_type`` became optional along with it: a recipe states its kind with a ``# modelopt-schema:`` comment, with ``metadata.recipe_type``, or by delegating to a recipe that does, and only a recipe that another file imports has to carry the schema comment. Whatever a recipe does state must be true: a schema comment and a ``recipe_type`` must agree, and so must a recipe and the recipe it delegates to. ``modelopt_recipes/models/`` uses this for checkpoint entries that a portable recipe already reproduces: the entry aliases that recipe instead of copying it. +- A recipe can now **delegate its whole body to another recipe** with a top-level ``$import``; any top-level key given alongside it overrides the imported one. ``metadata.recipe_type`` became optional along with it: a recipe states its kind with a ``# modelopt-schema:`` comment, with ``metadata.recipe_type``, or by delegating to a recipe that does, and only a recipe that another file imports has to carry the schema comment. Whatever a recipe does state must be true: a schema comment and a ``recipe_type`` must agree, and so must a recipe and the recipe it delegates to. ``modelopt_recipes/models/`` uses this for checkpoint entries that a portable recipe already reproduces: the entry aliases that recipe instead of copying it. - Backfill the recipes behind NVIDIA's already-published checkpoints under ``modelopt_recipes/models/``, so a released checkpoint's quantization scheme is reachable from its own model-hub path rather than only from the general tier. For example, ``moonshotai/Kimi-K2.6`` (published as ``nvidia/Kimi-K2.6-NVFP4``) and ``nvidia/Qwen3.5-397B-A17B`` (``nvidia/Qwen3.5-397B-A17B-NVFP4-V2``) each alias a portable recipe wholesale -- the general expert-only NVFP4 recipe and the ``qwen3_5_moe`` architecture recipe respectively -- rather than copying its body; other checkpoints follow in separate changes. A release is only backfilled when its model card supports a post-training quantization recipe -- one whose card documents quantization-aware distillation after PTQ is deliberately left out, since no PTQ recipe reproduces it. - Add ``layerwise.export_dir``: layerwise calibration writes each decoder layer to its own quantized checkpoint shard as it finishes, so no separate ``export_hf_checkpoint()`` pass is needed and, with ``layerwise.checkpoint_dir``, an interrupted run resumes without redoing finished layers. Calibration writes the layer shards; ``finalize()`` on the exporter left on the model adds the tail shard, the index and the config artifacts, and the checkpoint does not load until it runs. ``examples/hf_ptq`` does this for you. Supports FP8 and NVFP4 on single-process models, resident or offloaded, including multimodal models and models with MTP layers; other formats and placements raise ``NotImplementedError`` before calibration starts. - Add support for quantizing and calibrating enabled operators outside the transformer layers, such as ``lm_head``, when using layerwise calibration. diff --git a/docs/source/guides/10_recipes.rst b/docs/source/guides/10_recipes.rst index bf1131e2157..0d5a38a682a 100644 --- a/docs/source/guides/10_recipes.rst +++ b/docs/source/guides/10_recipes.rst @@ -380,7 +380,9 @@ Reusable snippets are stored under ``modelopt_recipes/configs/``: Metadata section ================ -Every recipe must contain a ``metadata`` mapping with at least a ``recipe_type`` field: +Every recipe contains a ``metadata`` mapping. ``recipe_type`` is optional and +**deprecated** -- a recipe may instead declare its kind with a ``# modelopt-schema:`` +comment, or inherit it from the recipe it delegates to; see `Declaring a recipe's kind`_: .. list-table:: :header-rows: 1 @@ -403,7 +405,7 @@ Type-specific configuration sections ===================================== Each recipe type defines its own configuration section. The section name and -schema depend on the ``recipe_type`` value in the metadata. +schema depend on the recipe's kind, however it is declared. PTQ (``recipe_type: ptq``) -------------------------- @@ -515,14 +517,6 @@ General PTQ recipes are model-agnostic and apply to any supported architecture: - NVFP4 for output projection + MLP layers, FP8 KV cache * - ``general/ptq/nvfp4_weight_only-kv_fp8_cast`` - NVFP4 W4A16 weight-only, FP8 KV cache with constant amax - * - ``general/ptq/fp8_default-kv_fp16`` - - FP8 per-tensor W8A8, KV cache left unquantized - * - ``general/ptq/nvfp4_default-kv_fp16`` - - NVFP4 W4A4, KV cache left unquantized - * - ``general/ptq/nvfp4_mlp_only-kv_fp16`` - - NVFP4 for MLP layers only, KV cache left unquantized - * - ``general/ptq/nvfp4_experts_only-kv_fp16`` - - NVFP4 for MoE expert layers only, KV cache left unquantized See `modelopt_recipes/ptq.md `_ for the full list and for guidance on choosing between them. @@ -582,6 +576,8 @@ inherited unchanged: metadata: description: What this checkpoint uses the base recipe for. +.. _Declaring a recipe's kind: + Note the missing ``recipe_type``. A recipe states its kind in whichever of these it likes, and the loader takes the first that answers: a ``# modelopt-schema:`` comment naming its schema class, ``metadata.recipe_type``, or -- as here -- the recipe it @@ -608,7 +604,7 @@ Python API Use :func:`~modelopt.recipe.load_recipe` to load a recipe. The path is resolved against the built-in library first, then the filesystem. The returned object's -type depends on the ``recipe_type`` in the metadata: +type depends on the recipe's kind, however it is declared: .. code-block:: python diff --git a/tests/unit/recipe/test_loader.py b/tests/unit/recipe/test_loader.py index 87c255d2b27..94f940d17b6 100644 --- a/tests/unit/recipe/test_loader.py +++ b/tests/unit/recipe/test_loader.py @@ -39,7 +39,13 @@ RecipeMetadataConfig, RecipeType, ) -from modelopt.recipe.loader import _apply_dotlist, _resolve_recipe_path, load_config, load_recipe +from modelopt.recipe.loader import ( + _apply_dotlist, + _peek_recipe_type, + _resolve_recipe_path, + load_config, + load_recipe, +) from modelopt.torch.opt.config_loader import ( _MODELOPT_SCHEMA_RE, _alias_builtin_recipe_prefix, @@ -327,13 +333,10 @@ def _all_shipped_ptq_recipe_paths(): # List-shaped fragments (layer-pattern units) are not recipes. if not isinstance(raw, dict): continue - declared = peek_declared_schema(path) - delegates = "$import" in raw # a checkpoint alias inherits its kind from its base - if ( - declared == _PTQ_SCHEMA - or (raw.get("metadata") or {}).get("recipe_type") == "ptq" - or delegates - ): + # Ask the loader's own resolver rather than re-deriving the rules here: a + # top-level ``$import`` does not imply PTQ (an alias of a speculative recipe + # delegates the same way), and this cannot drift from what load_recipe does. + if _peek_recipe_type(path) == RecipeType.PTQ: paths.append(str(rel.with_suffix(""))) return paths diff --git a/tests/unit/recipe/test_recipe_docs.py b/tests/unit/recipe/test_recipe_docs.py index ae3080e6534..cf82b4c3622 100644 --- a/tests/unit/recipe/test_recipe_docs.py +++ b/tests/unit/recipe/test_recipe_docs.py @@ -86,6 +86,43 @@ def test_general_ptq_recipe_count_in_ptq_md(): ) +def test_documented_recipe_paths_resolve(): + """Every ``general/ptq/`` a doc names must exist on disk. + + The existing checks run doc-from-disk: they catch a recipe that no doc mentions. + This is the other direction -- a doc naming a recipe that was never landed, or was + moved to another branch after the doc row was written. A recipe path is the + user-facing ``--recipe`` interface, so a phantom row sends users to + ``Recipe path '...' is not a valid YAML file or directory``. + """ + docs = { + "modelopt_recipes/ptq.md": PTQ_MD, + "docs/source/guides/10_recipes.rst": Path(__file__).resolve().parents[3] + / "docs" + / "source" + / "guides" + / "10_recipes.rst", + } + missing = [] + for label, path in docs.items(): + if not path.is_file(): + continue + for name in sorted(set(re.findall(r"general/ptq/([A-Za-z0-9._-]+)", path.read_text()))): + stem = name.removesuffix(".yaml").removesuffix(".yml") + if (GENERAL_PTQ_DIR / f"{stem}.yaml").is_file(): + continue + # Prose also names a recipe *family* -- e.g. ``general/ptq/nvfp4_mlp_only`` + # standing for its -kv_* variants -- which is not a phantom path. + if any(GENERAL_PTQ_DIR.glob(f"{stem}-*.yaml")): + continue + missing.append(f"{label} -> general/ptq/{name}") + assert not missing, ( + "Docs name general/ptq recipes that do not exist on disk:\n " + + "\n ".join(missing) + + "\nAdd the recipe, or remove the row if it belongs to a different change." + ) + + def test_every_model_specific_ptq_dir_is_mentioned(): """Every model-specific PTQ recipe must be identifiable in ptq.md. diff --git a/tools/precommit/check_modelopt_recipes.py b/tools/precommit/check_modelopt_recipes.py index db0ee31ae5f..a54ef385a80 100644 --- a/tools/precommit/check_modelopt_recipes.py +++ b/tools/precommit/check_modelopt_recipes.py @@ -199,6 +199,11 @@ def _is_recipe_file(path: Path) -> bool: return False # not a recipe file at all if _declares_recipe_schema(path): return True + if "$import" in data: + # A delegating alias declares neither a schema comment nor a recipe_type: its + # kind comes from the recipe it imports. Validate it so a typo in ``imports:`` + # or a ``$import`` naming an undeclared import fails here rather than at use. + return True metadata = data.get("metadata") if not isinstance(metadata, dict) or "recipe_type" not in metadata: return False # not a recipe file at all From c4e359293e149076486d9ad0c54cc49e46421e90 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Thu, 17 Sep 2026 23:29:22 +0000 Subject: [PATCH 5/5] Sweep recipe_type out of the recipes main added in this merge The merge brought in three new recipes; two still carried the deprecated metadata.recipe_type. general/ptq/nvfp4_default-kv_none-local_hessian declared nothing else, so it also gets the schema comment; the Qwen3.8-27B local-hessian recipe already had one, leaving the field redundant. Both load unchanged (RecipeType.PTQ, 27 and 11 quant_cfg entries), and the tree is back to zero uses of the deprecated field. The third file, models/Qwen/Qwen3.8-27B/ptq/nvfp4_w4a4_mlp_fp8_attn_max.yaml, is left alone: it is a QuantizeConfig fragment rather than a recipe, so it correctly declares a non-recipe schema and neither the loader nor the pre-commit hook treats it as one. This is the third merge in a row to reintroduce recipe_type, which is what the deprecation costs until the field is actually removed. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Shengliang Xu --- .../general/ptq/nvfp4_default-kv_none-local_hessian.yaml | 2 +- .../Qwen3.8-27B/ptq/nvfp4_w4a4_mlp_fp8_attn_local_hessian.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modelopt_recipes/general/ptq/nvfp4_default-kv_none-local_hessian.yaml b/modelopt_recipes/general/ptq/nvfp4_default-kv_none-local_hessian.yaml index abf9745cf83..7fee808ff7d 100644 --- a/modelopt_recipes/general/ptq/nvfp4_default-kv_none-local_hessian.yaml +++ b/modelopt_recipes/general/ptq/nvfp4_default-kv_none-local_hessian.yaml @@ -14,12 +14,12 @@ # limitations under the License. # PTQ recipe wrapping the shipped NVFP4 W4A4 local-Hessian preset. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: preset: configs/ptq/presets/model/nvfp4_w4a4_weight_local_hessian metadata: - recipe_type: ptq description: >- Applies NVFP4 W4A4 with static weight scales, dynamic inputs, KV quantizers disabled, and local-Hessian weight calibration with an FP8-scale sweep. diff --git a/modelopt_recipes/models/Qwen/Qwen3.8-27B/ptq/nvfp4_w4a4_mlp_fp8_attn_local_hessian.yaml b/modelopt_recipes/models/Qwen/Qwen3.8-27B/ptq/nvfp4_w4a4_mlp_fp8_attn_local_hessian.yaml index e007fe5d464..90decca0902 100644 --- a/modelopt_recipes/models/Qwen/Qwen3.8-27B/ptq/nvfp4_w4a4_mlp_fp8_attn_local_hessian.yaml +++ b/modelopt_recipes/models/Qwen/Qwen3.8-27B/ptq/nvfp4_w4a4_mlp_fp8_attn_local_hessian.yaml @@ -18,7 +18,6 @@ imports: base: models/Qwen/Qwen3.8-27B/ptq/nvfp4_w4a4_mlp_fp8_attn_max metadata: - recipe_type: ptq description: >- Qwen3.8-27B local-Hessian PTQ using the NVFP4 W4A4 and FP8 W8A8 assignment exported by the 5.5-bit NVFP4-max AutoQuantize sweep.