Prepare torch 2.12.1 / CUDA 13.2 / triton 3.7.1 / transformers 5.14.1 SFT compatibility

.pre-commit-config.yaml

@@ -1,77 +1,116 @@
+# Offline-friendly config: all hooks are `repo: local` and install via pip
+# (internal PyPI / mirrors). Avoids cloning github.com which is unreachable
+# on many GPU clusters.
 files: >
   (?x)^(
     xtuner/v1/.* |
     autotest/.*
   )$
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+  - repo: local
     hooks:
       - id: check-yaml
+        name: check-yaml
+        entry: check-yaml
+        language: python
+        types: [yaml]
+        additional_dependencies: ["pre-commit-hooks==5.0.0"]
       - id: requirements-txt-fixer
+        name: requirements-txt-fixer
+        entry: requirements-txt-fixer
+        language: python
+        types: [text]
+        files: requirements.*\.txt$
+        additional_dependencies: ["pre-commit-hooks==5.0.0"]
       - id: check-merge-conflict
+        name: check-merge-conflict
+        entry: check-merge-conflict
+        language: python
+        additional_dependencies: ["pre-commit-hooks==5.0.0"]
       - id: mixed-line-ending
+        name: mixed-line-ending
+        entry: mixed-line-ending
+        language: python
         args: ["--fix=lf"]
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.1
-    hooks:
+        additional_dependencies: ["pre-commit-hooks==5.0.0"]
+
       - id: codespell
-  - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.9
-    hooks:
+        name: codespell
+        entry: codespell
+        language: python
+        types: [text]
+        additional_dependencies: ["codespell==2.2.1"]
+
       - id: mdformat
+        name: mdformat
+        entry: mdformat
+        language: python
+        types: [markdown]
         args: ["--number"]
         additional_dependencies:
+          - mdformat==0.7.9
           - mdformat-openmmlab
           - mdformat_frontmatter
           - linkify-it-py
         exclude: 'docs/zh_cn/user_guides/sequence_parallel.md'
-  - repo: https://github.com/myint/docformatter
-    rev: 06907d0
-    hooks:
+
       - id: docformatter
+        name: docformatter
+        entry: docformatter
+        language: python
+        types: [python]
         args: ["--in-place", "--wrap-descriptions", "119"]
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.20.0
-    hooks:
+        additional_dependencies: ["docformatter==1.7.5"]
+
       - id: pyupgrade
+        name: pyupgrade
+        entry: pyupgrade
+        language: python
+        types: [python]
         args: ["--py36-plus"]
+        additional_dependencies: ["pyupgrade==3.20.0"]
 
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.9.10
-    hooks:
-      # Run the linter.
-      - id: ruff
-        types_or: [ python, pyi ]
-        args: [ --diff ]
       - id: ruff
-        types_or: [ python, pyi ]
-        args: [ --fix ]
-      # Run the formatter.
-      - id: ruff-format
-        types_or: [ python, pyi ]
-        args: [ --diff ]
+        name: ruff
+        entry: ruff check --force-exclude
+        language: python
+        types_or: [python, pyi]
+        args: [--diff]
+        additional_dependencies: ["ruff==0.9.10"]
+      - id: ruff-fix
+        name: ruff-fix
+        entry: ruff check --force-exclude
+        language: python
+        types_or: [python, pyi]
+        args: [--fix]
+        additional_dependencies: ["ruff==0.9.10"]
+      - id: ruff-format-diff
+        name: ruff-format
+        entry: ruff format --force-exclude
+        language: python
+        types_or: [python, pyi]
+        args: [--diff]
+        additional_dependencies: ["ruff==0.9.10"]
       - id: ruff-format
-        types_or: [ python, pyi ]
-  - repo: local
-  # We do not use pre-commit/mirrors-mypy, 
-  # as it comes with opinionated defaults 
-  # (like --ignore-missing-imports)
-  # and is difficult to configure to run 
-  # with the dependencies correctly installed.
-    hooks:
+        name: ruff-format
+        entry: ruff format --force-exclude
+        language: python
+        types_or: [python, pyi]
+        additional_dependencies: ["ruff==0.9.10"]
+
+      # We do not use pre-commit/mirrors-mypy (opinionated defaults).
+      # Torch is intentionally not pinned here: installing it into the hook
+      # env is heavy; missing third-party imports are ignored via pyproject.
       - id: mypy
         name: mypy
         entry: .dev_scripts/mypy_entrypoint.sh
         language: python
-        # For 3rd party lib, we only do type check for torch
-        additional_dependencies: ["mypy==1.16.1", "torch==2.6.0", "types-requests"]
+        additional_dependencies: ["mypy==1.16.1", "types-requests"]
         require_serial: true
         verbose: true
         types: [python]
       - id: pydantic-extra-check
-        name: pydantic-extra-check 
+        name: pydantic-extra-check
         language: system
         entry: .dev_scripts/test_pydantic.py
         verbose: false

Dockerfile

@@ -1,6 +1,7 @@
 # syntax=docker/dockerfile:1.10.0
 # builder
-ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:25.03-py3
+# 26.03-py3 = CUDA 13.2.0（匹配 pt121 实测栈）；由 image_build.sh 覆盖传入
+ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:26.03-py3
 
 ## build base env
 FROM ${BASE_IMAGE} AS setup_env
@@ -14,7 +15,7 @@ RUN sed -i "s@http://.*.ubuntu.com@${PPA_SOURCE}@g" /etc/apt/sources.list.d/ubun
     apt install --no-install-recommends build-essential sudo -y && \
     apt install --no-install-recommends git curl pkg-config tree unzip tmux \
     openssh-server openssh-client dnsutils iproute2 lsof net-tools zsh rclone \
-    iputils-ping telnet netcat-openbsd htop bubblewrap socat -y && \
+    iputils-ping telnet netcat-openbsd htop bubblewrap socat ffmpeg -y && \
     apt clean && rm -rf /var/lib/apt/lists/*
 
 RUN if [ -d /etc/pip ] && [ -f /etc/pip/constraint.txt ]; then echo > /etc/pip/constraint.txt; fi
@@ -27,9 +28,8 @@ ARG PYTORCH_WHEELS_URL
 RUN --mount=type=secret,id=HTTPS_PROXY,env=https_proxy \
     --mount=type=secret,id=NO_PROXY,env=no_proxy \
     if [ -n "${TORCH_VERSION}" ]; then \
-        pip install torchvision torch==${TORCH_VERSION} \
-        -i ${PYTORCH_WHEELS_URL}/cu128 \
-        --extra-index-url ${PYTORCH_WHEELS_URL}/cu126 \
+        pip install torchvision==0.27.1 torch==${TORCH_VERSION} \
+        -i ${PYTORCH_WHEELS_URL}/cu132 \
         --no-cache-dir; \
     fi
 # set reasonable default for CUDA architectures when building ngc image
@@ -67,8 +67,8 @@ RUN --mount=type=secret,id=HTTPS_PROXY,env=https_proxy \
 
 WORKDIR ${CODESPACE}/flash-attention
 
+# 只编译 FA3(hopper)，与 pt121 conda 一致（环境只装了 flash_attn_3，无 flash_attn 2.x）。
 RUN cd hopper && FLASH_ATTENTION_FORCE_BUILD=TRUE pip wheel -w ${FLASH_ATTN3_DIR} -v --no-deps .
-RUN FLASH_ATTENTION_FORCE_BUILD=TRUE pip wheel -w ${FLASH_ATTN_DIR} -v --no-deps .
 
 # compile adaptive_gemm
 FROM setup_env AS adaptive_gemm
@@ -85,6 +85,17 @@ RUN --mount=type=secret,id=HTTPS_PROXY,env=https_proxy \
 
 WORKDIR ${CODESPACE}/AdaptiveGEMM
 
+# Blocker1(GLM-5.2/cu13): CUDA 13 从 <cudaTypedefs.h> 移除了无版本号的
+# PFN_cuTensorMapEncodeTiled，只留 _v12000。adaptive_gemm 的 JIT 头文件(tma_utils.cuh)
+# 仍用旧名，在 cu13.2 下 nvcc 会 "identifier undefined"。这里在打 wheel 前把旧名别名回
+# 版本化符号（#if CUDA_VERSION>=13000 才生效，cu12 无副作用）。补丁进头文件、随 wheel 分发、
+# 运行时 JIT 读取。10411e0 本身不含此补丁，故必须在此注入。
+RUN sed -i '/^namespace adaptive_gemm {/i\
+#if (CUDA_VERSION >= 13000) && !defined(PFN_cuTensorMapEncodeTiled)\
+#define PFN_cuTensorMapEncodeTiled PFN_cuTensorMapEncodeTiled_v12000\
+#endif' adaptive_gemm/include/adaptive_gemm/tma_utils.cuh && \
+    grep -q "PFN_cuTensorMapEncodeTiled_v12000" adaptive_gemm/include/adaptive_gemm/tma_utils.cuh
+
 RUN pip wheel -w ${ADAPTIVE_GEMM_DIR} -v --no-deps .
 
 # compile grouped_gemm(permute and unpermute)
@@ -191,15 +202,13 @@ ARG DEEP_GEMM_DIR
 ARG CAUSAL_CONV1D_DIR
 
 COPY --from=flash_attn ${FLASH_ATTN3_DIR} ${FLASH_ATTN3_DIR}
-COPY --from=flash_attn ${FLASH_ATTN_DIR} ${FLASH_ATTN_DIR}
 COPY --from=adaptive_gemm ${ADAPTIVE_GEMM_DIR} ${ADAPTIVE_GEMM_DIR}
 COPY --from=grouped_gemm ${GROUPED_GEMM_DIR} ${GROUPED_GEMM_DIR}
 COPY --from=deep_ep ${DEEP_EP_DIR} ${DEEP_EP_DIR}
 # COPY --from=deep_ep ${NVSHMEM_PREFIX} ${NVSHMEM_PREFIX}
 COPY --from=deep_gemm ${DEEP_GEMM_DIR} ${DEEP_GEMM_DIR}
 COPY --from=causal_conv1d ${CAUSAL_CONV1D_DIR} ${CAUSAL_CONV1D_DIR}
 
-RUN unzip ${FLASH_ATTN_DIR}/*.whl -d ${PYTHON_SITE_PACKAGE_PATH}
 RUN unzip ${FLASH_ATTN3_DIR}/*.whl -d ${PYTHON_SITE_PACKAGE_PATH}
 RUN unzip ${ADAPTIVE_GEMM_DIR}/*.whl -d ${PYTHON_SITE_PACKAGE_PATH}
 RUN unzip ${GROUPED_GEMM_DIR}/*.whl -d ${PYTHON_SITE_PACKAGE_PATH}
@@ -224,7 +233,7 @@ RUN --mount=type=secret,id=HTTPS_PROXY,env=https_proxy \
         partial_json_parser 'ray[default]<3' shortuuid uvicorn pybase64 \
         tilelang==0.1.11 \
         'pydantic>2' openai_harmony dlblas --no-cache-dir -i ${DEFAULT_PYPI_URL} && \
-    pip install xgrammar==0.1.32 timm!=1.0.23 --no-cache-dir -i ${DEFAULT_PYPI_URL} --no-deps && \
+    pip install xgrammar==0.2.3 timm==1.0.28 --no-cache-dir -i ${DEFAULT_PYPI_URL} --no-deps && \
     if [ -n "${LMDEPLOY_VERSION}" ]; then \
         pip install lmdeploy==${LMDEPLOY_VERSION} --no-deps --no-cache-dir -i ${DEFAULT_PYPI_URL}; \
     else \

autotest/config.yaml

@@ -1035,7 +1035,7 @@ case:
                 output_path: /mnt/shared-storage-user/llmrazor-share/qa-llm-cicd/test_output
             resource:
                 memory_per_task: 1200
-                pip_package: pip install sglang==0.5.10.post1 --timeout 600 --retries 5;pip install transformers==5.2.0 apache-tvm-ffi==0.1.9 nvidia-cudnn-cu12==9.15.1.9;pip uninstall -y nvidia-cutlass-dsl;rm -rf /usr/local/lib/python3.12/dist-packages/nvidia_cutlass_dsl
+                pip_package: pip install sglang==0.5.10.post1 --timeout 600 --retries 5;pip install transformers==5.14.1 apache-tvm-ffi==0.1.9 nvidia-cudnn-cu12==9.15.1.9;pip uninstall -y nvidia-cutlass-dsl;rm -rf /usr/local/lib/python3.12/dist-packages/nvidia_cutlass_dsl
                 envs:
                     - MODEL_PATH=/mnt/shared-storage-user/llmrazor-share/model/Qwen3-30B-A3B
                     - DATA_PATH=/mnt/shared-storage-user/llmrazor-share/data/gsm8k/train-mini.jsonl

image_build.sh

@@ -1,17 +1,19 @@
 export HTTPS_PROXY=$HTTPS_PROXY
 
-export BASE_IMAGE=nvcr.io/nvidia/pytorch:25.03-py3
+# NGC 26.03-py3 = CUDA 13.2.0.046 / cuDNN 9.20 / NCCL 2.29.7，匹配 pt121 实测栈（torch2.12.1+cu132）。
+# 基镜像自带 torch 2.11a 会在下方被 pip 装的 torch==2.12.1(cu132) 覆盖；基镜像只需提供 cu13.2 工具链供各扩展编译。
+export BASE_IMAGE=${BASE_IMAGE:-"nvcr.io/nvidia/pytorch:26.03-py3"}
 export XTUNER_COMMIT=$(git rev-parse HEAD)
 export XTUNER_URL=https://github.com/InternLM/xtuner@${XTUNER_COMMIT}
-export FLASH_ATTN_URL=https://github.com/Dao-AILab/flash-attention@060c9188beec3a8b62b33a3bfa6d5d2d44975fab
-export ADAPTIVE_GEMM_URL=https://github.com/InternLM/AdaptiveGEMM@10411e08b182e853d0f3ecec4c68bf90c90e309f # fix fp8 dw k_grouped_gemm bug
-export GROUPED_GEMM_URL=https://github.com/InternLM/GroupedGEMM@aa5ffb21cb626d6cd61d99fc42958127b0b99be7
+export FLASH_ATTN_URL=https://github.com/Dao-AILab/flash-attention@8a8b2f10ddca88fd46db406c3e143e1ab0af977f # FA3 对齐 pt121 conda（flash_attn_3 8a8b2f, 2026-05-16）；FA2 2.x 在 Dockerfile 里不再编译/安装以匹配环境
+export ADAPTIVE_GEMM_URL=https://github.com/InternLM/AdaptiveGEMM@10411e08b182e853d0f3ecec4c68bf90c90e309f # #7 fix: make k_grouped_gemm_dw deterministic for varlen MoE backward（含 dw 修复；Dockerfile 会补 cu13 头补丁）
+export GROUPED_GEMM_URL=https://github.com/InternLM/GroupedGEMM@21c199dee72b0fb96025751e0dbc4ad35ef5a94f # #2 radix sort using torch current stream；对齐 pt121 conda（grouped_gemm 1.1.4）
 export DEEP_EP_URL=https://github.com/deepseek-ai/DeepEP@9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee # v1.2.1
 export DEEP_GEMM_URL=https://github.com/deepseek-ai/DeepGEMM@c9f8b34dcdacc20aa746b786f983492c51072870 # v2.1.1.post3
 export CAUSAL_CONV1D_URL=https://github.com/Dao-AILab/causal-conv1d@da6dbaa9fd5a919967f14d3fd031da1288ad5025 # v1.6.0
-export FLA_URL="${FLA_URL-https://github.com/HAOCHENYE/flash-linear-attention@tmp-tensor-cache}"
+export FLA_URL="${FLA_URL-https://github.com/HAOCHENYE/flash-linear-attention@72d2a8f3a06cefda6a3fc79b2fcbd0b41c34f238}" # 钉住 tmp-tensor-cache 分支到 pt121 conda 实装 commit（fla 0.4.2, 2026-05-22 "using queue for tensor cache"）
 
-export TORCH_VERSION=${TORCH_VERSION:-"2.9.1"}
+export TORCH_VERSION=${TORCH_VERSION:-"2.12.1"}
 # export LMDEPLOY_VERSION="0.13.0dev"
 export LMDEPLOY_URL=https://github.com/InternLM/lmdeploy@efe3b88607756a7ad9411b89627b5ac6ebaa540e
 export PPA_SOURCE="https://mirrors.aliyun.com"

pyproject.toml

@@ -24,7 +24,6 @@ classifiers = [
 ]
 dependencies = [
   "astor",
-  "bitsandbytes==0.45.0",
   "datasets<4.0.0",
   "einops",
   "loguru",
@@ -38,7 +37,7 @@ dependencies = [
   "tilelang==0.1.11",
   "torch>=2.6.0",
   "torchvision",
-  "transformers==5.2.0",
+  "transformers==5.14.1",
   "cyclopts",
   "transformers_stream_generator",
   "opencv-python-headless<=4.12.0.88",
@@ -76,11 +75,13 @@ rl = [
   "pylatexenc"
 ]
 video = [
+  "torchcodec==0.15.0",
   "decord",
   "av",
 ]
 all = [
   "jsonlines",
+  "torchcodec==0.15.0",
   "decord",
   "av",
   "ray[default]",

requirements/runtime.txt

@@ -1,4 +1,3 @@
-bitsandbytes==0.45.0
 datasets<4.0.0
 einops
 loguru
@@ -11,7 +10,7 @@ SentencePiece
 tiktoken
 torch>=2.6.0
 torchvision
-transformers==4.56.0
+transformers==5.14.1
 cyclopts
 transformers_stream_generator
 opencv-python-headless

tests/conftest.py

@@ -19,6 +19,43 @@
 from xtuner._testing.patch_rollout_config import patch_rollout_config_dist_port_base
 
 
+def _patch_parametrize_namespace_scan() -> None:
+    """Avoid lazy imports mutating module dictionaries during collection."""
+    parametrize_module = importlib.import_module("parametrize.parametrize")
+    original = parametrize_module._find_possible_decorators
+
+    def find_possible_decorators(namespace, search_in_modules=True):
+        return original(dict(namespace), search_in_modules=search_in_modules)
+
+    parametrize_module._find_possible_decorators = find_possible_decorators
+
+
+def _isolate_pytest_compiler_cache() -> None:
+    """Give the pytest parent process its own Triton/Inductor cache dirs.
+
+    Distributed ranks already isolate via ``DeterministicDDPTestCase``; this
+    covers non-DDP tests and the parent process so serial full-suite runs do not
+    corrupt a shared ``/tmp/.triton`` under torch 2.12 / triton 3.7.
+    """
+    import tempfile
+
+    base = os.environ.get("TRITON_CACHE_DIR") or os.path.join(tempfile.gettempdir(), ".triton")
+    leaf = os.path.basename(base)
+    if leaf.startswith("pytest_p") or leaf.startswith("r"):
+        cache_dir = base
+    else:
+        cache_dir = os.path.join(base, f"pytest_p{os.getpid()}")
+    os.makedirs(cache_dir, exist_ok=True)
+    os.environ["TRITON_CACHE_DIR"] = cache_dir
+    inductor_dir = cache_dir + "_inductor"
+    os.makedirs(inductor_dir, exist_ok=True)
+    os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_dir
+
+
+_patch_parametrize_namespace_scan()
+_isolate_pytest_compiler_cache()
+
+
 _HF_DYNAMIC_MODULE_PREFIX = "transformers_modules"
 _HF_PATCH_MODULES_CACHE_PREFIX = "modules_cache"
 

tests/datasets/test_qwen35_vl_tokenize_fn.py

@@ -5,7 +5,7 @@
 import json
 import torch
 import parametrize
-from xtuner.v1.utils.test_utils import add_video_root
+from xtuner.v1.utils.test_utils import add_video_root, normalize_hf_qwen3_vl_video_input_ids
 from packaging.version import Version
 from transformers import __version__ as transformers_version
 import unittest
@@ -262,6 +262,8 @@ def test_qwen3_vl_sft_video(self, add_vision_id):
                                                              return_dict=True, add_vision_id=add_vision_id,
                                                              return_tensors="pt")
                     input_ids_hf = ret['input_ids'][0]
+                    # transformers>=5.14 wraps video_pad with an extra vision_start/end pair.
+                    input_ids_hf = normalize_hf_qwen3_vl_video_input_ids(input_ids_hf, self.tokenizer)
                     pixel_values_hf = ret['pixel_values_videos']
                     image_grid_thw_hf = ret['video_grid_thw']
 
@@ -283,7 +285,7 @@ def test_qwen3_vl_sft_video(self, add_vision_id):
                         if i == 7:
                             self.assertEqual(len(input_ids_xtuner), len(input_ids_hf))
                         else:
-                            self.assertEqual(input_ids_xtuner, input_ids_hf.tolist())
+                            self.assertEqual(input_ids_xtuner, input_ids_hf)
                         self.assertTrue('seconds>' in text)
                         self.assertTrue(torch.allclose(pixel_values_xtuner, pixel_values_hf))
                         self.assertTrue(torch.allclose(image_grid_thw_xtuner, image_grid_thw_hf))