diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index c587825fa167..f0e0b062f69e 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -98,7 +98,9 @@ slurm_wait_all_ranks() { touch "$readyDir/rank_${SLURM_PROCID}.ready" # Bounded so a dead rank fails the stage loudly instead of hanging until the - # partition walltime kills it; the ceiling exceeds the 2700s pip3 retry budget + # partition walltime kills it. This bounds arrival skew between ranks (each + # rank's deadline starts after its own install finished) -- comfortably above + # the ~10min skew seen in the bug - not any single install-phase timeout; # in slurm_install.sh so a merely slow rank still releases the barrier. local timeoutSecs=3600 local deadline=$((SECONDS + timeoutSecs)) @@ -117,10 +119,7 @@ slurm_wait_all_ranks() { "all $numRanks ranks to be ready; ready: $ready/$numRanks" return 1 fi - # One rank reports progress; all of them would spam the log every 10s. - if [ "$SLURM_PROCID" -eq 0 ]; then - echo "(Waiting for all $numRanks ranks to be ready) ready: $ready/$numRanks" - fi + echo "(Waiting for all $numRanks ranks to be ready) ready: $ready/$numRanks" sleep 10 done } diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 625beece3457..f19cd5516d34 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -21,7 +21,6 @@ accuracy/test_llm_api_autodeploy.py::TestNemotronSuperV3::test_mtp[nvfp4_ws8_80g accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SKIP (https://nvbugs/6507114) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp] SKIP (https://nvbugs/6428101) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp_trtllm] SKIP (https://nvbugs/6426868) -accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency] SKIP (https://nvbugs/6561778) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen_adp_lmtp] SKIP (https://nvbugs/6655987) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/6481323) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1LongBenchV2::test_fp8_8gpus SKIP (https://nvbugs/6621358)