diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index fc53e2975e14..10615a0fba60 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -2600,10 +2600,15 @@ def _ring_broadcast_sample_state( if not self.dist.is_last_pp_rank: # Receive tokens from previous pp rank (w.r.t model forward direction) with nvtx_range("recv_sample_state"): - sample_state.host, py_result_diffs = self.dist.recv_object( - src=self.dist.prev_pp_rank, - tag=tag, - ) + sample_state.host, py_result_diffs, use_host_stop_criteria = \ + self.dist.recv_object(src=self.dist.prev_pp_rank, tag=tag) + + # The last PP rank's fast host-stop path leaves + # host.finish_reasons=None; without this flag, non-last ranks would + # take the finish_reasons-indexing branch in update_requests and + # raise IndexError on the empty list. + if hasattr(sample_state, "use_host_stop_criteria"): + sample_state.use_host_stop_criteria = use_host_stop_criteria for request, py_result_diff in zip(requests, py_result_diffs): request.py_result.apply_diff(py_result_diff) @@ -2621,7 +2626,8 @@ def _ring_broadcast_sample_state( self.wait_on_pp_send_handles(self.send_handles, microbatch_id) with nvtx_range("send_sample_state"): self.send_handles[microbatch_id] = self.dist.isend_object( - (sample_state.host, py_result_diffs), + (sample_state.host, py_result_diffs, + getattr(sample_state, "use_host_stop_criteria", False)), dest=self.dist.next_pp_rank, tag=tag, )