diff --git a/changelog/fixed/snuggly-cancellation-otter.md b/changelog/fixed/snuggly-cancellation-otter.md new file mode 100644 index 000000000..c69978042 --- /dev/null +++ b/changelog/fixed/snuggly-cancellation-otter.md @@ -0,0 +1 @@ +Add opt-in workflow cancellation fixes that respect shielding in `workflow.wait_condition` and preserve task cancellation when a condition or a Temporal operation completes concurrently. diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 94d71983a..e66f73b0e 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -500,13 +500,19 @@ def activate( self._is_replaying = act.is_replaying self._current_thread_id = threading.get_ident() self._current_internal_flags = act.available_internal_flags - # The original flag also changed cancellation behavior. Histories using - # it must retain those semantics; the new flag only changes scheduling. - self._single_batch_cancellation = self._workflow_logic_flag_enabled( + self._legacy_single_batch = self._workflow_logic_flag_enabled( _WorkflowLogicFlag.PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH ) + # The task-local wait condition fix must not change the behavior of + # histories that use the original single-batch flag. + self._preserve_task_cancellation = ( + not self._legacy_single_batch + and self._workflow_logic_flag_enabled( + _WorkflowLogicFlag.PRESERVE_TASK_CANCELLATION + ) + ) single_batch_activation = ( - self._single_batch_cancellation + self._legacy_single_batch or self._workflow_logic_flag_enabled( _WorkflowLogicFlag.APPLY_ACTIVATION_JOBS_BEFORE_RUNNING_TASKS ) @@ -708,7 +714,7 @@ def _apply_cancel_workflow( # workflow the ability to receive the cancellation, so we must defer # this cancellation to the next iteration of the event loop. self.call_soon(self._primary_task.cancel) - elif self._single_batch_cancellation: + elif self._legacy_single_batch or self._preserve_task_cancellation: # Initialization is the only job that creates the primary task, so # retain a same-activation cancellation until that task exists. self._cancel_primary_task_pending = True @@ -2036,7 +2042,7 @@ async def workflow_wait_condition( # cancellation and keep going. def cancellation_arrived() -> bool: return ( - self._single_batch_cancellation + self._legacy_single_batch and not cancellation_requested_before and self._cancel_reason is not None ) @@ -2053,12 +2059,17 @@ def cancellation_arrived() -> bool: else None ) markers = _capture_event_group_markers(event_groups) - # asyncio.wait_for() starts its timer from this task, so the options are - # set in the task's context and reset once the wait ends + # Scope the timer options so later timers in this task don't inherit them. token = _TimerOptionsCtxVar.set( _TimerOptions(user_metadata=user_metadata, event_group_markers=markers) ) try: + if self._preserve_task_cancellation: + # A direct await lets the waiting task arbitrate cancellation + # and readiness without bypassing shielding or using wait_for. + async with asyncio.timeout(timeout): + await fut + return await asyncio.wait_for(fut, timeout) except asyncio.TimeoutError: if cancellation_arrived(): @@ -2431,7 +2442,7 @@ async def _await_temporal_operation( if fut.done(): # Retrying the shield after both futures become ready would # return the result and erase the task cancellation. - if self._single_batch_cancellation: + if self._legacy_single_batch or self._preserve_task_cancellation: raise if completed_cancellation_flag is not None and ( not self._is_replaying @@ -4275,6 +4286,7 @@ class _WorkflowLogicFlag(IntEnum): RAISE_ON_CANCELLING_COMPLETED_ACTIVITY = 1 PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH = 2 APPLY_ACTIVATION_JOBS_BEFORE_RUNNING_TASKS = 3 + PRESERVE_TASK_CANCELLATION = 4 _DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS: frozenset[_WorkflowLogicFlag] = frozenset() diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index f2acbc768..63d6dbb4c 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -1521,6 +1521,7 @@ async def run(self) -> str: _BATCH_ONLY_WORKFLOW_LOGIC_FLAG = ( _WorkflowLogicFlag.APPLY_ACTIVATION_JOBS_BEFORE_RUNNING_TASKS ) +_PRESERVE_TASK_CANCELLATION_FLAG = _WorkflowLogicFlag.PRESERVE_TASK_CANCELLATION @pytest.mark.parametrize( @@ -1622,8 +1623,12 @@ async def ready() -> bool: ).replay_workflow(history) -@pytest.mark.parametrize("single_batch", [False, True]) -async def test_workflow_cancel_before_run(client: Client, single_batch: bool): +@pytest.mark.parametrize( + "flag", [None, _SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, _PRESERVE_TASK_CANCELLATION_FLAG] +) +async def test_workflow_cancel_before_run( + client: Client, flag: _WorkflowLogicFlag | None +): # Start the workflow _and_ send cancel before even starting the workflow task_queue = str(uuid.uuid4()) handle = await client.start_workflow( @@ -1634,9 +1639,8 @@ async def test_workflow_cancel_before_run(client: Client, single_batch: bool): await handle.cancel() # Start worker and wait for result worker = new_worker(client, TrapCancelWorkflow, task_queue=task_queue) - worker._set_default_workflow_logic_flag( - _SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, enabled=single_batch - ) + if flag is not None: + worker._set_default_workflow_logic_flag(flag, enabled=True) async with worker: assert "cancelled" == await handle.result() @@ -1681,9 +1685,19 @@ def waiting_after_cancel(self) -> bool: return self._waiting_after_cancel +@pytest.mark.parametrize( + "flags", + [ + pytest.param((_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG,), id="original-single-batch"), + pytest.param( + (_BATCH_ONLY_WORKFLOW_LOGIC_FLAG, _PRESERVE_TASK_CANCELLATION_FLAG), + id="batching-with-task-cancellation", + ), + ], +) @pytest.mark.parametrize("timeout", [False, True]) async def test_workflow_cancel_and_condition_ready_in_same_activation( - client: Client, timeout: bool + client: Client, timeout: bool, flags: tuple[_WorkflowLogicFlag, ...] ): task_queue = str(uuid.uuid4()) runner = CustomWorkflowRunner() @@ -1704,9 +1718,8 @@ async def test_workflow_cancel_and_condition_ready_in_same_activation( # the normal queue. max_cached_workflows=0, ) - worker._set_default_workflow_logic_flag( - _SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, enabled=True - ) + for flag in flags: + worker._set_default_workflow_logic_flag(flag, enabled=True) async with worker: async def ready() -> bool: @@ -1725,9 +1738,8 @@ async def ready() -> bool: task_queue=task_queue, workflow_runner=runner, ) - worker._set_default_workflow_logic_flag( - _SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, enabled=True - ) + for flag in flags: + worker._set_default_workflow_logic_flag(flag, enabled=True) async with worker: async def waiting_after_cancel() -> bool: @@ -1739,10 +1751,12 @@ async def waiting_after_cancel() -> bool: await handle.signal(CancelAtWaitConditionWorkflow.finish_after_cancel) assert await handle.result() == "cancelled" - assert any( - _SINGLE_BATCH_WORKFLOW_LOGIC_FLAG in completion.successful.used_internal_flags + used_flags = { + flag for _, completion in runner._pairs - ) + for flag in completion.successful.used_internal_flags + } + assert used_flags.issuperset(flags) assert any( {"signal_workflow", "cancel_workflow"}.issubset( {job.WhichOneof("variant") for job in activation.jobs} @@ -1754,6 +1768,98 @@ async def waiting_after_cancel() -> bool: ) +@workflow.defn +class ShieldedWaitConditionWorkflow: + def __init__(self) -> None: + self._confirmed = False + self._waiting = False + self._caught_cancel = False + + async def _cleanup(self, timeout: bool) -> None: + self._waiting = True + await workflow.wait_condition( + lambda: self._confirmed, timeout=1000 if timeout else None + ) + await workflow.sleep(0.01, summary="shielded cleanup") + + @workflow.run + async def run(self, timeout: bool) -> None: + cleanup = asyncio.create_task(self._cleanup(timeout)) + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + self._caught_cancel = True + await cleanup + raise + + @workflow.signal + def confirm(self) -> None: + self._confirmed = True + + @workflow.query + def state(self) -> tuple[bool, bool]: + return self._waiting, self._caught_cancel + + +@pytest.mark.parametrize("timeout", [False, True]) +@pytest.mark.parametrize( + ("batch_flag", "cleanup_completes"), + [ + pytest.param(_BATCH_ONLY_WORKFLOW_LOGIC_FLAG, True, id="task-cancellation"), + pytest.param( + _SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, False, id="original-behavior-preserved" + ), + ], +) +async def test_workflow_shielded_wait_condition_after_cancel( + client: Client, + timeout: bool, + batch_flag: _WorkflowLogicFlag, + cleanup_completes: bool, +) -> None: + worker = new_worker(client, ShieldedWaitConditionWorkflow) + worker._set_default_workflow_logic_flag(batch_flag, enabled=True) + worker._set_default_workflow_logic_flag( + _PRESERVE_TASK_CANCELLATION_FLAG, enabled=True + ) + async with worker: + handle = await client.start_workflow( + ShieldedWaitConditionWorkflow.run, + timeout, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + + async def state() -> tuple[bool, bool]: + return await handle.query(ShieldedWaitConditionWorkflow.state) + + await assert_eq_eventually((True, False), state) + await handle.cancel() + await assert_eq_eventually((True, True), state) + await handle.signal(ShieldedWaitConditionWorkflow.confirm) + with pytest.raises(WorkflowFailureError) as err: + await handle.result() + assert isinstance(err.value.cause, CancelledError) + + history = await handle.fetch_history() + assert ( + any( + event.event_type == EventType.EVENT_TYPE_TIMER_FIRED + for event in history.events + ) + is cleanup_completes + ) + assert ( + any( + _PRESERVE_TASK_CANCELLATION_FLAG + in event.workflow_task_completed_event_attributes.sdk_metadata.lang_used_flags + for event in history.events + ) + is cleanup_completes + ) + await Replayer(workflows=[ShieldedWaitConditionWorkflow]).replay_workflow(history) + + @workflow.defn class CompleteChildOnSignalWorkflow: def __init__(self) -> None: @@ -1794,6 +1900,7 @@ def child_started(self) -> bool: [ pytest.param(_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, id="with-cancellation-changes"), pytest.param(_BATCH_ONLY_WORKFLOW_LOGIC_FLAG, id="batch-only"), + pytest.param(_PRESERVE_TASK_CANCELLATION_FLAG, id="cancellation-only"), ], ) async def test_workflow_cancel_and_child_completion_in_same_activation( @@ -8427,14 +8534,19 @@ async def run(self) -> None: await asyncio.sleep(0.01) +@pytest.mark.parametrize("task_cancellation", [False, True]) async def test_wait_condition_timer_options_not_used_by_later_timers( - client: Client, env: WorkflowEnvironment + client: Client, env: WorkflowEnvironment, task_cancellation: bool ): if env.supports_time_skipping_v1: pytest.skip( "Java test server: https://github.com/temporalio/sdk-java/issues/2219" ) - async with new_worker(client, WaitConditionThenSleepWorkflow) as worker: + worker = new_worker(client, WaitConditionThenSleepWorkflow) + worker._set_default_workflow_logic_flag( + _PRESERVE_TASK_CANCELLATION_FLAG, enabled=task_cancellation + ) + async with worker: handle = await client.start_workflow( WaitConditionThenSleepWorkflow.run, id=f"workflow-{uuid.uuid4()}",