Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/fixed/snuggly-cancellation-otter.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add opt-in workflow cancellation fixes that respect shielding in `workflow.wait_condition` and preserve task cancellation when a condition or a Temporal operation completes concurrently.
30 changes: 21 additions & 9 deletions temporalio/worker/_workflow_instance.py
Original file line number Diff line number Diff line change
Expand Up @@ -500,13 +500,19 @@ def activate(
self._is_replaying = act.is_replaying
self._current_thread_id = threading.get_ident()
self._current_internal_flags = act.available_internal_flags
# The original flag also changed cancellation behavior. Histories using
# it must retain those semantics; the new flag only changes scheduling.
self._single_batch_cancellation = self._workflow_logic_flag_enabled(
self._legacy_single_batch = self._workflow_logic_flag_enabled(
_WorkflowLogicFlag.PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH
)
# The task-local wait condition fix must not change the behavior of
# histories that use the original single-batch flag.
self._preserve_task_cancellation = (
not self._legacy_single_batch
and self._workflow_logic_flag_enabled(
_WorkflowLogicFlag.PRESERVE_TASK_CANCELLATION
)
)
single_batch_activation = (
self._single_batch_cancellation
self._legacy_single_batch
or self._workflow_logic_flag_enabled(
_WorkflowLogicFlag.APPLY_ACTIVATION_JOBS_BEFORE_RUNNING_TASKS
)
Expand Down Expand Up @@ -708,7 +714,7 @@ def _apply_cancel_workflow(
# workflow the ability to receive the cancellation, so we must defer
# this cancellation to the next iteration of the event loop.
self.call_soon(self._primary_task.cancel)
elif self._single_batch_cancellation:
elif self._legacy_single_batch or self._preserve_task_cancellation:
# Initialization is the only job that creates the primary task, so
# retain a same-activation cancellation until that task exists.
self._cancel_primary_task_pending = True
Expand Down Expand Up @@ -2036,7 +2042,7 @@ async def workflow_wait_condition(
# cancellation and keep going.
def cancellation_arrived() -> bool:
return (
self._single_batch_cancellation
self._legacy_single_batch
and not cancellation_requested_before
and self._cancel_reason is not None
)
Expand All @@ -2053,12 +2059,17 @@ def cancellation_arrived() -> bool:
else None
)
markers = _capture_event_group_markers(event_groups)
# asyncio.wait_for() starts its timer from this task, so the options are
# set in the task's context and reset once the wait ends
# Scope the timer options so later timers in this task don't inherit them.
token = _TimerOptionsCtxVar.set(
_TimerOptions(user_metadata=user_metadata, event_group_markers=markers)
)
try:
if self._preserve_task_cancellation:
# A direct await lets the waiting task arbitrate cancellation
# and readiness without bypassing shielding or using wait_for.
async with asyncio.timeout(timeout):
await fut
return
await asyncio.wait_for(fut, timeout)
except asyncio.TimeoutError:
if cancellation_arrived():
Expand Down Expand Up @@ -2431,7 +2442,7 @@ async def _await_temporal_operation(
if fut.done():
# Retrying the shield after both futures become ready would
# return the result and erase the task cancellation.
if self._single_batch_cancellation:
if self._legacy_single_batch or self._preserve_task_cancellation:
raise
if completed_cancellation_flag is not None and (
not self._is_replaying
Expand Down Expand Up @@ -4275,6 +4286,7 @@ class _WorkflowLogicFlag(IntEnum):
RAISE_ON_CANCELLING_COMPLETED_ACTIVITY = 1
PROCESS_WORKFLOW_ACTIVATION_JOBS_AS_SINGLE_BATCH = 2
APPLY_ACTIVATION_JOBS_BEFORE_RUNNING_TASKS = 3
PRESERVE_TASK_CANCELLATION = 4


_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS: frozenset[_WorkflowLogicFlag] = frozenset()
146 changes: 129 additions & 17 deletions tests/worker/test_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1521,6 +1521,7 @@ async def run(self) -> str:
_BATCH_ONLY_WORKFLOW_LOGIC_FLAG = (
_WorkflowLogicFlag.APPLY_ACTIVATION_JOBS_BEFORE_RUNNING_TASKS
)
_PRESERVE_TASK_CANCELLATION_FLAG = _WorkflowLogicFlag.PRESERVE_TASK_CANCELLATION


@pytest.mark.parametrize(
Expand Down Expand Up @@ -1622,8 +1623,12 @@ async def ready() -> bool:
).replay_workflow(history)


@pytest.mark.parametrize("single_batch", [False, True])
async def test_workflow_cancel_before_run(client: Client, single_batch: bool):
@pytest.mark.parametrize(
"flag", [None, _SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, _PRESERVE_TASK_CANCELLATION_FLAG]
)
async def test_workflow_cancel_before_run(
client: Client, flag: _WorkflowLogicFlag | None
):
# Start the workflow _and_ send cancel before even starting the workflow
task_queue = str(uuid.uuid4())
handle = await client.start_workflow(
Expand All @@ -1634,9 +1639,8 @@ async def test_workflow_cancel_before_run(client: Client, single_batch: bool):
await handle.cancel()
# Start worker and wait for result
worker = new_worker(client, TrapCancelWorkflow, task_queue=task_queue)
worker._set_default_workflow_logic_flag(
_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, enabled=single_batch
)
if flag is not None:
worker._set_default_workflow_logic_flag(flag, enabled=True)
async with worker:
assert "cancelled" == await handle.result()

Expand Down Expand Up @@ -1681,9 +1685,19 @@ def waiting_after_cancel(self) -> bool:
return self._waiting_after_cancel


@pytest.mark.parametrize(
"flags",
[
pytest.param((_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG,), id="original-single-batch"),
pytest.param(
(_BATCH_ONLY_WORKFLOW_LOGIC_FLAG, _PRESERVE_TASK_CANCELLATION_FLAG),
id="batching-with-task-cancellation",
),
],
)
@pytest.mark.parametrize("timeout", [False, True])
async def test_workflow_cancel_and_condition_ready_in_same_activation(
client: Client, timeout: bool
client: Client, timeout: bool, flags: tuple[_WorkflowLogicFlag, ...]
):
task_queue = str(uuid.uuid4())
runner = CustomWorkflowRunner()
Expand All @@ -1704,9 +1718,8 @@ async def test_workflow_cancel_and_condition_ready_in_same_activation(
# the normal queue.
max_cached_workflows=0,
)
worker._set_default_workflow_logic_flag(
_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, enabled=True
)
for flag in flags:
worker._set_default_workflow_logic_flag(flag, enabled=True)
async with worker:

async def ready() -> bool:
Expand All @@ -1725,9 +1738,8 @@ async def ready() -> bool:
task_queue=task_queue,
workflow_runner=runner,
)
worker._set_default_workflow_logic_flag(
_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, enabled=True
)
for flag in flags:
worker._set_default_workflow_logic_flag(flag, enabled=True)
async with worker:

async def waiting_after_cancel() -> bool:
Expand All @@ -1739,10 +1751,12 @@ async def waiting_after_cancel() -> bool:
await handle.signal(CancelAtWaitConditionWorkflow.finish_after_cancel)
assert await handle.result() == "cancelled"

assert any(
_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG in completion.successful.used_internal_flags
used_flags = {
flag
for _, completion in runner._pairs
)
for flag in completion.successful.used_internal_flags
}
assert used_flags.issuperset(flags)
assert any(
{"signal_workflow", "cancel_workflow"}.issubset(
{job.WhichOneof("variant") for job in activation.jobs}
Expand All @@ -1754,6 +1768,98 @@ async def waiting_after_cancel() -> bool:
)


@workflow.defn
class ShieldedWaitConditionWorkflow:
def __init__(self) -> None:
self._confirmed = False
self._waiting = False
self._caught_cancel = False

async def _cleanup(self, timeout: bool) -> None:
self._waiting = True
await workflow.wait_condition(
lambda: self._confirmed, timeout=1000 if timeout else None
)
await workflow.sleep(0.01, summary="shielded cleanup")

@workflow.run
async def run(self, timeout: bool) -> None:
cleanup = asyncio.create_task(self._cleanup(timeout))
try:
await asyncio.shield(cleanup)
except asyncio.CancelledError:
self._caught_cancel = True
await cleanup
raise

@workflow.signal
def confirm(self) -> None:
self._confirmed = True

@workflow.query
def state(self) -> tuple[bool, bool]:
return self._waiting, self._caught_cancel


@pytest.mark.parametrize("timeout", [False, True])
@pytest.mark.parametrize(
("batch_flag", "cleanup_completes"),
[
pytest.param(_BATCH_ONLY_WORKFLOW_LOGIC_FLAG, True, id="task-cancellation"),
pytest.param(
_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, False, id="original-behavior-preserved"
),
],
)
async def test_workflow_shielded_wait_condition_after_cancel(
client: Client,
timeout: bool,
batch_flag: _WorkflowLogicFlag,
cleanup_completes: bool,
) -> None:
worker = new_worker(client, ShieldedWaitConditionWorkflow)
worker._set_default_workflow_logic_flag(batch_flag, enabled=True)
worker._set_default_workflow_logic_flag(
_PRESERVE_TASK_CANCELLATION_FLAG, enabled=True
)
async with worker:
handle = await client.start_workflow(
ShieldedWaitConditionWorkflow.run,
timeout,
id=f"workflow-{uuid.uuid4()}",
task_queue=worker.task_queue,
)

async def state() -> tuple[bool, bool]:
return await handle.query(ShieldedWaitConditionWorkflow.state)

await assert_eq_eventually((True, False), state)
await handle.cancel()
await assert_eq_eventually((True, True), state)
await handle.signal(ShieldedWaitConditionWorkflow.confirm)
with pytest.raises(WorkflowFailureError) as err:
await handle.result()
assert isinstance(err.value.cause, CancelledError)

history = await handle.fetch_history()
assert (
any(
event.event_type == EventType.EVENT_TYPE_TIMER_FIRED
for event in history.events
)
is cleanup_completes
)
assert (
any(
_PRESERVE_TASK_CANCELLATION_FLAG
in event.workflow_task_completed_event_attributes.sdk_metadata.lang_used_flags
for event in history.events
)
is cleanup_completes
)
await Replayer(workflows=[ShieldedWaitConditionWorkflow]).replay_workflow(history)


@workflow.defn
class CompleteChildOnSignalWorkflow:
def __init__(self) -> None:
Expand Down Expand Up @@ -1794,6 +1900,7 @@ def child_started(self) -> bool:
[
pytest.param(_SINGLE_BATCH_WORKFLOW_LOGIC_FLAG, id="with-cancellation-changes"),
pytest.param(_BATCH_ONLY_WORKFLOW_LOGIC_FLAG, id="batch-only"),
pytest.param(_PRESERVE_TASK_CANCELLATION_FLAG, id="cancellation-only"),
],
)
async def test_workflow_cancel_and_child_completion_in_same_activation(
Expand Down Expand Up @@ -8427,14 +8534,19 @@ async def run(self) -> None:
await asyncio.sleep(0.01)


@pytest.mark.parametrize("task_cancellation", [False, True])
async def test_wait_condition_timer_options_not_used_by_later_timers(
client: Client, env: WorkflowEnvironment
client: Client, env: WorkflowEnvironment, task_cancellation: bool
):
if env.supports_time_skipping_v1:
pytest.skip(
"Java test server: https://github.com/temporalio/sdk-java/issues/2219"
)
async with new_worker(client, WaitConditionThenSleepWorkflow) as worker:
worker = new_worker(client, WaitConditionThenSleepWorkflow)
worker._set_default_workflow_logic_flag(
_PRESERVE_TASK_CANCELLATION_FLAG, enabled=task_cancellation
)
async with worker:
handle = await client.start_workflow(
WaitConditionThenSleepWorkflow.run,
id=f"workflow-{uuid.uuid4()}",
Expand Down
Loading