diff --git a/docs/source/design/execution.rst b/docs/source/design/execution.rst index 22683fb8b2..9db0807c40 100644 --- a/docs/source/design/execution.rst +++ b/docs/source/design/execution.rst @@ -21,3 +21,26 @@ The point of running this script, or rather the reason for having any Flyte-rela * Capture and handle error reporting: Exceptions thrown in the course of task execution are captured and uploaded to the Flyte control plane for display on the Console. * Set up helper utilities like the ``statsd`` handle, logging and logging levels, etc. * Ensure configuration options about the Flyte backend, which are passed through by the Flyte engine, are properly loaded in Python memory. + +Recovering completed results after a shutdown crash +================================================== + +A native library can crash during interpreter shutdown after Flytekit has already +uploaded a task's results. By default, ``pyflyte-fast-execute`` reports the child +process's failure even when those results are available. + +Set ``FLYTE_RECOVER_SHUTDOWN_CRASH=true`` in the task environment to let +``pyflyte-fast-execute`` restore the completed SDK exit status after ``SIGSEGV``, +``SIGABRT``, or ``SIGBUS``. Recovery requires a completion record from the direct +child, written after result metadata and deck handling have finished. Missing or +invalid records, crashes before upload, ordinary nonzero exits, ``SIGTERM``, and +``SIGKILL`` are not recovered. The launcher also refuses recovery after forwarding +``SIGTERM`` to the child. + +Recovery preserves error reporting: an uploaded error document still describes a +failed task, and ``FLYTE_FAIL_ON_ERROR=true`` still produces exit status 1. The +launcher logs each recovered crash and does not skip shutdown hooks. A crash can +prevent remaining cleanup or hooks from finishing, so enable this option only +when the uploaded results are sufficient and unfinished cleanup is acceptable. +This option does not repair the native library or apply to standalone +``pyflyte-execute`` processes. diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 3be1fe0d69..f976e034e9 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -14,7 +14,8 @@ import uuid import warnings from sys import exit -from typing import Callable, Dict, List, Optional +from types import FrameType +from typing import Callable, Dict, Iterator, List, Optional import click from flyteidl.core import literals_pb2 as _literals_pb2 @@ -31,7 +32,7 @@ from flytekit.core import utils from flytekit.core.base_task import IgnoreOutputs, PythonTask from flytekit.core.checkpointer import SyncCheckpoint -from flytekit.core.constants import FLYTE_FAIL_ON_ERROR, RUNTIME_PACKAGES_ENV_NAME +from flytekit.core.constants import FLYTE_FAIL_ON_ERROR, FLYTE_RECOVER_SHUTDOWN_CRASH, RUNTIME_PACKAGES_ENV_NAME from flytekit.core.context_manager import ( ExecutionParameters, ExecutionState, @@ -62,6 +63,8 @@ from flytekit.tools.module_loader import load_object_from_module from flytekit.utils.pbhash import compute_hash_string +_TASK_COMPLETION_PATH: str = "_FLYTE_TASK_COMPLETION_PATH" + def get_version_message(): import flytekit @@ -69,16 +72,45 @@ def get_version_message(): return f"Welcome to Flyte! Version: {flytekit.__version__}" -def _run_subprocess(cmd: List[str], env: Optional[dict] = None) -> int: +def _run_subprocess( + cmd: list[str], env: dict[str, str] | None = None, completion_path: pathlib.Path | None = None +) -> int: """Run cmd with proper SIGTERM handling.""" - p = subprocess.Popen(cmd, env=env) + p = subprocess.Popen(args=cmd, env=env) + termination_forwarded: bool = False - def handle_sigterm(signum, frame): + def handle_sigterm(signum: int, frame: FrameType | None) -> None: + nonlocal termination_forwarded + termination_forwarded = True logger.info(f"passing signum {signum} [frame={frame}] to subprocess") p.send_signal(signum) - signal.signal(signal.SIGTERM, handle_sigterm) - return p.wait() + previous_handler: int | Callable[[int, FrameType | None], None] | None = signal.signal( + signalnum=signal.SIGTERM, handler=handle_sigterm + ) + try: + returncode: int = p.wait() + finally: + signal.signal(signalnum=signal.SIGTERM, handler=previous_handler) + + if ( + completion_path is None + or termination_forwarded + or returncode not in (-signal.SIGSEGV, -signal.SIGABRT, -signal.SIGBUS) + ): + return returncode + try: + child_pid, completed_exit_code = map(int, completion_path.read_text().split()) + except (OSError, ValueError): + logger.warning("Completed task status is unavailable or invalid; retaining the subprocess failure.") + return returncode + if child_pid != p.pid or completed_exit_code not in (0, 1): + return returncode + logger.warning( + f"Task subprocess crashed with signal {-returncode} after output upload; " + f"restoring completed SDK exit status {completed_exit_code}." + ) + return completed_exit_code def _compute_array_job_index(): @@ -354,7 +386,18 @@ def _dispatch_execute( logger.debug("Finished _dispatch_execute") - if str2bool(os.getenv(FLYTE_FAIL_ON_ERROR)) and error_file_name in output_file_dict: + exit_code: int = int(str2bool(os.getenv(FLYTE_FAIL_ON_ERROR)) and error_file_name in output_file_dict) + if ctx.user_space_params is not None and ctx.user_space_params.has_attr(_TASK_COMPLETION_PATH): + completion_path: str | None = ctx.user_space_params.get(_TASK_COMPLETION_PATH) + if completion_path: + try: + pathlib.Path(completion_path).write_text(f"{os.getpid()} {exit_code}\n") + except OSError: + logger.warning( + "Could not record completed task status; shutdown recovery is unavailable.", exc_info=True + ) + + if exit_code: """ If the environment variable FLYTE_FAIL_ON_ERROR is set to true, the task execution will fail if an error file is generated. This environment variable is set to true by the plugin author if they want the task to fail on error. @@ -362,7 +405,7 @@ def _dispatch_execute( For example, you can see the task fails on Databricks or AWS batch UI by setting this environment variable to true. """ - exit(1) + exit(exit_code) def get_traceback_str(e: Exception) -> str: @@ -426,7 +469,7 @@ def setup_execution( prev_checkpoint: Optional[str] = None, dynamic_addl_distro: Optional[str] = None, dynamic_dest_dir: Optional[str] = None, -): +) -> Iterator[FlyteContext]: """ :param raw_output_data_prefix: Where to write offloaded data (files, directories, dataframes). @@ -438,6 +481,7 @@ def setup_execution( :param dynamic_dest_dir: See above. :return: """ + completion_path: str | None = os.environ.pop(_TASK_COMPLETION_PATH, None) exe_project = get_one_of("FLYTE_INTERNAL_EXECUTION_PROJECT", "_F_PRJ") exe_domain = get_one_of("FLYTE_INTERNAL_EXECUTION_DOMAIN", "_F_DM") exe_name = get_one_of("FLYTE_INTERNAL_EXECUTION_ID", "_F_NM") @@ -498,6 +542,7 @@ def setup_execution( output_metadata_prefix=output_metadata_prefix, checkpoint=checkpointer, task_id=_identifier.Identifier(_identifier.ResourceType.TASK, tk_project, tk_domain, tk_name, tk_version), + **{_TASK_COMPLETION_PATH: completion_path}, ) metadata = { @@ -762,7 +807,7 @@ def execute_task_cmd( @click.option("--additional-distribution", required=False) @click.option("--dest-dir", required=False) @click.argument("task-execute-cmd", nargs=-1, type=click.UNPROCESSED) -def fast_execute_task_cmd(additional_distribution: str, dest_dir: str, task_execute_cmd: List[str]): +def fast_execute_task_cmd(additional_distribution: str, dest_dir: str, task_execute_cmd: List[str]) -> None: """ Downloads a compressed code distribution specified by additional-distribution and then calls the underlying task execute command for the updated code. @@ -795,7 +840,13 @@ def fast_execute_task_cmd(additional_distribution: str, dest_dir: str, task_exec env["PYTHONPATH"] += os.pathsep + dest_dir_resolved else: env["PYTHONPATH"] = dest_dir_resolved - returncode = _run_subprocess(cmd, env) + if str2bool(os.getenv(FLYTE_RECOVER_SHUTDOWN_CRASH)): + with tempfile.TemporaryDirectory(prefix="flyte-completion-") as completion_dir: + completion_path = pathlib.Path(completion_dir) / "completion" + env[_TASK_COMPLETION_PATH] = str(completion_path) + returncode = _run_subprocess(cmd=cmd, env=env, completion_path=completion_path) + else: + returncode = _run_subprocess(cmd=cmd, env=env) exit(returncode) diff --git a/flytekit/core/constants.py b/flytekit/core/constants.py index 4ea432bc96..ca56d85bec 100644 --- a/flytekit/core/constants.py +++ b/flytekit/core/constants.py @@ -25,6 +25,8 @@ # Set this environment variable to true to force the task to return non-zero exit code on failure. FLYTE_FAIL_ON_ERROR = "FLYTE_FAIL_ON_ERROR" +FLYTE_RECOVER_SHUTDOWN_CRASH: str = "FLYTE_RECOVER_SHUTDOWN_CRASH" + # Executions launched by the current eager task will be tagged with this key:current_eager_exec_name EAGER_TAG_KEY = "eager-exec" diff --git a/tests/flytekit/unit/bin/test_shutdown_recovery.py b/tests/flytekit/unit/bin/test_shutdown_recovery.py new file mode 100644 index 0000000000..5b5c1ad8c2 --- /dev/null +++ b/tests/flytekit/unit/bin/test_shutdown_recovery.py @@ -0,0 +1,249 @@ +import os +import signal +import subprocess +import sys +import tarfile +import textwrap +import time +from pathlib import Path + +import pytest +from flyteidl.core.errors_pb2 import ErrorDocument +from flyteidl.core.literals_pb2 import LiteralMap + +from flytekit.core.constants import ERROR_FILE_NAME, OUTPUT_FILE_NAME +from flytekit.loggers import LOGGING_FMT_ENV_VAR, LOGGING_RICH_FMT_ENV_VAR + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="Task entrypoints require POSIX signals.") + + +@pytest.fixture +def execution_command(tmp_path: Path) -> list[str]: + inputs = tmp_path / "inputs.pb" + inputs.write_bytes(LiteralMap().SerializeToString()) + task_module = tmp_path / "shutdown_task.py" + task_module.write_text( + textwrap.dedent( + """\ + import atexit + import os + import resource + import signal + import time + from pathlib import Path + from types import FrameType + + from flytekit import current_context, task + from flytekit.core.base_task import IgnoreOutputs + + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + + def receive_termination(signum: int, frame: FrameType | None) -> None: + Path(os.environ["SHUTDOWN_TEST_ROOT"], "termination-received").touch() + print("CHILD_SIGTERM_RECEIVED", flush=True) + + def shutdown(completion_path: str | None) -> None: + root = Path(os.environ["SHUTDOWN_TEST_ROOT"]) + mode = os.environ["SHUTDOWN_TEST_MODE"] + (root / "exit-hook-ran").touch() + print(f"EXIT_HOOK mode={mode}", flush=True) + if completion_path and mode in ("corrupt_record", "wrong_pid", "invalid_status"): + replacement = { + "corrupt_record": "incomplete", + "wrong_pid": f"{os.getppid()} 0", + "invalid_status": f"{os.getpid()} 2", + }[mode] + Path(completion_path).write_text(replacement) + if mode == "forwarded_sigterm": + signal.signal(signal.SIGTERM, receive_termination) + (root / "ready").touch() + deadline = time.monotonic() + 20 + while not (root / "termination-received").exists(): + if time.monotonic() >= deadline: + raise RuntimeError("Parent did not forward termination") + time.sleep(0.01) + if mode == "ordinary_exit": + os._exit(7) + if mode in ("task_error", "record_write_failure"): + return + crash_signal = { + "sigabrt": signal.SIGABRT, + "sigbus": signal.SIGBUS, + "sigterm": signal.SIGTERM, + "sigkill": signal.SIGKILL, + }.get(mode, signal.SIGSEGV) + os.kill(os.getpid(), crash_signal) + + @task + def value() -> int: + mode = os.environ["SHUTDOWN_TEST_MODE"] + assert "_FLYTE_TASK_COMPLETION_PATH" not in os.environ + params = current_context() + completion_path: str | None = None + if params.has_attr("_FLYTE_TASK_COMPLETION_PATH"): + completion_path = params.get("_FLYTE_TASK_COMPLETION_PATH") + if mode == "record_write_failure" and completion_path: + Path(completion_path).mkdir() + if mode != "normal": + atexit.register(shutdown, completion_path) + if mode == "early_crash": + os.kill(os.getpid(), signal.SIGSEGV) + if mode.startswith("task_error"): + raise ValueError("shutdown recovery task failure") + if mode == "ignore_outputs": + raise IgnoreOutputs() + return 42 + """ + ) + ) + startup_module = tmp_path / "sitecustomize.py" + startup_module.write_text( + textwrap.dedent( + """\ + import atexit + import os + import resource + import signal + from pathlib import Path + + def crash_after_dry_run() -> None: + root = Path(os.environ["SHUTDOWN_TEST_ROOT"]) + (root / "exit-hook-ran").touch() + print("EXIT_HOOK mode=dry_run", flush=True) + os.kill(os.getpid(), signal.SIGSEGV) + + if os.environ.get("SHUTDOWN_TEST_MODE") == "dry_run": + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + atexit.register(crash_after_dry_run) + """ + ) + ) + distribution = tmp_path / "code.tar.gz" + with tarfile.open(name=distribution, mode="w:gz") as archive: + archive.add(name=task_module, arcname=task_module.name) + archive.add(name=startup_module, arcname=startup_module.name) + destination = tmp_path / "code" + destination.mkdir() + return [ + sys.executable, + "-m", + "flytekit.bin.entrypoint", + "pyflyte-fast-execute", + "--additional-distribution", + str(distribution), + "--dest-dir", + str(destination), + "--", + sys.executable, + "-m", + "flytekit.bin.entrypoint", + "pyflyte-execute", + "--inputs", + str(inputs), + "--output-prefix", + str(tmp_path / "outputs"), + "--raw-output-data-prefix", + str(tmp_path / "raw"), + "--resolver", + "flytekit.core.python_auto_container.default_task_resolver", + "--", + "task-module", + "shutdown_task", + "task-name", + "value", + ] + + +@pytest.mark.parametrize( + ("mode", "recovery_enabled", "expected_exit", "artifact"), + [ + pytest.param("normal", False, 0, "output", id="normal-default"), + pytest.param("normal", True, 0, "output", id="normal-enabled"), + pytest.param("sigsegv", False, 245, "output", id="segv-default"), + pytest.param("sigsegv", True, 0, "output", id="recover-segv"), + pytest.param("sigabrt", True, 0, "output", id="recover-abrt"), + pytest.param("sigbus", True, 0, "output", id="recover-bus"), + pytest.param("early_crash", True, 245, None, id="early-crash"), + pytest.param("upload_failure", True, 245, None, id="upload-failure"), + pytest.param("sigterm", True, 241, "output", id="direct-term"), + pytest.param("sigkill", True, 247, "output", id="direct-kill"), + pytest.param("forwarded_sigterm", True, 245, "output", id="forwarded-term"), + pytest.param("ordinary_exit", True, 7, "output", id="ordinary-exit"), + pytest.param("task_error", True, 1, "error", id="task-error"), + pytest.param("task_error_crash", True, 1, "error", id="failed-task-crash"), + pytest.param("task_error_protocol", True, 0, "error", id="error-document-protocol"), + pytest.param("ignore_outputs", True, 245, None, id="ignored-outputs"), + pytest.param("dry_run", True, 245, None, id="dry-run"), + pytest.param("corrupt_record", True, 245, "output", id="corrupt-record"), + pytest.param("wrong_pid", True, 245, "output", id="wrong-child"), + pytest.param("invalid_status", True, 245, "output", id="invalid-status"), + pytest.param("record_write_failure", True, 0, "output", id="record-write-failure"), + ], +) +def test_shutdown_recovery( + tmp_path: Path, + execution_command: list[str], + mode: str, + recovery_enabled: bool, + expected_exit: int, + artifact: str | None, +) -> None: + if mode == "upload_failure": + (tmp_path / "outputs").write_text("The upload destination is a file, not a directory.") + if mode == "dry_run": + execution_command.insert(execution_command.index("--resolver"), "--test") + env: dict[str, str] = { + **os.environ, + "SHUTDOWN_TEST_ROOT": str(tmp_path), + "SHUTDOWN_TEST_MODE": mode, + "FLYTE_RECOVER_SHUTDOWN_CRASH": str(recovery_enabled), + "FLYTE_FAIL_ON_ERROR": str(mode != "task_error_protocol"), + LOGGING_FMT_ENV_VAR: "json", + LOGGING_RICH_FMT_ENV_VAR: "0", + } + with subprocess.Popen( + args=execution_command, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) as process: + try: + if mode == "forwarded_sigterm": + deadline: float = time.monotonic() + 30 + while not (tmp_path / "ready").exists(): + assert process.poll() is None, "Launcher exited before the shutdown hook was ready" + assert time.monotonic() < deadline, "Shutdown hook did not become ready" + time.sleep(0.01) + process.send_signal(signal.SIGTERM) + output: str = process.communicate(timeout=45)[0] + finally: + if process.poll() is None: + process.kill() + process.communicate(timeout=10) + (tmp_path / "launcher.log").write_text(output) + output_file = tmp_path / "outputs" / OUTPUT_FILE_NAME + error_file = tmp_path / "outputs" / ERROR_FILE_NAME + print(f"mode={mode} recovery={recovery_enabled} exit={process.returncode}") + for line in output.splitlines(): + if "EXIT_HOOK" in line or "CHILD_SIGTERM_RECEIVED" in line or "restoring" in line.lower(): + print(line) + if artifact == "output": + result = LiteralMap.FromString(output_file.read_bytes()) + assert result.literals["o0"].scalar.primitive.integer == 42 + assert not error_file.exists() + print("outputs.pb: o0=42; error.pb: absent") + elif artifact == "error": + error = ErrorDocument.FromString(error_file.read_bytes()) + assert "shutdown recovery task failure" in error.error.message + assert not output_file.exists() + print(f"error.pb: code={error.error.code}; outputs.pb: absent") + else: + assert not output_file.exists() + assert not error_file.exists() + print("outputs.pb: absent; error.pb: absent") + if mode == "forwarded_sigterm": + assert (tmp_path / "termination-received").exists() + if mode not in ("normal", "early_crash"): + assert (tmp_path / "exit-hook-ran").exists() + assert process.returncode == expected_exit, output