From fe2d59ea6e5326520418fa55980d5f5e9b043aef Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 16 Nov 2026 15:30:36 -0600 Subject: [PATCH 01/14] PYTHON-6266 Make the synchro hook atomic and write only real changes --- .gitignore | 1 + CONTRIBUTING.md | 8 +- tools/synchro.py | 164 ++++++++++++++++++++++--------- tools/test_synchro.py | 217 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 338 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index c113d3ec21..b4db6a497a 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ expansion.yml specifications/ results.json .synchro-modified +.synchro-staging/ .evergreen/atlas_x509_dev_client_certificate.pem .evergreen/atlas_x509_sfp_client_certificate.pem diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 007b73cfd7..155d2dcba8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -616,10 +616,10 @@ To prevent the `synchro` hook from accidentally overwriting code, it first check of a file is changing and not its async counterpart, and will fail. In the unlikely scenario that you want to override this behavior, first export `OVERRIDE_SYNCHRO_CHECK=1`. -Sometimes, the `synchro` hook will fail and introduce changes many previously unmodified files. This is due to static -Python errors, such as missing imports, incorrect syntax, or other fatal typos. To resolve these issues, -run `pre-commit run --all-files --hook-stage manual ruff` and fix all reported errors before running the `synchro` -hook again. +The `synchro` hook generates the synchronous files in a staging area and copies them into the working tree only when +their content changes, so a failed run leaves the working tree unchanged and nothing is staged when there is nothing +to regenerate. If the hook reports that ruff failed on the generated output, fix the reported error in the +corresponding `pymongo/asynchronous` source file and run the hook again. ## CPU profiling diff --git a/tools/synchro.py b/tools/synchro.py index 7cb68d4ca7..0c9fc22f16 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -22,6 +22,7 @@ import os import re +import shutil import subprocess import sys from pathlib import Path @@ -29,6 +30,7 @@ from unasync import Rule, unasync_files # type: ignore[import-untyped,import-not-found] MANIFEST = ".synchro-modified" +STAGING = ".synchro-staging" replacements = { "AsyncCollection": "Collection", @@ -343,6 +345,72 @@ def process_ignores(lines: list[str]) -> list[str]: return lines +def copy_changed_files(pairs: list[tuple[Path, Path]]) -> list[Path]: + """Copy staged files into the working tree when their content differs. + + If a write fails midway, destinations already written are restored so + the working tree is left unchanged. + """ + written: list[tuple[Path, bytes | None]] = [] + try: + for staged, dest in pairs: + new = staged.read_bytes() + old = dest.read_bytes() if dest.is_file() else None + if old == new: + continue + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(new) + written.append((dest, old)) + except OSError: + for dest, old in written: + if old is None: + dest.unlink(missing_ok=True) + else: + dest.write_bytes(old) + raise + return [dest for dest, _ in written] + + +def prepare_staging() -> None: + """Recreate the out-of-tree staging area used to generate sync files. + + A copy of pyproject.toml at the staging root keeps ruff's + per-file-ignores (e.g. ``test/*.py``) and first-party import + classification identical to a run against the repo root. + """ + shutil.rmtree(STAGING, ignore_errors=True) + for base in (_pymongo_dest_base, _gridfs_dest_base, _test_dest_base): + Path(STAGING, base[2:]).mkdir(parents=True, exist_ok=True) + shutil.copyfile("pyproject.toml", f"{STAGING}/pyproject.toml") + config = Path(STAGING, "pyproject.toml") + lines = config.read_text().splitlines(keepends=True) + for i, line in enumerate(lines): + if line.startswith("isort.required-imports"): + lines.insert(i + 1, 'isort.known-first-party = ["pymongo", "gridfs", "bson", "test"]\n') + break + else: + print( + "Synchro: no isort.required-imports line in pyproject.toml, so " + "isort.known-first-party was not added to the staged ruff config and " + "import classification may differ from the repo root.", + file=sys.stderr, + ) + config.write_text("".join(lines)) + + +def mirrored_test_files(filtered_tests: list[str]) -> list[str]: + """Return the generated test paths for the given async test files. + + A test file is mirrored only when its synchronous counterpart already + exists in ``test/``. + """ + return [ + _test_dest_base + Path(f).name + for f in filtered_tests + if (Path(_test_dest_base) / Path(f).name).is_file() + ] + + def unasync_directory(files: list[str], src: str, dest: str, replacements: dict[str, str]) -> None: unasync_files( files, @@ -390,58 +458,66 @@ def main() -> None: check=True, ) - unasync_directory(filtered_async, _pymongo_base, _pymongo_dest_base, replacements) - unasync_directory(filtered_gridfs, _gridfs_base, _gridfs_dest_base, replacements) - unasync_directory(filtered_tests, _test_base, _test_dest_base, replacements) - - # Derive generated output paths directly from filtered source paths. - generated_pymongo = [_pymongo_dest_base + Path(f).name for f in filtered_async] - generated_gridfs = [_gridfs_dest_base + Path(f).name for f in filtered_gridfs] - generated_tests = [ - _test_dest_base + Path(f).name - for f in filtered_tests - if (Path(_test_dest_base) / Path(f).name).is_file() - ] - - docstring_translate_files = generated_pymongo + generated_gridfs + generated_tests - - process_files( - generated_pymongo + generated_gridfs + generated_tests, - docstring_translate_files, - generated_tests, - ) + try: + prepare_staging() + generated_pymongo = [_pymongo_dest_base + Path(f).name for f in filtered_async] + generated_gridfs = [_gridfs_dest_base + Path(f).name for f in filtered_gridfs] + generated_tests = mirrored_test_files(filtered_tests) + generated_names = generated_pymongo + generated_gridfs + generated_tests + staged_pymongo = [f"{STAGING}/{name[2:]}" for name in generated_pymongo] + staged_gridfs = [f"{STAGING}/{name[2:]}" for name in generated_gridfs] + staged_tests = [f"{STAGING}/{name[2:]}" for name in generated_tests] + staged_names = staged_pymongo + staged_gridfs + staged_tests + + unasync_directory( + filtered_async, _pymongo_base, f"{STAGING}/{_pymongo_dest_base[2:]}", replacements + ) + unasync_directory( + filtered_gridfs, _gridfs_base, f"{STAGING}/{_gridfs_dest_base[2:]}", replacements + ) + unasync_directory( + filtered_tests, _test_base, f"{STAGING}/{_test_dest_base[2:]}", replacements + ) - generated_files = generated_pymongo + generated_gridfs + generated_tests + process_files(staged_names, staged_names, staged_tests) - if is_ci and generated_files: - print(f"Synchro generated {len(generated_files)} file(s):") - for f in generated_files: - print(f" {f}") + if staged_names: + try: + subprocess.run( # noqa: S603 + [sys.executable, "-m", "ruff", "check", *staged_names, "--fix", *ruff_extra], + check=True, + ) + subprocess.run( # noqa: S603 + [sys.executable, "-m", "ruff", "format", *staged_names, *ruff_extra], + check=True, + ) + except subprocess.CalledProcessError: + print( + "Synchro: ruff failed on the generated sync output. Fix the error in " + "the corresponding async source file and run the synchro hook again; " + "the working tree was left unchanged.", + file=sys.stderr, + ) + sys.exit(1) - if generated_files: - subprocess.run( # noqa: S603 - [sys.executable, "-m", "ruff", "check", *generated_files, "--fix", *ruff_extra], - check=is_ci, - ) - subprocess.run( # noqa: S603 - [sys.executable, "-m", "ruff", "format", *generated_files, *ruff_extra], - check=is_ci, + changed = copy_changed_files( + [ + (Path(staged), Path(name)) + for staged, name in zip(staged_names, generated_names, strict=True) + ] ) + changed_names = [f"./{dest.as_posix()}" for dest in changed] - if is_ci and generated_files: - result = subprocess.run( # noqa: S603 - ["git", "diff", "--name-only", "--", *generated_files], # noqa: S607 - capture_output=True, - text=True, - check=True, - ) - if result.stdout.strip(): - print("Sync files are out of date. Run `just lint --all-files synchro` to regenerate:") - for f in result.stdout.strip().splitlines(): + if is_ci and changed_names: + print(f"Synchro generated {len(changed_names)} file(s):") + for f in changed_names: print(f" {f}") + print("Sync files are out of date. Run `just lint --all-files synchro` to regenerate:") sys.exit(1) - Path(MANIFEST).write_text("\n".join(generated_files) + ("\n" if generated_files else "")) + Path(MANIFEST).write_text("\n".join(changed_names) + ("\n" if changed_names else "")) + finally: + shutil.rmtree(STAGING, ignore_errors=True) if __name__ == "__main__": diff --git a/tools/test_synchro.py b/tools/test_synchro.py index e6a1cc3b8f..d2e654f7c8 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -1,17 +1,18 @@ -"""Smoke test for tools/synchro.py.""" +"""Tests for tools/synchro.py.""" from __future__ import annotations +import subprocess import sys from pathlib import Path -from typing import TYPE_CHECKING from unittest.mock import patch -if TYPE_CHECKING: - import pytest +import pytest sys.path.insert(0, str(Path(__file__).parent)) +import synchro + def test_synchro(monkeypatch: pytest.MonkeyPatch) -> None: """Synchro processes only the async files passed as arguments. @@ -31,6 +32,8 @@ def capture(files: list[str], *args: object, **kwargs: object) -> None: with ( patch("synchro.unasync_directory", side_effect=capture), patch("synchro.process_files"), + patch("synchro.prepare_staging"), + patch("synchro.copy_changed_files", return_value=[]), patch("subprocess.run"), patch("pathlib.Path.write_text"), ): @@ -40,3 +43,209 @@ def capture(files: list[str], *args: object, **kwargs: object) -> None: assert "./pymongo/asynchronous/mongo_client.py" in processed assert "./pymongo/asynchronous/collection.py" not in processed + + +def test_copy_changed_files_skips_identical(tmp_path: Path) -> None: + """A destination whose content already matches is neither rewritten nor reported.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"same") + dest.write_bytes(b"same") + before = dest.stat().st_mtime_ns + + changed = synchro.copy_changed_files([(staged, dest)]) + + assert changed == [] + assert dest.read_bytes() == b"same" + assert dest.stat().st_mtime_ns == before + + +def test_copy_changed_files_updates_differing(tmp_path: Path) -> None: + """A destination whose content differs is overwritten and reported.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"new") + dest.write_bytes(b"old") + + changed = synchro.copy_changed_files([(staged, dest)]) + + assert changed == [dest] + assert dest.read_bytes() == b"new" + + +def test_copy_changed_files_creates_missing_dest(tmp_path: Path) -> None: + """A missing destination is created (with parents) and reported.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "subdir" / "dest.py" + staged.write_bytes(b"new") + + changed = synchro.copy_changed_files([(staged, dest)]) + + assert changed == [dest] + assert dest.read_bytes() == b"new" + + +def test_copy_changed_files_rolls_back_on_failure(tmp_path: Path) -> None: + """A failed write restores destinations that were already written.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "blocking-dir" / "b_dest.py" + staged_b.write_bytes(b"new-b") + # A file occupies b_dest.py's parent directory, so the mkdir fails. + (tmp_path / "blocking-dir").write_bytes(b"not a directory") + + with pytest.raises(FileExistsError): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + + +ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" +SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" + + +def setup_tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Create a minimal repo-like tree and point synchro at it.""" + (tmp_path / "pymongo" / "asynchronous").mkdir(parents=True) + (tmp_path / "pymongo" / "synchronous").mkdir(parents=True) + (tmp_path / "test" / "asynchronous").mkdir(parents=True) + (tmp_path / "pyproject.toml").write_text("") + (tmp_path / "pymongo" / "asynchronous" / "foo.py").write_text(ASYNC_SOURCE) + monkeypatch.setattr(sys, "argv", ["synchro.py"]) + monkeypatch.setattr(synchro, "async_files", ["./pymongo/asynchronous/foo.py"]) + monkeypatch.setattr(synchro, "gridfs_files", []) + monkeypatch.setattr(synchro, "test_files", []) + monkeypatch.delenv("CI", raising=False) + monkeypatch.chdir(tmp_path) + return tmp_path / "pymongo" / "synchronous" / "foo.py" + + +def test_noop_run_leaves_tree_unchanged(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A run with nothing to regenerate writes no files and stages nothing.""" + dest = setup_tree(tmp_path, monkeypatch) + dest.write_text(SYNC_SOURCE) + before = dest.stat().st_mtime_ns + + with patch("subprocess.run"): + synchro.main() + + assert dest.read_text() == SYNC_SOURCE + assert dest.stat().st_mtime_ns == before + assert (tmp_path / ".synchro-modified").read_text() == "" + assert not (tmp_path / ".synchro-staging").exists() + + +def test_regenerated_file_is_copied_and_manifested( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Only files whose content changed are copied and listed in the manifest.""" + dest = setup_tree(tmp_path, monkeypatch) + dest.write_text("_IS_SYNC = True\n# stale\n") + + with patch("subprocess.run"): + synchro.main() + + assert dest.read_text() == SYNC_SOURCE + assert (tmp_path / ".synchro-modified").read_text() == "./pymongo/synchronous/foo.py\n" + assert not (tmp_path / ".synchro-staging").exists() + + +def test_async_test_without_counterpart_is_not_mirrored( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An async test file is mirrored only when test/.py already exists.""" + setup_tree(tmp_path, monkeypatch) + (tmp_path / "test" / "asynchronous" / "test_bar.py").write_text(ASYNC_SOURCE) + monkeypatch.setattr(synchro, "test_files", ["./test/asynchronous/test_bar.py"]) + + with patch("subprocess.run"): + synchro.main() + + assert not (tmp_path / "test" / "test_bar.py").exists() + + +def test_ruff_failure_leaves_tree_unchanged( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A ruff failure on generated output exits nonzero without touching the tree.""" + dest = setup_tree(tmp_path, monkeypatch) + dest.write_text("_IS_SYNC = True\n# stale\n") + + def failing_ruff( + cmd: list[str], *args: object, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + if "--fix" in cmd: + raise subprocess.CalledProcessError(1, cmd) + return subprocess.CompletedProcess(cmd, 0) + + with patch("subprocess.run", side_effect=failing_ruff): + with pytest.raises(SystemExit) as excinfo: + synchro.main() + + assert excinfo.value.code == 1 + assert dest.read_text() == "_IS_SYNC = True\n# stale\n" + assert not (tmp_path / ".synchro-modified").exists() + assert not (tmp_path / ".synchro-staging").exists() + + +def test_ci_reports_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """In CI, changed generated files are copied so the drift is visible, then fail.""" + dest = setup_tree(tmp_path, monkeypatch) + dest.write_text("_IS_SYNC = True\n# stale\n") + monkeypatch.setenv("CI", "true") + + with patch("subprocess.run"): + with pytest.raises(SystemExit) as excinfo: + synchro.main() + + assert excinfo.value.code == 1 + assert dest.read_text() == SYNC_SOURCE + + +def test_mirrored_test_files_requires_sync_counterpart( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Async tests are mirrored only when the sync counterpart exists.""" + async_dir = tmp_path / "test" / "asynchronous" + async_dir.mkdir(parents=True) + (async_dir / "test_a.py").write_text("") + (async_dir / "test_b.py").write_text("") + (tmp_path / "test" / "test_a.py").write_text("") + monkeypatch.setattr(synchro, "_test_dest_base", f"{tmp_path / 'test'}/") + + filtered = [str(async_dir / "test_a.py"), str(async_dir / "test_b.py")] + + assert synchro.mirrored_test_files(filtered) == [f"{tmp_path / 'test' / 'test_a.py'}"] + + +def test_prepare_staging_pins_first_party_packages( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The staged pyproject copy pins the first-party packages for isort.""" + (tmp_path / "pyproject.toml").write_text( + '[tool.ruff.lint]\nisort.required-imports = ["from __future__ import annotations"]\n' + ) + monkeypatch.chdir(tmp_path) + + synchro.prepare_staging() + + staged = (tmp_path / synchro.STAGING / "pyproject.toml").read_text() + assert 'isort.known-first-party = ["pymongo", "gridfs", "bson", "test"]' in staged + assert "isort.required-imports" in staged + + +def test_prepare_staging_warns_when_anchor_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A pyproject without the isort anchor gets no insert and a stderr warning.""" + (tmp_path / "pyproject.toml").write_text("") + monkeypatch.chdir(tmp_path) + + synchro.prepare_staging() + + staged = (tmp_path / synchro.STAGING / "pyproject.toml").read_text() + assert "known-first-party" not in staged + assert "known-first-party" in capsys.readouterr().err From 3120b783937db7d4253a4609044ececa8305ef7b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 06:43:04 -0600 Subject: [PATCH 02/14] address review --- tools/synchro.py | 22 ++++++++++------- tools/test_synchro.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index 0c9fc22f16..6bbf6fcceb 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -348,8 +348,11 @@ def process_ignores(lines: list[str]) -> list[str]: def copy_changed_files(pairs: list[tuple[Path, Path]]) -> list[Path]: """Copy staged files into the working tree when their content differs. - If a write fails midway, destinations already written are restored so - the working tree is left unchanged. + Each destination is written through a temporary sibling that then + atomically replaces it, so a write that fails midway (for example when + the disk fills) leaves that destination untouched. If a later step + fails, destinations already written are restored so the working tree + is left unchanged. """ written: list[tuple[Path, bytes | None]] = [] try: @@ -359,7 +362,12 @@ def copy_changed_files(pairs: list[tuple[Path, Path]]) -> list[Path]: if old == new: continue dest.parent.mkdir(parents=True, exist_ok=True) - dest.write_bytes(new) + tmp = dest.with_name(dest.name + ".synchro-tmp") + try: + tmp.write_bytes(new) + tmp.replace(dest) + finally: + tmp.unlink(missing_ok=True) written.append((dest, old)) except OSError: for dest, old in written: @@ -448,13 +456,11 @@ def main() -> None: filtered_gridfs = [f for f in gridfs_files if not modified_set or f in modified_set] filtered_tests = [f for f in test_files if not modified_set or f in modified_set] - ruff_extra = [] if is_ci else ["--silent"] - # Check async source files for problems before generating sync output. async_sources = filtered_async + filtered_gridfs + filtered_tests if async_sources: subprocess.run( # noqa: S603 - [sys.executable, "-m", "ruff", "check", *async_sources, *ruff_extra], + [sys.executable, "-m", "ruff", "check", *async_sources], check=True, ) @@ -484,11 +490,11 @@ def main() -> None: if staged_names: try: subprocess.run( # noqa: S603 - [sys.executable, "-m", "ruff", "check", *staged_names, "--fix", *ruff_extra], + [sys.executable, "-m", "ruff", "check", *staged_names, "--fix"], check=True, ) subprocess.run( # noqa: S603 - [sys.executable, "-m", "ruff", "format", *staged_names, *ruff_extra], + [sys.executable, "-m", "ruff", "format", *staged_names], check=True, ) except subprocess.CalledProcessError: diff --git a/tools/test_synchro.py b/tools/test_synchro.py index d2e654f7c8..fe847827df 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -103,6 +103,41 @@ def test_copy_changed_files_rolls_back_on_failure(tmp_path: Path) -> None: assert dest_a.read_bytes() == b"old-a" +def test_copy_changed_files_failing_write_leaves_dest_unchanged(tmp_path: Path) -> None: + """A write that fails midway leaves the failing destination unchanged.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + + real_write = Path.write_bytes + + def failing_write(self: Path, data: bytes) -> None: + if self.name.startswith(dest_b.name): + # Simulate ENOSPC: some bytes land before the write fails. + real_write(self, b"ne") + raise OSError(28, "No space left on device") + real_write(self, data) + + with patch.object(Path, "write_bytes", failing_write): + with pytest.raises(OSError, match="No space left on device"): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + # No temporary files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "b_dest.py", + "b_staged.py", + ] + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" @@ -191,6 +226,26 @@ def failing_ruff( assert not (tmp_path / ".synchro-staging").exists() +def test_ruff_commands_keep_diagnostics_visible( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Ruff runs without --silent so fatal failures show their diagnostics.""" + setup_tree(tmp_path, monkeypatch) + + commands: list[list[str]] = [] + + def capture(cmd: list[str], *args: object, **kwargs: object) -> None: + commands.append(cmd) + + with patch("subprocess.run", side_effect=capture): + synchro.main() + + ruff_cmds = [cmd for cmd in commands if "ruff" in cmd] + assert ruff_cmds + for cmd in ruff_cmds: + assert "--silent" not in cmd + + def test_ci_reports_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """In CI, changed generated files are copied so the drift is visible, then fail.""" dest = setup_tree(tmp_path, monkeypatch) From b7ad845c85771d6843fc462299104812af12a93d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 07:22:13 -0600 Subject: [PATCH 03/14] Restore rollback files in reverse to release space first --- tools/synchro.py | 3 ++- tools/test_synchro.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/tools/synchro.py b/tools/synchro.py index 6bbf6fcceb..c2ef0c094c 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -370,7 +370,8 @@ def copy_changed_files(pairs: list[tuple[Path, Path]]) -> list[Path]: tmp.unlink(missing_ok=True) written.append((dest, old)) except OSError: - for dest, old in written: + # Restore in reverse so each restore releases space for the next. + for dest, old in reversed(written): if old is None: dest.unlink(missing_ok=True) else: diff --git a/tools/test_synchro.py b/tools/test_synchro.py index fe847827df..9611d498d1 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -103,6 +103,43 @@ def test_copy_changed_files_rolls_back_on_failure(tmp_path: Path) -> None: assert dest_a.read_bytes() == b"old-a" +def test_copy_changed_files_rollback_restores_in_reverse_order(tmp_path: Path) -> None: + """Rollback frees later allocations first so earlier restores fit.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + staged_c = tmp_path / "c_staged.py" + dest_c = tmp_path / "blocking-dir" / "c_dest.py" + staged_c.write_bytes(b"new-c") + # A file occupies c_dest.py's parent directory, so the mkdir fails. + (tmp_path / "blocking-dir").write_bytes(b"not a directory") + + real_write = Path.write_bytes + restored: list[bytes] = [] + + def tracking_write(self: Path, data: bytes) -> None: + if data == b"old-a" and b"old-b" not in restored: + # Simulate ENOSPC: old-a does not fit until b's larger staged + # content has been released by its own restore. + raise OSError(28, "No space left on device") + real_write(self, data) + if data.startswith(b"old"): + restored.append(data) + + with patch.object(Path, "write_bytes", tracking_write): + with pytest.raises(FileExistsError): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b), (staged_c, dest_c)]) + + assert restored == [b"old-b", b"old-a"] + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + + def test_copy_changed_files_failing_write_leaves_dest_unchanged(tmp_path: Path) -> None: """A write that fails midway leaves the failing destination unchanged.""" staged_a = tmp_path / "a_staged.py" From 570d1c3f78ed12f2e68bfcd78caff4f4b7523033 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 08:13:57 -0600 Subject: [PATCH 04/14] Stage untracked generated outputs and roll back manifest failures --- tools/synchro.py | 58 +++++++++++++++++++++++++----- tools/test_synchro.py | 84 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 129 insertions(+), 13 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index c2ef0c094c..326bfdc7e4 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -345,21 +345,54 @@ def process_ignores(lines: list[str]) -> list[str]: return lines -def copy_changed_files(pairs: list[tuple[Path, Path]]) -> list[Path]: +def git_tracked_files(dests: list[Path]) -> set[Path] | None: + """Return the subset of dests git tracks, or None when git is unusable. + + A ``None`` result (no git, or outside a repository, as in the test + tree) makes callers treat every destination as tracked, which skips + unchanged files as before. + """ + if not dests: + return set() + command = ["git", "ls-files", "--", *(str(dest) for dest in dests)] + try: + result = subprocess.run( # noqa: S603 + command, + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError): + return None + return {Path(line) for line in result.stdout.splitlines()} + + +def copy_changed_files( + pairs: list[tuple[Path, Path]], + manifest: Path | None = None, + tracked: set[Path] | None = None, +) -> list[Path]: """Copy staged files into the working tree when their content differs. Each destination is written through a temporary sibling that then atomically replaces it, so a write that fails midway (for example when the disk fills) leaves that destination untouched. If a later step - fails, destinations already written are restored so the working tree - is left unchanged. + fails, destinations already written are restored, so the working tree + is left unchanged; publishing ``manifest`` inside that same rollback + scope extends the guarantee to manifest write failures. An unchanged + destination that ``tracked`` does not contain is listed in the + manifest without being rewritten, so a newly generated module reaches + the staging hook on its first commit. """ written: list[tuple[Path, bytes | None]] = [] + included: list[Path] = [] try: for staged, dest in pairs: new = staged.read_bytes() old = dest.read_bytes() if dest.is_file() else None if old == new: + if tracked is not None and dest not in tracked: + included.append(dest) continue dest.parent.mkdir(parents=True, exist_ok=True) tmp = dest.with_name(dest.name + ".synchro-tmp") @@ -369,6 +402,10 @@ def copy_changed_files(pairs: list[tuple[Path, Path]]) -> list[Path]: finally: tmp.unlink(missing_ok=True) written.append((dest, old)) + included.append(dest) + if manifest is not None: + names = [f"./{dest.as_posix()}" for dest in included] + manifest.write_text("\n".join(names) + ("\n" if names else "")) except OSError: # Restore in reverse so each restore releases space for the next. for dest, old in reversed(written): @@ -507,11 +544,16 @@ def main() -> None: ) sys.exit(1) + pairs = [ + (Path(staged), Path(name)) + for staged, name in zip(staged_names, generated_names, strict=True) + ] + # In CI the drift itself is the report, so neither the manifest nor + # untracked detection runs and the copied drift is left in place. changed = copy_changed_files( - [ - (Path(staged), Path(name)) - for staged, name in zip(staged_names, generated_names, strict=True) - ] + pairs, + manifest=None if is_ci else Path(MANIFEST), + tracked=None if is_ci else git_tracked_files([dest for _, dest in pairs]), ) changed_names = [f"./{dest.as_posix()}" for dest in changed] @@ -521,8 +563,6 @@ def main() -> None: print(f" {f}") print("Sync files are out of date. Run `just lint --all-files synchro` to regenerate:") sys.exit(1) - - Path(MANIFEST).write_text("\n".join(changed_names) + ("\n" if changed_names else "")) finally: shutil.rmtree(STAGING, ignore_errors=True) diff --git a/tools/test_synchro.py b/tools/test_synchro.py index 9611d498d1..1b5837e23d 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -175,6 +175,56 @@ def failing_write(self: Path, data: bytes) -> None: ] +def test_copy_changed_files_manifests_untracked_unchanged( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An unchanged untracked destination is listed in the manifest, not rewritten.""" + monkeypatch.chdir(tmp_path) + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"same") + dest.write_bytes(b"same") + manifest = tmp_path / "manifest.txt" + before = dest.stat().st_mtime_ns + + changed = synchro.copy_changed_files( + [(staged, Path("dest.py"))], manifest=manifest, tracked=set() + ) + + assert changed == [] + assert manifest.read_text() == "./dest.py\n" + assert dest.read_bytes() == b"same" + assert dest.stat().st_mtime_ns == before + + +def test_copy_changed_files_manifest_failure_rolls_back(tmp_path: Path) -> None: + """A manifest write failure restores destinations already replaced.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + manifest = tmp_path / "manifest.txt" + + real_write_text = Path.write_text + + def failing_write_text( + self: Path, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + if self == manifest: + raise OSError(28, "No space left on device") + return real_write_text(self, data, encoding, errors, newline) + + with patch.object(Path, "write_text", failing_write_text): + with pytest.raises(OSError, match="No space left on device"): + synchro.copy_changed_files([(staged_a, dest_a)], manifest=manifest) + + assert dest_a.read_bytes() == b"old-a" + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" @@ -201,7 +251,10 @@ def test_noop_run_leaves_tree_unchanged(tmp_path: Path, monkeypatch: pytest.Monk dest.write_text(SYNC_SOURCE) before = dest.stat().st_mtime_ns - with patch("subprocess.run"): + with ( + patch("subprocess.run"), + patch("synchro.git_tracked_files", return_value=None), + ): synchro.main() assert dest.read_text() == SYNC_SOURCE @@ -217,7 +270,10 @@ def test_regenerated_file_is_copied_and_manifested( dest = setup_tree(tmp_path, monkeypatch) dest.write_text("_IS_SYNC = True\n# stale\n") - with patch("subprocess.run"): + with ( + patch("subprocess.run"), + patch("synchro.git_tracked_files", return_value=None), + ): synchro.main() assert dest.read_text() == SYNC_SOURCE @@ -233,7 +289,10 @@ def test_async_test_without_counterpart_is_not_mirrored( (tmp_path / "test" / "asynchronous" / "test_bar.py").write_text(ASYNC_SOURCE) monkeypatch.setattr(synchro, "test_files", ["./test/asynchronous/test_bar.py"]) - with patch("subprocess.run"): + with ( + patch("subprocess.run"), + patch("synchro.git_tracked_files", return_value=None), + ): synchro.main() assert not (tmp_path / "test" / "test_bar.py").exists() @@ -274,7 +333,10 @@ def test_ruff_commands_keep_diagnostics_visible( def capture(cmd: list[str], *args: object, **kwargs: object) -> None: commands.append(cmd) - with patch("subprocess.run", side_effect=capture): + with ( + patch("subprocess.run", side_effect=capture), + patch("synchro.git_tracked_files", return_value=None), + ): synchro.main() ruff_cmds = [cmd for cmd in commands if "ruff" in cmd] @@ -313,6 +375,20 @@ def test_mirrored_test_files_requires_sync_counterpart( assert synchro.mirrored_test_files(filtered) == [f"{tmp_path / 'test' / 'test_a.py'}"] +def test_git_tracked_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """git_tracked_files reports tracked paths and None outside a repository.""" + monkeypatch.chdir(tmp_path) + assert synchro.git_tracked_files([Path("a.py")]) is None + + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / "a.py").write_bytes(b"a") + (tmp_path / "b.py").write_bytes(b"b") + # git ls-files reports the index, so staging a.py marks it tracked. + subprocess.run(["git", "add", "a.py"], cwd=tmp_path, check=True) + + assert synchro.git_tracked_files([Path("a.py"), Path("b.py")]) == {Path("a.py")} + + def test_prepare_staging_pins_first_party_packages( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From f5e409e9e1292ab321d29acdafd04f09211a7c3e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 09:30:57 -0600 Subject: [PATCH 05/14] Publish the synchro manifest through a temp sibling --- tools/synchro.py | 7 ++++++- tools/test_synchro.py | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index 326bfdc7e4..d250cbb884 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -405,7 +405,12 @@ def copy_changed_files( included.append(dest) if manifest is not None: names = [f"./{dest.as_posix()}" for dest in included] - manifest.write_text("\n".join(names) + ("\n" if names else "")) + tmp_manifest = manifest.with_name(manifest.name + ".synchro-tmp") + try: + tmp_manifest.write_text("\n".join(names) + ("\n" if names else "")) + tmp_manifest.replace(manifest) + finally: + tmp_manifest.unlink(missing_ok=True) except OSError: # Restore in reverse so each restore releases space for the next. for dest, old in reversed(written): diff --git a/tools/test_synchro.py b/tools/test_synchro.py index 1b5837e23d..400277a60a 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -198,12 +198,13 @@ def test_copy_changed_files_manifests_untracked_unchanged( def test_copy_changed_files_manifest_failure_rolls_back(tmp_path: Path) -> None: - """A manifest write failure restores destinations already replaced.""" + """A manifest write failure restores destinations and leaves the manifest alone.""" staged_a = tmp_path / "a_staged.py" dest_a = tmp_path / "a_dest.py" staged_a.write_bytes(b"new-a") dest_a.write_bytes(b"old-a") manifest = tmp_path / "manifest.txt" + manifest.write_text("stale\n") real_write_text = Path.write_text @@ -214,7 +215,9 @@ def failing_write_text( errors: str | None = None, newline: str | None = None, ) -> int: - if self == manifest: + if self.name.startswith(manifest.name): + # Simulate ENOSPC: partial bytes land before the write fails. + real_write_text(self, data[:4], encoding, errors, newline) raise OSError(28, "No space left on device") return real_write_text(self, data, encoding, errors, newline) @@ -223,6 +226,14 @@ def failing_write_text( synchro.copy_changed_files([(staged_a, dest_a)], manifest=manifest) assert dest_a.read_bytes() == b"old-a" + # Only a successful publication replaces the manifest. + assert manifest.read_text() == "stale\n" + # No temporary files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "manifest.txt", + ] ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" From a75dccfd29afc05372b116560c5aa6411e678ae0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 10:04:05 -0600 Subject: [PATCH 06/14] Restore the working tree even when interrupted --- tools/synchro.py | 10 ++++++---- tools/test_synchro.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index d250cbb884..224d4a6d6f 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -377,9 +377,10 @@ def copy_changed_files( Each destination is written through a temporary sibling that then atomically replaces it, so a write that fails midway (for example when the disk fills) leaves that destination untouched. If a later step - fails, destinations already written are restored, so the working tree - is left unchanged; publishing ``manifest`` inside that same rollback - scope extends the guarantee to manifest write failures. An unchanged + fails or is interrupted, destinations already written are restored, so + the working tree is left unchanged; publishing ``manifest`` inside + that same rollback scope extends the guarantee to manifest write + failures. An unchanged destination that ``tracked`` does not contain is listed in the manifest without being rewritten, so a newly generated module reaches the staging hook on its first commit. @@ -411,8 +412,9 @@ def copy_changed_files( tmp_manifest.replace(manifest) finally: tmp_manifest.unlink(missing_ok=True) - except OSError: + except BaseException: # Restore in reverse so each restore releases space for the next. + # Any exception, including Ctrl-C, must restore the working tree. for dest, old in reversed(written): if old is None: dest.unlink(missing_ok=True) diff --git a/tools/test_synchro.py b/tools/test_synchro.py index 400277a60a..c3575e4845 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -236,6 +236,39 @@ def failing_write_text( ] +def test_copy_changed_files_interrupt_rolls_back(tmp_path: Path) -> None: + """An interrupt mid-publication still restores destinations already replaced.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + + real_write = Path.write_bytes + + def interrupting_write(self: Path, data: bytes) -> None: + if self.name.startswith(dest_b.name): + raise KeyboardInterrupt + real_write(self, data) + + with patch.object(Path, "write_bytes", interrupting_write): + with pytest.raises(KeyboardInterrupt): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + # No temporary files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "b_dest.py", + "b_staged.py", + ] + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" From 03c1127ec1ce730b734e8a43530eb87d0d1c5cdb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 10:30:37 -0600 Subject: [PATCH 07/14] Track rollback entries before replacing destinations --- tools/synchro.py | 31 ++++++++++++++++++++----------- tools/test_synchro.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index 224d4a6d6f..a15e665bed 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -380,12 +380,12 @@ def copy_changed_files( fails or is interrupted, destinations already written are restored, so the working tree is left unchanged; publishing ``manifest`` inside that same rollback scope extends the guarantee to manifest write - failures. An unchanged - destination that ``tracked`` does not contain is listed in the - manifest without being rewritten, so a newly generated module reaches - the staging hook on its first commit. + failures. An unchanged destination that ``tracked`` does not contain + is listed in the manifest without being rewritten, so a newly + generated module reaches the staging hook on its first commit. """ written: list[tuple[Path, bytes | None]] = [] + replaced: list[Path] = [] included: list[Path] = [] try: for staged, dest in pairs: @@ -396,13 +396,15 @@ def copy_changed_files( included.append(dest) continue dest.parent.mkdir(parents=True, exist_ok=True) + # Track before replacing so an interrupt cannot skip the rollback. + written.append((dest, old)) tmp = dest.with_name(dest.name + ".synchro-tmp") try: tmp.write_bytes(new) tmp.replace(dest) finally: tmp.unlink(missing_ok=True) - written.append((dest, old)) + replaced.append(dest) included.append(dest) if manifest is not None: names = [f"./{dest.as_posix()}" for dest in included] @@ -414,14 +416,21 @@ def copy_changed_files( tmp_manifest.unlink(missing_ok=True) except BaseException: # Restore in reverse so each restore releases space for the next. - # Any exception, including Ctrl-C, must restore the working tree. + # Any exception, including Ctrl-C, must restore the working tree, + # and a failing restore must not strand the other destinations. for dest, old in reversed(written): - if old is None: - dest.unlink(missing_ok=True) - else: - dest.write_bytes(old) + try: + if old is None: + dest.unlink(missing_ok=True) + else: + dest.write_bytes(old) + except BaseException as restore_error: + print( + f"Synchro: could not restore {dest}: {restore_error!r}", + file=sys.stderr, + ) raise - return [dest for dest, _ in written] + return replaced def prepare_staging() -> None: diff --git a/tools/test_synchro.py b/tools/test_synchro.py index c3575e4845..6034f7d9ff 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -154,7 +154,7 @@ def test_copy_changed_files_failing_write_leaves_dest_unchanged(tmp_path: Path) real_write = Path.write_bytes def failing_write(self: Path, data: bytes) -> None: - if self.name.startswith(dest_b.name): + if data == b"new-b": # Simulate ENOSPC: some bytes land before the write fails. real_write(self, b"ne") raise OSError(28, "No space left on device") @@ -269,6 +269,33 @@ def interrupting_write(self: Path, data: bytes) -> None: ] +def test_copy_changed_files_interrupt_after_replace_rolls_back(tmp_path: Path) -> None: + """An interrupt just after a replacement still rolls that destination back.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + + real_replace = Path.replace + + def replacing_interrupt(self: Path, target: Path) -> Path: + real_replace(self, target) + if target == dest_b: + raise KeyboardInterrupt + return target + + with patch.object(Path, "replace", replacing_interrupt): + with pytest.raises(KeyboardInterrupt): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" From 5abe39dfae0a82f4acee27306956275bd3225627 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 11:21:53 -0600 Subject: [PATCH 08/14] Back up originals and restore rollback by rename --- tools/synchro.py | 41 ++++++++++++++++++++++++++++++----------- tools/test_synchro.py | 20 ++++++++++++++------ 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index a15e665bed..9883a13000 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -376,15 +376,17 @@ def copy_changed_files( Each destination is written through a temporary sibling that then atomically replaces it, so a write that fails midway (for example when - the disk fills) leaves that destination untouched. If a later step - fails or is interrupted, destinations already written are restored, so - the working tree is left unchanged; publishing ``manifest`` inside - that same rollback scope extends the guarantee to manifest write - failures. An unchanged destination that ``tracked`` does not contain - is listed in the manifest without being rewritten, so a newly - generated module reaches the staging hook on its first commit. + the disk fills) leaves that destination untouched. The original is + hard-linked aside first, so the rollback restores it by rename and + needs no disk space. If a later step fails or is interrupted, + destinations already written are restored, so the working tree is left + unchanged; publishing ``manifest`` inside that same rollback scope + extends the guarantee to manifest write failures. An unchanged + destination that ``tracked`` does not contain is listed in the + manifest without being rewritten, so a newly generated module reaches + the staging hook on its first commit. """ - written: list[tuple[Path, bytes | None]] = [] + written: list[tuple[Path, bytes | None, Path | None]] = [] replaced: list[Path] = [] included: list[Path] = [] try: @@ -396,8 +398,17 @@ def copy_changed_files( included.append(dest) continue dest.parent.mkdir(parents=True, exist_ok=True) + backup: Path | None = None + if old is not None: + backup = dest.with_name(dest.name + ".synchro-backup") + try: + backup.unlink(missing_ok=True) + os.link(dest, backup) + except OSError: + # No hard-link support: fall back to the in-memory restore. + backup = None # Track before replacing so an interrupt cannot skip the rollback. - written.append((dest, old)) + written.append((dest, old, backup)) tmp = dest.with_name(dest.name + ".synchro-tmp") try: tmp.write_bytes(new) @@ -418,9 +429,11 @@ def copy_changed_files( # Restore in reverse so each restore releases space for the next. # Any exception, including Ctrl-C, must restore the working tree, # and a failing restore must not strand the other destinations. - for dest, old in reversed(written): + for dest, old, backup in reversed(written): try: - if old is None: + if backup is not None: + backup.replace(dest) + elif old is None: dest.unlink(missing_ok=True) else: dest.write_bytes(old) @@ -430,6 +443,12 @@ def copy_changed_files( file=sys.stderr, ) raise + finally: + # Drop the backups: consumed by the rollback above, or not needed + # because publication succeeded. + for _, _, backup in written: + if backup is not None: + backup.unlink(missing_ok=True) return replaced diff --git a/tools/test_synchro.py b/tools/test_synchro.py index 6034f7d9ff..f79adf21d7 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -71,6 +71,8 @@ def test_copy_changed_files_updates_differing(tmp_path: Path) -> None: assert changed == [dest] assert dest.read_bytes() == b"new" + # The backup is dropped once publication succeeds. + assert sorted(p.name for p in tmp_path.iterdir()) == ["dest.py", "staged.py"] def test_copy_changed_files_creates_missing_dest(tmp_path: Path) -> None: @@ -131,9 +133,14 @@ def tracking_write(self: Path, data: bytes) -> None: if data.startswith(b"old"): restored.append(data) - with patch.object(Path, "write_bytes", tracking_write): - with pytest.raises(FileExistsError): - synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b), (staged_c, dest_c)]) + # No hard-linkable backups: restores rewrite the files, so the + # space-releasing order above is what the restores depend on. + with patch("os.link", side_effect=OSError(1, "link not supported")): + with patch.object(Path, "write_bytes", tracking_write): + with pytest.raises(FileExistsError): + synchro.copy_changed_files( + [(staged_a, dest_a), (staged_b, dest_b), (staged_c, dest_c)] + ) assert restored == [b"old-b", b"old-a"] assert dest_a.read_bytes() == b"old-a" @@ -154,8 +161,9 @@ def test_copy_changed_files_failing_write_leaves_dest_unchanged(tmp_path: Path) real_write = Path.write_bytes def failing_write(self: Path, data: bytes) -> None: - if data == b"new-b": - # Simulate ENOSPC: some bytes land before the write fails. + if self.name.startswith(dest_b.name): + # Simulate persistent ENOSPC: every write under dest_b's name, + # including a rollback restore, lands partial bytes then fails. real_write(self, b"ne") raise OSError(28, "No space left on device") real_write(self, data) @@ -166,7 +174,7 @@ def failing_write(self: Path, data: bytes) -> None: assert dest_a.read_bytes() == b"old-a" assert dest_b.read_bytes() == b"old-b" - # No temporary files are left behind. + # No temporary or backup files are left behind. assert sorted(p.name for p in tmp_path.iterdir()) == [ "a_dest.py", "a_staged.py", From fcd3662763f72f052ce1e43131eeb2adaed09036 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 12:17:12 -0600 Subject: [PATCH 09/14] Skip rollback for destinations whose replacement never landed --- tools/synchro.py | 13 +++++++++++++ tools/test_synchro.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/tools/synchro.py b/tools/synchro.py index 9883a13000..70d65eefa8 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -413,6 +413,19 @@ def copy_changed_files( try: tmp.write_bytes(new) tmp.replace(dest) + except BaseException: + try: + landed = dest.read_bytes() == new + except OSError: + landed = True # cannot verify; keep the entry and attempt the restore + if not landed: + # The rename is atomic: nothing was published, so the + # destination still holds its original and the rollback + # must not rewrite it. + written.pop() + if backup is not None: + backup.unlink(missing_ok=True) + raise finally: tmp.unlink(missing_ok=True) replaced.append(dest) diff --git a/tools/test_synchro.py b/tools/test_synchro.py index f79adf21d7..485350e355 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -304,6 +304,44 @@ def replacing_interrupt(self: Path, target: Path) -> Path: assert dest_b.read_bytes() == b"old-b" +def test_copy_changed_files_never_published_dest_not_restored(tmp_path: Path) -> None: + """A destination whose replacement never lands is not rewritten by rollback.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + + real_write = Path.write_bytes + + def failing_write(self: Path, data: bytes) -> None: + if self.name.startswith(dest_b.name): + # Simulate persistent ENOSPC: every write under dest_b's name, + # including a rollback restore, lands partial bytes then fails. + real_write(self, b"ne") + raise OSError(28, "No space left on device") + real_write(self, data) + + # No hard-link support: rollback restores would have to rewrite files. + with patch("os.link", side_effect=OSError(1, "link not supported")): + with patch.object(Path, "write_bytes", failing_write): + with pytest.raises(OSError, match="No space left on device"): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + # No temporary or backup files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "b_dest.py", + "b_staged.py", + ] + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" From c2c1dfd08f018a360219dcba5edcae52fd4dfca7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 13:06:01 -0600 Subject: [PATCH 10/14] Remove the manifest on rollback so failed runs stage nothing --- tools/synchro.py | 14 ++++++++++++-- tools/test_synchro.py | 13 ++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index 70d65eefa8..5398af9ba5 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -379,8 +379,9 @@ def copy_changed_files( the disk fills) leaves that destination untouched. The original is hard-linked aside first, so the rollback restores it by rename and needs no disk space. If a later step fails or is interrupted, - destinations already written are restored, so the working tree is left - unchanged; publishing ``manifest`` inside that same rollback scope + destinations already written are restored and the manifest is removed, + so the working tree is left unchanged and nothing is claimed for + staging; publishing ``manifest`` inside that same rollback scope extends the guarantee to manifest write failures. An unchanged destination that ``tracked`` does not contain is listed in the manifest without being rewritten, so a newly generated module reaches @@ -439,6 +440,15 @@ def copy_changed_files( finally: tmp_manifest.unlink(missing_ok=True) except BaseException: + # A failed run claims nothing to stage, whatever happens next. + if manifest is not None: + try: + manifest.unlink(missing_ok=True) + except OSError as unlink_error: + print( + f"Synchro: could not remove {manifest}: {unlink_error!r}", + file=sys.stderr, + ) # Restore in reverse so each restore releases space for the next. # Any exception, including Ctrl-C, must restore the working tree, # and a failing restore must not strand the other destinations. diff --git a/tools/test_synchro.py b/tools/test_synchro.py index 485350e355..e9c95f80e4 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -98,11 +98,15 @@ def test_copy_changed_files_rolls_back_on_failure(tmp_path: Path) -> None: staged_b.write_bytes(b"new-b") # A file occupies b_dest.py's parent directory, so the mkdir fails. (tmp_path / "blocking-dir").write_bytes(b"not a directory") + manifest = tmp_path / "manifest.txt" + manifest.write_text("stale\n") with pytest.raises(FileExistsError): - synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)], manifest=manifest) assert dest_a.read_bytes() == b"old-a" + # A failed run claims nothing to stage. + assert not manifest.exists() def test_copy_changed_files_rollback_restores_in_reverse_order(tmp_path: Path) -> None: @@ -206,7 +210,7 @@ def test_copy_changed_files_manifests_untracked_unchanged( def test_copy_changed_files_manifest_failure_rolls_back(tmp_path: Path) -> None: - """A manifest write failure restores destinations and leaves the manifest alone.""" + """A manifest write failure restores destinations and claims nothing to stage.""" staged_a = tmp_path / "a_staged.py" dest_a = tmp_path / "a_dest.py" staged_a.write_bytes(b"new-a") @@ -234,13 +238,12 @@ def failing_write_text( synchro.copy_changed_files([(staged_a, dest_a)], manifest=manifest) assert dest_a.read_bytes() == b"old-a" - # Only a successful publication replaces the manifest. - assert manifest.read_text() == "stale\n" + # A failed run claims nothing to stage. + assert not manifest.exists() # No temporary files are left behind. assert sorted(p.name for p in tmp_path.iterdir()) == [ "a_dest.py", "a_staged.py", - "manifest.txt", ] From d09e4815f70dc75c2ef34f34d6ae09e6b5c098c9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 13:26:09 -0600 Subject: [PATCH 11/14] Keep failed-restores' backups and clear the manifest at entry --- tools/synchro.py | 32 ++++++++++++--------- tools/test_synchro.py | 67 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index 5398af9ba5..d87fade566 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -367,6 +367,14 @@ def git_tracked_files(dests: list[Path]) -> set[Path] | None: return {Path(line) for line in result.stdout.splitlines()} +def clear_manifest(manifest: Path) -> None: + """Remove the staging manifest so nothing is staged for this run.""" + try: + manifest.unlink(missing_ok=True) + except OSError as unlink_error: + print(f"Synchro: could not remove {manifest}: {unlink_error!r}", file=sys.stderr) + + def copy_changed_files( pairs: list[tuple[Path, Path]], manifest: Path | None = None, @@ -442,13 +450,7 @@ def copy_changed_files( except BaseException: # A failed run claims nothing to stage, whatever happens next. if manifest is not None: - try: - manifest.unlink(missing_ok=True) - except OSError as unlink_error: - print( - f"Synchro: could not remove {manifest}: {unlink_error!r}", - file=sys.stderr, - ) + clear_manifest(manifest) # Restore in reverse so each restore releases space for the next. # Any exception, including Ctrl-C, must restore the working tree, # and a failing restore must not strand the other destinations. @@ -466,12 +468,12 @@ def copy_changed_files( file=sys.stderr, ) raise - finally: - # Drop the backups: consumed by the rollback above, or not needed - # because publication succeeded. - for _, _, backup in written: - if backup is not None: - backup.unlink(missing_ok=True) + # Publication succeeded, so the backups are obsolete. A restore that + # failed above kept its backup as the only copy of the original; it is + # reclaimed losslessly on the next run, before the new backup is linked. + for _, _, backup in written: + if backup is not None: + backup.unlink(missing_ok=True) return replaced @@ -530,6 +532,10 @@ def unasync_directory(files: list[str], src: str, dest: str, replacements: dict[ def main() -> None: is_ci = bool(os.environ.get("CI")) + # Any run that fails before publication must claim nothing to stage, so + # the previous run's manifest is dropped up front; a successful run + # publishes a fresh one below. + clear_manifest(Path(MANIFEST)) modified_files = [f"./{f}" for f in sys.argv[1:]] errored = False for fname in async_files + gridfs_files + test_files: diff --git a/tools/test_synchro.py b/tools/test_synchro.py index e9c95f80e4..db57a190cd 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -345,6 +345,53 @@ def failing_write(self: Path, data: bytes) -> None: ] +def test_copy_changed_files_failed_restore_keeps_backup(tmp_path: Path) -> None: + """A restore that fails keeps its backup as the recoverable copy.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + manifest = tmp_path / "manifest.txt" + manifest.write_text("stale\n") + + real_replace = Path.replace + real_write_text = Path.write_text + + def failing_restore(self: Path, target: Path) -> Path: + if self.name.endswith(".synchro-backup") and target == dest_b: + raise OSError(13, "Permission denied") + return real_replace(self, target) + + def failing_write_text( + self: Path, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + if self.name.startswith(manifest.name): + raise OSError(28, "No space left on device") + return real_write_text(self, data, encoding, errors, newline) + + with ( + patch.object(Path, "replace", failing_restore), + patch.object(Path, "write_text", failing_write_text), + ): + with pytest.raises(OSError, match="No space left on device"): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)], manifest=manifest) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"new-b" + # The failed restore's backup survives as the only copy of the original. + backup_b = tmp_path / "b_dest.py.synchro-backup" + assert backup_b.read_bytes() == b"old-b" + assert not manifest.exists() + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" @@ -424,6 +471,7 @@ def test_ruff_failure_leaves_tree_unchanged( """A ruff failure on generated output exits nonzero without touching the tree.""" dest = setup_tree(tmp_path, monkeypatch) dest.write_text("_IS_SYNC = True\n# stale\n") + (tmp_path / ".synchro-modified").write_text("./pymongo/synchronous/foo.py\n") def failing_ruff( cmd: list[str], *args: object, **kwargs: object @@ -442,6 +490,25 @@ def failing_ruff( assert not (tmp_path / ".synchro-staging").exists() +def test_early_failure_claims_nothing_to_stage( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A failure before publication leaves no manifest to stage.""" + setup_tree(tmp_path, monkeypatch) + (tmp_path / ".synchro-modified").write_text("./pymongo/synchronous/foo.py\n") + + def failing_first_ruff( + cmd: list[str], *args: object, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + raise subprocess.CalledProcessError(1, cmd) + + with patch("subprocess.run", side_effect=failing_first_ruff): + with pytest.raises(subprocess.CalledProcessError): + synchro.main() + + assert not (tmp_path / ".synchro-modified").exists() + + def test_ruff_commands_keep_diagnostics_visible( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 8262b1f0f4d2b160eb584545a7363a075859c637 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 14:09:22 -0600 Subject: [PATCH 12/14] Refuse publication while an unrecovered backup exists --- CONTRIBUTING.md | 4 +++- tools/synchro.py | 9 +++++++++ tools/test_synchro.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 155d2dcba8..35f6f0456f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -619,7 +619,9 @@ In the unlikely scenario that you want to override this behavior, first export ` The `synchro` hook generates the synchronous files in a staging area and copies them into the working tree only when their content changes, so a failed run leaves the working tree unchanged and nothing is staged when there is nothing to regenerate. If the hook reports that ruff failed on the generated output, fix the reported error in the -corresponding `pymongo/asynchronous` source file and run the hook again. +corresponding `pymongo/asynchronous` source file and run the hook again. If the hook refuses to run because a +`.synchro-backup` file holds an unrecovered original from a failed run, inspect that file, restore or delete it +manually, and run the hook again. ## CPU profiling diff --git a/tools/synchro.py b/tools/synchro.py index d87fade566..18a9a64a66 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -410,6 +410,15 @@ def copy_changed_files( backup: Path | None = None if old is not None: backup = dest.with_name(dest.name + ".synchro-backup") + if backup.exists() and not backup.samefile(dest): + # A failed run left this backup holding the only copy of + # the original; refuse to overwrite it. Kept outside the + # try below so the refusal is not swallowed. + raise OSError( + f"Synchro: {backup} holds the unrecovered original of " + f"{dest}; inspect and remove it manually, then re-run " + "the hook." + ) try: backup.unlink(missing_ok=True) os.link(dest, backup) diff --git a/tools/test_synchro.py b/tools/test_synchro.py index db57a190cd..8486aef62d 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import subprocess import sys from pathlib import Path @@ -392,6 +393,37 @@ def failing_write_text( assert not manifest.exists() +def test_copy_changed_files_unrecovered_backup_blocks_publication(tmp_path: Path) -> None: + """An existing backup holding an unrecovered original refuses publication.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"new") + dest.write_bytes(b"current") + # A failed run left this backup holding the only copy of the original. + (tmp_path / "dest.py.synchro-backup").write_bytes(b"original") + + with pytest.raises(OSError, match="unrecovered original"): + synchro.copy_changed_files([(staged, dest)]) + + assert dest.read_bytes() == b"current" + assert (tmp_path / "dest.py.synchro-backup").read_bytes() == b"original" + + +def test_copy_changed_files_clears_stale_twin_backup(tmp_path: Path) -> None: + """A stale backup that still matches the destination is safely replaced.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"new") + dest.write_bytes(b"old") + os.link(dest, tmp_path / "dest.py.synchro-backup") + + changed = synchro.copy_changed_files([(staged, dest)]) + + assert changed == [dest] + assert dest.read_bytes() == b"new" + assert sorted(p.name for p in tmp_path.iterdir()) == ["dest.py", "staged.py"] + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" From 5e02d2573ea4784f06537be8df8eb18351dd713d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 15:48:51 -0600 Subject: [PATCH 13/14] Make backup cleanup best-effort after publication --- .gitignore | 1 + tools/synchro.py | 11 ++++++++++- tools/test_synchro.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b4db6a497a..d04dfa3c09 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ specifications/ results.json .synchro-modified .synchro-staging/ +*.synchro-backup .evergreen/atlas_x509_dev_client_certificate.pem .evergreen/atlas_x509_sfp_client_certificate.pem diff --git a/tools/synchro.py b/tools/synchro.py index 18a9a64a66..96c2ab1048 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -482,7 +482,16 @@ def copy_changed_files( # reclaimed losslessly on the next run, before the new backup is linked. for _, _, backup in written: if backup is not None: - backup.unlink(missing_ok=True) + try: + backup.unlink(missing_ok=True) + except OSError as unlink_error: + # A backup left behind makes the next run refuse publication + # with recovery guidance; a completed publication must not + # fail over the cleanup. + print( + f"Synchro: could not remove {backup}: {unlink_error!r}", + file=sys.stderr, + ) return replaced diff --git a/tools/test_synchro.py b/tools/test_synchro.py index 8486aef62d..386e1aefcd 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -424,6 +424,41 @@ def test_copy_changed_files_clears_stale_twin_backup(tmp_path: Path) -> None: assert sorted(p.name for p in tmp_path.iterdir()) == ["dest.py", "staged.py"] +def test_copy_changed_files_backup_cleanup_failure_is_not_fatal( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A backup that cannot be swept after publication does not fail the run.""" + monkeypatch.chdir(tmp_path) + staged = tmp_path / "staged.py" + dest = Path("dest.py") + staged.write_bytes(b"new") + dest.write_bytes(b"old") + manifest = tmp_path / "manifest.txt" + + real_unlink = Path.unlink + backup_unlinks = 0 + + def failing_unlink(self: Path, missing_ok: bool = False) -> None: + nonlocal backup_unlinks + if self.name.endswith(".synchro-backup"): + backup_unlinks += 1 + if backup_unlinks > 1: + # The first unlink is the stale-backup clear at creation; + # the second is the success sweep. + raise OSError(13, "Permission denied") + real_unlink(self, missing_ok=missing_ok) + + with patch.object(Path, "unlink", failing_unlink): + changed = synchro.copy_changed_files([(staged, dest)], manifest=manifest) + + assert changed == [dest] + assert dest.read_bytes() == b"new" + assert manifest.read_text() == "./dest.py\n" + # The backup remains, with a warning on stderr. + assert (tmp_path / "dest.py.synchro-backup").exists() + assert "could not remove" in capsys.readouterr().err + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" From ea3deb6c98becac186c8b48eed8875f31c8de0f1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Tue, 17 Nov 2026 18:44:47 -0600 Subject: [PATCH 14/14] Restore without a backup through a temp sibling --- tools/synchro.py | 22 ++++++++++++++------ tools/test_synchro.py | 48 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/tools/synchro.py b/tools/synchro.py index 96c2ab1048..f3604a1d6e 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -386,11 +386,13 @@ def copy_changed_files( atomically replaces it, so a write that fails midway (for example when the disk fills) leaves that destination untouched. The original is hard-linked aside first, so the rollback restores it by rename and - needs no disk space. If a later step fails or is interrupted, - destinations already written are restored and the manifest is removed, - so the working tree is left unchanged and nothing is claimed for - staging; publishing ``manifest`` inside that same rollback scope - extends the guarantee to manifest write failures. An unchanged + needs no disk space. Without hard-link support the rollback rewrites + the file through the same temporary sibling, so a restore that fails + leaves the published content in place. If a later step fails or is + interrupted, destinations already written are restored and the manifest + is removed, so the working tree is left unchanged and nothing is + claimed for staging; publishing ``manifest`` inside that same rollback + scope extends the guarantee to manifest write failures. An unchanged destination that ``tracked`` does not contain is listed in the manifest without being rewritten, so a newly generated module reaches the staging hook on its first commit. @@ -470,7 +472,15 @@ def copy_changed_files( elif old is None: dest.unlink(missing_ok=True) else: - dest.write_bytes(old) + # No backup: rewrite the file. The temporary sibling + # keeps a failing restore from truncating the + # destination, which keeps its published content. + tmp_restore = dest.with_name(dest.name + ".synchro-tmp") + try: + tmp_restore.write_bytes(old) + tmp_restore.replace(dest) + finally: + tmp_restore.unlink(missing_ok=True) except BaseException as restore_error: print( f"Synchro: could not restore {dest}: {restore_error!r}", diff --git a/tools/test_synchro.py b/tools/test_synchro.py index 386e1aefcd..3dc84886ca 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -459,6 +459,54 @@ def failing_unlink(self: Path, missing_ok: bool = False) -> None: assert "could not remove" in capsys.readouterr().err +def test_copy_changed_files_failed_fallback_restore_is_clean(tmp_path: Path) -> None: + """A fallback restore that fails leaves the published content in place.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + staged_c = tmp_path / "c_staged.py" + dest_c = tmp_path / "blocking-dir" / "c_dest.py" + staged_c.write_bytes(b"new-c") + # A file occupies c_dest.py's parent directory, so the mkdir fails. + (tmp_path / "blocking-dir").write_bytes(b"not a directory") + + real_write = Path.write_bytes + + def failing_restore(self: Path, data: bytes) -> None: + if data == b"old-b": + # Simulate persistent ENOSPC: the restore lands partial bytes + # then fails. + real_write(self, b"ol") + raise OSError(28, "No space left on device") + real_write(self, data) + + # No hard-link support: restores must rewrite the files. + with patch("os.link", side_effect=OSError(1, "link not supported")): + with patch.object(Path, "write_bytes", failing_restore): + with pytest.raises(FileExistsError): + synchro.copy_changed_files( + [(staged_a, dest_a), (staged_b, dest_b), (staged_c, dest_c)] + ) + + assert dest_a.read_bytes() == b"old-a" + # The failed restore leaves the published content, uncorrupted. + assert dest_b.read_bytes() == b"new-b" + # No temporary or backup files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "b_dest.py", + "b_staged.py", + "blocking-dir", + "c_staged.py", + ] + + ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n"