diff --git a/.gitignore b/.gitignore index c113d3ec21..d04dfa3c09 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,8 @@ expansion.yml specifications/ results.json .synchro-modified +.synchro-staging/ +*.synchro-backup .evergreen/atlas_x509_dev_client_certificate.pem .evergreen/atlas_x509_sfp_client_certificate.pem diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 007b73cfd7..35f6f0456f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -616,10 +616,12 @@ To prevent the `synchro` hook from accidentally overwriting code, it first check of a file is changing and not its async counterpart, and will fail. In the unlikely scenario that you want to override this behavior, first export `OVERRIDE_SYNCHRO_CHECK=1`. -Sometimes, the `synchro` hook will fail and introduce changes many previously unmodified files. This is due to static -Python errors, such as missing imports, incorrect syntax, or other fatal typos. To resolve these issues, -run `pre-commit run --all-files --hook-stage manual ruff` and fix all reported errors before running the `synchro` -hook again. +The `synchro` hook generates the synchronous files in a staging area and copies them into the working tree only when +their content changes, so a failed run leaves the working tree unchanged and nothing is staged when there is nothing +to regenerate. If the hook reports that ruff failed on the generated output, fix the reported error in the +corresponding `pymongo/asynchronous` source file and run the hook again. If the hook refuses to run because a +`.synchro-backup` file holds an unrecovered original from a failed run, inspect that file, restore or delete it +manually, and run the hook again. ## CPU profiling diff --git a/tools/synchro.py b/tools/synchro.py index 7cb68d4ca7..f3604a1d6e 100644 --- a/tools/synchro.py +++ b/tools/synchro.py @@ -22,6 +22,7 @@ import os import re +import shutil import subprocess import sys from pathlib import Path @@ -29,6 +30,7 @@ from unasync import Rule, unasync_files # type: ignore[import-untyped,import-not-found] MANIFEST = ".synchro-modified" +STAGING = ".synchro-staging" replacements = { "AsyncCollection": "Collection", @@ -343,6 +345,206 @@ def process_ignores(lines: list[str]) -> list[str]: return lines +def git_tracked_files(dests: list[Path]) -> set[Path] | None: + """Return the subset of dests git tracks, or None when git is unusable. + + A ``None`` result (no git, or outside a repository, as in the test + tree) makes callers treat every destination as tracked, which skips + unchanged files as before. + """ + if not dests: + return set() + command = ["git", "ls-files", "--", *(str(dest) for dest in dests)] + try: + result = subprocess.run( # noqa: S603 + command, + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError): + return None + return {Path(line) for line in result.stdout.splitlines()} + + +def clear_manifest(manifest: Path) -> None: + """Remove the staging manifest so nothing is staged for this run.""" + try: + manifest.unlink(missing_ok=True) + except OSError as unlink_error: + print(f"Synchro: could not remove {manifest}: {unlink_error!r}", file=sys.stderr) + + +def copy_changed_files( + pairs: list[tuple[Path, Path]], + manifest: Path | None = None, + tracked: set[Path] | None = None, +) -> list[Path]: + """Copy staged files into the working tree when their content differs. + + Each destination is written through a temporary sibling that then + atomically replaces it, so a write that fails midway (for example when + the disk fills) leaves that destination untouched. The original is + hard-linked aside first, so the rollback restores it by rename and + needs no disk space. Without hard-link support the rollback rewrites + the file through the same temporary sibling, so a restore that fails + leaves the published content in place. If a later step fails or is + interrupted, destinations already written are restored and the manifest + is removed, so the working tree is left unchanged and nothing is + claimed for staging; publishing ``manifest`` inside that same rollback + scope extends the guarantee to manifest write failures. An unchanged + destination that ``tracked`` does not contain is listed in the + manifest without being rewritten, so a newly generated module reaches + the staging hook on its first commit. + """ + written: list[tuple[Path, bytes | None, Path | None]] = [] + replaced: list[Path] = [] + included: list[Path] = [] + try: + for staged, dest in pairs: + new = staged.read_bytes() + old = dest.read_bytes() if dest.is_file() else None + if old == new: + if tracked is not None and dest not in tracked: + included.append(dest) + continue + dest.parent.mkdir(parents=True, exist_ok=True) + backup: Path | None = None + if old is not None: + backup = dest.with_name(dest.name + ".synchro-backup") + if backup.exists() and not backup.samefile(dest): + # A failed run left this backup holding the only copy of + # the original; refuse to overwrite it. Kept outside the + # try below so the refusal is not swallowed. + raise OSError( + f"Synchro: {backup} holds the unrecovered original of " + f"{dest}; inspect and remove it manually, then re-run " + "the hook." + ) + try: + backup.unlink(missing_ok=True) + os.link(dest, backup) + except OSError: + # No hard-link support: fall back to the in-memory restore. + backup = None + # Track before replacing so an interrupt cannot skip the rollback. + written.append((dest, old, backup)) + tmp = dest.with_name(dest.name + ".synchro-tmp") + try: + tmp.write_bytes(new) + tmp.replace(dest) + except BaseException: + try: + landed = dest.read_bytes() == new + except OSError: + landed = True # cannot verify; keep the entry and attempt the restore + if not landed: + # The rename is atomic: nothing was published, so the + # destination still holds its original and the rollback + # must not rewrite it. + written.pop() + if backup is not None: + backup.unlink(missing_ok=True) + raise + finally: + tmp.unlink(missing_ok=True) + replaced.append(dest) + included.append(dest) + if manifest is not None: + names = [f"./{dest.as_posix()}" for dest in included] + tmp_manifest = manifest.with_name(manifest.name + ".synchro-tmp") + try: + tmp_manifest.write_text("\n".join(names) + ("\n" if names else "")) + tmp_manifest.replace(manifest) + finally: + tmp_manifest.unlink(missing_ok=True) + except BaseException: + # A failed run claims nothing to stage, whatever happens next. + if manifest is not None: + clear_manifest(manifest) + # Restore in reverse so each restore releases space for the next. + # Any exception, including Ctrl-C, must restore the working tree, + # and a failing restore must not strand the other destinations. + for dest, old, backup in reversed(written): + try: + if backup is not None: + backup.replace(dest) + elif old is None: + dest.unlink(missing_ok=True) + else: + # No backup: rewrite the file. The temporary sibling + # keeps a failing restore from truncating the + # destination, which keeps its published content. + tmp_restore = dest.with_name(dest.name + ".synchro-tmp") + try: + tmp_restore.write_bytes(old) + tmp_restore.replace(dest) + finally: + tmp_restore.unlink(missing_ok=True) + except BaseException as restore_error: + print( + f"Synchro: could not restore {dest}: {restore_error!r}", + file=sys.stderr, + ) + raise + # Publication succeeded, so the backups are obsolete. A restore that + # failed above kept its backup as the only copy of the original; it is + # reclaimed losslessly on the next run, before the new backup is linked. + for _, _, backup in written: + if backup is not None: + try: + backup.unlink(missing_ok=True) + except OSError as unlink_error: + # A backup left behind makes the next run refuse publication + # with recovery guidance; a completed publication must not + # fail over the cleanup. + print( + f"Synchro: could not remove {backup}: {unlink_error!r}", + file=sys.stderr, + ) + return replaced + + +def prepare_staging() -> None: + """Recreate the out-of-tree staging area used to generate sync files. + + A copy of pyproject.toml at the staging root keeps ruff's + per-file-ignores (e.g. ``test/*.py``) and first-party import + classification identical to a run against the repo root. + """ + shutil.rmtree(STAGING, ignore_errors=True) + for base in (_pymongo_dest_base, _gridfs_dest_base, _test_dest_base): + Path(STAGING, base[2:]).mkdir(parents=True, exist_ok=True) + shutil.copyfile("pyproject.toml", f"{STAGING}/pyproject.toml") + config = Path(STAGING, "pyproject.toml") + lines = config.read_text().splitlines(keepends=True) + for i, line in enumerate(lines): + if line.startswith("isort.required-imports"): + lines.insert(i + 1, 'isort.known-first-party = ["pymongo", "gridfs", "bson", "test"]\n') + break + else: + print( + "Synchro: no isort.required-imports line in pyproject.toml, so " + "isort.known-first-party was not added to the staged ruff config and " + "import classification may differ from the repo root.", + file=sys.stderr, + ) + config.write_text("".join(lines)) + + +def mirrored_test_files(filtered_tests: list[str]) -> list[str]: + """Return the generated test paths for the given async test files. + + A test file is mirrored only when its synchronous counterpart already + exists in ``test/``. + """ + return [ + _test_dest_base + Path(f).name + for f in filtered_tests + if (Path(_test_dest_base) / Path(f).name).is_file() + ] + + def unasync_directory(files: list[str], src: str, dest: str, replacements: dict[str, str]) -> None: unasync_files( files, @@ -358,6 +560,10 @@ def unasync_directory(files: list[str], src: str, dest: str, replacements: dict[ def main() -> None: is_ci = bool(os.environ.get("CI")) + # Any run that fails before publication must claim nothing to stage, so + # the previous run's manifest is dropped up front; a successful run + # publishes a fresh one below. + clear_manifest(Path(MANIFEST)) modified_files = [f"./{f}" for f in sys.argv[1:]] errored = False for fname in async_files + gridfs_files + test_files: @@ -380,68 +586,77 @@ def main() -> None: filtered_gridfs = [f for f in gridfs_files if not modified_set or f in modified_set] filtered_tests = [f for f in test_files if not modified_set or f in modified_set] - ruff_extra = [] if is_ci else ["--silent"] - # Check async source files for problems before generating sync output. async_sources = filtered_async + filtered_gridfs + filtered_tests if async_sources: subprocess.run( # noqa: S603 - [sys.executable, "-m", "ruff", "check", *async_sources, *ruff_extra], + [sys.executable, "-m", "ruff", "check", *async_sources], check=True, ) - unasync_directory(filtered_async, _pymongo_base, _pymongo_dest_base, replacements) - unasync_directory(filtered_gridfs, _gridfs_base, _gridfs_dest_base, replacements) - unasync_directory(filtered_tests, _test_base, _test_dest_base, replacements) - - # Derive generated output paths directly from filtered source paths. - generated_pymongo = [_pymongo_dest_base + Path(f).name for f in filtered_async] - generated_gridfs = [_gridfs_dest_base + Path(f).name for f in filtered_gridfs] - generated_tests = [ - _test_dest_base + Path(f).name - for f in filtered_tests - if (Path(_test_dest_base) / Path(f).name).is_file() - ] - - docstring_translate_files = generated_pymongo + generated_gridfs + generated_tests - - process_files( - generated_pymongo + generated_gridfs + generated_tests, - docstring_translate_files, - generated_tests, - ) - - generated_files = generated_pymongo + generated_gridfs + generated_tests - - if is_ci and generated_files: - print(f"Synchro generated {len(generated_files)} file(s):") - for f in generated_files: - print(f" {f}") - - if generated_files: - subprocess.run( # noqa: S603 - [sys.executable, "-m", "ruff", "check", *generated_files, "--fix", *ruff_extra], - check=is_ci, + try: + prepare_staging() + generated_pymongo = [_pymongo_dest_base + Path(f).name for f in filtered_async] + generated_gridfs = [_gridfs_dest_base + Path(f).name for f in filtered_gridfs] + generated_tests = mirrored_test_files(filtered_tests) + generated_names = generated_pymongo + generated_gridfs + generated_tests + staged_pymongo = [f"{STAGING}/{name[2:]}" for name in generated_pymongo] + staged_gridfs = [f"{STAGING}/{name[2:]}" for name in generated_gridfs] + staged_tests = [f"{STAGING}/{name[2:]}" for name in generated_tests] + staged_names = staged_pymongo + staged_gridfs + staged_tests + + unasync_directory( + filtered_async, _pymongo_base, f"{STAGING}/{_pymongo_dest_base[2:]}", replacements ) - subprocess.run( # noqa: S603 - [sys.executable, "-m", "ruff", "format", *generated_files, *ruff_extra], - check=is_ci, + unasync_directory( + filtered_gridfs, _gridfs_base, f"{STAGING}/{_gridfs_dest_base[2:]}", replacements + ) + unasync_directory( + filtered_tests, _test_base, f"{STAGING}/{_test_dest_base[2:]}", replacements ) - if is_ci and generated_files: - result = subprocess.run( # noqa: S603 - ["git", "diff", "--name-only", "--", *generated_files], # noqa: S607 - capture_output=True, - text=True, - check=True, + process_files(staged_names, staged_names, staged_tests) + + if staged_names: + try: + subprocess.run( # noqa: S603 + [sys.executable, "-m", "ruff", "check", *staged_names, "--fix"], + check=True, + ) + subprocess.run( # noqa: S603 + [sys.executable, "-m", "ruff", "format", *staged_names], + check=True, + ) + except subprocess.CalledProcessError: + print( + "Synchro: ruff failed on the generated sync output. Fix the error in " + "the corresponding async source file and run the synchro hook again; " + "the working tree was left unchanged.", + file=sys.stderr, + ) + sys.exit(1) + + pairs = [ + (Path(staged), Path(name)) + for staged, name in zip(staged_names, generated_names, strict=True) + ] + # In CI the drift itself is the report, so neither the manifest nor + # untracked detection runs and the copied drift is left in place. + changed = copy_changed_files( + pairs, + manifest=None if is_ci else Path(MANIFEST), + tracked=None if is_ci else git_tracked_files([dest for _, dest in pairs]), ) - if result.stdout.strip(): - print("Sync files are out of date. Run `just lint --all-files synchro` to regenerate:") - for f in result.stdout.strip().splitlines(): + changed_names = [f"./{dest.as_posix()}" for dest in changed] + + if is_ci and changed_names: + print(f"Synchro generated {len(changed_names)} file(s):") + for f in changed_names: print(f" {f}") + print("Sync files are out of date. Run `just lint --all-files synchro` to regenerate:") sys.exit(1) - - Path(MANIFEST).write_text("\n".join(generated_files) + ("\n" if generated_files else "")) + finally: + shutil.rmtree(STAGING, ignore_errors=True) if __name__ == "__main__": diff --git a/tools/test_synchro.py b/tools/test_synchro.py index e6a1cc3b8f..3dc84886ca 100644 --- a/tools/test_synchro.py +++ b/tools/test_synchro.py @@ -1,17 +1,19 @@ -"""Smoke test for tools/synchro.py.""" +"""Tests for tools/synchro.py.""" from __future__ import annotations +import os +import subprocess import sys from pathlib import Path -from typing import TYPE_CHECKING from unittest.mock import patch -if TYPE_CHECKING: - import pytest +import pytest sys.path.insert(0, str(Path(__file__).parent)) +import synchro + def test_synchro(monkeypatch: pytest.MonkeyPatch) -> None: """Synchro processes only the async files passed as arguments. @@ -31,6 +33,8 @@ def capture(files: list[str], *args: object, **kwargs: object) -> None: with ( patch("synchro.unasync_directory", side_effect=capture), patch("synchro.process_files"), + patch("synchro.prepare_staging"), + patch("synchro.copy_changed_files", return_value=[]), patch("subprocess.run"), patch("pathlib.Path.write_text"), ): @@ -40,3 +44,678 @@ def capture(files: list[str], *args: object, **kwargs: object) -> None: assert "./pymongo/asynchronous/mongo_client.py" in processed assert "./pymongo/asynchronous/collection.py" not in processed + + +def test_copy_changed_files_skips_identical(tmp_path: Path) -> None: + """A destination whose content already matches is neither rewritten nor reported.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"same") + dest.write_bytes(b"same") + before = dest.stat().st_mtime_ns + + changed = synchro.copy_changed_files([(staged, dest)]) + + assert changed == [] + assert dest.read_bytes() == b"same" + assert dest.stat().st_mtime_ns == before + + +def test_copy_changed_files_updates_differing(tmp_path: Path) -> None: + """A destination whose content differs is overwritten and reported.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"new") + dest.write_bytes(b"old") + + changed = synchro.copy_changed_files([(staged, dest)]) + + assert changed == [dest] + assert dest.read_bytes() == b"new" + # The backup is dropped once publication succeeds. + assert sorted(p.name for p in tmp_path.iterdir()) == ["dest.py", "staged.py"] + + +def test_copy_changed_files_creates_missing_dest(tmp_path: Path) -> None: + """A missing destination is created (with parents) and reported.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "subdir" / "dest.py" + staged.write_bytes(b"new") + + changed = synchro.copy_changed_files([(staged, dest)]) + + assert changed == [dest] + assert dest.read_bytes() == b"new" + + +def test_copy_changed_files_rolls_back_on_failure(tmp_path: Path) -> None: + """A failed write restores destinations that were already written.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "blocking-dir" / "b_dest.py" + staged_b.write_bytes(b"new-b") + # A file occupies b_dest.py's parent directory, so the mkdir fails. + (tmp_path / "blocking-dir").write_bytes(b"not a directory") + manifest = tmp_path / "manifest.txt" + manifest.write_text("stale\n") + + with pytest.raises(FileExistsError): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)], manifest=manifest) + + assert dest_a.read_bytes() == b"old-a" + # A failed run claims nothing to stage. + assert not manifest.exists() + + +def test_copy_changed_files_rollback_restores_in_reverse_order(tmp_path: Path) -> None: + """Rollback frees later allocations first so earlier restores fit.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + staged_c = tmp_path / "c_staged.py" + dest_c = tmp_path / "blocking-dir" / "c_dest.py" + staged_c.write_bytes(b"new-c") + # A file occupies c_dest.py's parent directory, so the mkdir fails. + (tmp_path / "blocking-dir").write_bytes(b"not a directory") + + real_write = Path.write_bytes + restored: list[bytes] = [] + + def tracking_write(self: Path, data: bytes) -> None: + if data == b"old-a" and b"old-b" not in restored: + # Simulate ENOSPC: old-a does not fit until b's larger staged + # content has been released by its own restore. + raise OSError(28, "No space left on device") + real_write(self, data) + if data.startswith(b"old"): + restored.append(data) + + # No hard-linkable backups: restores rewrite the files, so the + # space-releasing order above is what the restores depend on. + with patch("os.link", side_effect=OSError(1, "link not supported")): + with patch.object(Path, "write_bytes", tracking_write): + with pytest.raises(FileExistsError): + synchro.copy_changed_files( + [(staged_a, dest_a), (staged_b, dest_b), (staged_c, dest_c)] + ) + + assert restored == [b"old-b", b"old-a"] + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + + +def test_copy_changed_files_failing_write_leaves_dest_unchanged(tmp_path: Path) -> None: + """A write that fails midway leaves the failing destination unchanged.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + + real_write = Path.write_bytes + + def failing_write(self: Path, data: bytes) -> None: + if self.name.startswith(dest_b.name): + # Simulate persistent ENOSPC: every write under dest_b's name, + # including a rollback restore, lands partial bytes then fails. + real_write(self, b"ne") + raise OSError(28, "No space left on device") + real_write(self, data) + + with patch.object(Path, "write_bytes", failing_write): + with pytest.raises(OSError, match="No space left on device"): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + # No temporary or backup files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "b_dest.py", + "b_staged.py", + ] + + +def test_copy_changed_files_manifests_untracked_unchanged( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An unchanged untracked destination is listed in the manifest, not rewritten.""" + monkeypatch.chdir(tmp_path) + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"same") + dest.write_bytes(b"same") + manifest = tmp_path / "manifest.txt" + before = dest.stat().st_mtime_ns + + changed = synchro.copy_changed_files( + [(staged, Path("dest.py"))], manifest=manifest, tracked=set() + ) + + assert changed == [] + assert manifest.read_text() == "./dest.py\n" + assert dest.read_bytes() == b"same" + assert dest.stat().st_mtime_ns == before + + +def test_copy_changed_files_manifest_failure_rolls_back(tmp_path: Path) -> None: + """A manifest write failure restores destinations and claims nothing to stage.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + manifest = tmp_path / "manifest.txt" + manifest.write_text("stale\n") + + real_write_text = Path.write_text + + def failing_write_text( + self: Path, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + if self.name.startswith(manifest.name): + # Simulate ENOSPC: partial bytes land before the write fails. + real_write_text(self, data[:4], encoding, errors, newline) + raise OSError(28, "No space left on device") + return real_write_text(self, data, encoding, errors, newline) + + with patch.object(Path, "write_text", failing_write_text): + with pytest.raises(OSError, match="No space left on device"): + synchro.copy_changed_files([(staged_a, dest_a)], manifest=manifest) + + assert dest_a.read_bytes() == b"old-a" + # A failed run claims nothing to stage. + assert not manifest.exists() + # No temporary files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + ] + + +def test_copy_changed_files_interrupt_rolls_back(tmp_path: Path) -> None: + """An interrupt mid-publication still restores destinations already replaced.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + + real_write = Path.write_bytes + + def interrupting_write(self: Path, data: bytes) -> None: + if self.name.startswith(dest_b.name): + raise KeyboardInterrupt + real_write(self, data) + + with patch.object(Path, "write_bytes", interrupting_write): + with pytest.raises(KeyboardInterrupt): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + # No temporary files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "b_dest.py", + "b_staged.py", + ] + + +def test_copy_changed_files_interrupt_after_replace_rolls_back(tmp_path: Path) -> None: + """An interrupt just after a replacement still rolls that destination back.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + + real_replace = Path.replace + + def replacing_interrupt(self: Path, target: Path) -> Path: + real_replace(self, target) + if target == dest_b: + raise KeyboardInterrupt + return target + + with patch.object(Path, "replace", replacing_interrupt): + with pytest.raises(KeyboardInterrupt): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + + +def test_copy_changed_files_never_published_dest_not_restored(tmp_path: Path) -> None: + """A destination whose replacement never lands is not rewritten by rollback.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + + real_write = Path.write_bytes + + def failing_write(self: Path, data: bytes) -> None: + if self.name.startswith(dest_b.name): + # Simulate persistent ENOSPC: every write under dest_b's name, + # including a rollback restore, lands partial bytes then fails. + real_write(self, b"ne") + raise OSError(28, "No space left on device") + real_write(self, data) + + # No hard-link support: rollback restores would have to rewrite files. + with patch("os.link", side_effect=OSError(1, "link not supported")): + with patch.object(Path, "write_bytes", failing_write): + with pytest.raises(OSError, match="No space left on device"): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)]) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"old-b" + # No temporary or backup files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "b_dest.py", + "b_staged.py", + ] + + +def test_copy_changed_files_failed_restore_keeps_backup(tmp_path: Path) -> None: + """A restore that fails keeps its backup as the recoverable copy.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + manifest = tmp_path / "manifest.txt" + manifest.write_text("stale\n") + + real_replace = Path.replace + real_write_text = Path.write_text + + def failing_restore(self: Path, target: Path) -> Path: + if self.name.endswith(".synchro-backup") and target == dest_b: + raise OSError(13, "Permission denied") + return real_replace(self, target) + + def failing_write_text( + self: Path, + data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: + if self.name.startswith(manifest.name): + raise OSError(28, "No space left on device") + return real_write_text(self, data, encoding, errors, newline) + + with ( + patch.object(Path, "replace", failing_restore), + patch.object(Path, "write_text", failing_write_text), + ): + with pytest.raises(OSError, match="No space left on device"): + synchro.copy_changed_files([(staged_a, dest_a), (staged_b, dest_b)], manifest=manifest) + + assert dest_a.read_bytes() == b"old-a" + assert dest_b.read_bytes() == b"new-b" + # The failed restore's backup survives as the only copy of the original. + backup_b = tmp_path / "b_dest.py.synchro-backup" + assert backup_b.read_bytes() == b"old-b" + assert not manifest.exists() + + +def test_copy_changed_files_unrecovered_backup_blocks_publication(tmp_path: Path) -> None: + """An existing backup holding an unrecovered original refuses publication.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"new") + dest.write_bytes(b"current") + # A failed run left this backup holding the only copy of the original. + (tmp_path / "dest.py.synchro-backup").write_bytes(b"original") + + with pytest.raises(OSError, match="unrecovered original"): + synchro.copy_changed_files([(staged, dest)]) + + assert dest.read_bytes() == b"current" + assert (tmp_path / "dest.py.synchro-backup").read_bytes() == b"original" + + +def test_copy_changed_files_clears_stale_twin_backup(tmp_path: Path) -> None: + """A stale backup that still matches the destination is safely replaced.""" + staged = tmp_path / "staged.py" + dest = tmp_path / "dest.py" + staged.write_bytes(b"new") + dest.write_bytes(b"old") + os.link(dest, tmp_path / "dest.py.synchro-backup") + + changed = synchro.copy_changed_files([(staged, dest)]) + + assert changed == [dest] + assert dest.read_bytes() == b"new" + assert sorted(p.name for p in tmp_path.iterdir()) == ["dest.py", "staged.py"] + + +def test_copy_changed_files_backup_cleanup_failure_is_not_fatal( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A backup that cannot be swept after publication does not fail the run.""" + monkeypatch.chdir(tmp_path) + staged = tmp_path / "staged.py" + dest = Path("dest.py") + staged.write_bytes(b"new") + dest.write_bytes(b"old") + manifest = tmp_path / "manifest.txt" + + real_unlink = Path.unlink + backup_unlinks = 0 + + def failing_unlink(self: Path, missing_ok: bool = False) -> None: + nonlocal backup_unlinks + if self.name.endswith(".synchro-backup"): + backup_unlinks += 1 + if backup_unlinks > 1: + # The first unlink is the stale-backup clear at creation; + # the second is the success sweep. + raise OSError(13, "Permission denied") + real_unlink(self, missing_ok=missing_ok) + + with patch.object(Path, "unlink", failing_unlink): + changed = synchro.copy_changed_files([(staged, dest)], manifest=manifest) + + assert changed == [dest] + assert dest.read_bytes() == b"new" + assert manifest.read_text() == "./dest.py\n" + # The backup remains, with a warning on stderr. + assert (tmp_path / "dest.py.synchro-backup").exists() + assert "could not remove" in capsys.readouterr().err + + +def test_copy_changed_files_failed_fallback_restore_is_clean(tmp_path: Path) -> None: + """A fallback restore that fails leaves the published content in place.""" + staged_a = tmp_path / "a_staged.py" + dest_a = tmp_path / "a_dest.py" + staged_a.write_bytes(b"new-a") + dest_a.write_bytes(b"old-a") + staged_b = tmp_path / "b_staged.py" + dest_b = tmp_path / "b_dest.py" + staged_b.write_bytes(b"new-b") + dest_b.write_bytes(b"old-b") + staged_c = tmp_path / "c_staged.py" + dest_c = tmp_path / "blocking-dir" / "c_dest.py" + staged_c.write_bytes(b"new-c") + # A file occupies c_dest.py's parent directory, so the mkdir fails. + (tmp_path / "blocking-dir").write_bytes(b"not a directory") + + real_write = Path.write_bytes + + def failing_restore(self: Path, data: bytes) -> None: + if data == b"old-b": + # Simulate persistent ENOSPC: the restore lands partial bytes + # then fails. + real_write(self, b"ol") + raise OSError(28, "No space left on device") + real_write(self, data) + + # No hard-link support: restores must rewrite the files. + with patch("os.link", side_effect=OSError(1, "link not supported")): + with patch.object(Path, "write_bytes", failing_restore): + with pytest.raises(FileExistsError): + synchro.copy_changed_files( + [(staged_a, dest_a), (staged_b, dest_b), (staged_c, dest_c)] + ) + + assert dest_a.read_bytes() == b"old-a" + # The failed restore leaves the published content, uncorrupted. + assert dest_b.read_bytes() == b"new-b" + # No temporary or backup files are left behind. + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "a_dest.py", + "a_staged.py", + "b_dest.py", + "b_staged.py", + "blocking-dir", + "c_staged.py", + ] + + +ASYNC_SOURCE = "_IS_SYNC = False\n\n\ndef f() -> None:\n pass\n" +SYNC_SOURCE = "_IS_SYNC = True\n\n\ndef f() -> None:\n pass\n" + + +def setup_tree(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Create a minimal repo-like tree and point synchro at it.""" + (tmp_path / "pymongo" / "asynchronous").mkdir(parents=True) + (tmp_path / "pymongo" / "synchronous").mkdir(parents=True) + (tmp_path / "test" / "asynchronous").mkdir(parents=True) + (tmp_path / "pyproject.toml").write_text("") + (tmp_path / "pymongo" / "asynchronous" / "foo.py").write_text(ASYNC_SOURCE) + monkeypatch.setattr(sys, "argv", ["synchro.py"]) + monkeypatch.setattr(synchro, "async_files", ["./pymongo/asynchronous/foo.py"]) + monkeypatch.setattr(synchro, "gridfs_files", []) + monkeypatch.setattr(synchro, "test_files", []) + monkeypatch.delenv("CI", raising=False) + monkeypatch.chdir(tmp_path) + return tmp_path / "pymongo" / "synchronous" / "foo.py" + + +def test_noop_run_leaves_tree_unchanged(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A run with nothing to regenerate writes no files and stages nothing.""" + dest = setup_tree(tmp_path, monkeypatch) + dest.write_text(SYNC_SOURCE) + before = dest.stat().st_mtime_ns + + with ( + patch("subprocess.run"), + patch("synchro.git_tracked_files", return_value=None), + ): + synchro.main() + + assert dest.read_text() == SYNC_SOURCE + assert dest.stat().st_mtime_ns == before + assert (tmp_path / ".synchro-modified").read_text() == "" + assert not (tmp_path / ".synchro-staging").exists() + + +def test_regenerated_file_is_copied_and_manifested( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Only files whose content changed are copied and listed in the manifest.""" + dest = setup_tree(tmp_path, monkeypatch) + dest.write_text("_IS_SYNC = True\n# stale\n") + + with ( + patch("subprocess.run"), + patch("synchro.git_tracked_files", return_value=None), + ): + synchro.main() + + assert dest.read_text() == SYNC_SOURCE + assert (tmp_path / ".synchro-modified").read_text() == "./pymongo/synchronous/foo.py\n" + assert not (tmp_path / ".synchro-staging").exists() + + +def test_async_test_without_counterpart_is_not_mirrored( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An async test file is mirrored only when test/.py already exists.""" + setup_tree(tmp_path, monkeypatch) + (tmp_path / "test" / "asynchronous" / "test_bar.py").write_text(ASYNC_SOURCE) + monkeypatch.setattr(synchro, "test_files", ["./test/asynchronous/test_bar.py"]) + + with ( + patch("subprocess.run"), + patch("synchro.git_tracked_files", return_value=None), + ): + synchro.main() + + assert not (tmp_path / "test" / "test_bar.py").exists() + + +def test_ruff_failure_leaves_tree_unchanged( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A ruff failure on generated output exits nonzero without touching the tree.""" + dest = setup_tree(tmp_path, monkeypatch) + dest.write_text("_IS_SYNC = True\n# stale\n") + (tmp_path / ".synchro-modified").write_text("./pymongo/synchronous/foo.py\n") + + def failing_ruff( + cmd: list[str], *args: object, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + if "--fix" in cmd: + raise subprocess.CalledProcessError(1, cmd) + return subprocess.CompletedProcess(cmd, 0) + + with patch("subprocess.run", side_effect=failing_ruff): + with pytest.raises(SystemExit) as excinfo: + synchro.main() + + assert excinfo.value.code == 1 + assert dest.read_text() == "_IS_SYNC = True\n# stale\n" + assert not (tmp_path / ".synchro-modified").exists() + assert not (tmp_path / ".synchro-staging").exists() + + +def test_early_failure_claims_nothing_to_stage( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A failure before publication leaves no manifest to stage.""" + setup_tree(tmp_path, monkeypatch) + (tmp_path / ".synchro-modified").write_text("./pymongo/synchronous/foo.py\n") + + def failing_first_ruff( + cmd: list[str], *args: object, **kwargs: object + ) -> subprocess.CompletedProcess[str]: + raise subprocess.CalledProcessError(1, cmd) + + with patch("subprocess.run", side_effect=failing_first_ruff): + with pytest.raises(subprocess.CalledProcessError): + synchro.main() + + assert not (tmp_path / ".synchro-modified").exists() + + +def test_ruff_commands_keep_diagnostics_visible( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Ruff runs without --silent so fatal failures show their diagnostics.""" + setup_tree(tmp_path, monkeypatch) + + commands: list[list[str]] = [] + + def capture(cmd: list[str], *args: object, **kwargs: object) -> None: + commands.append(cmd) + + with ( + patch("subprocess.run", side_effect=capture), + patch("synchro.git_tracked_files", return_value=None), + ): + synchro.main() + + ruff_cmds = [cmd for cmd in commands if "ruff" in cmd] + assert ruff_cmds + for cmd in ruff_cmds: + assert "--silent" not in cmd + + +def test_ci_reports_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """In CI, changed generated files are copied so the drift is visible, then fail.""" + dest = setup_tree(tmp_path, monkeypatch) + dest.write_text("_IS_SYNC = True\n# stale\n") + monkeypatch.setenv("CI", "true") + + with patch("subprocess.run"): + with pytest.raises(SystemExit) as excinfo: + synchro.main() + + assert excinfo.value.code == 1 + assert dest.read_text() == SYNC_SOURCE + + +def test_mirrored_test_files_requires_sync_counterpart( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Async tests are mirrored only when the sync counterpart exists.""" + async_dir = tmp_path / "test" / "asynchronous" + async_dir.mkdir(parents=True) + (async_dir / "test_a.py").write_text("") + (async_dir / "test_b.py").write_text("") + (tmp_path / "test" / "test_a.py").write_text("") + monkeypatch.setattr(synchro, "_test_dest_base", f"{tmp_path / 'test'}/") + + filtered = [str(async_dir / "test_a.py"), str(async_dir / "test_b.py")] + + assert synchro.mirrored_test_files(filtered) == [f"{tmp_path / 'test' / 'test_a.py'}"] + + +def test_git_tracked_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """git_tracked_files reports tracked paths and None outside a repository.""" + monkeypatch.chdir(tmp_path) + assert synchro.git_tracked_files([Path("a.py")]) is None + + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + (tmp_path / "a.py").write_bytes(b"a") + (tmp_path / "b.py").write_bytes(b"b") + # git ls-files reports the index, so staging a.py marks it tracked. + subprocess.run(["git", "add", "a.py"], cwd=tmp_path, check=True) + + assert synchro.git_tracked_files([Path("a.py"), Path("b.py")]) == {Path("a.py")} + + +def test_prepare_staging_pins_first_party_packages( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The staged pyproject copy pins the first-party packages for isort.""" + (tmp_path / "pyproject.toml").write_text( + '[tool.ruff.lint]\nisort.required-imports = ["from __future__ import annotations"]\n' + ) + monkeypatch.chdir(tmp_path) + + synchro.prepare_staging() + + staged = (tmp_path / synchro.STAGING / "pyproject.toml").read_text() + assert 'isort.known-first-party = ["pymongo", "gridfs", "bson", "test"]' in staged + assert "isort.required-imports" in staged + + +def test_prepare_staging_warns_when_anchor_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A pyproject without the isort anchor gets no insert and a stderr warning.""" + (tmp_path / "pyproject.toml").write_text("") + monkeypatch.chdir(tmp_path) + + synchro.prepare_staging() + + staged = (tmp_path / synchro.STAGING / "pyproject.toml").read_text() + assert "known-first-party" not in staged + assert "known-first-party" in capsys.readouterr().err