Add Windows long-path proof

Author: harminius

.github/scripts/long_path_proof.py

@@ -0,0 +1,221 @@
+r"""
+Windows long-path proof for python-api-client.
+
+Builds a project whose absolute paths exceed MAX_PATH (260 chars) and exercises it in three layers:
+  python  - mergin's own file handling (open project, metadata, inspect_files, checksum)
+  geodiff - geodiff operations called with plain paths, exactly as mergin calls them
+  e2e     - get_push_changes(): both layers together must produce a diff for a modified gpkg,
+            and the payload sent to the server must not contain the \\?\ prefix
+
+Meant to run on Windows with OS long-path support OFF (the Windows default). Elsewhere it runs as a
+smoke test of the harness only, since long_path() is a no-op off Windows.
+
+Exit codes: 0 = every layer behaved as expected, 1 = mismatch, 2 = inconclusive (long paths enabled).
+"""
+
+import argparse
+import ctypes
+import json
+import os
+import shutil
+import sys
+import tempfile
+
+PREFIX = "\\\\?\\"
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--code", required=True, help="checkout of python-api-client to test")
+parser.add_argument("--expect-python", choices=["pass", "fail"], default="pass")
+parser.add_argument("--expect-geodiff", choices=["pass", "fail"], default="pass")
+args = parser.parse_args()
+
+code = os.path.abspath(args.code)
+sys.path.insert(0, code)
+import pygeodiff  # noqa: E402  (site-packages; mergin/deps is never present in a fresh checkout)
+from mergin.merginproject import MerginProject  # noqa: E402
+from mergin.utils import generate_checksum  # noqa: E402
+
+ON_WINDOWS = os.name == "nt"
+
+
+def ext(path):
+    """Harness-only prefixing, so fixtures can be created no matter what the code under test does."""
+    if not ON_WINDOWS or path.startswith(PREFIX):
+        return path
+    return PREFIX + os.path.abspath(path)
+
+
+def long_paths_enabled():
+    if not ON_WINDOWS:
+        return None
+    fn = ctypes.WinDLL("ntdll").RtlAreLongPathsEnabled
+    fn.restype = ctypes.c_ubyte
+    return bool(fn())
+
+
+# fixture: every folder name < 255 chars, full paths > 260 chars
+root = tempfile.mkdtemp(prefix="lp_")
+proj = os.path.join(root, "a" * 120, "b" * 120)
+data = os.path.join(code, "mergin", "test", "test_data")
+base_src = os.path.join(data, "base.gpkg")
+modified_src = os.path.join(data, "inserted_1_A.gpkg")
+live = os.path.join(proj, "survey", "points.gpkg")
+basefile = os.path.join(proj, ".mergin", "survey", "points.gpkg")
+os.makedirs(ext(os.path.dirname(live)))
+shutil.copy(ext(modified_src), ext(live))
+
+print(f"python {sys.version.split()[0]} | pygeodiff {pygeodiff.GeoDiff().version()} | code {code}")
+print(f"OS long-path support enabled for this process: {long_paths_enabled()}")
+print(f"longest fixture path: {len(basefile)} chars (MAX_PATH is 260)\n")
+
+results = []
+state = {}
+
+
+def step(layer, name, fn):
+    try:
+        detail = fn() or ""
+        results.append((layer, name, True, detail))
+    except Exception as e:  # noqa: BLE001 - every failure is a result here
+        results.append((layer, name, False, f"{type(e).__name__}: {e}"[:200]))
+
+
+def need_project():
+    if "mp" not in state:
+        raise RuntimeError("skipped: project could not be opened")
+    return state["mp"]
+
+
+def open_project():
+    state["mp"] = MerginProject(proj)
+
+
+def place_basefile():
+    # mergin creates .mergin itself (step above); the harness only drops the basefile in
+    os.makedirs(ext(os.path.dirname(basefile)), exist_ok=True)
+    shutil.copy(ext(base_src), ext(basefile))
+
+
+def write_read_metadata():
+    meta = {
+        "name": "proof",
+        "namespace": "test",
+        "version": "v1",
+        "files": [
+            {
+                "path": "survey/points.gpkg",
+                "checksum": generate_checksum(ext(base_src)),
+                "size": os.path.getsize(ext(base_src)),
+                "mtime": "2026-01-01T00:00:00+00:00",
+                "version": "v1",
+            }
+        ],
+    }
+    MerginProject.write_metadata(proj, meta)
+    assert need_project().metadata["name"] == "proof"
+
+
+def inspect():
+    files = need_project().inspect_files()
+    paths = [f["path"] for f in files]
+    assert paths == ["survey/points.gpkg"], paths
+    return f"paths={paths}"
+
+
+def checksum():
+    assert generate_checksum(live) == generate_checksum(ext(live))
+
+
+def gd_copy():
+    copy = os.path.join(proj, ".mergin", "copy.gpkg")  # inside .mergin so inspect_files ignores it
+    pygeodiff.GeoDiff().make_copy_sqlite(live, copy)
+    assert os.path.exists(ext(copy))
+
+
+def gd_diff():
+    diff = basefile + "-diff-proof"
+    gd = pygeodiff.GeoDiff()
+    gd.create_changeset(basefile, live, diff)
+    assert gd.has_changes(diff), "changeset has no changes"
+
+
+def push_changes():
+    changes = need_project().get_push_changes()
+    updated = [f for f in changes["updated"] if f["path"] == "survey/points.gpkg"]
+    assert updated, f"gpkg not detected as updated: {changes}"
+    assert "diff" in updated[0], "no diff created - would fall back to full upload"
+    leaked = [s for s in strings(changes) if PREFIX in s]
+    assert not leaked, f"prefix leaked into server payload: {leaked}"
+    return f"diff={updated[0]['diff']['path']}"
+
+
+def sparse_write():
+    need_project().write_file_filter({"include": None, "exclude": ["notes/*"]})
+
+
+def sparse_read():
+    # harness writes the filter and a decoy, so this checks the read side on its own:
+    # an unprefixed exists() on Windows returns False instead of raising, silently dropping the filter
+    flt = {"include": None, "exclude": ["notes/*"]}
+    with open(ext(os.path.join(proj, ".mergin", "file_filter.json")), "w") as f:
+        json.dump(flt, f)
+    decoy = os.path.join(proj, "notes", "skip.txt")
+    os.makedirs(ext(os.path.dirname(decoy)), exist_ok=True)
+    with open(ext(decoy), "w") as f:
+        f.write("out of scope")
+    mp = need_project()
+    assert mp.file_filter() == flt, f"filter not read back: {mp.file_filter()}"
+    paths = [f["path"] for f in mp.inspect_files()]
+    assert "notes/skip.txt" not in paths, f"out-of-scope file not filtered: {paths}"
+    assert "survey/points.gpkg" in paths, f"in-scope file missing: {paths}"
+
+
+def strings(obj):
+    if isinstance(obj, str):
+        yield obj
+    elif isinstance(obj, dict):
+        for v in obj.values():
+            yield from strings(v)
+    elif isinstance(obj, (list, tuple)):
+        for v in obj:
+            yield from strings(v)
+
+
+step("python", "open project (dirs + log file)", open_project)
+place_basefile()
+step("python", "write + read mergin.json", write_read_metadata)
+step("python", "inspect_files (walk, size, mtime)", inspect)
+step("python", "generate_checksum", checksum)
+step("geodiff", "make_copy_sqlite", gd_copy)
+step("geodiff", "create_changeset + has_changes", gd_diff)
+step("e2e", "get_push_changes -> diff, no prefix in payload", push_changes)
+step("python", "sparse checkout: write_file_filter", sparse_write)
+step("python", "sparse checkout: filter read back + honoured", sparse_read)
+
+if "mp" in state:
+    state["mp"].remove_logging_handler()  # release the log file so cleanup works on Windows
+shutil.rmtree(ext(root), ignore_errors=True)
+
+layer_ok = {}
+for layer, name, ok, detail in results:
+    layer_ok[layer] = layer_ok.get(layer, True) and ok
+    print(f"  [{'PASS' if ok else 'FAIL'}] {layer:7} {name}" + (f"  -- {detail}" if detail else ""))
+
+expected = {"python": args.expect_python == "pass", "geodiff": args.expect_geodiff == "pass"}
+expected["e2e"] = expected["python"] and expected["geodiff"]
+print()
+mismatch = False
+for layer in ("python", "geodiff", "e2e"):
+    got, want = layer_ok[layer], expected[layer]
+    verdict = "as expected" if (got == want or not ON_WINDOWS) else "UNEXPECTED"
+    mismatch |= verdict == "UNEXPECTED"
+    print(f"{layer:7}: {'pass' if got else 'fail'} (expected {'pass' if want else 'fail'}) -> {verdict}")
+
+if not ON_WINDOWS:
+    ok = all(layer_ok.values())
+    print("\nNot Windows: harness smoke test only (long_path() is a no-op here).", "OK" if ok else "HARNESS BROKEN")
+    sys.exit(0 if ok else 1)
+if long_paths_enabled() and not all(expected.values()):
+    print("\nINCONCLUSIVE: OS long-path support is on, so the 'fail' cases cannot reproduce.")
+    sys.exit(2)
+sys.exit(1 if mismatch else 0)

.github/workflows/long_path_windows.yml

@@ -0,0 +1,46 @@
+name: Windows long-path proof
+
+on:
+  push:
+    branches: [test/long-path-windows]
+  workflow_dispatch:
+
+jobs:
+  proof:
+    name: ${{ matrix.code }} + geodiff ${{ matrix.geodiff }}
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { code: master, geodiff: "2.3.1", expect_python: fail, expect_geodiff: fail }
+          - { code: master, geodiff: "2.3.2", expect_python: fail, expect_geodiff: pass }
+          - { code: pr, geodiff: "2.3.1", expect_python: pass, expect_geodiff: fail }
+          - { code: pr, geodiff: "2.3.2", expect_python: pass, expect_geodiff: pass }
+    steps:
+      - name: Turn OS long-path support off, as on a default Windows install
+        shell: pwsh
+        run: Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name LongPathsEnabled -Value 0 -Type DWord
+
+      - uses: actions/checkout@v4
+        with:
+          path: pr
+
+      - uses: actions/checkout@v4
+        with:
+          ref: master
+          path: master
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: python -m pip install python-dateutil pytz click truststore "pygeodiff==${{ matrix.geodiff }}"
+
+      - name: Run proof
+        run: >
+          python pr/.github/scripts/long_path_proof.py
+          --code ${{ matrix.code }}
+          --expect-python ${{ matrix.expect_python }}
+          --expect-geodiff ${{ matrix.expect_geodiff }}