From 44dca9af1d80ed72383a29283081631849d9ef4e Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 9 Aug 2026 03:39:33 +0000 Subject: [PATCH 01/11] test(venv): expand importlib_metadata_test to verify all files exist and are readable Expand importlib_metadata_test to assert that all files returned by importlib.metadata.files() can be found on disk and read both as binary and UTF-8 text. Update test fixture RECORD files to list installed paths. --- .../whl_with_data1-1.0.dist-info/RECORD | 26 ++----- .../whl_with_data2-1.0.dist-info/RECORD | 19 ++--- tests/venv_site_packages_libs/BUILD.bazel | 1 + .../importlib_metadata_test.py | 71 ++++++++++++++++--- 4 files changed, 75 insertions(+), 42 deletions(-) diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD index 10307c76a0..fe2c012ba1 100644 --- a/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD @@ -1,19 +1,7 @@ -whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt,sha256=123,123 -whl_with_data1-1.0.data/scripts/whl_with_data1_script,sha256=123,123 -whl_with_data1-1.0.data/scripts/whl_script.sh,sha256=123,123 -whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h,sha256=123,123 -whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt,sha256=123,123 -whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt,sha256=123,123 -whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt,sha256=123,123 -whl_with_data1-1.0.data/data/overlap/both.txt,sha256=123,123 -whl_with_data1-1.0.data/data/overlap/data1.txt,sha256=123,123 -whl_with_data1-1.0.data/scripts/overlap/both.sh,sha256=123,123 -whl_with_data1-1.0.data/scripts/overlap/script1.sh,sha256=123,123 -whl_with_data1-1.0.data/headers/overlap/both.h,sha256=123,123 -whl_with_data1-1.0.data/headers/overlap/header1.h,sha256=123,123 -whl_with_data1-1.0.data/scripts/data_overlap.sh,sha256=123,123 -whl_with_data1-1.0.data/data/bin/data_overlap.sh,sha256=123,123 -whl_with_data1-1.0.data/headers/data_overlap.h,sha256=123,123 -whl_with_data1-1.0.data/data/include/data_overlap.h,sha256=123,123 -whl_with_data1-1.0.data/purelib/data_overlap.py,sha256=123,123 -whl_with_data1-1.0.data/data/site-packages/data_overlap.py,sha256=123,123 +data_overlap.py,sha256=123,123 +whl_with_data1/__init__.py,sha256=123,123 +whl_with_data1/data_file.txt,sha256=123,123 +whl_with_data1/platlib_file.txt,sha256=123,123 +whl_with_data1-1.0.dist-info/METADATA,sha256=123,123 +whl_with_data1-1.0.dist-info/WHEEL,sha256=123,123 +whl_with_data1-1.0.dist-info/RECORD,, diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD index 55c70740c8..0beab9c33b 100644 --- a/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD @@ -1,14 +1,7 @@ -whl_with_data2-1.0.data/platlib/whl_with_data2/platlib_file.txt,sha256=123,123 -whl_with_data2-1.0.data/scripts/whl_script.sh,sha256=123,123 -whl_with_data2-1.0.data/headers/whl_with_data2/header_file.h,sha256=123,123 -whl_with_data2-1.0.data/purelib/whl_with_data2/data_file.txt,sha256=123,123 -whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt,sha256=123,123 -whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt,sha256=123,123 -whl_with_data2-1.0.data/data/overlap/both.txt,sha256=123,123 -whl_with_data2-1.0.data/data/overlap/data2.txt,sha256=123,123 -whl_with_data2-1.0.data/scripts/overlap/both.sh,sha256=123,123 -whl_with_data2-1.0.data/scripts/overlap/script2.sh,sha256=123,123 -whl_with_data2-1.0.data/headers/overlap/both.h,sha256=123,123 -whl_with_data2-1.0.data/headers/overlap/header2.h,sha256=123,123 -whl_with_data2-1.0.data/purelib/whl_with_data2/__init__.py,sha256=123,123 +whl_with_data2/platlib_file.txt,sha256=123,123 +whl_with_data2/data_file.txt,sha256=123,123 +whl_with_data2/__init__.py,sha256=123,123 +whl_with_data2-1.0.dist-info/METADATA,sha256=123,123 +whl_with_data2-1.0.dist-info/WHEEL,sha256=123,123 whl_with_data2-1.0.dist-info/entry_points.txt,sha256=123,123 +whl_with_data2-1.0.dist-info/RECORD,, diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel index 6a7b3b9e12..b5f84ae693 100644 --- a/tests/venv_site_packages_libs/BUILD.bazel +++ b/tests/venv_site_packages_libs/BUILD.bazel @@ -97,5 +97,6 @@ py_reconfig_test( venvs_site_packages = "yes", deps = [ "@whl_with_data1//:pkg", + "@whl_with_data2//:pkg", ], ) diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index 963d43b6e0..7f223e9067 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -3,20 +3,71 @@ class ImportlibMetadataTest(unittest.TestCase): - def test_importlib_metadata_files(self): - files = importlib.metadata.files("whl-with-data1") - self.assertIsNotNone(files, "importlib.metadata.files returned None") + def _assert_distribution_files(self, dist_name, expected_files): + files = importlib.metadata.files(dist_name) + self.assertIsNotNone( + files, f"importlib.metadata.files({dist_name!r}) returned None" + ) self.assertGreater( - len(files), 0, "importlib.metadata.files returned empty list" + len(files), + 0, + f"importlib.metadata.files({dist_name!r}) returned empty list", ) - # Verify it contains some expected files. - # The RECORD file lists paths relative to the installation root (site-packages). - # whl_with_data1-1.0.data/purelib/data_overlap.py should be installed as data_overlap.py - # whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt should be whl_with_data1/platlib_file.txt - + posix_paths = [str(f).replace("\\", "/") for f in files] file_names = [f.name for f in files] - self.assertIn("data_overlap.py", file_names) + + for expected in expected_files: + self.assertTrue( + expected in posix_paths or expected in file_names, + f"Expected {expected!r} to be in distribution files: {posix_paths}", + ) + + for f in files: + resolved = f.locate() + self.assertTrue( + resolved.exists(), + f"Expected file {f} (resolved to {resolved}) to exist", + ) + self.assertTrue( + resolved.is_file(), + f"Expected {resolved} to be a regular file", + ) + + # Verify file content can be read both as binary and as text + binary_content = f.read_binary() + self.assertIsInstance(binary_content, bytes) + + text_content = f.read_text(encoding="utf-8") + self.assertIsInstance(text_content, str) + + def test_whl_with_data1_files(self): + self._assert_distribution_files( + "whl-with-data1", + [ + "data_overlap.py", + "whl_with_data1/__init__.py", + "whl_with_data1/data_file.txt", + "whl_with_data1/platlib_file.txt", + "whl_with_data1-1.0.dist-info/METADATA", + "whl_with_data1-1.0.dist-info/WHEEL", + "whl_with_data1-1.0.dist-info/RECORD", + ], + ) + + def test_whl_with_data2_files(self): + self._assert_distribution_files( + "whl-with-data2", + [ + "whl_with_data2/__init__.py", + "whl_with_data2/data_file.txt", + "whl_with_data2/platlib_file.txt", + "whl_with_data2-1.0.dist-info/METADATA", + "whl_with_data2-1.0.dist-info/WHEEL", + "whl_with_data2-1.0.dist-info/RECORD", + "whl_with_data2-1.0.dist-info/entry_points.txt", + ], + ) if __name__ == "__main__": From caeb2092f6b15bb04c93787a82b886ee8ad11885 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 9 Aug 2026 03:43:56 +0000 Subject: [PATCH 02/11] test(venv): update importlib_metadata_test to verify all files are found and read Assert that each file returned by importlib.metadata.files() exists and can be read, reproducing the issue with unrewritten RECORD files. --- .../whl_with_data1-1.0.dist-info/RECORD | 26 ++++++--- .../whl_with_data2-1.0.dist-info/RECORD | 19 +++++-- tests/venv_site_packages_libs/BUILD.bazel | 1 - .../importlib_metadata_test.py | 57 +++---------------- 4 files changed, 40 insertions(+), 63 deletions(-) diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD index fe2c012ba1..10307c76a0 100644 --- a/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD @@ -1,7 +1,19 @@ -data_overlap.py,sha256=123,123 -whl_with_data1/__init__.py,sha256=123,123 -whl_with_data1/data_file.txt,sha256=123,123 -whl_with_data1/platlib_file.txt,sha256=123,123 -whl_with_data1-1.0.dist-info/METADATA,sha256=123,123 -whl_with_data1-1.0.dist-info/WHEEL,sha256=123,123 -whl_with_data1-1.0.dist-info/RECORD,, +whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt,sha256=123,123 +whl_with_data1-1.0.data/scripts/whl_with_data1_script,sha256=123,123 +whl_with_data1-1.0.data/scripts/whl_script.sh,sha256=123,123 +whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h,sha256=123,123 +whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt,sha256=123,123 +whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt,sha256=123,123 +whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt,sha256=123,123 +whl_with_data1-1.0.data/data/overlap/both.txt,sha256=123,123 +whl_with_data1-1.0.data/data/overlap/data1.txt,sha256=123,123 +whl_with_data1-1.0.data/scripts/overlap/both.sh,sha256=123,123 +whl_with_data1-1.0.data/scripts/overlap/script1.sh,sha256=123,123 +whl_with_data1-1.0.data/headers/overlap/both.h,sha256=123,123 +whl_with_data1-1.0.data/headers/overlap/header1.h,sha256=123,123 +whl_with_data1-1.0.data/scripts/data_overlap.sh,sha256=123,123 +whl_with_data1-1.0.data/data/bin/data_overlap.sh,sha256=123,123 +whl_with_data1-1.0.data/headers/data_overlap.h,sha256=123,123 +whl_with_data1-1.0.data/data/include/data_overlap.h,sha256=123,123 +whl_with_data1-1.0.data/purelib/data_overlap.py,sha256=123,123 +whl_with_data1-1.0.data/data/site-packages/data_overlap.py,sha256=123,123 diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD index 0beab9c33b..55c70740c8 100644 --- a/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD @@ -1,7 +1,14 @@ -whl_with_data2/platlib_file.txt,sha256=123,123 -whl_with_data2/data_file.txt,sha256=123,123 -whl_with_data2/__init__.py,sha256=123,123 -whl_with_data2-1.0.dist-info/METADATA,sha256=123,123 -whl_with_data2-1.0.dist-info/WHEEL,sha256=123,123 +whl_with_data2-1.0.data/platlib/whl_with_data2/platlib_file.txt,sha256=123,123 +whl_with_data2-1.0.data/scripts/whl_script.sh,sha256=123,123 +whl_with_data2-1.0.data/headers/whl_with_data2/header_file.h,sha256=123,123 +whl_with_data2-1.0.data/purelib/whl_with_data2/data_file.txt,sha256=123,123 +whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt,sha256=123,123 +whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt,sha256=123,123 +whl_with_data2-1.0.data/data/overlap/both.txt,sha256=123,123 +whl_with_data2-1.0.data/data/overlap/data2.txt,sha256=123,123 +whl_with_data2-1.0.data/scripts/overlap/both.sh,sha256=123,123 +whl_with_data2-1.0.data/scripts/overlap/script2.sh,sha256=123,123 +whl_with_data2-1.0.data/headers/overlap/both.h,sha256=123,123 +whl_with_data2-1.0.data/headers/overlap/header2.h,sha256=123,123 +whl_with_data2-1.0.data/purelib/whl_with_data2/__init__.py,sha256=123,123 whl_with_data2-1.0.dist-info/entry_points.txt,sha256=123,123 -whl_with_data2-1.0.dist-info/RECORD,, diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel index b5f84ae693..6a7b3b9e12 100644 --- a/tests/venv_site_packages_libs/BUILD.bazel +++ b/tests/venv_site_packages_libs/BUILD.bazel @@ -97,6 +97,5 @@ py_reconfig_test( venvs_site_packages = "yes", deps = [ "@whl_with_data1//:pkg", - "@whl_with_data2//:pkg", ], ) diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index 7f223e9067..309dbe3e7f 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -3,26 +3,13 @@ class ImportlibMetadataTest(unittest.TestCase): - def _assert_distribution_files(self, dist_name, expected_files): - files = importlib.metadata.files(dist_name) - self.assertIsNotNone( - files, f"importlib.metadata.files({dist_name!r}) returned None" - ) + def test_importlib_metadata_files(self): + files = importlib.metadata.files("whl-with-data1") + self.assertIsNotNone(files, "importlib.metadata.files returned None") self.assertGreater( - len(files), - 0, - f"importlib.metadata.files({dist_name!r}) returned empty list", + len(files), 0, "importlib.metadata.files returned empty list" ) - posix_paths = [str(f).replace("\\", "/") for f in files] - file_names = [f.name for f in files] - - for expected in expected_files: - self.assertTrue( - expected in posix_paths or expected in file_names, - f"Expected {expected!r} to be in distribution files: {posix_paths}", - ) - for f in files: resolved = f.locate() self.assertTrue( @@ -35,39 +22,11 @@ def _assert_distribution_files(self, dist_name, expected_files): ) # Verify file content can be read both as binary and as text - binary_content = f.read_binary() - self.assertIsInstance(binary_content, bytes) + content = f.read_binary() + self.assertIsNotNone(content) - text_content = f.read_text(encoding="utf-8") - self.assertIsInstance(text_content, str) - - def test_whl_with_data1_files(self): - self._assert_distribution_files( - "whl-with-data1", - [ - "data_overlap.py", - "whl_with_data1/__init__.py", - "whl_with_data1/data_file.txt", - "whl_with_data1/platlib_file.txt", - "whl_with_data1-1.0.dist-info/METADATA", - "whl_with_data1-1.0.dist-info/WHEEL", - "whl_with_data1-1.0.dist-info/RECORD", - ], - ) - - def test_whl_with_data2_files(self): - self._assert_distribution_files( - "whl-with-data2", - [ - "whl_with_data2/__init__.py", - "whl_with_data2/data_file.txt", - "whl_with_data2/platlib_file.txt", - "whl_with_data2-1.0.dist-info/METADATA", - "whl_with_data2-1.0.dist-info/WHEEL", - "whl_with_data2-1.0.dist-info/RECORD", - "whl_with_data2-1.0.dist-info/entry_points.txt", - ], - ) + text = f.read_text(encoding="utf-8") + self.assertIsNotNone(text) if __name__ == "__main__": From 866c9eb06819d6236f9617de10c20586f6836647 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 9 Aug 2026 03:46:02 +0000 Subject: [PATCH 03/11] test(venv): assert exact expected file paths in importlib_metadata_test Verify that the list of files returned by importlib.metadata.files() matches the expected set of paths from the distribution's RECORD. --- .../importlib_metadata_test.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index 309dbe3e7f..e5ae3fe121 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -10,6 +10,30 @@ def test_importlib_metadata_files(self): len(files), 0, "importlib.metadata.files returned empty list" ) + expected_paths = [ + "whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt", + "whl_with_data1-1.0.data/scripts/whl_with_data1_script", + "whl_with_data1-1.0.data/scripts/whl_script.sh", + "whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h", + "whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt", + "whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt", + "whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt", + "whl_with_data1-1.0.data/data/overlap/both.txt", + "whl_with_data1-1.0.data/data/overlap/data1.txt", + "whl_with_data1-1.0.data/scripts/overlap/both.sh", + "whl_with_data1-1.0.data/scripts/overlap/script1.sh", + "whl_with_data1-1.0.data/headers/overlap/both.h", + "whl_with_data1-1.0.data/headers/overlap/header1.h", + "whl_with_data1-1.0.data/scripts/data_overlap.sh", + "whl_with_data1-1.0.data/data/bin/data_overlap.sh", + "whl_with_data1-1.0.data/headers/data_overlap.h", + "whl_with_data1-1.0.data/data/include/data_overlap.h", + "whl_with_data1-1.0.data/purelib/data_overlap.py", + "whl_with_data1-1.0.data/data/site-packages/data_overlap.py", + ] + file_paths = [str(f).replace("\\", "/") for f in files] + self.assertEqual(file_paths, expected_paths) + for f in files: resolved = f.locate() self.assertTrue( From bd0da7ea8f6f47e7418ffa45cc91a5234bc62e46 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 9 Aug 2026 03:57:14 +0000 Subject: [PATCH 04/11] test(venv): sort expected paths in importlib_metadata_test Sort expected_paths list alphabetically in source for improved readability. --- .../importlib_metadata_test.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index e5ae3fe121..f7bff529cf 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -11,27 +11,27 @@ def test_importlib_metadata_files(self): ) expected_paths = [ - "whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt", - "whl_with_data1-1.0.data/scripts/whl_with_data1_script", - "whl_with_data1-1.0.data/scripts/whl_script.sh", - "whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h", - "whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt", - "whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt", - "whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt", + "whl_with_data1-1.0.data/data/bin/data_overlap.sh", + "whl_with_data1-1.0.data/data/include/data_overlap.h", "whl_with_data1-1.0.data/data/overlap/both.txt", "whl_with_data1-1.0.data/data/overlap/data1.txt", - "whl_with_data1-1.0.data/scripts/overlap/both.sh", - "whl_with_data1-1.0.data/scripts/overlap/script1.sh", + "whl_with_data1-1.0.data/data/site-packages/data_overlap.py", + "whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt", + "whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt", + "whl_with_data1-1.0.data/headers/data_overlap.h", "whl_with_data1-1.0.data/headers/overlap/both.h", "whl_with_data1-1.0.data/headers/overlap/header1.h", - "whl_with_data1-1.0.data/scripts/data_overlap.sh", - "whl_with_data1-1.0.data/data/bin/data_overlap.sh", - "whl_with_data1-1.0.data/headers/data_overlap.h", - "whl_with_data1-1.0.data/data/include/data_overlap.h", + "whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h", + "whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt", "whl_with_data1-1.0.data/purelib/data_overlap.py", - "whl_with_data1-1.0.data/data/site-packages/data_overlap.py", + "whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt", + "whl_with_data1-1.0.data/scripts/data_overlap.sh", + "whl_with_data1-1.0.data/scripts/overlap/both.sh", + "whl_with_data1-1.0.data/scripts/overlap/script1.sh", + "whl_with_data1-1.0.data/scripts/whl_script.sh", + "whl_with_data1-1.0.data/scripts/whl_with_data1_script", ] - file_paths = [str(f).replace("\\", "/") for f in files] + file_paths = sorted(str(f).replace("\\", "/") for f in files) self.assertEqual(file_paths, expected_paths) for f in files: From a3995750b1acf0d1b2db6661dbca67548a6f0ba5 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 10 Aug 2026 05:45:10 +0000 Subject: [PATCH 05/11] fix(pypi): rewrite RECORD file entries for extracted .data contents When extracting wheels containing .data directories (purelib, platlib, scripts, headers, data), the files are moved into their respective target scheme directories and .data is deleted. Rewrite .dist-info/RECORD entries so that archive member paths prefixed with .data/ are updated to their installed paths relative to site-packages, allowing tools like importlib.metadata.files() to locate and read them. Also add Starlark unit tests for the RECORD rewriting logic. --- python/private/pypi/whl_extract.bzl | 72 ++++++++++++++ tests/pypi/whl_extract/BUILD.bazel | 3 + tests/pypi/whl_extract/whl_extract_tests.bzl | 93 +++++++++++++++++++ .../importlib_metadata_test.py | 38 ++++---- 4 files changed, 187 insertions(+), 19 deletions(-) create mode 100644 tests/pypi/whl_extract/BUILD.bazel create mode 100644 tests/pypi/whl_extract/whl_extract_tests.bzl diff --git a/python/private/pypi/whl_extract.bzl b/python/private/pypi/whl_extract.bzl index 0d61b9a07b..fd48c5d29f 100644 --- a/python/private/pypi/whl_extract.bzl +++ b/python/private/pypi/whl_extract.bzl @@ -61,9 +61,81 @@ def whl_extract(rctx, *, whl_path, logger): logger.debug(lambda: "Renaming: {} -> {}".format(src, dest)) repo_utils.rename(rctx, src, dest) + _rewrite_record(rctx, dist_info_dir, data_dir.basename) + # Ensure that there is no data dir left rctx.delete(data_dir) +_DATA_PREFIX_REWRITES = { + "data/": "../../../", + "headers/": "../../../include/", + "platlib/": "", + "purelib/": "", + "scripts/": "../../../bin/", +} + +# Visible for testing +def rewrite_record_content(content, data_dir_basename): + """Rewrite RECORD file content to reflect extracted paths of .data contents. + + In a wheel archive, files destined for different installation schemes are + stored under the `{distribution}-{version}.data/` directory (e.g. `purelib`, + `platlib`, `scripts`, `headers`, `data`), and their archive member paths are + recorded in `.dist-info/RECORD` with the `.data/` prefix. + + Per PEP 427 (https://peps.python.org/pep-0427/#the-data-directory) and + PEP 376 (https://peps.python.org/pep-0376/#record), when a wheel is + installed, files in `.data/` are unpacked into their target installation + scheme locations (`purelib` and `platlib` into `site-packages`, `scripts` + into `bin`, `headers` into `include`, and `data` into `data`/sys.prefix), and + the `.data` directory is removed. The `RECORD` file is updated to list the + installed paths relative to the directory containing `.dist-info` (i.e. + `site-packages`). + + Tools such as `importlib.metadata.files()` resolve paths in `RECORD` + relative to `site-packages`. Without rewriting `RECORD`, these tools attempt + to locate files under the deleted `.data/` path and fail. + + Args: + content: {type}`str` The original RECORD file content. + data_dir_basename: {type}`str` The basename of the .data directory + (e.g., "foo-1.0.data"). + + Returns: + {type}`str` The rewritten RECORD file content. + """ + data_prefix = data_dir_basename + "/" + quoted_data_prefix = '"' + data_prefix + + new_lines = [] + for line in content.splitlines(): + if not line: + continue + if line.startswith(data_prefix): + rest = line[len(data_prefix):] + for category, replacement in _DATA_PREFIX_REWRITES.items(): + if rest.startswith(category): + line = replacement + rest[len(category):] + break + elif line.startswith(quoted_data_prefix): + rest = line[len(quoted_data_prefix):] + for category, replacement in _DATA_PREFIX_REWRITES.items(): + if rest.startswith(category): + line = '"' + replacement + rest[len(category):] + break + new_lines.append(line) + + return "\n".join(new_lines) + "\n" + +def _rewrite_record(rctx, dist_info_dir, data_dir_basename): + record_file = dist_info_dir.get_child("RECORD") + if not record_file.exists: + return + + content = rctx.read(record_file) + new_content = rewrite_record_content(content, data_dir_basename) + rctx.file(record_file, new_content) + def merge_trees(src, dest): """Merge src into the destination path. diff --git a/tests/pypi/whl_extract/BUILD.bazel b/tests/pypi/whl_extract/BUILD.bazel new file mode 100644 index 0000000000..9270c75b77 --- /dev/null +++ b/tests/pypi/whl_extract/BUILD.bazel @@ -0,0 +1,3 @@ +load(":whl_extract_tests.bzl", "whl_extract_test_suite") + +whl_extract_test_suite(name = "whl_extract_tests") diff --git a/tests/pypi/whl_extract/whl_extract_tests.bzl b/tests/pypi/whl_extract/whl_extract_tests.bzl new file mode 100644 index 0000000000..5a1eec2b3f --- /dev/null +++ b/tests/pypi/whl_extract/whl_extract_tests.bzl @@ -0,0 +1,93 @@ +"""Tests for whl_extract.""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:whl_extract.bzl", "rewrite_record_content") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_purelib_and_platlib_prefix_stripped(env): + record = """\ +foo-1.0.data/purelib/pkg/__init__.py,sha256=abc,100 +foo-1.0.data/purelib/pkg/module.py,sha256=def,200 +foo-1.0.data/platlib/pkg/_ext.so,sha256=ghi,300 +foo-1.0.dist-info/METADATA,sha256=jkl,400 +foo-1.0.dist-info/RECORD,, +""" + result = rewrite_record_content(record, "foo-1.0.data") + env.expect.that_str(result).equals("""\ +pkg/__init__.py,sha256=abc,100 +pkg/module.py,sha256=def,200 +pkg/_ext.so,sha256=ghi,300 +foo-1.0.dist-info/METADATA,sha256=jkl,400 +foo-1.0.dist-info/RECORD,, +""") + +_tests.append(_test_purelib_and_platlib_prefix_stripped) + +def _test_data_headers_scripts_prefix_rewritten(env): + record = """\ +foo-1.0.data/data/pkg/data.txt,sha256=111,10 +foo-1.0.data/headers/pkg/header.h,sha256=222,20 +foo-1.0.data/scripts/my_script.sh,sha256=333,30 +""" + result = rewrite_record_content(record, "foo-1.0.data") + env.expect.that_str(result).equals("""\ +../../../pkg/data.txt,sha256=111,10 +../../../include/pkg/header.h,sha256=222,20 +../../../bin/my_script.sh,sha256=333,30 +""") + +_tests.append(_test_data_headers_scripts_prefix_rewritten) + +def _test_quoted_paths_preserved(env): + record = """\ +"foo-1.0.data/purelib/pkg/my file.py",sha256=abc,100 +"foo-1.0.data/scripts/my tool",sha256=def,200 +"foo-1.0.data/headers/my header.h",sha256=ghi,300 +"foo-1.0.data/data/my data.txt",sha256=jkl,400 +""" + result = rewrite_record_content(record, "foo-1.0.data") + env.expect.that_str(result).equals("""\ +"pkg/my file.py",sha256=abc,100 +"../../../bin/my tool",sha256=def,200 +"../../../include/my header.h",sha256=ghi,300 +"../../../my data.txt",sha256=jkl,400 +""") + +_tests.append(_test_quoted_paths_preserved) + +def _test_non_data_and_dist_info_entries_unchanged(env): + record = """\ +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/WHEEL,sha256=ccc,70 +foo-1.0.dist-info/RECORD,, +""" + result = rewrite_record_content(record, "foo-1.0.data") + env.expect.that_str(result).equals("""\ +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/WHEEL,sha256=ccc,70 +foo-1.0.dist-info/RECORD,, +""") + +_tests.append(_test_non_data_and_dist_info_entries_unchanged) + +def _test_unrecognized_data_category_preserved(env): + record = """\ +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +""" + result = rewrite_record_content(record, "foo-1.0.data") + env.expect.that_str(result).equals("""\ +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +""") + +_tests.append(_test_unrecognized_data_category_preserved) + +def whl_extract_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index f7bff529cf..f1853d1269 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -11,25 +11,25 @@ def test_importlib_metadata_files(self): ) expected_paths = [ - "whl_with_data1-1.0.data/data/bin/data_overlap.sh", - "whl_with_data1-1.0.data/data/include/data_overlap.h", - "whl_with_data1-1.0.data/data/overlap/both.txt", - "whl_with_data1-1.0.data/data/overlap/data1.txt", - "whl_with_data1-1.0.data/data/site-packages/data_overlap.py", - "whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt", - "whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt", - "whl_with_data1-1.0.data/headers/data_overlap.h", - "whl_with_data1-1.0.data/headers/overlap/both.h", - "whl_with_data1-1.0.data/headers/overlap/header1.h", - "whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h", - "whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt", - "whl_with_data1-1.0.data/purelib/data_overlap.py", - "whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt", - "whl_with_data1-1.0.data/scripts/data_overlap.sh", - "whl_with_data1-1.0.data/scripts/overlap/both.sh", - "whl_with_data1-1.0.data/scripts/overlap/script1.sh", - "whl_with_data1-1.0.data/scripts/whl_script.sh", - "whl_with_data1-1.0.data/scripts/whl_with_data1_script", + "../../../bin/data_overlap.sh", + "../../../bin/data_overlap.sh", + "../../../bin/overlap/both.sh", + "../../../bin/overlap/script1.sh", + "../../../bin/whl_script.sh", + "../../../bin/whl_with_data1_script", + "../../../include/data_overlap.h", + "../../../include/data_overlap.h", + "../../../include/overlap/both.h", + "../../../include/overlap/header1.h", + "../../../include/whl_with_data1/header_file.h", + "../../../overlap/both.txt", + "../../../overlap/data1.txt", + "../../../site-packages/data_overlap.py", + "../../../whl_with_data1/data_data_file.txt", + "../../../whl_with_data1/data_data_file.txt", + "data_overlap.py", + "whl_with_data1/data_file.txt", + "whl_with_data1/platlib_file.txt", ] file_paths = sorted(str(f).replace("\\", "/") for f in files) self.assertEqual(file_paths, expected_paths) From cd2f6e4fff51f1db99b35e28fce2604515e586f6 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 10 Aug 2026 06:15:13 +0000 Subject: [PATCH 06/11] refactor(pypi): unify wheel .data extraction and RECORD rewrite mapping Unify extraction destinations and RECORD path rewrite prefixes into a single _DATA_CATEGORIES mapping in whl_extract.bzl. Also add a news entry fragment for the bug fix in #4025. --- news/4025.fixed.md | 3 ++ python/private/pypi/whl_extract.bzl | 60 ++++++++++++++--------------- 2 files changed, 33 insertions(+), 30 deletions(-) create mode 100644 news/4025.fixed.md diff --git a/news/4025.fixed.md b/news/4025.fixed.md new file mode 100644 index 0000000000..80e81333f6 --- /dev/null +++ b/news/4025.fixed.md @@ -0,0 +1,3 @@ +(pypi) Fixed {obj}`RECORD` file paths for extracted `.data` directory contents +so that {obj}`importlib.metadata.files()` correctly locates installed +distribution files ([#4025](https://github.com/bazel-contrib/rules_python/pull/4025)). diff --git a/python/private/pypi/whl_extract.bzl b/python/private/pypi/whl_extract.bzl index fd48c5d29f..e1491362ff 100644 --- a/python/private/pypi/whl_extract.bzl +++ b/python/private/pypi/whl_extract.bzl @@ -36,20 +36,7 @@ def whl_extract(rctx, *, whl_path, logger): # Get the .dist_info dir name data_dir = dist_info_dir.dirname.get_child(dist_info_dir.basename[:-len(".dist-info")] + ".data") if data_dir.exists: - for prefix, dest_prefix in { - # https://docs.python.org/3/library/sysconfig.html#posix-prefix - # We are taking this from the legacy whl installer config - "data": "data", - "headers": "include", - # In theory there may be directory collisions here, so it would be best to - # merge the paths here. We are doing for quite a few levels deep. What is - # more, this code has to be reasonably efficient because some packages like - # to not put everything to the top level, but to indicate explicitly if - # something is in `platlib` or `purelib` (e.g. libclang wheel). - "platlib": "site-packages", - "purelib": "site-packages", - "scripts": "bin", - }.items(): + for prefix, (dest_prefix, _) in _DATA_CATEGORIES.items(): src = data_dir.get_child(prefix) if not src.exists: # The prefix does not exist in the wheel, we can continue @@ -66,12 +53,23 @@ def whl_extract(rctx, *, whl_path, logger): # Ensure that there is no data dir left rctx.delete(data_dir) -_DATA_PREFIX_REWRITES = { - "data/": "../../../", - "headers/": "../../../include/", - "platlib/": "", - "purelib/": "", - "scripts/": "../../../bin/", +# Mapping of wheel .data categories to their extraction destination (relative to +# repository root) and their installed relative path prefix in RECORD (relative +# to site-packages/, where .dist-info is located in a standard venv layout: +# /lib/pythonX.Y/site-packages/). +# See https://docs.python.org/3/library/sysconfig.html#posix-prefix and +# https://peps.python.org/pep-0427/#the-data-directory. +_DATA_CATEGORIES = { + # category: (repo_dest_dir, record_relative_prefix) + "data": ("data", "../../../"), + "headers": ("include", "../../../include/"), + # In theory there may be directory collisions in platlib/purelib, so it is + # best to merge the paths here. What is more, this code has to be reasonably + # efficient because some packages like to explicitly indicate if something + # is in `platlib` or `purelib` (e.g. libclang wheel). + "platlib": ("site-packages", ""), + "purelib": ("site-packages", ""), + "scripts": ("bin", "../../../bin/"), } # Visible for testing @@ -87,10 +85,10 @@ def rewrite_record_content(content, data_dir_basename): PEP 376 (https://peps.python.org/pep-0376/#record), when a wheel is installed, files in `.data/` are unpacked into their target installation scheme locations (`purelib` and `platlib` into `site-packages`, `scripts` - into `bin`, `headers` into `include`, and `data` into `data`/sys.prefix), and - the `.data` directory is removed. The `RECORD` file is updated to list the - installed paths relative to the directory containing `.dist-info` (i.e. - `site-packages`). + into `bin`, `headers` into `include`, and `data` into `data`/sys.prefix), + and the `.data` directory is removed. The `RECORD` file is updated to list + the installed paths relative to the directory containing `.dist-info` + (i.e. `site-packages`). Tools such as `importlib.metadata.files()` resolve paths in `RECORD` relative to `site-packages`. Without rewriting `RECORD`, these tools attempt @@ -113,15 +111,17 @@ def rewrite_record_content(content, data_dir_basename): continue if line.startswith(data_prefix): rest = line[len(data_prefix):] - for category, replacement in _DATA_PREFIX_REWRITES.items(): - if rest.startswith(category): - line = replacement + rest[len(category):] + for category, (_, replacement) in _DATA_CATEGORIES.items(): + cat_slash = category + "/" + if rest.startswith(cat_slash): + line = replacement + rest[len(cat_slash):] break elif line.startswith(quoted_data_prefix): rest = line[len(quoted_data_prefix):] - for category, replacement in _DATA_PREFIX_REWRITES.items(): - if rest.startswith(category): - line = '"' + replacement + rest[len(category):] + for category, (_, replacement) in _DATA_CATEGORIES.items(): + cat_slash = category + "/" + if rest.startswith(cat_slash): + line = '"' + replacement + rest[len(cat_slash):] break new_lines.append(line) From b7206677d10fca3b5740200e199cc4ccfd15bce4 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 10 Aug 2026 06:15:58 +0000 Subject: [PATCH 07/11] test(venv): handle Windows venv directory depth in importlib_metadata_test On Windows, virtual environments have a 2-level directory depth (Lib/site-packages) compared to POSIX 3-level depth (lib/pythonX.Y/site-packages), so relative paths in RECORD starting with .. cannot resolve via locate() on Windows. Skip locate() on Windows for files outside site-packages. --- .../importlib_metadata_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index f1853d1269..8836e944c4 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -1,4 +1,5 @@ import importlib.metadata +import sys import unittest @@ -34,7 +35,17 @@ def test_importlib_metadata_files(self): file_paths = sorted(str(f).replace("\\", "/") for f in files) self.assertEqual(file_paths, expected_paths) + is_windows = sys.platform == "win32" for f in files: + # On Windows, virtual environments have a 2-level directory depth + # (Lib/site-packages) while POSIX virtual environments have a + # 3-level depth (lib/pythonX.Y/site-packages). Since wheel + # extraction writes POSIX-standard relative paths (../../../) in + # RECORD, files outside site-packages cannot be resolved via + # locate() on Windows. + if is_windows and str(f).startswith(".."): + continue + resolved = f.locate() self.assertTrue( resolved.exists(), From d54bedeb861fcfbff8f4158edd65e8da52542b7d Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 10 Aug 2026 07:37:07 +0000 Subject: [PATCH 08/11] Generate platform-specific RECORD files at build time Virtual environment directory structure depth differs between Windows (`Lib/site-packages`) and POSIX (`lib/pythonX.Y/site-packages`). Because the host platform during repo extraction cannot determine the target platform, RECORD files must be generated per target platform. Move extracted RECORD files to `rewrite-record/` and rewrite relative paths for `.data` directories at build time via `gen_wheel_record`. --- python/private/pypi/BUILD.bazel | 18 ++ python/private/pypi/gen_wheel_record.bzl | 83 +++++++++ python/private/pypi/wheel_record_rewriter.ps1 | 66 +++++++ python/private/pypi/wheel_record_rewriter.sh | 60 ++++++ python/private/pypi/whl_extract.bzl | 92 ++-------- python/private/pypi/whl_library_targets.bzl | 14 ++ tests/pypi/whl_extract/BUILD.bazel | 8 + .../whl_extract/wheel_record_rewriter_test.sh | 74 ++++++++ tests/pypi/whl_extract/whl_extract_tests.bzl | 172 ++++++++++-------- .../whl_library_targets_tests.bzl | 9 +- .../importlib_metadata_test.py | 63 +++---- 11 files changed, 477 insertions(+), 182 deletions(-) create mode 100644 python/private/pypi/gen_wheel_record.bzl create mode 100644 python/private/pypi/wheel_record_rewriter.ps1 create mode 100755 python/private/pypi/wheel_record_rewriter.sh create mode 100755 tests/pypi/whl_extract/wheel_record_rewriter_test.sh diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index bb84ff9280..7b8349e174 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -41,6 +41,15 @@ alias( visibility = ["//visibility:public"], ) +alias( + name = "wheel_record_rewriter", + actual = select({ + "@platforms//os:windows": "wheel_record_rewriter.ps1", + "//conditions:default": "wheel_record_rewriter.sh", + }), + visibility = ["//visibility:public"], +) + exports_files( srcs = ["deps.bzl"], visibility = ["//tools/private/update_deps:__pkg__"], @@ -490,11 +499,20 @@ bzl_library( ], ) +bzl_library( + name = "gen_wheel_record", + srcs = ["gen_wheel_record.bzl"], + deps = [ + "//python/private:common", + ], +) + bzl_library( name = "whl_library_targets", srcs = ["whl_library_targets.bzl"], deps = [ ":env_marker_setting", + ":gen_wheel_record", ":labels", ":namespace_pkgs", ":pep508_deps", diff --git a/python/private/pypi/gen_wheel_record.bzl b/python/private/pypi/gen_wheel_record.bzl new file mode 100644 index 0000000000..e709614b91 --- /dev/null +++ b/python/private/pypi/gen_wheel_record.bzl @@ -0,0 +1,83 @@ +"""Rule for generating platform-specific RECORD files.""" + +load("//python/private:common.bzl", "is_windows_platform") + +def _gen_wheel_record_impl(ctx): + is_windows = is_windows_platform(ctx) + rewriter_file = ctx.files._wheel_record_rewriter[0] + out_files = [] + + for in_file in ctx.files.srcs: + dist_info_name = in_file.dirname.rpartition("/")[2] + if dist_info_name: + if dist_info_name.endswith(".dist-info"): + data_dir_basename = ( + dist_info_name[:-len(".dist-info")] + ".data" + ) + else: + data_dir_basename = dist_info_name + ".data" + out_file = ctx.actions.declare_file( + "site-packages/{}/RECORD".format(dist_info_name), + ) + else: + data_dir_basename = "data" + out_file = ctx.actions.declare_file("site-packages/RECORD") + + out_files.append(out_file) + + action_args = ctx.actions.args() + inputs = depset([in_file, rewriter_file]) + + if rewriter_file.path.endswith(".ps1"): + action_exe = "powershell.exe" + action_args.add_all([ + "-ExecutionPolicy", + "Bypass", + "-NoProfile", + "-File", + rewriter_file, + ]) + else: + action_exe = ( + ctx.attr._wheel_record_rewriter[DefaultInfo].files_to_run + ) + + action_args.add(in_file) + action_args.add(out_file) + action_args.add("windows" if is_windows else "unix") + action_args.add(data_dir_basename) + + ctx.actions.run( + inputs = inputs, + outputs = [out_file], + executable = action_exe, + arguments = [action_args], + mnemonic = "PyGenWheelRecord", + progress_message = "Generating wheel RECORD %{output}", + toolchain = None, + ) + + return [ + DefaultInfo(files = depset(out_files)), + ] + +gen_wheel_record = rule( + implementation = _gen_wheel_record_impl, + attrs = { + "srcs": attr.label_list( + doc = "The original RECORD files to rewrite.", + mandatory = True, + allow_files = True, + ), + "_wheel_record_rewriter": attr.label( + default = "//python/private/pypi:wheel_record_rewriter", + allow_files = True, + cfg = "exec", + ), + "_windows_constraints": attr.label_list( + default = [ + "@platforms//os:windows", + ], + ), + }, +) diff --git a/python/private/pypi/wheel_record_rewriter.ps1 b/python/private/pypi/wheel_record_rewriter.ps1 new file mode 100644 index 0000000000..acf664f833 --- /dev/null +++ b/python/private/pypi/wheel_record_rewriter.ps1 @@ -0,0 +1,66 @@ +[CmdletBinding()] +param( + [Parameter(Position=0, Mandatory=$true)] + [string]$InFile, + + [Parameter(Position=1, Mandatory=$true)] + [string]$OutFile, + + [Parameter(Position=2, Mandatory=$true)] + [string]$TargetOs, + + [Parameter(Position=3, Mandatory=$true)] + [string]$DataDirBasename +) + +$ErrorActionPreference = "Stop" + +$dataPrefix = "$DataDirBasename/" +$quotedDataPrefix = "`"$DataDirBasename/" + +if ($TargetOs -eq "windows") { + $dataRepl = "../../" + $headersRepl = "../../Include/" + $platlibRepl = "" + $purelibRepl = "" + $scriptsRepl = "../../Scripts/" +} else { + $dataRepl = "../../../" + $headersRepl = "../../../include/" + $platlibRepl = "" + $purelibRepl = "" + $scriptsRepl = "../../../bin/" +} + +$lines = Get-Content -Path $InFile +$outLines = [System.Collections.Generic.List[string]]::new() +$Utf8NoBom = New-Object System.Text.UTF8Encoding $False + +foreach ($line in $lines) { + if ($line.StartsWith($quotedDataPrefix)) { + $quote = "`"" + $rest = $line.Substring($quotedDataPrefix.Length) + } elseif ($line.StartsWith($dataPrefix)) { + $quote = "" + $rest = $line.Substring($dataPrefix.Length) + } else { + $outLines.Add($line) + continue + } + + if ($rest.StartsWith("purelib/")) { + $outLines.Add($quote + $purelibRepl + $rest.Substring(8)) + } elseif ($rest.StartsWith("platlib/")) { + $outLines.Add($quote + $platlibRepl + $rest.Substring(8)) + } elseif ($rest.StartsWith("scripts/")) { + $outLines.Add($quote + $scriptsRepl + $rest.Substring(8)) + } elseif ($rest.StartsWith("headers/")) { + $outLines.Add($quote + $headersRepl + $rest.Substring(8)) + } elseif ($rest.StartsWith("data/")) { + $outLines.Add($quote + $dataRepl + $rest.Substring(5)) + } else { + $outLines.Add($line) + } +} + +[System.IO.File]::WriteAllLines($OutFile, $outLines, $Utf8NoBom) diff --git a/python/private/pypi/wheel_record_rewriter.sh b/python/private/pypi/wheel_record_rewriter.sh new file mode 100755 index 0000000000..10189435e1 --- /dev/null +++ b/python/private/pypi/wheel_record_rewriter.sh @@ -0,0 +1,60 @@ +#!/bin/sh +set -eu + +IN="$1" +OUT="$2" +TARGET_OS="$3" +DATA_DIR_BASENAME="$4" + +DATA_PREFIX="${DATA_DIR_BASENAME}/" +QUOTED_DATA_PREFIX="\"${DATA_DIR_BASENAME}/" + +if [ "$TARGET_OS" = "windows" ]; then + DATA_REPL="../../" + HEADERS_REPL="../../Include/" + PLATLIB_REPL="" + PURELIB_REPL="" + SCRIPTS_REPL="../../Scripts/" +else + DATA_REPL="../../../" + HEADERS_REPL="../../../include/" + PLATLIB_REPL="" + PURELIB_REPL="" + SCRIPTS_REPL="../../../bin/" +fi + +awk -v data_prefix="$DATA_PREFIX" \ + -v quoted_data_prefix="$QUOTED_DATA_PREFIX" \ + -v data_repl="$DATA_REPL" \ + -v headers_repl="$HEADERS_REPL" \ + -v platlib_repl="$PLATLIB_REPL" \ + -v purelib_repl="$PURELIB_REPL" \ + -v scripts_repl="$SCRIPTS_REPL" ' +{ + line = $0 + quote = "" + if (substr(line, 1, length(quoted_data_prefix)) == quoted_data_prefix) { + quote = "\"" + rest = substr(line, length(quoted_data_prefix) + 1) + } else if (substr(line, 1, length(data_prefix)) == data_prefix) { + rest = substr(line, length(data_prefix) + 1) + } else { + print line + next + } + + if (substr(rest, 1, 8) == "purelib/") { + print quote purelib_repl substr(rest, 9) + } else if (substr(rest, 1, 8) == "platlib/") { + print quote platlib_repl substr(rest, 9) + } else if (substr(rest, 1, 8) == "scripts/") { + print quote scripts_repl substr(rest, 9) + } else if (substr(rest, 1, 8) == "headers/") { + print quote headers_repl substr(rest, 9) + } else if (substr(rest, 1, 5) == "data/") { + print quote data_repl substr(rest, 6) + } else { + print line + } +} +' "$IN" > "$OUT" diff --git a/python/private/pypi/whl_extract.bzl b/python/private/pypi/whl_extract.bzl index e1491362ff..5dd6d218f8 100644 --- a/python/private/pypi/whl_extract.bzl +++ b/python/private/pypi/whl_extract.bzl @@ -36,7 +36,7 @@ def whl_extract(rctx, *, whl_path, logger): # Get the .dist_info dir name data_dir = dist_info_dir.dirname.get_child(dist_info_dir.basename[:-len(".dist-info")] + ".data") if data_dir.exists: - for prefix, (dest_prefix, _) in _DATA_CATEGORIES.items(): + for prefix, dest_prefix in _DATA_CATEGORIES.items(): src = data_dir.get_child(prefix) if not src.exists: # The prefix does not exist in the wheel, we can continue @@ -48,94 +48,32 @@ def whl_extract(rctx, *, whl_path, logger): logger.debug(lambda: "Renaming: {} -> {}".format(src, dest)) repo_utils.rename(rctx, src, dest) - _rewrite_record(rctx, dist_info_dir, data_dir.basename) + # Move RECORD to rewrite-record so gen_wheel_record can generate + # the platform-specific RECORD file at build time. + record_file = dist_info_dir.get_child("RECORD") + if record_file.exists: + rewrite_record_dir = rctx.path("rewrite-record/" + dist_info_dir.basename) + repo_utils.mkdir(rctx, rewrite_record_dir) + repo_utils.rename(rctx, record_file, rewrite_record_dir.get_child("RECORD")) # Ensure that there is no data dir left rctx.delete(data_dir) # Mapping of wheel .data categories to their extraction destination (relative to -# repository root) and their installed relative path prefix in RECORD (relative -# to site-packages/, where .dist-info is located in a standard venv layout: -# /lib/pythonX.Y/site-packages/). -# See https://docs.python.org/3/library/sysconfig.html#posix-prefix and -# https://peps.python.org/pep-0427/#the-data-directory. +# repository root). _DATA_CATEGORIES = { - # category: (repo_dest_dir, record_relative_prefix) - "data": ("data", "../../../"), - "headers": ("include", "../../../include/"), + # category: repo_dest_dir + "data": "data", + "headers": "include", # In theory there may be directory collisions in platlib/purelib, so it is # best to merge the paths here. What is more, this code has to be reasonably # efficient because some packages like to explicitly indicate if something # is in `platlib` or `purelib` (e.g. libclang wheel). - "platlib": ("site-packages", ""), - "purelib": ("site-packages", ""), - "scripts": ("bin", "../../../bin/"), + "platlib": "site-packages", + "purelib": "site-packages", + "scripts": "bin", } -# Visible for testing -def rewrite_record_content(content, data_dir_basename): - """Rewrite RECORD file content to reflect extracted paths of .data contents. - - In a wheel archive, files destined for different installation schemes are - stored under the `{distribution}-{version}.data/` directory (e.g. `purelib`, - `platlib`, `scripts`, `headers`, `data`), and their archive member paths are - recorded in `.dist-info/RECORD` with the `.data/` prefix. - - Per PEP 427 (https://peps.python.org/pep-0427/#the-data-directory) and - PEP 376 (https://peps.python.org/pep-0376/#record), when a wheel is - installed, files in `.data/` are unpacked into their target installation - scheme locations (`purelib` and `platlib` into `site-packages`, `scripts` - into `bin`, `headers` into `include`, and `data` into `data`/sys.prefix), - and the `.data` directory is removed. The `RECORD` file is updated to list - the installed paths relative to the directory containing `.dist-info` - (i.e. `site-packages`). - - Tools such as `importlib.metadata.files()` resolve paths in `RECORD` - relative to `site-packages`. Without rewriting `RECORD`, these tools attempt - to locate files under the deleted `.data/` path and fail. - - Args: - content: {type}`str` The original RECORD file content. - data_dir_basename: {type}`str` The basename of the .data directory - (e.g., "foo-1.0.data"). - - Returns: - {type}`str` The rewritten RECORD file content. - """ - data_prefix = data_dir_basename + "/" - quoted_data_prefix = '"' + data_prefix - - new_lines = [] - for line in content.splitlines(): - if not line: - continue - if line.startswith(data_prefix): - rest = line[len(data_prefix):] - for category, (_, replacement) in _DATA_CATEGORIES.items(): - cat_slash = category + "/" - if rest.startswith(cat_slash): - line = replacement + rest[len(cat_slash):] - break - elif line.startswith(quoted_data_prefix): - rest = line[len(quoted_data_prefix):] - for category, (_, replacement) in _DATA_CATEGORIES.items(): - cat_slash = category + "/" - if rest.startswith(cat_slash): - line = '"' + replacement + rest[len(cat_slash):] - break - new_lines.append(line) - - return "\n".join(new_lines) + "\n" - -def _rewrite_record(rctx, dist_info_dir, data_dir_basename): - record_file = dist_info_dir.get_child("RECORD") - if not record_file.exists: - return - - content = rctx.read(record_file) - new_content = rewrite_record_content(content, data_dir_basename) - rctx.file(record_file, new_content) - def merge_trees(src, dest): """Merge src into the destination path. diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index ee5c781b3d..b7fdbd55e9 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -19,6 +19,7 @@ load("//python:py_binary.bzl", "py_binary") load("//python:py_library.bzl", "py_library") load("//python/private:normalize_name.bzl", "normalize_name") load(":env_marker_setting.bzl", "env_marker_setting") +load(":gen_wheel_record.bzl", "gen_wheel_record") load( ":labels.bzl", "DATA_LABEL", @@ -159,6 +160,7 @@ def whl_library_srcs( py_library = py_library, venv_entry_point = venv_entry_point, venv_rewrite_shebang = venv_rewrite_shebang, + gen_wheel_record = gen_wheel_record, env_marker_setting = env_marker_setting, create_inits = _create_inits, )): @@ -225,6 +227,16 @@ def whl_library_srcs( bins_for_data_label.append(rewrite_target_name) data.append(rewrite_target_name) + record_srcs = native.glob(["rewrite-record/*/RECORD"], allow_empty = True) + record_target_name = "record" + if record_srcs: + rules.gen_wheel_record( + name = record_target_name, + srcs = record_srcs, + tags = ["manual"], + ) + data.append(record_target_name) + if filegroups == None: filegroups = { EXTRACTED_WHEEL_FILES: dict( @@ -248,6 +260,8 @@ def whl_library_srcs( srcs = native.glob(**glob_kwargs) if filegroup_name == DATA_LABEL: srcs = srcs + bins_for_data_label + if filegroup_name == DIST_INFO_LABEL and record_srcs: + srcs = srcs + [record_target_name] native.filegroup( name = filegroup_name, srcs = srcs, diff --git a/tests/pypi/whl_extract/BUILD.bazel b/tests/pypi/whl_extract/BUILD.bazel index 9270c75b77..3983477f2b 100644 --- a/tests/pypi/whl_extract/BUILD.bazel +++ b/tests/pypi/whl_extract/BUILD.bazel @@ -1,3 +1,11 @@ +load("@rules_shell//shell:sh_test.bzl", "sh_test") load(":whl_extract_tests.bzl", "whl_extract_test_suite") whl_extract_test_suite(name = "whl_extract_tests") + +sh_test( + name = "wheel_record_rewriter_test", + srcs = ["wheel_record_rewriter_test.sh"], + args = ["$(location //python/private/pypi:wheel_record_rewriter)"], + data = ["//python/private/pypi:wheel_record_rewriter"], +) diff --git a/tests/pypi/whl_extract/wheel_record_rewriter_test.sh b/tests/pypi/whl_extract/wheel_record_rewriter_test.sh new file mode 100755 index 0000000000..fc0f78cd92 --- /dev/null +++ b/tests/pypi/whl_extract/wheel_record_rewriter_test.sh @@ -0,0 +1,74 @@ +#!/bin/sh +set -eu + +REWRITER="$1" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +INPUT="$TMP_DIR/input_RECORD" +cat <<'EOF' > "$INPUT" +foo-1.0.data/purelib/pkg/__init__.py,sha256=abc,100 +foo-1.0.data/purelib/pkg/module.py,sha256=def,200 +foo-1.0.data/platlib/pkg/_ext.so,sha256=ghi,300 +foo-1.0.data/data/pkg/data.txt,sha256=111,10 +foo-1.0.data/headers/pkg/header.h,sha256=222,20 +foo-1.0.data/scripts/my_script.sh,sha256=333,30 +"foo-1.0.data/purelib/pkg/my file.py",sha256=abc,100 +"foo-1.0.data/scripts/my tool",sha256=def,200 +"foo-1.0.data/headers/my header.h",sha256=ghi,300 +"foo-1.0.data/data/my data.txt",sha256=jkl,400 +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/RECORD,, +EOF + +# Test Unix rewrite +UNIX_OUT="$TMP_DIR/unix_RECORD" +"$REWRITER" "$INPUT" "$UNIX_OUT" "unix" "foo-1.0.data" + +EXPECTED_UNIX="$TMP_DIR/expected_unix" +cat <<'EOF' > "$EXPECTED_UNIX" +pkg/__init__.py,sha256=abc,100 +pkg/module.py,sha256=def,200 +pkg/_ext.so,sha256=ghi,300 +../../../pkg/data.txt,sha256=111,10 +../../../include/pkg/header.h,sha256=222,20 +../../../bin/my_script.sh,sha256=333,30 +"pkg/my file.py",sha256=abc,100 +"../../../bin/my tool",sha256=def,200 +"../../../include/my header.h",sha256=ghi,300 +"../../../my data.txt",sha256=jkl,400 +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/RECORD,, +EOF + +diff -u "$EXPECTED_UNIX" "$UNIX_OUT" + +# Test Windows rewrite +WIN_OUT="$TMP_DIR/win_RECORD" +"$REWRITER" "$INPUT" "$WIN_OUT" "windows" "foo-1.0.data" + +EXPECTED_WIN="$TMP_DIR/expected_win" +cat <<'EOF' > "$EXPECTED_WIN" +pkg/__init__.py,sha256=abc,100 +pkg/module.py,sha256=def,200 +pkg/_ext.so,sha256=ghi,300 +../../pkg/data.txt,sha256=111,10 +../../Include/pkg/header.h,sha256=222,20 +../../Scripts/my_script.sh,sha256=333,30 +"pkg/my file.py",sha256=abc,100 +"../../Scripts/my tool",sha256=def,200 +"../../Include/my header.h",sha256=ghi,300 +"../../my data.txt",sha256=jkl,400 +foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 +top_level/__init__.py,sha256=aaa,50 +foo-1.0.dist-info/METADATA,sha256=bbb,60 +foo-1.0.dist-info/RECORD,, +EOF + +diff -u "$EXPECTED_WIN" "$WIN_OUT" + +echo "All rewriter assertions passed!" diff --git a/tests/pypi/whl_extract/whl_extract_tests.bzl b/tests/pypi/whl_extract/whl_extract_tests.bzl index 5a1eec2b3f..f7e1e6d2cf 100644 --- a/tests/pypi/whl_extract/whl_extract_tests.bzl +++ b/tests/pypi/whl_extract/whl_extract_tests.bzl @@ -1,88 +1,114 @@ -"""Tests for whl_extract.""" +"""Tests for whl_extract and gen_wheel_record.""" +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:whl_extract.bzl", "rewrite_record_content") # buildifier: disable=bzl-visibility +load("@rules_testing//lib:util.bzl", rt_util = "util") +load( + "//python/private/pypi:gen_wheel_record.bzl", # buildifier: disable=bzl-visibility + "gen_wheel_record", +) +load( + "//tests/support/platforms:platforms.bzl", # buildifier: disable=bzl-visibility + "platform_targets", +) _tests = [] -def _test_purelib_and_platlib_prefix_stripped(env): - record = """\ -foo-1.0.data/purelib/pkg/__init__.py,sha256=abc,100 -foo-1.0.data/purelib/pkg/module.py,sha256=def,200 -foo-1.0.data/platlib/pkg/_ext.so,sha256=ghi,300 -foo-1.0.dist-info/METADATA,sha256=jkl,400 -foo-1.0.dist-info/RECORD,, -""" - result = rewrite_record_content(record, "foo-1.0.data") - env.expect.that_str(result).equals("""\ -pkg/__init__.py,sha256=abc,100 -pkg/module.py,sha256=def,200 -pkg/_ext.so,sha256=ghi,300 -foo-1.0.dist-info/METADATA,sha256=jkl,400 -foo-1.0.dist-info/RECORD,, -""") +def _test_gen_wheel_record(name): + rt_util.helper_target( + native.genrule, + name = name + "_src", + outs = [name + "_orig/alpha-1.0.dist-info/RECORD"], + cmd = "echo 'alpha-1.0.data/scripts/foo.sh' > $@", + ) + rt_util.helper_target( + gen_wheel_record, + name = name + "_subject", + srcs = [":" + name + "_src"], + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_gen_wheel_record_impl, + ) -_tests.append(_test_purelib_and_platlib_prefix_stripped) +_tests.append(_test_gen_wheel_record) -def _test_data_headers_scripts_prefix_rewritten(env): - record = """\ -foo-1.0.data/data/pkg/data.txt,sha256=111,10 -foo-1.0.data/headers/pkg/header.h,sha256=222,20 -foo-1.0.data/scripts/my_script.sh,sha256=333,30 -""" - result = rewrite_record_content(record, "foo-1.0.data") - env.expect.that_str(result).equals("""\ -../../../pkg/data.txt,sha256=111,10 -../../../include/pkg/header.h,sha256=222,20 -../../../bin/my_script.sh,sha256=333,30 -""") +def _test_gen_wheel_record_impl(env, target): + files = target[DefaultInfo].files.to_list() + env.expect.that_collection(files).has_size(1) + env.expect.that_str(files[0].short_path).contains( + "site-packages/alpha-1.0.dist-info/RECORD", + ) -_tests.append(_test_data_headers_scripts_prefix_rewritten) +def _test_gen_wheel_record_windows(name): + rt_util.helper_target( + native.genrule, + name = name + "_src", + outs = [name + "_orig/beta-1.0.dist-info/RECORD"], + cmd = "echo 'beta-1.0.data/scripts/foo.sh' > $@", + ) + rt_util.helper_target( + gen_wheel_record, + name = name + "_subject", + srcs = [":" + name + "_src"], + ) + analysis_test( + name = name, + target = name + "_subject", + config_settings = { + "//command_line_option:platforms": [ + platform_targets.WINDOWS_X86_64, + ], + }, + impl = _test_gen_wheel_record_windows_impl, + ) -def _test_quoted_paths_preserved(env): - record = """\ -"foo-1.0.data/purelib/pkg/my file.py",sha256=abc,100 -"foo-1.0.data/scripts/my tool",sha256=def,200 -"foo-1.0.data/headers/my header.h",sha256=ghi,300 -"foo-1.0.data/data/my data.txt",sha256=jkl,400 -""" - result = rewrite_record_content(record, "foo-1.0.data") - env.expect.that_str(result).equals("""\ -"pkg/my file.py",sha256=abc,100 -"../../../bin/my tool",sha256=def,200 -"../../../include/my header.h",sha256=ghi,300 -"../../../my data.txt",sha256=jkl,400 -""") +_tests.append(_test_gen_wheel_record_windows) -_tests.append(_test_quoted_paths_preserved) +def _test_gen_wheel_record_windows_impl(env, target): + files = target[DefaultInfo].files.to_list() + env.expect.that_collection(files).has_size(1) + env.expect.that_str(files[0].short_path).contains( + "site-packages/beta-1.0.dist-info/RECORD", + ) -def _test_non_data_and_dist_info_entries_unchanged(env): - record = """\ -top_level/__init__.py,sha256=aaa,50 -foo-1.0.dist-info/METADATA,sha256=bbb,60 -foo-1.0.dist-info/WHEEL,sha256=ccc,70 -foo-1.0.dist-info/RECORD,, -""" - result = rewrite_record_content(record, "foo-1.0.data") - env.expect.that_str(result).equals("""\ -top_level/__init__.py,sha256=aaa,50 -foo-1.0.dist-info/METADATA,sha256=bbb,60 -foo-1.0.dist-info/WHEEL,sha256=ccc,70 -foo-1.0.dist-info/RECORD,, -""") +def _test_gen_wheel_record_multiple_srcs(name): + rt_util.helper_target( + native.genrule, + name = name + "_src1", + outs = [name + "_orig1/gamma-1.0.dist-info/RECORD"], + cmd = "echo 'gamma-1.0.data/scripts/foo.sh' > $@", + ) + rt_util.helper_target( + native.genrule, + name = name + "_src2", + outs = [name + "_orig2/delta-2.0.dist-info/RECORD"], + cmd = "echo 'delta-2.0.data/scripts/bar.sh' > $@", + ) + rt_util.helper_target( + gen_wheel_record, + name = name + "_subject", + srcs = [":" + name + "_src1", ":" + name + "_src2"], + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_gen_wheel_record_multiple_srcs_impl, + ) -_tests.append(_test_non_data_and_dist_info_entries_unchanged) +_tests.append(_test_gen_wheel_record_multiple_srcs) -def _test_unrecognized_data_category_preserved(env): - record = """\ -foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 -""" - result = rewrite_record_content(record, "foo-1.0.data") - env.expect.that_str(result).equals("""\ -foo-1.0.data/custom_dir/custom.txt,sha256=xyz,123 -""") - -_tests.append(_test_unrecognized_data_category_preserved) +def _test_gen_wheel_record_multiple_srcs_impl(env, target): + files = target[DefaultInfo].files.to_list() + env.expect.that_collection(files).has_size(2) + paths = [f.short_path for f in files] + env.expect.that_bool( + any(["site-packages/gamma-1.0.dist-info/RECORD" in p for p in paths]), + ).equals(True) + env.expect.that_bool( + any(["site-packages/delta-2.0.dist-info/RECORD" in p for p in paths]), + ).equals(True) def whl_extract_test_suite(name): """Create the test suite. @@ -90,4 +116,4 @@ def whl_extract_test_suite(name): Args: name: the name of the test suite """ - test_suite(name = name, basic_tests = _tests) + test_suite(name = name, tests = _tests) diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl index d752159b32..3fe1b99768 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl @@ -30,7 +30,7 @@ def _test_filegroups(env): def glob(include, *, exclude = [], allow_empty): _ = exclude # @unused env.expect.that_bool(allow_empty).equals(True) - if include == ["rewrite-bin/*"] or include == ["bin/*"]: + if include in [["rewrite-bin/*"], ["bin/*"], ["rewrite-record/*/RECORD"]]: return [] return include @@ -42,6 +42,7 @@ def _test_filegroups(env): ), rules = struct( venv_rewrite_shebang = lambda **kwargs: None, + gen_wheel_record = lambda **kwargs: None, ), ) @@ -84,6 +85,7 @@ def _test_copy(env): rules = struct( copy_file = lambda **kwargs: calls.append(kwargs), venv_rewrite_shebang = lambda **kwargs: None, + gen_wheel_record = lambda **kwargs: None, ), ) @@ -242,6 +244,7 @@ def _test_sdist_excludes_record(env): m_glob = mocks.glob() m_glob.results.append([]) # bin m_glob.results.append([]) # rewrite-bin + m_glob.results.append([]) # rewrite-record m_glob.results.append([]) # srcs m_glob.results.append([]) # data m_glob.results.append([]) # pyi @@ -259,6 +262,7 @@ def _test_sdist_excludes_record(env): py_library = lambda **kwargs: py_library_calls.append(kwargs), create_inits = lambda **kwargs: [], venv_rewrite_shebang = lambda **kwargs: None, + gen_wheel_record = lambda **kwargs: None, ), ) @@ -284,6 +288,7 @@ def _test_exclude_bazel_files(env): m_glob = mocks.glob() m_glob.results.append([]) # bin m_glob.results.append([]) # rewrite-bin + m_glob.results.append([]) # rewrite-record m_glob.results.append([]) # extracted_whl_files m_glob.results.append([]) # dist_info m_glob.results.append([]) # data @@ -297,6 +302,7 @@ def _test_exclude_bazel_files(env): ), rules = struct( venv_rewrite_shebang = lambda **kwargs: None, + gen_wheel_record = lambda **kwargs: None, ), ) @@ -314,6 +320,7 @@ def _test_exclude_bazel_files(env): env.expect.that_collection(m_glob.calls).contains_exactly([ mocks.glob_call(["bin/*"], allow_empty = True), mocks.glob_call(["rewrite-bin/*"], allow_empty = True), + mocks.glob_call(["rewrite-record/*/RECORD"], allow_empty = True), mocks.glob_call( include = ["**"], exclude = expected_exclude, diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index 8836e944c4..9481a8d71d 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -11,41 +11,42 @@ def test_importlib_metadata_files(self): len(files), 0, "importlib.metadata.files returned empty list" ) - expected_paths = [ - "../../../bin/data_overlap.sh", - "../../../bin/data_overlap.sh", - "../../../bin/overlap/both.sh", - "../../../bin/overlap/script1.sh", - "../../../bin/whl_script.sh", - "../../../bin/whl_with_data1_script", - "../../../include/data_overlap.h", - "../../../include/data_overlap.h", - "../../../include/overlap/both.h", - "../../../include/overlap/header1.h", - "../../../include/whl_with_data1/header_file.h", - "../../../overlap/both.txt", - "../../../overlap/data1.txt", - "../../../site-packages/data_overlap.py", - "../../../whl_with_data1/data_data_file.txt", - "../../../whl_with_data1/data_data_file.txt", - "data_overlap.py", - "whl_with_data1/data_file.txt", - "whl_with_data1/platlib_file.txt", - ] + if sys.platform == "win32": + bin_prefix = "../../Scripts/" + include_prefix = "../../Include/" + data_prefix = "../../" + else: + bin_prefix = "../../../bin/" + include_prefix = "../../../include/" + data_prefix = "../../../" + + expected_paths = sorted( + [ + bin_prefix + "data_overlap.sh", + bin_prefix + "data_overlap.sh", + bin_prefix + "overlap/both.sh", + bin_prefix + "overlap/script1.sh", + bin_prefix + "whl_script.sh", + bin_prefix + "whl_with_data1_script", + include_prefix + "data_overlap.h", + include_prefix + "data_overlap.h", + include_prefix + "overlap/both.h", + include_prefix + "overlap/header1.h", + include_prefix + "whl_with_data1/header_file.h", + data_prefix + "overlap/both.txt", + data_prefix + "overlap/data1.txt", + data_prefix + "site-packages/data_overlap.py", + data_prefix + "whl_with_data1/data_data_file.txt", + data_prefix + "whl_with_data1/data_data_file.txt", + "data_overlap.py", + "whl_with_data1/data_file.txt", + "whl_with_data1/platlib_file.txt", + ] + ) file_paths = sorted(str(f).replace("\\", "/") for f in files) self.assertEqual(file_paths, expected_paths) - is_windows = sys.platform == "win32" for f in files: - # On Windows, virtual environments have a 2-level directory depth - # (Lib/site-packages) while POSIX virtual environments have a - # 3-level depth (lib/pythonX.Y/site-packages). Since wheel - # extraction writes POSIX-standard relative paths (../../../) in - # RECORD, files outside site-packages cannot be resolved via - # locate() on Windows. - if is_windows and str(f).startswith(".."): - continue - resolved = f.locate() self.assertTrue( resolved.exists(), From 56921ab824e431c335e74b7fa732a1f47ff05c66 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 10 Aug 2026 15:30:10 +0000 Subject: [PATCH 09/11] test(pypi): fix windows test runner and assertions for RECORD rewriting Update wheel_record_rewriter_test.sh to run .ps1 scripts via powershell.exe when executing under Windows MSYS2 bash. Update importlib_metadata_test.py to assert data_prefix ('../../') for files originating in .data/data/bin/ and .data/data/include/ on Windows. --- .../whl_extract/wheel_record_rewriter_test.sh | 17 ++++++++--- .../importlib_metadata_test.py | 30 +++++++++---------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/pypi/whl_extract/wheel_record_rewriter_test.sh b/tests/pypi/whl_extract/wheel_record_rewriter_test.sh index fc0f78cd92..ccf090b3f6 100755 --- a/tests/pypi/whl_extract/wheel_record_rewriter_test.sh +++ b/tests/pypi/whl_extract/wheel_record_rewriter_test.sh @@ -5,6 +5,17 @@ REWRITER="$1" TMP_DIR="$(mktemp -d)" trap 'rm -rf "$TMP_DIR"' EXIT +run_rewriter() { + case "$REWRITER" in + *.ps1) + powershell.exe -ExecutionPolicy Bypass -NoProfile -File "$REWRITER" "$@" + ;; + *) + "$REWRITER" "$@" + ;; + esac +} + INPUT="$TMP_DIR/input_RECORD" cat <<'EOF' > "$INPUT" foo-1.0.data/purelib/pkg/__init__.py,sha256=abc,100 @@ -25,7 +36,7 @@ EOF # Test Unix rewrite UNIX_OUT="$TMP_DIR/unix_RECORD" -"$REWRITER" "$INPUT" "$UNIX_OUT" "unix" "foo-1.0.data" +run_rewriter "$INPUT" "$UNIX_OUT" "unix" "foo-1.0.data" EXPECTED_UNIX="$TMP_DIR/expected_unix" cat <<'EOF' > "$EXPECTED_UNIX" @@ -49,7 +60,7 @@ diff -u "$EXPECTED_UNIX" "$UNIX_OUT" # Test Windows rewrite WIN_OUT="$TMP_DIR/win_RECORD" -"$REWRITER" "$INPUT" "$WIN_OUT" "windows" "foo-1.0.data" +run_rewriter "$INPUT" "$WIN_OUT" "windows" "foo-1.0.data" EXPECTED_WIN="$TMP_DIR/expected_win" cat <<'EOF' > "$EXPECTED_WIN" @@ -70,5 +81,3 @@ foo-1.0.dist-info/RECORD,, EOF diff -u "$EXPECTED_WIN" "$WIN_OUT" - -echo "All rewriter assertions passed!" diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index 9481a8d71d..87fa249752 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -12,27 +12,27 @@ def test_importlib_metadata_files(self): ) if sys.platform == "win32": - bin_prefix = "../../Scripts/" - include_prefix = "../../Include/" + scripts_prefix = "../../Scripts/" + headers_prefix = "../../Include/" data_prefix = "../../" else: - bin_prefix = "../../../bin/" - include_prefix = "../../../include/" + scripts_prefix = "../../../bin/" + headers_prefix = "../../../include/" data_prefix = "../../../" expected_paths = sorted( [ - bin_prefix + "data_overlap.sh", - bin_prefix + "data_overlap.sh", - bin_prefix + "overlap/both.sh", - bin_prefix + "overlap/script1.sh", - bin_prefix + "whl_script.sh", - bin_prefix + "whl_with_data1_script", - include_prefix + "data_overlap.h", - include_prefix + "data_overlap.h", - include_prefix + "overlap/both.h", - include_prefix + "overlap/header1.h", - include_prefix + "whl_with_data1/header_file.h", + scripts_prefix + "data_overlap.sh", + data_prefix + "bin/data_overlap.sh", + scripts_prefix + "overlap/both.sh", + scripts_prefix + "overlap/script1.sh", + scripts_prefix + "whl_script.sh", + scripts_prefix + "whl_with_data1_script", + headers_prefix + "data_overlap.h", + data_prefix + "include/data_overlap.h", + headers_prefix + "overlap/both.h", + headers_prefix + "overlap/header1.h", + headers_prefix + "whl_with_data1/header_file.h", data_prefix + "overlap/both.txt", data_prefix + "overlap/data1.txt", data_prefix + "site-packages/data_overlap.py", From e30d1b7a0cb1fd1d7764311ebf88cd1912746db6 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 10 Aug 2026 15:51:01 +0000 Subject: [PATCH 10/11] test(pypi): fix windows path resolution and diff crlf in tests Convert input/output paths using cygpath -w when running powershell in wheel_record_rewriter_test.sh and ignore CRLF differences in diff. Handle .bat extension on Windows venv bin scripts when validating located files in importlib_metadata_test.py. --- .../whl_extract/wheel_record_rewriter_test.sh | 14 ++++++-- .../importlib_metadata_test.py | 34 ++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/tests/pypi/whl_extract/wheel_record_rewriter_test.sh b/tests/pypi/whl_extract/wheel_record_rewriter_test.sh index ccf090b3f6..c5880520ad 100755 --- a/tests/pypi/whl_extract/wheel_record_rewriter_test.sh +++ b/tests/pypi/whl_extract/wheel_record_rewriter_test.sh @@ -8,7 +8,15 @@ trap 'rm -rf "$TMP_DIR"' EXIT run_rewriter() { case "$REWRITER" in *.ps1) - powershell.exe -ExecutionPolicy Bypass -NoProfile -File "$REWRITER" "$@" + in_file="$1" + out_file="$2" + platform_type="$3" + data_dir="$4" + if command -v cygpath >/dev/null 2>&1; then + in_file="$(cygpath -w "$in_file")" + out_file="$(cygpath -w "$out_file")" + fi + powershell.exe -ExecutionPolicy Bypass -NoProfile -File "$REWRITER" "$in_file" "$out_file" "$platform_type" "$data_dir" ;; *) "$REWRITER" "$@" @@ -56,7 +64,7 @@ foo-1.0.dist-info/METADATA,sha256=bbb,60 foo-1.0.dist-info/RECORD,, EOF -diff -u "$EXPECTED_UNIX" "$UNIX_OUT" +diff -u --strip-trailing-cr "$EXPECTED_UNIX" "$UNIX_OUT" # Test Windows rewrite WIN_OUT="$TMP_DIR/win_RECORD" @@ -80,4 +88,4 @@ foo-1.0.dist-info/METADATA,sha256=bbb,60 foo-1.0.dist-info/RECORD,, EOF -diff -u "$EXPECTED_WIN" "$WIN_OUT" +diff -u --strip-trailing-cr "$EXPECTED_WIN" "$WIN_OUT" diff --git a/tests/venv_site_packages_libs/importlib_metadata_test.py b/tests/venv_site_packages_libs/importlib_metadata_test.py index 87fa249752..8e73b53141 100644 --- a/tests/venv_site_packages_libs/importlib_metadata_test.py +++ b/tests/venv_site_packages_libs/importlib_metadata_test.py @@ -48,21 +48,29 @@ def test_importlib_metadata_files(self): for f in files: resolved = f.locate() - self.assertTrue( - resolved.exists(), - f"Expected file {f} (resolved to {resolved}) to exist", - ) - self.assertTrue( - resolved.is_file(), - f"Expected {resolved} to be a regular file", - ) + if resolved.exists(): + self.assertTrue( + resolved.is_file(), + f"Expected {resolved} to be a regular file", + ) - # Verify file content can be read both as binary and as text - content = f.read_binary() - self.assertIsNotNone(content) + # Verify file content can be read both as binary and as text + content = f.read_binary() + self.assertIsNotNone(content) - text = f.read_text(encoding="utf-8") - self.assertIsNotNone(text) + text = f.read_text(encoding="utf-8") + self.assertIsNotNone(text) + else: + # On Windows, venv bin scripts have a .bat extension appended. + bat_resolved = resolved.parent / (resolved.name + ".bat") + self.assertTrue( + bat_resolved.exists(), + f"Expected file {f} (resolved to {resolved} or {bat_resolved}) to exist", + ) + self.assertTrue( + bat_resolved.is_file(), + f"Expected {bat_resolved} to be a regular file", + ) if __name__ == "__main__": From ccb86603a1f085373cb09d99be3b729c8dc42622 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 10 Aug 2026 15:52:12 +0000 Subject: [PATCH 11/11] fix(pypi): emit lf line endings in powershell wheel record rewriter --- python/private/pypi/wheel_record_rewriter.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/wheel_record_rewriter.ps1 b/python/private/pypi/wheel_record_rewriter.ps1 index acf664f833..60dd0eca11 100644 --- a/python/private/pypi/wheel_record_rewriter.ps1 +++ b/python/private/pypi/wheel_record_rewriter.ps1 @@ -63,4 +63,4 @@ foreach ($line in $lines) { } } -[System.IO.File]::WriteAllLines($OutFile, $outLines, $Utf8NoBom) +[System.IO.File]::WriteAllText($OutFile, ($outLines -join "`n") + "`n", $Utf8NoBom)