fix: do not include transitive sources from pyi_deps in runtime runfiles

news/4178.fixed.md

@@ -0,0 +1,4 @@
+(rules) Fixed {obj}`py_library.pyi_deps` being included in runtime output and
+bloating output. Type-checking only information is now in
+{obj}`PyInfo.type_checking_info`
+([#4178](https://github.com/bazel-contrib/rules_python/pull/4178)).

python/private/common.bzl

@@ -412,7 +412,7 @@ def create_py_info(
     for target in ctx.attr.pyi_deps:
         # PyInfo may not be present e.g. cc_library rules.
         if PyInfo in target or (BuiltinPyInfo != None and BuiltinPyInfo in target):
-            py_info.merge(_get_py_info(target))
+            py_info.merge_type_checking(_get_py_info(target))
 
     py_info.transitive_sources.add(required_py_files)
 

python/private/py_info.bzl

@@ -375,6 +375,7 @@ def _PyInfo_init(
         transitive_original_sources = depset(),
         direct_pyi_files = depset(),
         transitive_pyi_files = depset(),
+        type_checking_info = None,
         venv_symlinks = depset()):
     _check_arg_type("transitive_sources", "depset", transitive_sources)
 
@@ -396,6 +397,8 @@ def _PyInfo_init(
 
     _check_arg_type("direct_pyi_files", "depset", direct_pyi_files)
     _check_arg_type("transitive_pyi_files", "depset", transitive_pyi_files)
+    if type_checking_info != None:
+        _check_arg_type("type_checking_info", "struct", type_checking_info)
     return {
         "direct_original_sources": direct_original_sources,
         "direct_pyc_files": direct_pyc_files,
@@ -409,6 +412,7 @@ def _PyInfo_init(
         "transitive_pyc_files": transitive_pyc_files,
         "transitive_pyi_files": transitive_pyi_files,
         "transitive_sources": transitive_sources,
+        "type_checking_info": type_checking_info,
         "uses_shared_libraries": uses_shared_libraries,
         "venv_symlinks": venv_symlinks,
     }
@@ -558,6 +562,17 @@ in this depset being **empty**.
 The files are considered necessary for downstream binaries to function;
 previously they were considerd informational and largely unused.
 ::::
+""",
+        "type_checking_info": """
+:type: PyInfo | None
+
+Additional `PyInfo` information needed only for static type checking (for
+example, from `pyi_deps`). This information is not included into the final
+output of a program. Type checkers should merge this into their information to
+augment the analyzed output.
+
+::::{versionadded} VERSION_NEXT_FEATURE
+::::
 """,
         "uses_shared_libraries": """
 :type: bool
@@ -627,17 +642,21 @@ def _PyInfoBuilder_typedef():
     :type: DepsetBuilder[File]
     :::
 
-    :::{field} venv_symlinks
-    :type: DepsetBuilder[tuple[str | None, str]]
-    """
+    :::{field} type_checking_info
+    :type: PyInfoBuilder | None
 
-def _PyInfoBuilder_new():
-    """Creates an instance.
+    Builder for {obj}`PyInfo.type_checking_info`. `None` on nested
+    type-checking builders.
 
-    Returns:
-        {type}`PyInfoBuilder`
+    ::::{versionadded} VERSION_NEXT_FEATURE
+    ::::
+    :::
+
+    :::{field} venv_symlinks
+    :type: DepsetBuilder[tuple[str | None, str]]
     """
 
+def _new_raw_py_info_builder(type_checking_info = None):
     # buildifier: disable=uninitialized
     self = struct(
         _has_py2_only_sources = [False],
@@ -660,6 +679,7 @@ def _PyInfoBuilder_new():
         merge_has_py3_only_sources = lambda *a, **k: _PyInfoBuilder_merge_has_py3_only_sources(self, *a, **k),
         merge_target = lambda *a, **k: _PyInfoBuilder_merge_target(self, *a, **k),
         merge_targets = lambda *a, **k: _PyInfoBuilder_merge_targets(self, *a, **k),
+        merge_type_checking = lambda *a, **k: _PyInfoBuilder_merge_type_checking(self, *a, **k),
         merge_uses_shared_libraries = lambda *a, **k: _PyInfoBuilder_merge_uses_shared_libraries(self, *a, **k),
         set_has_py2_only_sources = lambda *a, **k: _PyInfoBuilder_set_has_py2_only_sources(self, *a, **k),
         set_has_py3_only_sources = lambda *a, **k: _PyInfoBuilder_set_has_py3_only_sources(self, *a, **k),
@@ -670,10 +690,20 @@ def _PyInfoBuilder_new():
         transitive_pyc_files = builders.DepsetBuilder(),
         transitive_pyi_files = builders.DepsetBuilder(),
         transitive_sources = builders.DepsetBuilder(),
+        type_checking_info = type_checking_info,
         venv_symlinks = builders.DepsetBuilder(),
     )
     return self
 
+def _PyInfoBuilder_new():
+    """Creates an instance.
+
+    Returns:
+        {type}`PyInfoBuilder`
+    """
+    type_checking_info = _new_raw_py_info_builder(None)
+    return _new_raw_py_info_builder(type_checking_info)
+
 def _PyInfoBuilder_add_venv_symlink(self):
     """Create and return a new VenvSymlinkEntryBuilder.
 
@@ -819,19 +849,7 @@ def _PyInfoBuilder_merge(self, *infos, direct = []):
     """
     return self.merge_all(list(infos), direct = direct)
 
-def _PyInfoBuilder_merge_all(self, transitive, *, direct = []):
-    """Merge other PyInfos into this PyInfo.
-
-    Args:
-        self: implicitly added.
-        transitive: {type}`list[PyInfo]` objects to merge in, but only merge in
-            their information into this object's transitive fields.
-        direct: {type}`list[PyInfo]` objects to merge in, but also merge their
-            direct fields into this object's direct fields.
-
-    Returns:
-        {type}`PyInfoBuilder` self
-    """
+def _merge_py_info_fields(self, transitive, *, direct = []):
     for info in direct:
         # BuiltinPyInfo doesn't have this field
         if hasattr(info, "direct_pyc_files"):
@@ -855,6 +873,77 @@ def _PyInfoBuilder_merge_all(self, transitive, *, direct = []):
             self.transitive_pyi_files.add(info.transitive_pyi_files)
             self.venv_symlinks.add(info.venv_symlinks)
 
+def _PyInfoBuilder_merge_all(self, transitive, *, direct = []):
+    """Merge other PyInfos into this PyInfo.
+
+    Args:
+        self: implicitly added.
+        transitive: {type}`list[PyInfo]` objects to merge in, but only merge in
+            their information into this object's transitive fields.
+        direct: {type}`list[PyInfo]` objects to merge in, but also merge their
+            direct fields into this object's direct fields.
+
+    Returns:
+        {type}`PyInfoBuilder` self
+    """
+    _merge_py_info_fields(self, transitive, direct = direct)
+    tc_direct = [
+        info.type_checking_info
+        for info in direct
+        if getattr(info, "type_checking_info", None) != None
+    ]
+    tc_transitive = [
+        info.type_checking_info
+        for info in transitive
+        if getattr(info, "type_checking_info", None) != None
+    ]
+    _merge_py_info_fields(self.type_checking_info, tc_transitive, direct = tc_direct)
+
+    return self
+
+def _PyInfoBuilder_merge_type_checking(self, *infos, direct = []):
+    """Merge type-checking fields from other PyInfos into this PyInfo.
+
+    Merges `pyi_files` into this object's `direct_pyi_files` /
+    `transitive_pyi_files` and merges the full `PyInfo` (and any nested
+    `type_checking_info`) into {obj}`type_checking_info`, excluding runtime
+    fields like `imports` and `transitive_sources` from the top-level `PyInfo`.
+
+    :::{versionadded} VERSION_NEXT_FEATURE
+    :::
+
+    Args:
+        self: implicitly added.
+        *infos: {type}`PyInfo` objects to merge in, but only merge in their
+            information into this object's transitive fields.
+        direct: {type}`list[PyInfo]` objects to merge in, but also merge their
+            direct fields into this object's direct fields.
+
+    Returns:
+        {type}`PyInfoBuilder` self
+    """
+    for info in direct:
+        # BuiltinPyInfo doesn't have this field
+        if hasattr(info, "direct_pyi_files"):
+            self.direct_pyi_files.add(info.direct_pyi_files)
+
+    for info in direct + list(infos):
+        # BuiltinPyInfo doesn't have this field
+        if hasattr(info, "transitive_pyi_files"):
+            self.transitive_pyi_files.add(info.transitive_pyi_files)
+
+    tc_direct = direct + [
+        info.type_checking_info
+        for info in direct
+        if getattr(info, "type_checking_info", None) != None
+    ]
+    tc_transitive = list(infos) + [
+        info.type_checking_info
+        for info in infos
+        if getattr(info, "type_checking_info", None) != None
+    ]
+    _merge_py_info_fields(self.type_checking_info, tc_transitive, direct = tc_direct)
+
     return self
 
 def _PyInfoBuilder_merge_target(self, target):
@@ -892,15 +981,7 @@ def _PyInfoBuilder_merge_targets(self, targets):
         self.merge_target(t)
     return self
 
-def _PyInfoBuilder_build(self):
-    """Builds into a {obj}`PyInfo` object.
-
-    Args:
-        self: implicitly added.
-
-    Returns:
-        {type}`PyInfo`
-    """
+def _build_py_info_fields(self, type_checking_info = None):
     venv_symlinks = depset(
         direct = [b.build() for b in self._venv_symlink_builders],
         transitive = [self.venv_symlinks.build()],
@@ -920,9 +1001,22 @@ def _PyInfoBuilder_build(self):
         transitive_original_sources = self.transitive_original_sources.build(),
         transitive_pyc_files = self.transitive_pyc_files.build(),
         transitive_pyi_files = self.transitive_pyi_files.build(),
+        type_checking_info = type_checking_info,
         venv_symlinks = venv_symlinks,
     )
 
+def _PyInfoBuilder_build(self):
+    """Builds into a {obj}`PyInfo` object.
+
+    Args:
+        self: implicitly added.
+
+    Returns:
+        {type}`PyInfo`
+    """
+    type_checking_info = _build_py_info_fields(self.type_checking_info, None)
+    return _build_py_info_fields(self, type_checking_info)
+
 def _PyInfoBuilder_build_builtin_py_info(self):
     """Builds into a Bazel-builtin PyInfo object, if available.
 
@@ -961,6 +1055,7 @@ PyInfoBuilder = struct(
     merge_has_py3_only_sources = _PyInfoBuilder_merge_has_py3_only_sources,
     merge_target = _PyInfoBuilder_merge_target,
     merge_targets = _PyInfoBuilder_merge_targets,
+    merge_type_checking = _PyInfoBuilder_merge_type_checking,
     merge_uses_shared_libraries = _PyInfoBuilder_merge_uses_shared_libraries,
     set_has_py2_only_sources = _PyInfoBuilder_set_has_py2_only_sources,
     set_has_py3_only_sources = _PyInfoBuilder_set_has_py3_only_sources,

tests/base_rules/base_tests.bzl

@@ -72,6 +72,7 @@ def _test_py_info_populated(name, config):
         name = name + "_lib2",
         srcs = ["lib2.py"],
         pyi_srcs = ["lib2.pyi"],
+        imports = ["lib2_import"],
     )
 
     analysis_test(
@@ -90,8 +91,8 @@ def _test_py_info_populated_impl(env, target):
     ])
     info.transitive_original_sources().contains_exactly([
         "{package}/test_py_info_populated_subject.py",
-        "{package}/lib2.py",
     ])
+    info.imports().contains_exactly([])
 
     info.direct_pyi_files().contains_exactly([
         "{package}/subject.pyi",
@@ -100,6 +101,23 @@ def _test_py_info_populated_impl(env, target):
         "{package}/lib2.pyi",
         "{package}/subject.pyi",
     ])
+    info.transitive_sources().contains_exactly([
+        "{package}/test_py_info_populated_subject.py",
+    ])
+
+    tc_info = info.type_checking_info()
+    tc_info.transitive_sources().contains_exactly([
+        "{package}/lib2.py",
+    ])
+    tc_info.transitive_original_sources().contains_exactly([
+        "{package}/lib2.py",
+    ])
+    tc_info.imports().contains_exactly([
+        "{}/{}/lib2_import".format(env.ctx.workspace_name, target.label.package),
+    ])
+    tc_info.transitive_pyi_files().contains_exactly([
+        "{package}/lib2.pyi",
+    ])
 
 _tests.append(_test_py_info_populated)
 

tests/base_rules/py_executable_base_tests.bzl

@@ -519,6 +519,43 @@ def _test_py_runtime_info_provided_impl(env, target):
 
 _tests.append(_test_py_runtime_info_provided)
 
+def _test_pyi_deps_not_in_runfiles(name, config):
+    rt_util.helper_target(
+        py_library,
+        name = name + "_types",
+        srcs = [rt_util.empty_file(name + "_type_stub.py")],
+    )
+    rt_util.helper_target(
+        py_library,
+        name = name + "_lib",
+        srcs = [rt_util.empty_file(name + "_lib.py")],
+        pyi_deps = [name + "_types"],
+    )
+    rt_util.helper_target(
+        config.rule,
+        name = name + "_subject",
+        srcs = [name + "_main.py"],
+        main = name + "_main.py",
+        deps = [name + "_lib"],
+    )
+    analysis_test(
+        name = name,
+        impl = _test_pyi_deps_not_in_runfiles_impl,
+        target = name + "_subject",
+    )
+
+def _test_pyi_deps_not_in_runfiles_impl(env, target):
+    target = env.expect.that_target(target)
+    target.runfiles().contains_at_least([
+        "{workspace}/{package}/{test_name}_main.py",
+        "{workspace}/{package}/{test_name}_lib.py",
+    ])
+    target.runfiles().not_contains(
+        "{workspace}/{package}/{test_name}_type_stub.py",
+    )
+
+_tests.append(_test_pyi_deps_not_in_runfiles)
+
 def _test_venv_output_prefix_with_path_separators(name, config):
     rt_util.helper_target(
         config.rule,