Keep leaf nodes spread across files when pruning the candidate set

codewiki/src/be/dependency_analyzer/leaf_selection.py

@@ -1,4 +1,6 @@
-from typing import Dict, List, Set
+import re
+from collections import Counter, defaultdict
+from typing import Dict, Iterable, List, Set
 
 from codewiki.src.be.dependency_analyzer.models.core import Node
 
@@ -13,9 +15,67 @@
 OOP_MINORITY_RATIO = 0.2
 # Above this many candidate leaf nodes, prune to true graph leaves.
 LEAF_REDUCTION_THRESHOLD = 400
+# Pruning to true graph leaves is rejected when it drops below this fraction
+# of the source files the unpruned candidates covered. In call-graph shaped
+# repositories (C, Go) almost every function is called by something, so the
+# pruning can collapse the set to a handful of unreferenced functions and
+# leave most of the codebase without an entry point (issue #75).
+MIN_FILE_COVERAGE_AFTER_REDUCTION = 0.5
 
 OOP_TYPES = {"class", "interface", "struct"}
 
+# Error strings that occasionally reach leaf-node selection instead of an
+# identifier. Matched on word boundaries and only for entries that are not
+# known components, so that names like `handleInvalidInput` survive.
+ERROR_MESSAGE_RE = re.compile(
+    r"\b(error|exception|failed|invalid)\b", re.IGNORECASE)
+
+
+def files_covered(nodes: Iterable[str], components: Dict[str, Node]) -> Set[str]:
+    """Source files that the given nodes are defined in."""
+    return {
+        components[n].file_path
+        for n in nodes
+        if n in components and components[n].file_path
+    }
+
+
+def spread_over_files(
+    candidates: Iterable[str],
+    components: Dict[str, Node],
+    in_degree: Dict[str, int],
+    limit: int,
+) -> List[str]:
+    """
+    Pick at most `limit` candidates while touching as many files as possible.
+
+    Within a file the most depended-upon component comes first, since that is
+    the one a reader would start from. Files are then visited round-robin, so
+    a cap never silently drops whole files while another file contributes
+    dozens of entry points.
+    """
+    by_file: Dict[str, List[str]] = defaultdict(list)
+    for node in candidates:
+        path = components[node].file_path if node in components else ""
+        by_file[path].append(node)
+    for path in by_file:
+        by_file[path].sort(key=lambda n: (-in_degree.get(n, 0), n))
+
+    ordered: List[str] = []
+    depth = 0
+    while len(ordered) < limit:
+        added = False
+        for path in sorted(by_file):
+            if depth < len(by_file[path]):
+                ordered.append(by_file[path][depth])
+                added = True
+                if len(ordered) >= limit:
+                    break
+        if not added:
+            break
+        depth += 1
+    return ordered
+
 
 def compute_valid_leaf_types(components: Dict[str, Node]) -> Set[str]:
     """
@@ -61,7 +121,13 @@ def filter_leaf_nodes(
     keep_leaf_nodes = []
     for leaf_node in leaf_nodes:
         # Skip any leaf nodes that are clearly error strings or invalid identifiers
-        if not isinstance(leaf_node, str) or leaf_node.strip() == "" or any(err_keyword in leaf_node.lower() for err_keyword in ['error', 'exception', 'failed', 'invalid']):
+        if not isinstance(leaf_node, str) or leaf_node.strip() == "":
+            logger.debug(f"Skipping invalid leaf node identifier: '{leaf_node}'")
+            continue
+
+        # Only reject strings that are error messages, not identifiers that
+        # merely contain such a word (handleInvalidInput, ErrorLog, ...).
+        if leaf_node not in components and ERROR_MESSAGE_RE.search(leaf_node):
             logger.debug(f"Skipping invalid leaf node identifier: '{leaf_node}'")
             continue
 

codewiki/src/be/dependency_analyzer/topo_sort.py

@@ -13,8 +13,11 @@
 from codewiki.src.be.dependency_analyzer.models.core import Node
 from codewiki.src.be.dependency_analyzer.leaf_selection import (
     LEAF_REDUCTION_THRESHOLD,
+    MIN_FILE_COVERAGE_AFTER_REDUCTION,
     compute_valid_leaf_types,
+    files_covered,
     filter_leaf_nodes,
+    spread_over_files,
 )
 
 logger = logging.getLogger(__name__)
@@ -310,16 +313,52 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]:
 
     concise_leaf_nodes = concise_node(leaf_nodes)
     if len(concise_leaf_nodes) >= LEAF_REDUCTION_THRESHOLD:
-        logger.debug(f"Leaf nodes are too many ({len(concise_leaf_nodes)}), removing dependencies of other nodes")
+        logger.info(
+            "Leaf nodes are too many (%d); reducing to components nothing "
+            "depends on.",
+            len(concise_leaf_nodes),
+        )
         # Remove nodes that are dependencies of other nodes
+        reduced_input = set(leaf_nodes)
         for node, deps in acyclic_graph.items():
             for dep in deps:
-                leaf_nodes.discard(dep)
-
-        concise_leaf_nodes = concise_node(leaf_nodes)
-
+                reduced_input.discard(dep)
+        reduced = concise_node(reduced_input)
+
+        # In call-graph shaped repositories nearly every function is called by
+        # something, so the reduction above can collapse to a few unreferenced
+        # functions and leave most files without an entry point. Compare file
+        # coverage and fall back to a capped, file-spread selection instead.
+        before = files_covered(concise_leaf_nodes, components)
+        after = files_covered(reduced, components)
+        keeps_coverage = (
+            not before
+            or len(after) / len(before) >= MIN_FILE_COVERAGE_AFTER_REDUCTION
+        )
+
+        if reduced and keeps_coverage:
+            leaf_nodes = reduced_input
+            concise_leaf_nodes = reduced
+        else:
+            in_degree: Dict[str, int] = {}
+            for node, deps in acyclic_graph.items():
+                for dep in deps:
+                    in_degree[dep] = in_degree.get(dep, 0) + 1
+            capped = spread_over_files(
+                concise_leaf_nodes, components, in_degree,
+                LEAF_REDUCTION_THRESHOLD,
+            )
+            logger.info(
+                "Reduction would have covered only %d of %d source files; "
+                "keeping %d entry points spread across %d files instead.",
+                len(after), len(before), len(capped),
+                len(files_covered(capped, components)),
+            )
+            leaf_nodes = set(capped)
+            concise_leaf_nodes = capped
+
     if not leaf_nodes:
         logger.warning("No leaf nodes found in the graph")
         return []
-    
-    return concise_leaf_nodes 
+
+    return concise_leaf_nodes