From 25cdac44179ddc135974faca52bbafed970dbf53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Z=C3=B6rner?= Date: Thu, 3 Sep 2026 11:38:47 +0200 Subject: [PATCH] Keep leaf nodes spread across files when pruning the candidate set When there are more than LEAF_REDUCTION_THRESHOLD candidates, the node set is pruned down to components that nothing else depends on. In call-graph shaped repositories such as C or Go, almost every function is called from somewhere, so this can collapse the set to a handful of unreferenced functions. On umoria it left 27 of 765 functions, covering only 11 of 46 source files, and the resulting set was small enough to skip LLM clustering entirely. Compare file coverage before and after the pruning. If it drops below half, discard the pruned set and instead cap the full candidate list at LEAF_REDUCTION_THRESHOLD, picking entries round-robin across files so that no file is dropped while another contributes dozens of entries. Also raise the pruning log from DEBUG to INFO, since previously there was no indication at normal verbosity that a reduction had happened at all, and stop rejecting identifiers that merely contain a word like invalid. Refs #75 --- .../be/dependency_analyzer/leaf_selection.py | 70 ++++++++++++++++++- .../src/be/dependency_analyzer/topo_sort.py | 53 ++++++++++++-- 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/codewiki/src/be/dependency_analyzer/leaf_selection.py b/codewiki/src/be/dependency_analyzer/leaf_selection.py index fd37b512..41ddf373 100644 --- a/codewiki/src/be/dependency_analyzer/leaf_selection.py +++ b/codewiki/src/be/dependency_analyzer/leaf_selection.py @@ -1,4 +1,6 @@ -from typing import Dict, List, Set +import re +from collections import Counter, defaultdict +from typing import Dict, Iterable, List, Set from codewiki.src.be.dependency_analyzer.models.core import Node @@ -13,9 +15,67 @@ OOP_MINORITY_RATIO = 0.2 # Above this many candidate leaf nodes, prune to true graph leaves. LEAF_REDUCTION_THRESHOLD = 400 +# Pruning to true graph leaves is rejected when it drops below this fraction +# of the source files the unpruned candidates covered. In call-graph shaped +# repositories (C, Go) almost every function is called by something, so the +# pruning can collapse the set to a handful of unreferenced functions and +# leave most of the codebase without an entry point (issue #75). +MIN_FILE_COVERAGE_AFTER_REDUCTION = 0.5 OOP_TYPES = {"class", "interface", "struct"} +# Error strings that occasionally reach leaf-node selection instead of an +# identifier. Matched on word boundaries and only for entries that are not +# known components, so that names like `handleInvalidInput` survive. +ERROR_MESSAGE_RE = re.compile( + r"\b(error|exception|failed|invalid)\b", re.IGNORECASE) + + +def files_covered(nodes: Iterable[str], components: Dict[str, Node]) -> Set[str]: + """Source files that the given nodes are defined in.""" + return { + components[n].file_path + for n in nodes + if n in components and components[n].file_path + } + + +def spread_over_files( + candidates: Iterable[str], + components: Dict[str, Node], + in_degree: Dict[str, int], + limit: int, +) -> List[str]: + """ + Pick at most `limit` candidates while touching as many files as possible. + + Within a file the most depended-upon component comes first, since that is + the one a reader would start from. Files are then visited round-robin, so + a cap never silently drops whole files while another file contributes + dozens of entry points. + """ + by_file: Dict[str, List[str]] = defaultdict(list) + for node in candidates: + path = components[node].file_path if node in components else "" + by_file[path].append(node) + for path in by_file: + by_file[path].sort(key=lambda n: (-in_degree.get(n, 0), n)) + + ordered: List[str] = [] + depth = 0 + while len(ordered) < limit: + added = False + for path in sorted(by_file): + if depth < len(by_file[path]): + ordered.append(by_file[path][depth]) + added = True + if len(ordered) >= limit: + break + if not added: + break + depth += 1 + return ordered + def compute_valid_leaf_types(components: Dict[str, Node]) -> Set[str]: """ @@ -61,7 +121,13 @@ def filter_leaf_nodes( keep_leaf_nodes = [] for leaf_node in leaf_nodes: # Skip any leaf nodes that are clearly error strings or invalid identifiers - if not isinstance(leaf_node, str) or leaf_node.strip() == "" or any(err_keyword in leaf_node.lower() for err_keyword in ['error', 'exception', 'failed', 'invalid']): + if not isinstance(leaf_node, str) or leaf_node.strip() == "": + logger.debug(f"Skipping invalid leaf node identifier: '{leaf_node}'") + continue + + # Only reject strings that are error messages, not identifiers that + # merely contain such a word (handleInvalidInput, ErrorLog, ...). + if leaf_node not in components and ERROR_MESSAGE_RE.search(leaf_node): logger.debug(f"Skipping invalid leaf node identifier: '{leaf_node}'") continue diff --git a/codewiki/src/be/dependency_analyzer/topo_sort.py b/codewiki/src/be/dependency_analyzer/topo_sort.py index da9ae3a0..b4df57bb 100644 --- a/codewiki/src/be/dependency_analyzer/topo_sort.py +++ b/codewiki/src/be/dependency_analyzer/topo_sort.py @@ -13,8 +13,11 @@ from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.be.dependency_analyzer.leaf_selection import ( LEAF_REDUCTION_THRESHOLD, + MIN_FILE_COVERAGE_AFTER_REDUCTION, compute_valid_leaf_types, + files_covered, filter_leaf_nodes, + spread_over_files, ) logger = logging.getLogger(__name__) @@ -310,16 +313,52 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]: concise_leaf_nodes = concise_node(leaf_nodes) if len(concise_leaf_nodes) >= LEAF_REDUCTION_THRESHOLD: - logger.debug(f"Leaf nodes are too many ({len(concise_leaf_nodes)}), removing dependencies of other nodes") + logger.info( + "Leaf nodes are too many (%d); reducing to components nothing " + "depends on.", + len(concise_leaf_nodes), + ) # Remove nodes that are dependencies of other nodes + reduced_input = set(leaf_nodes) for node, deps in acyclic_graph.items(): for dep in deps: - leaf_nodes.discard(dep) - - concise_leaf_nodes = concise_node(leaf_nodes) - + reduced_input.discard(dep) + reduced = concise_node(reduced_input) + + # In call-graph shaped repositories nearly every function is called by + # something, so the reduction above can collapse to a few unreferenced + # functions and leave most files without an entry point. Compare file + # coverage and fall back to a capped, file-spread selection instead. + before = files_covered(concise_leaf_nodes, components) + after = files_covered(reduced, components) + keeps_coverage = ( + not before + or len(after) / len(before) >= MIN_FILE_COVERAGE_AFTER_REDUCTION + ) + + if reduced and keeps_coverage: + leaf_nodes = reduced_input + concise_leaf_nodes = reduced + else: + in_degree: Dict[str, int] = {} + for node, deps in acyclic_graph.items(): + for dep in deps: + in_degree[dep] = in_degree.get(dep, 0) + 1 + capped = spread_over_files( + concise_leaf_nodes, components, in_degree, + LEAF_REDUCTION_THRESHOLD, + ) + logger.info( + "Reduction would have covered only %d of %d source files; " + "keeping %d entry points spread across %d files instead.", + len(after), len(before), len(capped), + len(files_covered(capped, components)), + ) + leaf_nodes = set(capped) + concise_leaf_nodes = capped + if not leaf_nodes: logger.warning("No leaf nodes found in the graph") return [] - - return concise_leaf_nodes \ No newline at end of file + + return concise_leaf_nodes \ No newline at end of file