diff --git a/codewiki/src/be/dependency_analyzer/leaf_selection.py b/codewiki/src/be/dependency_analyzer/leaf_selection.py index fd37b512..41ddf373 100644 --- a/codewiki/src/be/dependency_analyzer/leaf_selection.py +++ b/codewiki/src/be/dependency_analyzer/leaf_selection.py @@ -1,4 +1,6 @@ -from typing import Dict, List, Set +import re +from collections import Counter, defaultdict +from typing import Dict, Iterable, List, Set from codewiki.src.be.dependency_analyzer.models.core import Node @@ -13,9 +15,67 @@ OOP_MINORITY_RATIO = 0.2 # Above this many candidate leaf nodes, prune to true graph leaves. LEAF_REDUCTION_THRESHOLD = 400 +# Pruning to true graph leaves is rejected when it drops below this fraction +# of the source files the unpruned candidates covered. In call-graph shaped +# repositories (C, Go) almost every function is called by something, so the +# pruning can collapse the set to a handful of unreferenced functions and +# leave most of the codebase without an entry point (issue #75). +MIN_FILE_COVERAGE_AFTER_REDUCTION = 0.5 OOP_TYPES = {"class", "interface", "struct"} +# Error strings that occasionally reach leaf-node selection instead of an +# identifier. Matched on word boundaries and only for entries that are not +# known components, so that names like `handleInvalidInput` survive. +ERROR_MESSAGE_RE = re.compile( + r"\b(error|exception|failed|invalid)\b", re.IGNORECASE) + + +def files_covered(nodes: Iterable[str], components: Dict[str, Node]) -> Set[str]: + """Source files that the given nodes are defined in.""" + return { + components[n].file_path + for n in nodes + if n in components and components[n].file_path + } + + +def spread_over_files( + candidates: Iterable[str], + components: Dict[str, Node], + in_degree: Dict[str, int], + limit: int, +) -> List[str]: + """ + Pick at most `limit` candidates while touching as many files as possible. + + Within a file the most depended-upon component comes first, since that is + the one a reader would start from. Files are then visited round-robin, so + a cap never silently drops whole files while another file contributes + dozens of entry points. + """ + by_file: Dict[str, List[str]] = defaultdict(list) + for node in candidates: + path = components[node].file_path if node in components else "" + by_file[path].append(node) + for path in by_file: + by_file[path].sort(key=lambda n: (-in_degree.get(n, 0), n)) + + ordered: List[str] = [] + depth = 0 + while len(ordered) < limit: + added = False + for path in sorted(by_file): + if depth < len(by_file[path]): + ordered.append(by_file[path][depth]) + added = True + if len(ordered) >= limit: + break + if not added: + break + depth += 1 + return ordered + def compute_valid_leaf_types(components: Dict[str, Node]) -> Set[str]: """ @@ -61,7 +121,13 @@ def filter_leaf_nodes( keep_leaf_nodes = [] for leaf_node in leaf_nodes: # Skip any leaf nodes that are clearly error strings or invalid identifiers - if not isinstance(leaf_node, str) or leaf_node.strip() == "" or any(err_keyword in leaf_node.lower() for err_keyword in ['error', 'exception', 'failed', 'invalid']): + if not isinstance(leaf_node, str) or leaf_node.strip() == "": + logger.debug(f"Skipping invalid leaf node identifier: '{leaf_node}'") + continue + + # Only reject strings that are error messages, not identifiers that + # merely contain such a word (handleInvalidInput, ErrorLog, ...). + if leaf_node not in components and ERROR_MESSAGE_RE.search(leaf_node): logger.debug(f"Skipping invalid leaf node identifier: '{leaf_node}'") continue diff --git a/codewiki/src/be/dependency_analyzer/topo_sort.py b/codewiki/src/be/dependency_analyzer/topo_sort.py index da9ae3a0..b4df57bb 100644 --- a/codewiki/src/be/dependency_analyzer/topo_sort.py +++ b/codewiki/src/be/dependency_analyzer/topo_sort.py @@ -13,8 +13,11 @@ from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.be.dependency_analyzer.leaf_selection import ( LEAF_REDUCTION_THRESHOLD, + MIN_FILE_COVERAGE_AFTER_REDUCTION, compute_valid_leaf_types, + files_covered, filter_leaf_nodes, + spread_over_files, ) logger = logging.getLogger(__name__) @@ -310,16 +313,52 @@ def concise_node(leaf_nodes: Set[str]) -> Set[str]: concise_leaf_nodes = concise_node(leaf_nodes) if len(concise_leaf_nodes) >= LEAF_REDUCTION_THRESHOLD: - logger.debug(f"Leaf nodes are too many ({len(concise_leaf_nodes)}), removing dependencies of other nodes") + logger.info( + "Leaf nodes are too many (%d); reducing to components nothing " + "depends on.", + len(concise_leaf_nodes), + ) # Remove nodes that are dependencies of other nodes + reduced_input = set(leaf_nodes) for node, deps in acyclic_graph.items(): for dep in deps: - leaf_nodes.discard(dep) - - concise_leaf_nodes = concise_node(leaf_nodes) - + reduced_input.discard(dep) + reduced = concise_node(reduced_input) + + # In call-graph shaped repositories nearly every function is called by + # something, so the reduction above can collapse to a few unreferenced + # functions and leave most files without an entry point. Compare file + # coverage and fall back to a capped, file-spread selection instead. + before = files_covered(concise_leaf_nodes, components) + after = files_covered(reduced, components) + keeps_coverage = ( + not before + or len(after) / len(before) >= MIN_FILE_COVERAGE_AFTER_REDUCTION + ) + + if reduced and keeps_coverage: + leaf_nodes = reduced_input + concise_leaf_nodes = reduced + else: + in_degree: Dict[str, int] = {} + for node, deps in acyclic_graph.items(): + for dep in deps: + in_degree[dep] = in_degree.get(dep, 0) + 1 + capped = spread_over_files( + concise_leaf_nodes, components, in_degree, + LEAF_REDUCTION_THRESHOLD, + ) + logger.info( + "Reduction would have covered only %d of %d source files; " + "keeping %d entry points spread across %d files instead.", + len(after), len(before), len(capped), + len(files_covered(capped, components)), + ) + leaf_nodes = set(capped) + concise_leaf_nodes = capped + if not leaf_nodes: logger.warning("No leaf nodes found in the graph") return [] - - return concise_leaf_nodes \ No newline at end of file + + return concise_leaf_nodes \ No newline at end of file