From 968bd09f3d7b8d63be21c168d895b424c0adc094 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 14 Aug 2025 15:21:21 -0500 Subject: [PATCH 01/26] precommit --- Snakefile | 4 ++-- spras/evaluation.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/Snakefile b/Snakefile index 1daa8dee9..518e7509e 100644 --- a/Snakefile +++ b/Snakefile @@ -525,7 +525,7 @@ rule evaluation_ensemble_pr_curve: run: node_table = Evaluation.from_file(input.gold_standard_file).node_table node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, output.pr_curve_png, output.pr_curve_file) + Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input.dataset_file,output.pr_curve_png, output.pr_curve_file) # Returns list of algorithm specific ensemble files per dataset def collect_ensemble_per_algo_per_dataset(wildcards): @@ -544,7 +544,7 @@ rule evaluation_per_algo_ensemble_pr_curve: run: node_table = Evaluation.from_file(input.gold_standard_file).node_table node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input.dataset_file,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) # Remove the output directory diff --git a/spras/evaluation.py b/spras/evaluation.py index 732e58576..0b99308c3 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -354,7 +354,7 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[ return node_ensembles_dict @staticmethod - def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, output_png: str | PathLike, + def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, dataset_file: str, output_png: str | PathLike, output_file: str | PathLike, aggregate_per_algorithm: bool = False): """ Plots precision-recall (PR) curves for a set of node ensembles evaluated against a gold standard. @@ -387,6 +387,34 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da if not node_ensemble.empty: y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() + + # TODO: add a new one here for y_scores_baseline where the sources and targets frequency are set to 1 + # set the scores for nodes in node_ensemble that are in both the gold_standard_nodes and sources/targets/prizes to 1.0 + # then make that into a new node_ensemble with altered values that are then plotted. + + pickle = Evaluation.from_file(dataset_file) + prizes_df = pickle.get_node_columns(["sources", "targets", "prize"]) + prizes = set(prizes_df['NODEID']) + prize_gold_intersection = prizes & gold_standard_nodes + prize_node_ensemble_df = node_ensemble.copy() + + # TODO what if the node_ensemble is all frequency = 0.0, that will be the new source/target/prize/ baseline? + + # Set frequency to 1.0 for matching nodes + prize_node_ensemble_df.loc[ + prize_node_ensemble_df['Node'].isin(prize_gold_intersection), + 'Frequency' + ] = 1.0 + print(prize_node_ensemble_df) + + y_scores_prizes = prize_node_ensemble_df['Frequency'].tolist() + + precision_prizes, recall_prizes, thresholds_prizes = precision_recall_curve(y_true, y_scores_prizes) + plt.plot(recall_prizes, precision_prizes, color=color_palette[label], marker='o', linestyle=':', + label=f'{label.capitalize()} with prizes') + + + precision, recall, thresholds = precision_recall_curve(y_true, y_scores) # avg precision summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold avg_precision = average_precision_score(y_true, y_scores) From 7202a990dfbf11920fe52a38bbc9b7727813ec7a Mon Sep 17 00:00:00 2001 From: ntalluri Date: Tue, 19 Aug 2025 17:34:23 -0500 Subject: [PATCH 02/26] updating the baseline --- spras/evaluation.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 0b99308c3..7c3117459 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -382,37 +382,36 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da metric_dfs = [] baseline = None + y_scores_input_nodes = None + for label, node_ensemble in node_ensembles.items(): if not node_ensemble.empty: y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() - # TODO: add a new one here for y_scores_baseline where the sources and targets frequency are set to 1 - # set the scores for nodes in node_ensemble that are in both the gold_standard_nodes and sources/targets/prizes to 1.0 - # then make that into a new node_ensemble with altered values that are then plotted. - - pickle = Evaluation.from_file(dataset_file) - prizes_df = pickle.get_node_columns(["sources", "targets", "prize"]) - prizes = set(prizes_df['NODEID']) - prize_gold_intersection = prizes & gold_standard_nodes - prize_node_ensemble_df = node_ensemble.copy() - - # TODO what if the node_ensemble is all frequency = 0.0, that will be the new source/target/prize/ baseline? + if y_scores_input_nodes is None: + pickle = Evaluation.from_file(dataset_file) + input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes = set(input_nodes_df['NODEID']) + input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs or the intersection with the gold standard for this baseline? + input_nodes_ensemble_df = node_ensemble.copy() - # Set frequency to 1.0 for matching nodes - prize_node_ensemble_df.loc[ - prize_node_ensemble_df['Node'].isin(prize_gold_intersection), - 'Frequency' - ] = 1.0 - print(prize_node_ensemble_df) + input_nodes_ensemble_df.loc[ + input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), + 'Frequency' + ] = 1.0 - y_scores_prizes = prize_node_ensemble_df['Frequency'].tolist() + input_nodes_ensemble_df.loc[ + ~input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), + 'Frequency' + ] = 0.0 - precision_prizes, recall_prizes, thresholds_prizes = precision_recall_curve(y_true, y_scores_prizes) - plt.plot(recall_prizes, precision_prizes, color=color_palette[label], marker='o', linestyle=':', - label=f'{label.capitalize()} with prizes') + y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() + precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) + plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', + label=f'Input Nodes Baseline') precision, recall, thresholds = precision_recall_curve(y_true, y_scores) @@ -472,6 +471,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da combined_prc_df = pd.concat(prc_dfs, ignore_index=True) combined_metrics_df = pd.concat(metric_dfs, ignore_index=True) combined_metrics_df['Baseline'] = baseline + # TODO add new input_node baseline to the txt # merge dfs and NaN out metric values except for first row of each Ensemble_Source complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left') From 1b3be75bd4460dac9edad4626d8df8cc9ee544d6 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Tue, 19 Aug 2025 17:37:55 -0500 Subject: [PATCH 03/26] make it all input nodes, not the intersection between gold standard and input nodes --- spras/evaluation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 7c3117459..3b66e70bc 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -394,16 +394,16 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da pickle = Evaluation.from_file(dataset_file) input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) input_nodes = set(input_nodes_df['NODEID']) - input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs or the intersection with the gold standard for this baseline? + # input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs or the intersection with the gold standard for this baseline? input_nodes_ensemble_df = node_ensemble.copy() input_nodes_ensemble_df.loc[ - input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), + input_nodes_ensemble_df['Node'].isin(input_nodes), 'Frequency' ] = 1.0 input_nodes_ensemble_df.loc[ - ~input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), + ~input_nodes_ensemble_df['Node'].isin(input_nodes), 'Frequency' ] = 0.0 From 8ca0e9ed10eb45b812d453640aaea9dc3c2e9030 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Wed, 20 Aug 2025 12:25:50 -0500 Subject: [PATCH 04/26] update to be the baseline with gold standard intersection, added data to the txt files --- spras/evaluation.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 3b66e70bc..f05423e47 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -380,9 +380,12 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] + prc_input_nodes_baseline_df = None baseline = None - y_scores_input_nodes = None + precision_input_nodes = None + recall_input_nodes = None + thresholds_input_nodes = None for label, node_ensemble in node_ensembles.items(): @@ -390,20 +393,20 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() - if y_scores_input_nodes is None: + if precision_input_nodes is None and recall_input_nodes is None and thresholds_input_nodes is None: pickle = Evaluation.from_file(dataset_file) input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) input_nodes = set(input_nodes_df['NODEID']) - # input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs or the intersection with the gold standard for this baseline? + input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? input_nodes_ensemble_df = node_ensemble.copy() input_nodes_ensemble_df.loc[ - input_nodes_ensemble_df['Node'].isin(input_nodes), + input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), 'Frequency' ] = 1.0 input_nodes_ensemble_df.loc[ - ~input_nodes_ensemble_df['Node'].isin(input_nodes), + ~input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), 'Frequency' ] = 0.0 @@ -413,6 +416,17 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') + prc_input_nodes_baseline_data = { + 'Threshold': thresholds_input_nodes, + 'Precision': precision_input_nodes[:-1], + 'Recall': recall_input_nodes[:-1], + } + + prc_input_nodes_baseline_data = {'Ensemble_Source': ["input_nodes_baseline"] * len(thresholds_input_nodes), **prc_input_nodes_baseline_data} + + prc_input_nodes_baseline_df = pd.DataFrame.from_dict(prc_input_nodes_baseline_data) + + precision, recall, thresholds = precision_recall_curve(y_true, y_scores) # avg precision summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold @@ -471,10 +485,9 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da combined_prc_df = pd.concat(prc_dfs, ignore_index=True) combined_metrics_df = pd.concat(metric_dfs, ignore_index=True) combined_metrics_df['Baseline'] = baseline - # TODO add new input_node baseline to the txt # merge dfs and NaN out metric values except for first row of each Ensemble_Source - complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left') + complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left').merge(prc_input_nodes_baseline_df, on=['Ensemble_Source', 'Threshold', 'Precision', 'Recall'], how='outer') not_last_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') complete_df.loc[not_last_rows, ['Average_Precision', 'Baseline']] = None complete_df.to_csv(output_file, index=False, sep='\t') From dda62645fe71b36ba1128ff1a87bf4178354f00b Mon Sep 17 00:00:00 2001 From: ntalluri Date: Wed, 20 Aug 2025 14:16:23 -0500 Subject: [PATCH 05/26] debugging error in test cases --- spras/evaluation.py | 4 ++++ test/evaluate/test_evaluate.py | 32 +++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index f05423e47..a22a31c3a 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -410,12 +410,16 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da 'Frequency' ] = 0.0 + print(input_nodes_ensemble_df) + y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') + print(precision_input_nodes) + print(recall_input_nodes) prc_input_nodes_baseline_data = { 'Threshold': thresholds_input_nodes, 'Precision': precision_input_nodes[:-1], diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index ce50350e5..71616496b 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -35,6 +35,8 @@ def setup_class(cls): 'other_files': [] }) + # TODO figure out why the input-nodes file is not being included in the data.pickle file + # it keeps coming up empty with open(out_dataset, 'wb') as f: pickle.dump(dataset, f) @@ -126,8 +128,8 @@ def test_node_ensemble(self): out_path_file = Path(OUT_DIR + 'node-ensemble.csv') out_path_file.unlink(missing_ok=True) ensemble_network = [INPUT_DIR + 'ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_network) + input_data = OUT_DIR + 'data.pickle' + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_data) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) @@ -135,9 +137,9 @@ def test_empty_node_ensemble(self): out_path_file = Path(OUT_DIR + 'empty-node-ensemble.csv') out_path_file.unlink(missing_ok=True) empty_ensemble_network = [INPUT_DIR + 'empty-ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' + input_data = OUT_DIR + 'data.pickle' node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, - input_network) + input_data) node_ensemble_dict['empty'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-empty-node-ensemble.csv', shallow=False) @@ -147,8 +149,8 @@ def test_multiple_node_ensemble(self): out_path_empty_file = Path(OUT_DIR + 'empty-node-ensemble.csv') out_path_empty_file.unlink(missing_ok=True) ensemble_networks = [INPUT_DIR + 'ensemble-network.tsv', INPUT_DIR + 'empty-ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_network) + input_data = OUT_DIR + 'data.pickle' + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_data) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) node_ensemble_dict['empty'].to_csv(out_path_empty_file, sep='\t', index=False) @@ -159,9 +161,19 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + + pickle = Evaluation.from_file(input_data) + input_nodes_df = pickle.get_node_columns(["prize", "active"]) + + print(input_nodes_df) + ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) + + print(ensemble_file) + node_ensembles_dict = {'ensemble': ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes.txt', shallow=False) @@ -171,9 +183,10 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes-empty.txt', shallow=False) @@ -183,10 +196,11 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, out_path_file, True) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-multiple-ensemble-nodes.txt', shallow=False) From d8a16fc6c4e5d73a97510a16434483cf0a011abd Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 21 Aug 2025 15:17:17 -0500 Subject: [PATCH 06/26] fix input-nodes file with tristan's update --- test/evaluate/input/input-nodes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/evaluate/input/input-nodes.txt b/test/evaluate/input/input-nodes.txt index 1ffd8bdff..ef08aff66 100644 --- a/test/evaluate/input/input-nodes.txt +++ b/test/evaluate/input/input-nodes.txt @@ -1,3 +1,3 @@ NODEID prize active dummy sources targets -N -C 5.7 True True +N +C 5.7 True True \ No newline at end of file From 7f3c7f63b27d9b94be2e6e6852b2a02c2a3b4e9b Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 21 Aug 2025 15:41:23 -0500 Subject: [PATCH 07/26] update test case --- spras/evaluation.py | 4 ---- .../expected-pr-curve-ensemble-nodes-empty.txt | 2 ++ .../expected/expected-pr-curve-ensemble-nodes.txt | 2 ++ .../expected-pr-curve-multiple-ensemble-nodes.txt | 2 ++ test/evaluate/input/input-nodes.txt | 3 ++- test/evaluate/test_evaluate.py | 11 ----------- 6 files changed, 8 insertions(+), 16 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index a22a31c3a..f05423e47 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -410,16 +410,12 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da 'Frequency' ] = 0.0 - print(input_nodes_ensemble_df) - y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') - print(precision_input_nodes) - print(recall_input_nodes) prc_input_nodes_baseline_data = { 'Threshold': thresholds_input_nodes, 'Precision': precision_input_nodes[:-1], diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt index c9f6561c6..b57d3adfa 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt @@ -1,2 +1,4 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline Aggregated 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 +input_nodes_baseline 0.0 0.15384615384615385 1.0 +input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt index b0e50594e..21063c107 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt @@ -3,3 +3,5 @@ Aggregated 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Aggregated 0.01 0.6666666666666666 1.0 Aggregated 0.5 0.75 0.75 Aggregated 0.66 0.5 0.25 +input_nodes_baseline 0.0 0.15384615384615385 1.0 +input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt index 630a89ceb..c38848d82 100644 --- a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt @@ -8,3 +8,5 @@ Ensemble2 0.01 0.6666666666666666 1.0 Ensemble2 0.5 0.75 0.75 Ensemble2 0.66 0.5 0.25 Ensemble3 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 +input_nodes_baseline 0.0 0.15384615384615385 1.0 +input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/input/input-nodes.txt b/test/evaluate/input/input-nodes.txt index ef08aff66..ec0e4058f 100644 --- a/test/evaluate/input/input-nodes.txt +++ b/test/evaluate/input/input-nodes.txt @@ -1,3 +1,4 @@ NODEID prize active dummy sources targets N -C 5.7 True True \ No newline at end of file +C 5.7 True True +A 5 True True \ No newline at end of file diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 71616496b..1178f427a 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -35,8 +35,6 @@ def setup_class(cls): 'other_files': [] }) - # TODO figure out why the input-nodes file is not being included in the data.pickle file - # it keeps coming up empty with open(out_dataset, 'wb') as f: pickle.dump(dataset, f) @@ -162,16 +160,7 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - - pickle = Evaluation.from_file(input_data) - input_nodes_df = pickle.get_node_columns(["prize", "active"]) - - print(input_nodes_df) - ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) - - print(ensemble_file) - node_ensembles_dict = {'ensemble': ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, out_path_file) From 8492f70ca3b151bae94e6e71ab67dc09c2ce29fa Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 22 Aug 2025 11:05:27 -0500 Subject: [PATCH 08/26] clean up code and outputs --- spras/evaluation.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index f05423e47..8d1913d88 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -354,7 +354,7 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[ return node_ensembles_dict @staticmethod - def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, dataset_file: str, output_png: str | PathLike, + def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, input_nodes: pd.DataFrame, output_png: str | PathLike, output_file: str | PathLike, aggregate_per_algorithm: bool = False): """ Plots precision-recall (PR) curves for a set of node ensembles evaluated against a gold standard. @@ -365,6 +365,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da @param node_ensembles: dict of the pre-computed node_ensemble(s) @param node_table: gold standard nodes + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset @param output_png: filename to save the precision and recall curves as a .png image @param output_file: filename to save the precision, recall, threshold values, average precision, and baseline average precision @@ -393,22 +394,18 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() - if precision_input_nodes is None and recall_input_nodes is None and thresholds_input_nodes is None: - pickle = Evaluation.from_file(dataset_file) - input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) - input_nodes = set(input_nodes_df['NODEID']) - input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? + # input nodes (sources, targets, prizes, actives) may be easier to recover but are still valid gold standard nodes; + # the Input_Nodes_Baseline PR curve highlights their overlap with the gold standard. + if prc_input_nodes_baseline_df is None: + # pickle = Evaluation.from_file(dataset_file) + # input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes_set = set(input_nodes['NODEID']) + input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection input_nodes_ensemble_df = node_ensemble.copy() - input_nodes_ensemble_df.loc[ - input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), - 'Frequency' - ] = 1.0 - - input_nodes_ensemble_df.loc[ - ~input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), - 'Frequency' - ] = 0.0 + input_nodes_ensemble_df["Frequency"] = ( + input_nodes_ensemble_df["Node"].isin(input_nodes_gold_intersection).astype(float) + ) y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() @@ -416,18 +413,16 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') + # Dropping last elements because scikit-learn adds (1, 0) to precision/recall for plotting, not tied to real thresholds prc_input_nodes_baseline_data = { 'Threshold': thresholds_input_nodes, 'Precision': precision_input_nodes[:-1], 'Recall': recall_input_nodes[:-1], } - prc_input_nodes_baseline_data = {'Ensemble_Source': ["input_nodes_baseline"] * len(thresholds_input_nodes), **prc_input_nodes_baseline_data} - + prc_input_nodes_baseline_data = {'Ensemble_Source': ["Input_Nodes_Baseline"] * len(thresholds_input_nodes), **prc_input_nodes_baseline_data} prc_input_nodes_baseline_df = pd.DataFrame.from_dict(prc_input_nodes_baseline_data) - - precision, recall, thresholds = precision_recall_curve(y_true, y_scores) # avg precision summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold avg_precision = average_precision_score(y_true, y_scores) @@ -488,6 +483,18 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # merge dfs and NaN out metric values except for first row of each Ensemble_Source complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left').merge(prc_input_nodes_baseline_df, on=['Ensemble_Source', 'Threshold', 'Precision', 'Recall'], how='outer') + + # for each Ensemble_Source, remove Average_Precision and Baseline in all but the first row not_last_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') complete_df.loc[not_last_rows, ['Average_Precision', 'Baseline']] = None + + # move Input_Nodes_Baseline to the top of the df + complete_df.sort_values( + by='Ensemble_Source', + # x.ne('Input_Nodes_Baseline'): returns a Series of booleans; True for all rows except Input_Nodes_Baseline. + # Since False < True, baseline rows sort to the top. + key=lambda x: x.ne('Input_Nodes_Baseline'), + inplace=True + ) + complete_df.to_csv(output_file, index=False, sep='\t') From 9b8e31ce92a36eca877f34ccdc9bb5b56d397e04 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 22 Aug 2025 11:16:25 -0500 Subject: [PATCH 09/26] update to use input nodes and interactome from the snakefile instead of the functions itself --- Snakefile | 12 ++++++++---- spras/evaluation.py | 19 +++++++------------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Snakefile b/Snakefile index 518e7509e..068b3fcb6 100644 --- a/Snakefile +++ b/Snakefile @@ -524,8 +524,10 @@ rule evaluation_ensemble_pr_curve: pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes.txt']), run: node_table = Evaluation.from_file(input.gold_standard_file).node_table - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input.dataset_file,output.pr_curve_png, output.pr_curve_file) + input_interactome = Evaluation.from_file(input.dataset_file).get_interactome() + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input_interactome) + input_nodes = Evaluation.from_file(input.dataset_file).get_node_columns(["sources", "targets", "prize", "active"]) + Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file) # Returns list of algorithm specific ensemble files per dataset def collect_ensemble_per_algo_per_dataset(wildcards): @@ -543,8 +545,10 @@ rule evaluation_per_algo_ensemble_pr_curve: pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes-per-algorithm.txt']), run: node_table = Evaluation.from_file(input.gold_standard_file).node_table - node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input.dataset_file,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) + input_interactome = Evaluation.from_file(input.dataset_file).get_interactome() + node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input_interactome) + input_nodes = Evaluation.from_file(input.dataset_file).get_node_columns(["sources", "targets", "prize", "active"]) + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) # Remove the output directory diff --git a/spras/evaluation.py b/spras/evaluation.py index 8d1913d88..628c46b33 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -288,7 +288,7 @@ def pca_chosen_pathway(coordinates_files: list[Union[str, PathLike]], pathway_su return rep_pathways @staticmethod - def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[Union[str, PathLike]], dataset_file: str) -> dict: + def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[Union[str, PathLike]], input_interactome: pd.DataFrame) -> dict: """ Generates a dictionary of node ensembles using edge frequency data from a list of ensemble files. A list of ensemble files can contain an aggregated ensemble or algorithm-specific ensembles per dataset @@ -308,28 +308,25 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[ @param node_table: dataFrame of gold standard nodes (column: NODEID) @param ensemble_files: list of file paths containing edge ensemble outputs - @param dataset_file: path to the dataset file used to load the interactome + @param input_interactome: the input interactome used for a specific dataset @return: dictionary mapping each ensemble source to its node ensemble DataFrame """ node_ensembles_dict = dict() - pickle = Evaluation.from_file(dataset_file) - interactome = pickle.get_interactome() - - if interactome.empty: + if input_interactome.empty: raise ValueError( - f"Cannot compute PR curve or generate node ensemble. Input network for dataset \"{dataset_file.split('-')[0]}\" is empty." + f"Cannot compute PR curve or generate node ensemble. The input network is empty." ) if node_table.empty: raise ValueError( - f"Cannot compute PR curve or generate node ensemble. Gold standard associated with dataset \"{dataset_file.split('-')[0]}\" is empty." + f"Cannot compute PR curve or generate node ensemble. The gold standard is empty." ) # set the initial default frequencies to 0 for all interactome and gold standard nodes - node1_interactome = interactome[['Interactor1']].rename(columns={'Interactor1': 'Node'}) + node1_interactome = input_interactome[['Interactor1']].rename(columns={'Interactor1': 'Node'}) node1_interactome['Frequency'] = 0.0 - node2_interactome = interactome[['Interactor2']].rename(columns={'Interactor2': 'Node'}) + node2_interactome = input_interactome[['Interactor2']].rename(columns={'Interactor2': 'Node'}) node2_interactome['Frequency'] = 0.0 gs_nodes = node_table[[Evaluation.NODE_ID]].rename(columns={Evaluation.NODE_ID: 'Node'}) gs_nodes['Frequency'] = 0.0 @@ -397,8 +394,6 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # input nodes (sources, targets, prizes, actives) may be easier to recover but are still valid gold standard nodes; # the Input_Nodes_Baseline PR curve highlights their overlap with the gold standard. if prc_input_nodes_baseline_df is None: - # pickle = Evaluation.from_file(dataset_file) - # input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) input_nodes_set = set(input_nodes['NODEID']) input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection input_nodes_ensemble_df = node_ensemble.copy() From 04ddc1ffe3b2aefc238fb934fadf06b6299faf25 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 22 Aug 2025 11:21:49 -0500 Subject: [PATCH 10/26] update test cases --- ...expected-pr-curve-ensemble-nodes-empty.txt | 4 ++-- .../expected-pr-curve-ensemble-nodes.txt | 4 ++-- ...ected-pr-curve-multiple-ensemble-nodes.txt | 4 ++-- test/evaluate/test_evaluate.py | 19 ++++++++++++------- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt index b57d3adfa..4517ceb0e 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt @@ -1,4 +1,4 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline +Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 +Input_Nodes_Baseline 1.0 1.0 0.25 Aggregated 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 -input_nodes_baseline 0.0 0.15384615384615385 1.0 -input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt index 21063c107..9a78dbe77 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt @@ -1,7 +1,7 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline +Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 +Input_Nodes_Baseline 1.0 1.0 0.25 Aggregated 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Aggregated 0.01 0.6666666666666666 1.0 Aggregated 0.5 0.75 0.75 Aggregated 0.66 0.5 0.25 -input_nodes_baseline 0.0 0.15384615384615385 1.0 -input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt index c38848d82..483e972d8 100644 --- a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt @@ -1,4 +1,6 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline +Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 +Input_Nodes_Baseline 1.0 1.0 0.25 Ensemble1 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Ensemble1 0.01 0.6666666666666666 1.0 Ensemble1 0.5 0.75 0.75 @@ -8,5 +10,3 @@ Ensemble2 0.01 0.6666666666666666 1.0 Ensemble2 0.5 0.75 0.75 Ensemble2 0.66 0.5 0.25 Ensemble3 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 -input_nodes_baseline 0.0 0.15384615384615385 1.0 -input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 1178f427a..3eb0ca03e 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -127,7 +127,8 @@ def test_node_ensemble(self): out_path_file.unlink(missing_ok=True) ensemble_network = [INPUT_DIR + 'ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_data) + input_interactome = Evaluation.from_file(input_data).get_interactome() + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_interactome) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) @@ -136,8 +137,8 @@ def test_empty_node_ensemble(self): out_path_file.unlink(missing_ok=True) empty_ensemble_network = [INPUT_DIR + 'empty-ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, - input_data) + input_interactome = Evaluation.from_file(input_data).get_interactome() + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, input_interactome) node_ensemble_dict['empty'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-empty-node-ensemble.csv', shallow=False) @@ -148,7 +149,8 @@ def test_multiple_node_ensemble(self): out_path_empty_file.unlink(missing_ok=True) ensemble_networks = [INPUT_DIR + 'ensemble-network.tsv', INPUT_DIR + 'empty-ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_data) + input_interactome = Evaluation.from_file(input_data).get_interactome() + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_interactome) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) node_ensemble_dict['empty'].to_csv(out_path_empty_file, sep='\t', index=False) @@ -160,9 +162,10 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' + input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes.txt', shallow=False) @@ -173,9 +176,10 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' + input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes-empty.txt', shallow=False) @@ -186,10 +190,11 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' + input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file, True) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-multiple-ensemble-nodes.txt', shallow=False) From e98d1e2e932c12b0f2f2757614bb7aa1eef805c9 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 22 Aug 2025 11:24:34 -0500 Subject: [PATCH 11/26] removed unnecessary code --- spras/evaluation.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 628c46b33..29749320d 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -379,12 +379,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] prc_input_nodes_baseline_df = None - baseline = None - precision_input_nodes = None - recall_input_nodes = None - thresholds_input_nodes = None - for label, node_ensemble in node_ensembles.items(): if not node_ensemble.empty: @@ -405,8 +400,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) - plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', - label=f'Input Nodes Baseline') + plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') # Dropping last elements because scikit-learn adds (1, 0) to precision/recall for plotting, not tied to real thresholds prc_input_nodes_baseline_data = { From 2004c9a61bcf9e6b8c7bc5e32b8c3358ebcc0674 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Mon, 25 Aug 2025 10:59:57 -0500 Subject: [PATCH 12/26] update code to use helper function for input nodes --- Snakefile | 10 ++++++---- spras/dataset.py | 9 +++++++++ spras/evaluation.py | 1 + 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Snakefile b/Snakefile index 068b3fcb6..7054ab6f8 100644 --- a/Snakefile +++ b/Snakefile @@ -523,10 +523,11 @@ rule evaluation_ensemble_pr_curve: pr_curve_png = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes.png']), pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes.txt']), run: + input_dataset = Evaluation.from_file(input.dataset_file) + input_interactome = input_dataset.get_interactome() + input_nodes = input_dataset.get_interesting_input_nodes() node_table = Evaluation.from_file(input.gold_standard_file).node_table - input_interactome = Evaluation.from_file(input.dataset_file).get_interactome() node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input_interactome) - input_nodes = Evaluation.from_file(input.dataset_file).get_node_columns(["sources", "targets", "prize", "active"]) Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file) # Returns list of algorithm specific ensemble files per dataset @@ -544,10 +545,11 @@ rule evaluation_per_algo_ensemble_pr_curve: pr_curve_png = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes-per-algorithm.png']), pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes-per-algorithm.txt']), run: + input_dataset = Evaluation.from_file(input.dataset_file) + input_interactome = input_dataset.get_interactome() + input_nodes = input_dataset.get_interesting_input_nodes() node_table = Evaluation.from_file(input.gold_standard_file).node_table - input_interactome = Evaluation.from_file(input.dataset_file).get_interactome() node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input_interactome) - input_nodes = Evaluation.from_file(input.dataset_file).get_node_columns(["sources", "targets", "prize", "active"]) Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) diff --git a/spras/dataset.py b/spras/dataset.py index 1346750e3..8b501733e 100644 --- a/spras/dataset.py +++ b/spras/dataset.py @@ -171,6 +171,15 @@ def contains_node_columns(self, col_names: list[str] | str): return False return True + def get_interesting_input_nodes(self) -> pd.DataFrame: + """ + Returns: a table listing the input nodes considered as starting points for pathway reconstruction algorithms, + restricted to nodes that have at least one of the specified attributes. + """ + interesting_input_node_columns = ["sources", "targets", "prize", "active"] + interesting_input_nodes = Dataset.get_node_columns(self, col_names = interesting_input_node_columns) + return interesting_input_nodes + def get_other_files(self): return self.other_files.copy() diff --git a/spras/evaluation.py b/spras/evaluation.py index 29749320d..af6ae965a 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -393,6 +393,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection input_nodes_ensemble_df = node_ensemble.copy() + # TODO: make a comment on what this is doing for future use input_nodes_ensemble_df["Frequency"] = ( input_nodes_ensemble_df["Node"].isin(input_nodes_gold_intersection).astype(float) ) From 82fbfce8562cd08ddd4f217b9ac3fce77ec87fb3 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Mon, 25 Aug 2025 11:04:51 -0500 Subject: [PATCH 13/26] update test case to use new helper function --- test/evaluate/test_evaluate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 3eb0ca03e..422660bd7 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -162,7 +162,7 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, @@ -176,7 +176,7 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, @@ -190,7 +190,7 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} From dcab7810586482c6861a770521cb641783cf0159 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Mon, 25 Aug 2025 11:06:37 -0500 Subject: [PATCH 14/26] update comment --- spras/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index af6ae965a..313727fb8 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -393,7 +393,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection input_nodes_ensemble_df = node_ensemble.copy() - # TODO: make a comment on what this is doing for future use + # set 'Frequency' to 1.0 if the input node is also in the gold standard intersection, else set to 0.0 input_nodes_ensemble_df["Frequency"] = ( input_nodes_ensemble_df["Node"].isin(input_nodes_gold_intersection).astype(float) ) From a57ef19e1be459dee4fd5f42e929a739d1cd42c5 Mon Sep 17 00:00:00 2001 From: Neha Talluri <78840540+ntalluri@users.noreply.github.com> Date: Tue, 26 Aug 2025 08:52:07 -0700 Subject: [PATCH 15/26] fix space formatting --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 7054ab6f8..f009821c1 100644 --- a/Snakefile +++ b/Snakefile @@ -528,7 +528,7 @@ rule evaluation_ensemble_pr_curve: input_nodes = input_dataset.get_interesting_input_nodes() node_table = Evaluation.from_file(input.gold_standard_file).node_table node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input_interactome) - Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file) + Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes, output.pr_curve_png, output.pr_curve_file) # Returns list of algorithm specific ensemble files per dataset def collect_ensemble_per_algo_per_dataset(wildcards): @@ -550,7 +550,7 @@ rule evaluation_per_algo_ensemble_pr_curve: input_nodes = input_dataset.get_interesting_input_nodes() node_table = Evaluation.from_file(input.gold_standard_file).node_table node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input_interactome) - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes, output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) # Remove the output directory From 58b4e044d189fc7632740d8738e1e8b8a2aa9885 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 30 Apr 2026 11:01:03 -0500 Subject: [PATCH 16/26] make config back to what it was --- config/config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index fc718e3cf..fe95ab177 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -217,7 +217,7 @@ analysis: include: true # Create Cytoscape session file with all pathway graphs for each dataset cytoscape: - include: true + include: false # Machine learning analysis (e.g. clustering) of the pathway output files for each dataset ml: # ml analysis per dataset @@ -228,7 +228,7 @@ analysis: # specify how many principal components to calculate components: 2 # boolean to show the labels on the pca graph - labels: true + labels: false # 'ward', 'complete', 'average', 'single' # if linkage: ward, must use metric: euclidean linkage: 'ward' @@ -244,7 +244,7 @@ analysis: evaluation: # evaluation per dataset-goldstandard pair # evaluation will not run unless ml include is set to true - include: true + include: false # adds evaluation per algorithm per dataset-goldstandard pair # evaluation per algorithm will not run unless ml include and ml aggregate_per_algorithm are set to true aggregate_per_algorithm: true From c9845dd7fdb5461343390a9906a9360e190d2793 Mon Sep 17 00:00:00 2001 From: Neha Talluri <78840540+ntalluri@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:39:42 -0500 Subject: [PATCH 17/26] Apply suggestions from code review Co-authored-by: Tristan F.-R. --- spras/evaluation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spras/evaluation.py b/spras/evaluation.py index c659e187f..a528590ef 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -475,6 +475,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') # Dropping last elements because scikit-learn adds (1, 0) to precision/recall for plotting, not tied to real thresholds + # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve:~:text=Returns%3A-,precision,predictions%20with%20score%20%3E%3D%20thresholds%5Bi%5D%20and%20the%20last%20element%20is%200.,-thresholds prc_input_nodes_baseline_data = { 'Threshold': thresholds_input_nodes, 'Precision': precision_input_nodes[:-1], From 0aef46e963d632dc40ca1e427593d1b799a76a13 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Mon, 6 Jul 2026 10:50:17 -0500 Subject: [PATCH 18/26] remove interesting terminology --- spras/dataset.py | 8 ++++---- spras/evaluation.py | 8 ++++---- test/evaluate/test_evaluate.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/spras/dataset.py b/spras/dataset.py index 7829f8558..f34cd01d0 100644 --- a/spras/dataset.py +++ b/spras/dataset.py @@ -226,14 +226,14 @@ def contains_node_columns(self, col_names: list[str] | str): return False return True - def get_interesting_input_nodes(self) -> pd.DataFrame: + def get_input_nodes(self) -> pd.DataFrame: """ Returns: a table listing the input nodes considered as starting points for pathway reconstruction algorithms, restricted to nodes that have at least one of the specified attributes. """ - interesting_input_node_columns = ["sources", "targets", "prize", "active"] - interesting_input_nodes = Dataset.get_node_columns(self, col_names = interesting_input_node_columns) - return interesting_input_nodes + input_node_columns = ["sources", "targets", "prize", "active"] # TODO: do we want to add dummy nodes? + input_nodes = Dataset.get_node_columns(self, col_names = input_node_columns) + return input_nodes def get_other_files(self): return self.other_files.copy() diff --git a/spras/evaluation.py b/spras/evaluation.py index a528590ef..2aff1dfab 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -34,10 +34,10 @@ class Evaluation: label: str datasets: list[str] - node_table: pd.DataFrame - mixed_edge_table: pd.DataFrame - undirected_edge_table: pd.DataFrame - directed_edge_table: pd.DataFrame + node_table: pd.DataFrame # the node gold standard + mixed_edge_table: pd.DataFrame # the edge gold standard + undirected_edge_table: pd.DataFrame # the edge gold standard fully undirected + directed_edge_table: pd.DataFrame # the edge gold standard fully directed @staticmethod def merge_gold_standard_input(gs_dict: GoldStandardDict, gs_file: str | os.PathLike): diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index e47dd2a3f..4e94f60c2 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -163,7 +163,7 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() + input_nodes = Evaluation.from_file(input_data).get_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, @@ -177,7 +177,7 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() + input_nodes = Evaluation.from_file(input_data).get_input_nodes() empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, @@ -191,7 +191,7 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() + input_nodes = Evaluation.from_file(input_data).get_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} From f7d98289c6190404568e9b145276ecebd229d3a6 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Tue, 7 Jul 2026 14:36:36 -0500 Subject: [PATCH 19/26] draft of baselines --- Snakefile | 35 +++++---- spras/evaluation.py | 183 ++++++++++++++++++++++---------------------- 2 files changed, 112 insertions(+), 106 deletions(-) diff --git a/Snakefile b/Snakefile index 5ad7aa185..5c278ed2c 100644 --- a/Snakefile +++ b/Snakefile @@ -430,13 +430,17 @@ def get_gold_standard_pickle_file(wildcards): gs = parts[1] return SEP.join([out_dir, f'gs-{gs}-merged.pickle']) +# Return the dataset pickle file for a specific dataset +def get_dataset_pickle_file(wildcards): + dataset_label = get_dataset_label(wildcards) + return SEP.join([out_dir, f'dataset-{dataset_label}-merged.pickle']) + # Returns the dataset corresponding to the gold standard pair def get_dataset_label(wildcards): parts = wildcards.dataset_gold_standard_pair.split('-') dataset = parts[0] return dataset - # Returns all pathways for a specific dataset def collect_pathways_per_dataset(wildcards): dataset_label = get_dataset_label(wildcards) @@ -446,14 +450,16 @@ def collect_pathways_per_dataset(wildcards): rule evaluation_pr_per_pathways: input: node_gold_standard_file = get_gold_standard_pickle_file, + dataset_file = get_dataset_pickle_file, pathways = collect_pathways_per_dataset output: node_pr_file = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', "pr-per-pathway-nodes.txt"]), node_pr_png = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-per-pathway-nodes.png']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pr_df = Evaluation.node_precision_and_recall(input.pathways, node_table) - Evaluation.precision_and_recall_per_pathway(pr_df, output.node_pr_file, output.node_pr_png) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, node_table, output.node_pr_file, output.node_pr_png) # Returns all pathways for a specific algorithm and dataset def collect_pathways_per_algo_per_dataset(wildcards): @@ -465,14 +471,16 @@ def collect_pathways_per_algo_per_dataset(wildcards): rule evaluation_per_algo_pr_per_pathways: input: node_gold_standard_file = get_gold_standard_pickle_file, + dataset_file = get_dataset_pickle_file, pathways = collect_pathways_per_algo_per_dataset, output: node_pr_file = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', "pr-per-pathway-for-{algorithm}-nodes.txt"]), node_pr_png = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-per-pathway-for-{algorithm}-nodes.png']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pr_df = Evaluation.node_precision_and_recall(input.pathways, node_table) - Evaluation.precision_and_recall_per_pathway(pr_df, output.node_pr_file, output.node_pr_png, include_aggregate_algo_eval) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, node_table, output.node_pr_file, output.node_pr_png, include_aggregate_algo_eval) # Return pathway summary file per dataset def collect_summary_statistics_per_dataset(wildcards): @@ -490,6 +498,7 @@ def collect_pca_coordinates_per_dataset(wildcards): rule evaluation_pca_chosen: input: node_gold_standard_file = get_gold_standard_pickle_file, + dataset_file = get_dataset_pickle_file, pca_coordinates_file = collect_pca_coordinates_per_dataset, pathway_summary_file = collect_summary_statistics_per_dataset output: @@ -497,9 +506,10 @@ rule evaluation_pca_chosen: node_pca_chosen_pr_png = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-pca-chosen-pathway-nodes.png']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pca_chosen_pathway = Evaluation.pca_chosen_pathway(input.pca_coordinates_file, input.pathway_summary_file, out_dir) pr_df = Evaluation.node_precision_and_recall(pca_chosen_pathway, node_table) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png) + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, node_table, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png, include_aggregate_algo_eval) # Returns pca coordinates for a specific algorithm and dataset def collect_pca_coordinates_per_algo_per_dataset(wildcards): @@ -511,6 +521,7 @@ def collect_pca_coordinates_per_algo_per_dataset(wildcards): rule evaluation_per_algo_pca_chosen: input: node_gold_standard_file = get_gold_standard_pickle_file, + dataset_file = get_dataset_pickle_file, pca_coordinates_file = collect_pca_coordinates_per_algo_per_dataset, pathway_summary_file = collect_summary_statistics_per_dataset output: @@ -518,14 +529,10 @@ rule evaluation_per_algo_pca_chosen: node_pca_chosen_pr_png = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-pca-chosen-pathway-per-algorithm-nodes.png']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pca_chosen_pathways = Evaluation.pca_chosen_pathway(input.pca_coordinates_file, input.pathway_summary_file, out_dir) pr_df = Evaluation.node_precision_and_recall(pca_chosen_pathways, node_table) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png, include_aggregate_algo_eval) - -# Return the dataset pickle file for a specific dataset -def get_dataset_pickle_file(wildcards): - dataset_label = get_dataset_label(wildcards) - return SEP.join([out_dir, f'dataset-{dataset_label}-merged.pickle']) + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, node_table, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png, include_aggregate_algo_eval) # Returns ensemble file for each dataset def collect_ensemble_per_dataset(wildcards): @@ -543,8 +550,9 @@ rule evaluation_ensemble_pr_curve: node_pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-curve-ensemble-nodes.txt']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, output.node_pr_curve_png, output.node_pr_curve_file) + Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes, output.node_pr_curve_png, output.node_pr_curve_file) # Returns list of algorithm specific ensemble files per dataset def collect_ensemble_per_algo_per_dataset(wildcards): @@ -562,8 +570,9 @@ rule evaluation_per_algo_ensemble_pr_curve: node_pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-curve-ensemble-nodes-per-algorithm-nodes.txt']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table - node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, output.node_pr_curve_png, output.node_pr_curve_file, include_aggregate_algo_eval) + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() + node_ensembles_dict= Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input.dataset_file) + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes, output.node_pr_curve_png, output.node_pr_curve_file, include_aggregate_algo_eval) rule evaluation_edge_dummy: input: diff --git a/spras/evaluation.py b/spras/evaluation.py index 2aff1dfab..4034cdf5b 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -15,6 +15,7 @@ ) from spras.analysis.ml import create_palette +from spras.dataset import Dataset from spras.interactome import ( convert_directed_to_undirected, convert_undirected_to_directed, @@ -32,8 +33,8 @@ class GoldStandardDict(TypedDict): class Evaluation: NODE_ID = 'NODEID' - label: str - datasets: list[str] + label: str # gold standard label + datasets: list[str] # datasets associated with the dataset_labels for the specific gold standard node_table: pd.DataFrame # the node gold standard mixed_edge_table: pd.DataFrame # the edge gold standard undirected_edge_table: pd.DataFrame # the edge gold standard fully undirected @@ -184,7 +185,7 @@ def node_precision_and_recall(file_paths: Iterable[Union[str, PathLike]], node_t return pr_df @staticmethod - def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, title: str): + def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, input_nodes: pd.DataFrame, node_table: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, title: str): """ Generates a scatter plot of precision and recall values for each pathway and saves both the plot and the data. @@ -194,7 +195,9 @@ def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, output_file: str | for each algorithm. @param pr_df: Dataframe of calculated precision and recall for each pathway file. - Must include a preprocessed 'Algorithm' column. + Must include a preprocessed 'ADataframelgorithm' column. + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset + @param node_table: the gold standard nodes @param output_file: the filename to save the precision and recall of each pathway @param output_png: the filename to plot the precision and recall of each pathway (not a PRC) @param title: The title to use for the plot @@ -220,6 +223,13 @@ def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, output_file: str | label=algorithm.capitalize() ) + gold_standard_nodes = set(node_table[Evaluation.NODE_ID]) + input_nodes_set = set(input_nodes['NODEID']) + input_nodes_tp = len(input_nodes_set & gold_standard_nodes) + precision_input = input_nodes_tp / len(input_nodes_set) + recall_input = input_nodes_tp / len(gold_standard_nodes) + plt.plot(recall_input, precision_input, color='red', markersize=12, marker='X', linestyle='None', label=f'Input Nodes vs Gold Standard Baseline (P={precision_input:.3f}, R={recall_input:.3f})') + plt.title(title) plt.xlabel('Recall') plt.ylabel('Precision') @@ -235,13 +245,15 @@ def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, output_file: str | pr_df.to_csv(output_file, sep='\t', index=False) @staticmethod - def precision_and_recall_per_pathway(pr_df: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, aggregate_per_algorithm: bool = False): + def precision_and_recall_per_pathway(pr_df: pd.DataFrame, input_nodes: pd.DataFrame, node_table: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, aggregate_per_algorithm: bool = False): """ Function for visualizing per pathway precision and recall across all algorithms. Each point in the plot represents a single pathway reconstruction. If `aggregate_per_algorithm` is set to True, the plot is restricted to a single algorithm and titled accordingly. @param pr_df: Dataframe of calculated precision and recall for each pathway file + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset + @param node_table: the gold standard nodes @param output_file: the filename to save the precision and recall of each pathway @param output_png: the filename to plot the precision and recall of each pathway (not a PRC) @param aggregate_per_algorithm: Boolean indicating if function is used per algorithm (Default False) @@ -256,7 +268,7 @@ def precision_and_recall_per_pathway(pr_df: pd.DataFrame, output_file: str | Pat else: title = "Precision and Recall Plot Per Pathway Per Algorithm" - Evaluation.visualize_precision_and_recall_plot(pr_df, output_file, output_png, title) + Evaluation.visualize_precision_and_recall_plot(pr_df, input_nodes, node_table, output_file, output_png, title) else: # this block should never be reached — having 0 pathways implies that no algorithms or parameter combinations were run, @@ -264,7 +276,7 @@ def precision_and_recall_per_pathway(pr_df: pd.DataFrame, output_file: str | Pat raise ValueError("No pathways were provided to evaluate and visulize on. This likely means no algorithms or parameter combinations were run.") @staticmethod - def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, aggregate_per_algorithm: bool = False): + def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, input_nodes: pd.DataFrame, node_table: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, aggregate_per_algorithm: bool = False): """ Function for visualizing the precision and recall of the single parameter combination selected via PCA, @@ -273,6 +285,8 @@ def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, output_file: st is True, the plot includes a pca chosen pathway per algorithm and titled accordingly. @param pr_df: Dataframe of calculated precision and recall for each pathway file + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset + @param node_table: the gold standard nodes @param output_file: the filename to save the precision and recall of each pathway @param output_png: the filename to plot the precision and recall of each pathway (not a PRC) @param aggregate_per_algorithm: Boolean indicating if function is used per algorithm (Default False) @@ -288,7 +302,7 @@ def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, output_file: st else: title = "PCA-Chosen Pathway Across All Algorithms Precision and Recall Plot" - Evaluation.visualize_precision_and_recall_plot(pr_df, output_file, output_png, title) + Evaluation.visualize_precision_and_recall_plot(pr_df, input_nodes, node_table, output_file, output_png, title) else: # Edge case: if all algorithms chosen use only 1 parameter combination @@ -377,12 +391,12 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: Itera @param node_table: dataFrame of gold standard nodes (column: NODEID) @param ensemble_files: list of file paths containing edge ensemble outputs @param dataset_file: path to the dataset file used to load the interactome - @return: dictionary mapping each ensemble source to its node ensemble DataFrame + @return: dictionary mapping each ensemble source to its node ensemble DataFrame and the input nodes (sources, targets, prizes, actives) """ node_ensembles_dict = dict() - pickle = Evaluation.from_file(dataset_file) + pickle = Dataset.from_file(dataset_file) interactome = pickle.get_interactome() if interactome.empty: @@ -422,8 +436,9 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: Itera return node_ensembles_dict @staticmethod - def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, input_nodes: pd.DataFrame, output_png: str | PathLike, - output_file: str | PathLike, aggregate_per_algorithm: bool = False): + def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, input_nodes: pd.DataFrame, + output_png: str | PathLike, output_file: str | PathLike, + aggregate_per_algorithm: bool = False): """ Plots precision-recall (PR) curves for a set of node ensembles evaluated against a gold standard. @@ -440,6 +455,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da @param aggregate_per_algorithm: Boolean indicating if function is used per algorithm (Default False) """ gold_standard_nodes = set(node_table[Evaluation.NODE_ID]) + input_nodes_set = set(input_nodes['NODEID']) # make color palette per ensemble label name label_names = list(node_ensembles.keys()) @@ -449,82 +465,61 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] - prc_input_nodes_baseline_df = None - baseline = None + gs_baseline = None + input_gold_baseline = None + input_baseline = None for label, node_ensemble in node_ensembles.items(): - if not node_ensemble.empty: - y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] - y_scores = node_ensemble['Frequency'].tolist() - - # input nodes (sources, targets, prizes, actives) may be easier to recover but are still valid gold standard nodes; - # the Input_Nodes_Baseline PR curve highlights their overlap with the gold standard. - if prc_input_nodes_baseline_df is None: - input_nodes_set = set(input_nodes['NODEID']) - input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection - input_nodes_ensemble_df = node_ensemble.copy() - - # set 'Frequency' to 1.0 if the input node is also in the gold standard intersection, else set to 0.0 - input_nodes_ensemble_df["Frequency"] = ( - input_nodes_ensemble_df["Node"].isin(input_nodes_gold_intersection).astype(float) - ) - - y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() - - precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) - plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') - - # Dropping last elements because scikit-learn adds (1, 0) to precision/recall for plotting, not tied to real thresholds - # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve:~:text=Returns%3A-,precision,predictions%20with%20score%20%3E%3D%20thresholds%5Bi%5D%20and%20the%20last%20element%20is%200.,-thresholds - prc_input_nodes_baseline_data = { - 'Threshold': thresholds_input_nodes, - 'Precision': precision_input_nodes[:-1], - 'Recall': recall_input_nodes[:-1], - } - - prc_input_nodes_baseline_data = {'Ensemble_Source': ["Input_Nodes_Baseline"] * len(thresholds_input_nodes), **prc_input_nodes_baseline_data} - prc_input_nodes_baseline_df = pd.DataFrame.from_dict(prc_input_nodes_baseline_data) - - precision, recall, thresholds = precision_recall_curve(y_true, y_scores) - # avg precision summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold - avg_precision = average_precision_score(y_true, y_scores) - - # only set baseline precision once - # the same for every algorithm per dataset/goldstandard pair - if baseline is None: - baseline = np.sum(y_true) / len(y_true) - plt.axhline(y=baseline, color='black', linestyle='--', label=f'Baseline: {baseline:.4f}') - - plt.plot(recall, precision, color=color_palette[label], marker='o', - label=f'{label.capitalize()} (AP: {avg_precision:.4f})') - - # Dropping last elements because scikit-learn adds (1, 0) to precision/recall for plotting, not tied to real thresholds - # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve:~:text=Returns%3A-,precision,predictions%20with%20score%20%3E%3D%20thresholds%5Bi%5D%20and%20the%20last%20element%20is%200.,-thresholds - prc_data = { - 'Threshold': thresholds, - 'Precision': precision[:-1], - 'Recall': recall[:-1], - } - - metric_data = { - 'Average_Precision': [avg_precision], - } - - ensemble_source = label.capitalize() if label != 'ensemble' else 'Aggregated' - prc_data = {'Ensemble_Source': [ensemble_source] * len(thresholds), **prc_data} - metric_data = {'Ensemble_Source': [ensemble_source], **metric_data} - - prc_df = pd.DataFrame.from_dict(prc_data) - prc_dfs.append(prc_df) - metric_df = pd.DataFrame.from_dict(metric_data) - metric_dfs.append(metric_df) - - else: + if node_ensemble.empty: raise ValueError( - "Cannot compute PR curve: the ensemble network is empty." - f"This should not happen unless the input network for pathway reconstruction is empty." + "Cannot compute PR curve: the ensemble network is empty. " + "This should not happen unless the input network for pathway reconstruction is empty." ) + y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] + y_scores = node_ensemble['Frequency'].tolist() + + precision, recall, thresholds = precision_recall_curve(y_true, y_scores) + # avg precision summarizes a PR curve as the weighted mean of precisions achieved at each threshold + avg_precision = average_precision_score(y_true, y_scores) + plt.plot(recall, precision, color=color_palette[label], marker='o', label=f'{label.capitalize()} (AP: {avg_precision:.4f})') + + # Different baselines + # Computed once; identical for every algorithm on a given dataset/gold-standard pair. + if gs_baseline is None and input_gold_baseline is None and input_baseline is None: + universe_size = len(y_scores) + + # gs_baseline = |gold_standard| / |universe|: random-predictor precision + # beat this to beat random + gs_baseline = np.sum(y_true) / universe_size + plt.axhline(y=gs_baseline, color='gray', linestyle='--', + label=f'Gold Standard Baseline (P={gs_baseline:.4f})') + + # input_gold_prev = |input and gold| / |universe|: fraction of the network that is both input and gold + # numerator counts only gold nodes that are also inputs, not all gold nodes. + input_gold_baseline = len(input_nodes_set & gold_standard_nodes) / universe_size + plt.axhline(y=input_gold_baseline, color='gray', linestyle=':', + label=f'Input Nodes and Gold Standard Baseline (P={input_gold_baseline:.4f})') + + # input_prevalence = |input| / |universe|: fraction of the network that is input nodes + input_baseline= len(input_nodes_set) / universe_size + plt.axhline(y=input_baseline, color='gray', linestyle='-.', + label=f'Input Nodes Baseline (P={input_baseline:.4f})') + + # Drop the last precision/recall element: sklearn appends (1, 0) for plotting, not tied to a real threshold. + # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html + ensemble_source = label.capitalize() if label != 'ensemble' else 'Aggregated' + prc_dfs.append(pd.DataFrame({ + 'Ensemble_Source': [ensemble_source] * len(thresholds), + 'Threshold': thresholds, + 'Precision': precision[:-1], + 'Recall': recall[:-1], + })) + metric_dfs.append(pd.DataFrame({ + 'Ensemble_Source': [ensemble_source], + 'Average_Precision': [avg_precision], + })) + if aggregate_per_algorithm: plt.title('Precision-Recall Curve Per Algorithm Specific Ensemble') else: @@ -541,21 +536,23 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da combined_prc_df = pd.concat(prc_dfs, ignore_index=True) combined_metrics_df = pd.concat(metric_dfs, ignore_index=True) - combined_metrics_df['Baseline'] = baseline + combined_metrics_df['Gold_Standard_Baseline'] = gs_baseline + combined_metrics_df['Input_Nodes_and _Gold_Standard_Baseline'] = gs_baseline + combined_metrics_df['Input_Nodes_Baseline'] = input_baseline + - # merge dfs and NaN out metric values except for first row of each Ensemble_Source - complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left').merge(prc_input_nodes_baseline_df, on=['Ensemble_Source', 'Threshold', 'Precision', 'Recall'], how='outer') + # merge curves with per-source metrics, then add the input-nodes baseline as its own row + complete_df = ( + combined_prc_df + .merge(combined_metrics_df, on='Ensemble_Source', how='left') + ) - # for each Ensemble_Source, remove Average_Precision and Baseline in all but the first row - not_last_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') - complete_df.loc[not_last_rows, ['Average_Precision', 'Baseline']] = None + # keep Average_Precision and Baseline only on the first row of each Ensemble_Source + not_first_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') + complete_df.loc[not_first_rows, ['Average_Precision', 'Gold_Standard_Baseline', 'Input_Nodes_and _Gold_Standard_Baseline', 'Input_Nodes_Baseline']] = None - # move Input_Nodes_Baseline to the top of the df complete_df.sort_values( by='Ensemble_Source', - # x.ne('Input_Nodes_Baseline'): returns a Series of booleans; True for all rows except Input_Nodes_Baseline. - # Since False < True, baseline rows sort to the top. - key=lambda x: x.ne('Input_Nodes_Baseline'), inplace=True ) From 1aa6a9c147a3e43f81045c2b7d798939ed52e7d1 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Wed, 8 Jul 2026 16:28:41 -0500 Subject: [PATCH 20/26] add the new baseline --- spras/evaluation.py | 57 ++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 4034cdf5b..3320b537e 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -465,9 +465,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] - gs_baseline = None - input_gold_baseline = None - input_baseline = None + baseline = None for label, node_ensemble in node_ensembles.items(): if node_ensemble.empty: @@ -486,25 +484,42 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # Different baselines # Computed once; identical for every algorithm on a given dataset/gold-standard pair. - if gs_baseline is None and input_gold_baseline is None and input_baseline is None: + if baseline is None: universe_size = len(y_scores) - # gs_baseline = |gold_standard| / |universe|: random-predictor precision - # beat this to beat random - gs_baseline = np.sum(y_true) / universe_size - plt.axhline(y=gs_baseline, color='gray', linestyle='--', - label=f'Gold Standard Baseline (P={gs_baseline:.4f})') + # baseline = |gold_standard| / |universe|: random-predictor precision + baseline = np.sum(y_true) / universe_size + plt.axhline(y=baseline, color='red', linestyle='--', + label=f'Baseline (P={baseline:.4f})') - # input_gold_prev = |input and gold| / |universe|: fraction of the network that is both input and gold - # numerator counts only gold nodes that are also inputs, not all gold nodes. - input_gold_baseline = len(input_nodes_set & gold_standard_nodes) / universe_size - plt.axhline(y=input_gold_baseline, color='gray', linestyle=':', - label=f'Input Nodes and Gold Standard Baseline (P={input_gold_baseline:.4f})') + # Input nodes PR curve: a 2-point curve built the same way as the algorithm + # ensembles, but with a synthetic frequency of 1 for input nodes and 0 for + # everything else. Since scores are binary, precision_recall_curve returns + # exactly two operating points: predicting only the input nodes as positive, + # and predicting the full universe as positive (equivalent to baseline). - # input_prevalence = |input| / |universe|: fraction of the network that is input nodes - input_baseline= len(input_nodes_set) / universe_size - plt.axhline(y=input_baseline, color='gray', linestyle='-.', - label=f'Input Nodes Baseline (P={input_baseline:.4f})') + input_node_ensemble = node_ensemble[['Node']].copy() # the full interactome is in this already + input_node_ensemble['Frequency'] = input_node_ensemble['Node'].isin(input_nodes_set).astype(float) + + input_y_true = [1 if node in gold_standard_nodes else 0 for node in input_node_ensemble['Node']] + input_y_scores = input_node_ensemble['Frequency'].tolist() + + input_precision, input_recall, input_thresholds = precision_recall_curve(input_y_true, input_y_scores) + input_baseline = average_precision_score(input_y_true, input_y_scores) + + plt.plot(input_recall, input_precision, color='red', marker='s', linestyle='--', + label=f'Input Nodes (AP: {input_baseline:.4f})') + + prc_dfs.append(pd.DataFrame({ + 'Ensemble_Source': ['Input Nodes'] * len(input_thresholds), + 'Threshold': input_thresholds, + 'Precision': input_precision[:-1], + 'Recall': input_recall[:-1], + })) + metric_dfs.append(pd.DataFrame({ + 'Ensemble_Source': ['Input Nodes'], + 'Average_Precision': [input_baseline], + })) # Drop the last precision/recall element: sklearn appends (1, 0) for plotting, not tied to a real threshold. # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html @@ -536,9 +551,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da combined_prc_df = pd.concat(prc_dfs, ignore_index=True) combined_metrics_df = pd.concat(metric_dfs, ignore_index=True) - combined_metrics_df['Gold_Standard_Baseline'] = gs_baseline - combined_metrics_df['Input_Nodes_and _Gold_Standard_Baseline'] = gs_baseline - combined_metrics_df['Input_Nodes_Baseline'] = input_baseline + combined_metrics_df['Baseline'] = baseline # merge curves with per-source metrics, then add the input-nodes baseline as its own row @@ -549,7 +562,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # keep Average_Precision and Baseline only on the first row of each Ensemble_Source not_first_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') - complete_df.loc[not_first_rows, ['Average_Precision', 'Gold_Standard_Baseline', 'Input_Nodes_and _Gold_Standard_Baseline', 'Input_Nodes_Baseline']] = None + complete_df.loc[not_first_rows, ['Average_Precision', 'Baseline']] = None complete_df.sort_values( by='Ensemble_Source', From 8ee40ba25b88841e16cb0bd060a6d5222bda0e51 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 9 Jul 2026 10:07:04 -0500 Subject: [PATCH 21/26] changes --- config/config.yaml | 6 +++--- config/egfr.yaml | 28 ++++++++++++++-------------- spras/evaluation.py | 1 - 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 83ccbd044..b95c185a0 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -233,7 +233,7 @@ reconstruction_settings: locations: #place the save path here # TODO move to global - reconstruction_dir: "output" + reconstruction_dir: "output/config" analysis: # Create one summary per pathway file and a single summary table for all pathways for each dataset @@ -241,7 +241,7 @@ analysis: include: true # Create Cytoscape session file with all pathway graphs for each dataset cytoscape: - include: false + include: true # Machine learning analysis (e.g. clustering) of the pathway output files for each dataset ml: # ml analysis per dataset @@ -268,7 +268,7 @@ analysis: evaluation: # evaluation per dataset-goldstandard pair # evaluation will not run unless ml include is set to true - include: false + include: true # adds evaluation per algorithm per dataset-goldstandard pair # evaluation per algorithm will not run unless ml include and ml aggregate_per_algorithm are set to true aggregate_per_algorithm: true diff --git a/config/egfr.yaml b/config/egfr.yaml index b93c593c4..83beb4f10 100644 --- a/config/egfr.yaml +++ b/config/egfr.yaml @@ -31,12 +31,12 @@ algorithms: include: true runs: run1: - k: - - 10 - - 20 - - 70 + k: range(10, 510, 20) + # - 10 + # - 20 + # - 70 - name: omicsintegrator1 - include: true + include: false runs: run1: b: @@ -44,13 +44,13 @@ algorithms: - 2 - 10 d: 10 - g: 1e-3 + g: 0.001 r: 0.01 w: 0.1 mu: 0.008 dummy_mode: ["file"] - name: omicsintegrator2 - include: true + include: false runs: run1: b: 4 @@ -59,7 +59,7 @@ algorithms: b: 2 g: 3 - name: meo - include: true + include: false runs: run1: local_search: true @@ -70,15 +70,15 @@ algorithms: max_path_length: 2 rand_restarts: 10 - name: allpairs - include: true + include: false - name: domino - include: true + include: false runs: run1: slice_threshold: 0.3 module_threshold: 0.05 - name: mincostflow - include: true + include: false runs: run1: capacity: 15 @@ -90,14 +90,14 @@ algorithms: capacity: 5 flow: 60 - name: "strwr" - include: true + include: false runs: run1: alpha: [0.85] threshold: [100, 200] - name: "rwr" - include: true + include: false runs: run1: alpha: [0.85] @@ -125,7 +125,7 @@ reconstruction_settings: reconstruction_dir: output/egfr analysis: cytoscape: - include: true + include: false summary: include: true ml: diff --git a/spras/evaluation.py b/spras/evaluation.py index 3320b537e..ba32c9127 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -497,7 +497,6 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # everything else. Since scores are binary, precision_recall_curve returns # exactly two operating points: predicting only the input nodes as positive, # and predicting the full universe as positive (equivalent to baseline). - input_node_ensemble = node_ensemble[['Node']].copy() # the full interactome is in this already input_node_ensemble['Frequency'] = input_node_ensemble['Node'].isin(input_nodes_set).astype(float) From b2a642bc1d1a99f95f7f1e544514f0462ddce36d Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 9 Jul 2026 10:12:05 -0500 Subject: [PATCH 22/26] revert configs --- config/config.yaml | 4 ++-- config/egfr.yaml | 29 +++++++++++++++-------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index b95c185a0..ef51de739 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -233,7 +233,7 @@ reconstruction_settings: locations: #place the save path here # TODO move to global - reconstruction_dir: "output/config" + reconstruction_dir: "output" analysis: # Create one summary per pathway file and a single summary table for all pathways for each dataset @@ -252,7 +252,7 @@ analysis: # specify how many principal components to calculate components: 2 # boolean to show the labels on the pca graph - labels: false + labels: true # 'ward', 'complete', 'average', 'single' # if linkage: ward, must use metric: euclidean linkage: 'ward' diff --git a/config/egfr.yaml b/config/egfr.yaml index 83beb4f10..a2c96ab82 100644 --- a/config/egfr.yaml +++ b/config/egfr.yaml @@ -31,12 +31,12 @@ algorithms: include: true runs: run1: - k: range(10, 510, 20) - # - 10 - # - 20 - # - 70 + k: + - 10 + - 20 + - 70 - name: omicsintegrator1 - include: false + include: true runs: run1: b: @@ -45,12 +45,13 @@ algorithms: - 10 d: 10 g: 0.001 + # g: 1e-3 r: 0.01 w: 0.1 mu: 0.008 dummy_mode: ["file"] - name: omicsintegrator2 - include: false + include: true runs: run1: b: 4 @@ -59,7 +60,7 @@ algorithms: b: 2 g: 3 - name: meo - include: false + include: true runs: run1: local_search: true @@ -70,15 +71,15 @@ algorithms: max_path_length: 2 rand_restarts: 10 - name: allpairs - include: false + include: true - name: domino - include: false + include: true runs: run1: slice_threshold: 0.3 module_threshold: 0.05 - name: mincostflow - include: false + include: true runs: run1: capacity: 15 @@ -90,21 +91,21 @@ algorithms: capacity: 5 flow: 60 - name: "strwr" - include: false + include: true runs: run1: alpha: [0.85] threshold: [100, 200] - name: "rwr" - include: false + include: true runs: run1: alpha: [0.85] threshold: [100, 200] - name: "bowtiebuilder" - include: false + include: true datasets: - data_dir: input edge_files: @@ -125,7 +126,7 @@ reconstruction_settings: reconstruction_dir: output/egfr analysis: cytoscape: - include: false + include: true summary: include: true ml: From 0c70db373898f90e1c204903edc07927efc2abaf Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 9 Jul 2026 10:44:26 -0500 Subject: [PATCH 23/26] clean up the code --- config/egfr.yaml | 2 +- spras/evaluation.py | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/config/egfr.yaml b/config/egfr.yaml index a2c96ab82..6f2aabde1 100644 --- a/config/egfr.yaml +++ b/config/egfr.yaml @@ -105,7 +105,7 @@ algorithms: threshold: [100, 200] - name: "bowtiebuilder" - include: true + include: false datasets: - data_dir: input edge_files: diff --git a/spras/evaluation.py b/spras/evaluation.py index c7d44d709..7e1b3365d 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -224,12 +224,13 @@ def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, input_nodes: pd.Dat label=algorithm.capitalize() ) + # input nodes baseline gold_standard_nodes = set(node_table[Evaluation.NODE_ID]) input_nodes_set = set(input_nodes['NODEID']) input_nodes_tp = len(input_nodes_set & gold_standard_nodes) precision_input = input_nodes_tp / len(input_nodes_set) recall_input = input_nodes_tp / len(gold_standard_nodes) - plt.plot(recall_input, precision_input, color='red', markersize=12, marker='X', linestyle='None', label=f'Input Nodes vs Gold Standard Baseline (P={precision_input:.3f}, R={recall_input:.3f})') + plt.plot(recall_input, precision_input, color='red', markersize=12, marker='X', linestyle='None', label=f'Input Nodes (P={precision_input:.3f}, R={recall_input:.3f})') plt.title(title) plt.xlabel('Recall') @@ -241,8 +242,14 @@ def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, input_nodes: pd.Dat plt.savefig(output_png) plt.close() - # save dataframe + # save dataframe with input node baseline pr_df.drop(columns=['Algorithm'], inplace=True) + input_nodes_row = pd.DataFrame({ + 'Pathway': ['Input Nodes'], + 'Precision': [precision_input], + 'Recall': [recall_input], + }) + pr_df = pd.concat([pr_df, input_nodes_row], ignore_index=True) pr_df.to_csv(output_file, sep='\t', index=False) @staticmethod @@ -473,6 +480,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] baseline = None + input_baseline = None for label, node_ensemble in node_ensembles.items(): if node_ensemble.empty: @@ -484,14 +492,9 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() - precision, recall, thresholds = precision_recall_curve(y_true, y_scores) - # avg precision summarizes a PR curve as the weighted mean of precisions achieved at each threshold - avg_precision = average_precision_score(y_true, y_scores) - plt.plot(recall, precision, color=color_palette[label], marker='o', label=f'{label.capitalize()} (AP: {avg_precision:.4f})') - # Different baselines # Computed once; identical for every algorithm on a given dataset/gold-standard pair. - if baseline is None: + if baseline is None and input_baseline is None: universe_size = len(y_scores) # baseline = |gold_standard| / |universe|: random-predictor precision @@ -527,6 +530,12 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da 'Average_Precision': [input_baseline], })) + # calculate and plot prc for node_ensemble + precision, recall, thresholds = precision_recall_curve(y_true, y_scores) + # avg precision summarizes a PR curve as the weighted mean of precisions achieved at each threshold + avg_precision = average_precision_score(y_true, y_scores) + plt.plot(recall, precision, color=color_palette[label], marker='o', label=f'{label.capitalize()} (AP: {avg_precision:.4f})') + # Drop the last precision/recall element: sklearn appends (1, 0) for plotting, not tied to a real threshold. # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html ensemble_source = label.capitalize() if label != 'ensemble' else 'Aggregated' @@ -559,7 +568,6 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da combined_metrics_df = pd.concat(metric_dfs, ignore_index=True) combined_metrics_df['Baseline'] = baseline - # merge curves with per-source metrics, then add the input-nodes baseline as its own row complete_df = ( combined_prc_df From b7b12d5ae3a078fb5eb796bc4d709a328d05b75a Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 9 Jul 2026 13:38:19 -0500 Subject: [PATCH 24/26] updated the test cases, fixed minor bug --- Snakefile | 2 +- spras/evaluation.py | 18 ++++++++--------- test/evaluate/test_evaluate.py | 37 ++++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/Snakefile b/Snakefile index 5c278ed2c..33b0b848d 100644 --- a/Snakefile +++ b/Snakefile @@ -509,7 +509,7 @@ rule evaluation_pca_chosen: input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pca_chosen_pathway = Evaluation.pca_chosen_pathway(input.pca_coordinates_file, input.pathway_summary_file, out_dir) pr_df = Evaluation.node_precision_and_recall(pca_chosen_pathway, node_table) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, node_table, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png, include_aggregate_algo_eval) + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, node_table, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png) # Returns pca coordinates for a specific algorithm and dataset def collect_pca_coordinates_per_algo_per_dataset(wildcards): diff --git a/spras/evaluation.py b/spras/evaluation.py index 7e1b3365d..42da75cf1 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -304,7 +304,6 @@ def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, input_nodes: pd if not pr_df.empty: pr_df['Algorithm'] = pr_df['Pathway'].apply(lambda p: Path(p).parent.name.split('-')[1]) pr_df.sort_values(by=['Recall', 'Pathway'], axis=0, ascending=True, inplace=True) - if aggregate_per_algorithm: title = "PCA-Chosen Pathway Per Algorithm Precision and Recall Plot" else: @@ -480,7 +479,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] baseline = None - input_baseline = None + input_ap = None for label, node_ensemble in node_ensembles.items(): if node_ensemble.empty: @@ -494,7 +493,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # Different baselines # Computed once; identical for every algorithm on a given dataset/gold-standard pair. - if baseline is None and input_baseline is None: + if baseline is None and input_ap is None: universe_size = len(y_scores) # baseline = |gold_standard| / |universe|: random-predictor precision @@ -504,9 +503,8 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # Input nodes PR curve: a 2-point curve built the same way as the algorithm # ensembles, but with a synthetic frequency of 1 for input nodes and 0 for - # everything else. Since scores are binary, precision_recall_curve returns - # exactly two operating points: predicting only the input nodes as positive, - # and predicting the full universe as positive (equivalent to baseline). + # everything else. Returns exactly two operating points: predicting only + # the input nodes as positive, and predicting the full universe as positive input_node_ensemble = node_ensemble[['Node']].copy() # the full interactome is in this already input_node_ensemble['Frequency'] = input_node_ensemble['Node'].isin(input_nodes_set).astype(float) @@ -514,10 +512,10 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da input_y_scores = input_node_ensemble['Frequency'].tolist() input_precision, input_recall, input_thresholds = precision_recall_curve(input_y_true, input_y_scores) - input_baseline = average_precision_score(input_y_true, input_y_scores) + input_ap = average_precision_score(input_y_true, input_y_scores) plt.plot(input_recall, input_precision, color='red', marker='s', linestyle='--', - label=f'Input Nodes (AP: {input_baseline:.4f})') + label=f'Input Nodes (AP: {input_ap:.4f})') prc_dfs.append(pd.DataFrame({ 'Ensemble_Source': ['Input Nodes'] * len(input_thresholds), @@ -527,7 +525,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da })) metric_dfs.append(pd.DataFrame({ 'Ensemble_Source': ['Input Nodes'], - 'Average_Precision': [input_baseline], + 'Average_Precision': [input_ap], })) # calculate and plot prc for node_ensemble @@ -578,11 +576,11 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da not_first_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') complete_df.loc[not_first_rows, ['Average_Precision', 'Baseline']] = None + # save df complete_df.sort_values( by='Ensemble_Source', inplace=True ) - complete_df.to_csv(output_file, index=False, sep='\t') @staticmethod diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 4e94f60c2..1e2b4f706 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -45,9 +45,11 @@ def test_node_precision_recall_per_pathway(self): output_png = Path(OUT_DIR + 'pr-per-pathway.png') output_file.unlink(missing_ok=True) output_png.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() pr_df = Evaluation.node_precision_and_recall(file_paths, GS_NODE_TABLE) - Evaluation.precision_and_recall_per_pathway(pr_df, output_file, output_png, True) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png, True) output = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-per-pathway.txt', sep='\t', header=0).round(8) @@ -62,9 +64,11 @@ def test_node_precision_recall_per_pathway_empty(self): output_png = Path(OUT_DIR + 'pr-per-pathway-empty.png') output_file.unlink(missing_ok=True) output_png.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() pr_df = Evaluation.node_precision_and_recall(file_paths, GS_NODE_TABLE) - Evaluation.precision_and_recall_per_pathway(pr_df, output_file, output_png, True) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png, True) output = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-per-pathway-empty.txt', sep='\t', header=0).round(8) @@ -75,21 +79,25 @@ def test_node_precision_recall_per_pathway_empty(self): def test_node_precision_recall_per_pathway_not_provided(self): output_file = OUT_DIR + 'pr-per-pathway-not-provided.txt' output_png = OUT_DIR + 'pr-per-pathway-not-provided.png' + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() file_paths = [] pr_df = Evaluation.node_precision_and_recall(file_paths, GS_NODE_TABLE) with pytest.raises(ValueError): - Evaluation.precision_and_recall_per_pathway(pr_df, output_file, output_png) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png) def test_node_precision_recall_pca_chosen_pathway_not_provided(self): output_file = Path( OUT_DIR + 'pr-per-pathway-pca-chosen-not-provided.txt') output_file.unlink(missing_ok=True) output_png = Path(OUT_DIR + 'pr-per-pathway-pca-chosen-not-provided.png') output_png.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() file_paths = [] pr_df = Evaluation.node_precision_and_recall(file_paths, GS_NODE_TABLE) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output_file, output_png) + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png) output = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-pca-chosen-not-provided.txt', sep='\t', header=0).round(8) @@ -104,6 +112,8 @@ def test_node_precision_recall_pca_chosen_pathway(self): output_png.unlink(missing_ok=True) output_coordinates = Path(OUT_DIR + 'pca-coordinates.tsv') output_coordinates.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() file_paths = [INPUT_DIR + 'data-test-params-123/pathway.txt', INPUT_DIR + 'data-test-params-456/pathway.txt', INPUT_DIR + 'data-test-params-789/pathway.txt', INPUT_DIR + 'data-test-params-empty/pathway.txt'] @@ -114,8 +124,7 @@ def test_node_precision_recall_pca_chosen_pathway(self): pathway = Evaluation.pca_chosen_pathway([output_coordinates], SUMMARY_FILE, INPUT_DIR) pr_df = Evaluation.node_precision_and_recall(pathway, GS_NODE_TABLE) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output_file, output_png, True) - + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output_file, input_nodes, GS_NODE_TABLE, output_png, True) chosen = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-per-pathway-pca-chosen.txt', sep='\t', header=0).round(8) @@ -128,8 +137,8 @@ def test_node_ensemble(self): out_path_file.unlink(missing_ok=True) ensemble_network = [INPUT_DIR + 'ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - input_interactome = Evaluation.from_file(input_data).get_interactome() - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_interactome) + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_data) + print(node_ensemble_dict) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) @@ -138,8 +147,7 @@ def test_empty_node_ensemble(self): out_path_file.unlink(missing_ok=True) empty_ensemble_network = [INPUT_DIR + 'empty-ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - input_interactome = Evaluation.from_file(input_data).get_interactome() - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, input_interactome) + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, input_data) node_ensemble_dict['empty'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-empty-node-ensemble.csv', shallow=False) @@ -150,8 +158,7 @@ def test_multiple_node_ensemble(self): out_path_empty_file.unlink(missing_ok=True) ensemble_networks = [INPUT_DIR + 'ensemble-network.tsv', INPUT_DIR + 'empty-ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - input_interactome = Evaluation.from_file(input_data).get_interactome() - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_interactome) + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_data) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) node_ensemble_dict['empty'].to_csv(out_path_empty_file, sep='\t', index=False) @@ -163,7 +170,7 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_input_nodes() + input_nodes = Dataset.from_file(input_data).get_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, @@ -177,7 +184,7 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_input_nodes() + input_nodes = Dataset.from_file(input_data).get_input_nodes() empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, @@ -191,7 +198,7 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_input_nodes() + input_nodes = Dataset.from_file(input_data).get_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} From 9929f199f1f7feda7d620d7fea5cfa5d052ddccf Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 9 Jul 2026 14:24:01 -0500 Subject: [PATCH 25/26] fix test cases --- .../expected/expected-pr-curve-ensemble-nodes-empty.txt | 4 ++-- test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt | 4 ++-- .../expected/expected-pr-curve-multiple-ensemble-nodes.txt | 4 ++-- test/evaluate/expected/expected-pr-per-pathway-empty.txt | 1 + test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt | 1 + test/evaluate/expected/expected-pr-per-pathway.txt | 1 + test/evaluate/test_evaluate.py | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt index 4517ceb0e..089b2ffa3 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt @@ -1,4 +1,4 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline -Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 -Input_Nodes_Baseline 1.0 1.0 0.25 Aggregated 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 +Input Nodes 0.0 0.15384615384615385 1.0 0.2403846153846154 0.15384615384615385 +Input Nodes 1.0 0.5 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt index 9a78dbe77..76b9e68ff 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt @@ -1,7 +1,7 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline -Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 -Input_Nodes_Baseline 1.0 1.0 0.25 Aggregated 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Aggregated 0.01 0.6666666666666666 1.0 Aggregated 0.5 0.75 0.75 Aggregated 0.66 0.5 0.25 +Input Nodes 0.0 0.15384615384615385 1.0 0.2403846153846154 0.15384615384615385 +Input Nodes 1.0 0.5 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt index 483e972d8..6313978a7 100644 --- a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt @@ -1,6 +1,4 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline -Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 -Input_Nodes_Baseline 1.0 1.0 0.25 Ensemble1 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Ensemble1 0.01 0.6666666666666666 1.0 Ensemble1 0.5 0.75 0.75 @@ -10,3 +8,5 @@ Ensemble2 0.01 0.6666666666666666 1.0 Ensemble2 0.5 0.75 0.75 Ensemble2 0.66 0.5 0.25 Ensemble3 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 +Input Nodes 0.0 0.15384615384615385 1.0 0.2403846153846154 0.15384615384615385 +Input Nodes 1.0 0.5 0.25 diff --git a/test/evaluate/expected/expected-pr-per-pathway-empty.txt b/test/evaluate/expected/expected-pr-per-pathway-empty.txt index 6c97ff7eb..da1068b5e 100644 --- a/test/evaluate/expected/expected-pr-per-pathway-empty.txt +++ b/test/evaluate/expected/expected-pr-per-pathway-empty.txt @@ -1,2 +1,3 @@ Pathway Precision Recall test/evaluate/input/data-test-params-empty/pathway.txt 0.0 0.0 +Input Nodes 0.5 0.25 diff --git a/test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt b/test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt index 97c13ebc3..74614f0aa 100644 --- a/test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt +++ b/test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt @@ -1,2 +1,3 @@ Pathway Precision Recall test/evaluate/input/data-test-params-123/pathway.txt 0.6666666666666666 0.5 +Input Nodes 0.5 0.25 \ No newline at end of file diff --git a/test/evaluate/expected/expected-pr-per-pathway.txt b/test/evaluate/expected/expected-pr-per-pathway.txt index 08732918b..de1c9615f 100644 --- a/test/evaluate/expected/expected-pr-per-pathway.txt +++ b/test/evaluate/expected/expected-pr-per-pathway.txt @@ -3,3 +3,4 @@ test/evaluate/input/data-test-params-456/pathway.txt 0.0 0.0 test/evaluate/input/data-test-params-empty/pathway.txt 0.0 0.0 test/evaluate/input/data-test-params-123/pathway.txt 0.6666666666666666 0.5 test/evaluate/input/data-test-params-789/pathway.txt 1.0 0.75 +Input Nodes 0.5 0.25 diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 1e2b4f706..2995a557d 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -124,7 +124,7 @@ def test_node_precision_recall_pca_chosen_pathway(self): pathway = Evaluation.pca_chosen_pathway([output_coordinates], SUMMARY_FILE, INPUT_DIR) pr_df = Evaluation.node_precision_and_recall(pathway, GS_NODE_TABLE) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output_file, input_nodes, GS_NODE_TABLE, output_png, True) + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png, True) chosen = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-per-pathway-pca-chosen.txt', sep='\t', header=0).round(8) From 441498ba4073474bcf08c14e7a1cc32e87c7f8df Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 10 Jul 2026 11:09:32 -0500 Subject: [PATCH 26/26] revert g parameter --- config/egfr.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/egfr.yaml b/config/egfr.yaml index 6f2aabde1..b93c593c4 100644 --- a/config/egfr.yaml +++ b/config/egfr.yaml @@ -44,8 +44,7 @@ algorithms: - 2 - 10 d: 10 - g: 0.001 - # g: 1e-3 + g: 1e-3 r: 0.01 w: 0.1 mu: 0.008