diff --git a/configs/common/hdbscan.json b/configs/common/hdbscan.json new file mode 100644 index 00000000..3b2fe12f --- /dev/null +++ b/configs/common/hdbscan.json @@ -0,0 +1,54 @@ +{ + "PARAMETERS_SETS": { + "hdbscan sklearn-ex[cpu] implementations": { + "algorithm": [ + { "library": "sklearn", "device": "cpu" }, + { "library": "sklearnex.preview.cluster", "device": "cpu" } + ] + }, + "common hdbscan parameters": { + "algorithm": { + "estimator": "HDBSCAN", + "estimator_params": { + "min_cluster_size": 5, + "min_samples": 5, + "metric": "euclidean", + "cluster_selection_method": "eom", + "allow_single_cluster": false, + "store_centers": null, + "copy": false + }, + "estimator_methods": { "training": "fit" } + }, + "data": { "format": "numpy", "order": "C", "dtype": "float64" }, + "bench": { "n_runs": 3, "time_limit": 1200 } + }, + "sklearn hdbscan parameters": { + "algorithm": { + "estimator_params": { "n_jobs": "[SPECIAL_VALUE]physical_cpus" } + } + }, + "hdbscan brute method": { + "algorithm": { "estimator_params": { "algorithm": "brute" } } + }, + "hdbscan kd_tree method": { + "algorithm": { "estimator_params": { "algorithm": "kd_tree", "leaf_size": 40 } } + }, + "hdbscan ball_tree method": { + "algorithm": { "estimator_params": { "algorithm": "ball_tree", "leaf_size": 40 } } + }, + "hdbscan tree methods": { + "algorithm": { + "estimator_params": { "algorithm": ["kd_tree", "ball_tree"], "leaf_size": 40 } + } + }, + "hdbscan all methods": { + "algorithm": { + "estimator_params": { + "algorithm": ["brute", "kd_tree", "ball_tree"], + "leaf_size": 40 + } + } + } + } +} diff --git a/configs/experiments/README.md b/configs/experiments/README.md index 2b6225c5..dc442c9b 100644 --- a/configs/experiments/README.md +++ b/configs/experiments/README.md @@ -2,4 +2,8 @@ `daal4py_svd`: tests performance scalability of `daal4py.svd` algorithm +`hdbscan_parameters`: sweeps the `HDBSCAN` parameter space (metrics, cluster selection, density thresholds, stored centers, dtypes and data formats) over the `sklearn` and `sklearnex` implementations. + +`hdbscan_scaling`: tests thread, NUMA, `n_samples` and `n_features` scalability of `HDBSCAN`. + `nearest_neighbors`: tests performance of neighbors search implementations from `sklearnex`, `sklearn`, `raft`, `faiss` and `svs`. diff --git a/configs/experiments/hdbscan_parameters.json b/configs/experiments/hdbscan_parameters.json new file mode 100644 index 00000000..fe546ec4 --- /dev/null +++ b/configs/experiments/hdbscan_parameters.json @@ -0,0 +1,169 @@ +{ + "INCLUDE": ["../common/hdbscan.json"], + "PARAMETERS_SETS": { + "hdbscan tree metrics": { + "algorithm": { + "estimator_params": { "metric": ["euclidean", "manhattan", "chebyshev"] } + } + }, + "hdbscan minkowski metric": { + "algorithm": { + "estimator_params": { "metric": "minkowski", "metric_params": { "p": 3 } } + } + }, + "hdbscan cosine metric": { + "algorithm": { + "estimator_params": { "metric": "cosine", "algorithm": "brute" } + } + }, + "hdbscan cluster selection sweep": { + "algorithm": { + "estimator_params": { "cluster_selection_method": ["eom", "leaf"] } + } + }, + "hdbscan density sweep": { + "algorithm": { + "estimator_params": { "min_cluster_size": [5, 25, 100], "min_samples": [5, 25] } + } + }, + "hdbscan store centers sweep": { + "algorithm": { + "estimator_params": { "store_centers": [null, "centroid", "medoid", "both"] } + } + }, + "hdbscan dtype sweep": { + "data": { "dtype": ["float32", "float64"] } + }, + "hdbscan data format sweep": { + "data": [ + { "format": "numpy", "order": "C" }, + { "format": "numpy", "order": "F" }, + { "format": "pandas", "order": "F" } + ] + }, + "hdbscan parameters data [tree]": { + "data": { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": 100000, + "n_features": 8, + "cluster_std": 2.0, + "random_state": 42 + }, + "split_kwargs": { "ignore": true } + } + }, + "hdbscan parameters data [brute]": { + "data": { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": 25000, + "n_features": 64, + "cluster_std": 4.0, + "random_state": 42 + }, + "split_kwargs": { "ignore": true } + } + }, + "hdbscan parameters data [real]": { + "data": { + "dataset": "skin_segmentation", + "split_kwargs": { "train_size": 30000 }, + "preprocessing_kwargs": { "normalize": "standard" } + } + } + }, + "TEMPLATES": { + "hdbscan metrics [tree]": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan tree methods", + "hdbscan tree metrics", + "hdbscan parameters data [tree]" + ] + }, + "hdbscan minkowski [tree]": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan kd_tree method", + "hdbscan minkowski metric", + "hdbscan parameters data [tree]" + ] + }, + "hdbscan metrics [brute]": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan brute method", + "hdbscan tree metrics", + "hdbscan parameters data [brute]" + ] + }, + "hdbscan cosine [brute]": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan cosine metric", + "hdbscan parameters data [brute]" + ] + }, + "hdbscan cluster selection": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan kd_tree method", + "hdbscan cluster selection sweep", + "hdbscan parameters data [tree]" + ] + }, + "hdbscan density": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan kd_tree method", + "hdbscan density sweep", + "hdbscan parameters data [tree]" + ] + }, + "hdbscan store centers": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan kd_tree method", + "hdbscan store centers sweep", + "hdbscan parameters data [tree]" + ] + }, + "hdbscan dtypes": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan all methods", + "hdbscan dtype sweep", + "hdbscan parameters data [real]" + ] + }, + "hdbscan data formats": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan kd_tree method", + "hdbscan data format sweep", + "hdbscan parameters data [tree]" + ] + } + } +} diff --git a/configs/experiments/hdbscan_scaling.json b/configs/experiments/hdbscan_scaling.json new file mode 100644 index 00000000..b1c48c3c --- /dev/null +++ b/configs/experiments/hdbscan_scaling.json @@ -0,0 +1,217 @@ +{ + "INCLUDE": ["../common/hdbscan.json"], + "PARAMETERS_SETS": { + "sklearnex hdbscan implementation": { + "algorithm": { "library": "sklearnex.preview.cluster", "device": "cpu" } + }, + "hdbscan thread scaling": [ + { + "bench": { "taskset": "0" }, + "algorithm": { "estimator_params": { "n_jobs": 1 } } + }, + { + "bench": { "taskset": "0-3" }, + "algorithm": { "estimator_params": { "n_jobs": 4 } } + }, + { + "bench": { "taskset": "0-7" }, + "algorithm": { "estimator_params": { "n_jobs": 8 } } + }, + { + "bench": { "taskset": "0-15" }, + "algorithm": { "estimator_params": { "n_jobs": 16 } } + }, + { + "bench": { "taskset": "0-27" }, + "algorithm": { "estimator_params": { "n_jobs": 28 } } + }, + { + "bench": { "taskset": "0-55" }, + "algorithm": { "estimator_params": { "n_jobs": 56 } } + }, + { + "bench": { "taskset": "0-111" }, + "algorithm": { "estimator_params": { "n_jobs": 112 } } + }, + { + "bench": { "taskset": "0-223" }, + "algorithm": { "estimator_params": { "n_jobs": 224 } } + } + ], + "hdbscan numa scaling": [ + { + "bench": { "taskset": "0-55" }, + "algorithm": { "estimator_params": { "n_jobs": 56 } } + }, + { + "bench": { "taskset": "0-27,56-83" }, + "algorithm": { "estimator_params": { "n_jobs": 56 } } + }, + { + "bench": { "taskset": "0-111" }, + "algorithm": { "estimator_params": { "n_jobs": 112 } } + } + ], + "hdbscan scaling profiling": { + "bench": { + "n_runs": 3, + "time_limit": 1200, + "cpu_profile": true, + "memory_profile": true + } + }, + "hdbscan brute scaling data": { + "data": { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": 50000, + "n_features": 256, + "cluster_std": 4.0, + "random_state": 42 + }, + "split_kwargs": { "ignore": true } + } + }, + "hdbscan tree scaling data": { + "data": { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": 1000000, + "n_features": 16, + "cluster_std": 2.0, + "random_state": 42 + }, + "split_kwargs": { "ignore": true } + } + }, + "hdbscan n_samples sweep [brute]": { + "data": { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": "[RANGE]mul:5000:40000:2", + "n_features": 64, + "cluster_std": 4.0, + "random_state": 42 + }, + "split_kwargs": { "ignore": true } + } + }, + "hdbscan n_samples sweep [tree]": { + "data": { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": "[RANGE]mul:25000:400000:2", + "n_features": 8, + "cluster_std": 2.0, + "random_state": 42 + }, + "split_kwargs": { "ignore": true } + } + }, + "hdbscan n_samples sweep [tree, sklearnex only]": { + "data": { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": "[RANGE]mul:500000:8000000:2", + "n_features": 8, + "cluster_std": 2.0, + "random_state": 42 + }, + "split_kwargs": { "ignore": true } + } + }, + "hdbscan n_features sweep": { + "data": { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": 20000, + "n_features": "[RANGE]pow:2:2:10", + "cluster_std": 4.0, + "random_state": 42 + }, + "split_kwargs": { "ignore": true } + } + }, + "hdbscan size sweep limits": { + "bench": { "n_runs": 3, "time_limit": 300 } + } + }, + "TEMPLATES": { + "hdbscan thread scaling [brute]": { + "SETS": [ + "sklearnex hdbscan implementation", + "common hdbscan parameters", + "hdbscan scaling profiling", + "hdbscan brute method", + "hdbscan thread scaling", + "hdbscan brute scaling data" + ] + }, + "hdbscan thread scaling [kd_tree]": { + "SETS": [ + "sklearnex hdbscan implementation", + "common hdbscan parameters", + "hdbscan scaling profiling", + "hdbscan kd_tree method", + "hdbscan thread scaling", + "hdbscan tree scaling data" + ] + }, + "hdbscan numa scaling [brute]": { + "SETS": [ + "sklearnex hdbscan implementation", + "common hdbscan parameters", + "hdbscan scaling profiling", + "hdbscan brute method", + "hdbscan numa scaling", + "hdbscan brute scaling data" + ] + }, + "hdbscan n_samples scaling [brute]": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan brute method", + "hdbscan n_samples sweep [brute]", + "hdbscan size sweep limits" + ] + }, + "hdbscan n_samples scaling [kd_tree]": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan kd_tree method", + "hdbscan n_samples sweep [tree]", + "hdbscan size sweep limits" + ] + }, + "hdbscan n_samples scaling [kd_tree, beyond sklearn]": { + "SETS": [ + "sklearnex hdbscan implementation", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan kd_tree method", + "hdbscan n_samples sweep [tree, sklearnex only]", + "hdbscan size sweep limits" + ] + }, + "hdbscan n_features scaling": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan all methods", + "hdbscan n_features sweep", + "hdbscan size sweep limits" + ] + } + } +} diff --git a/configs/regular/hdbscan.json b/configs/regular/hdbscan.json new file mode 100644 index 00000000..e96fb314 --- /dev/null +++ b/configs/regular/hdbscan.json @@ -0,0 +1,63 @@ +{ + "INCLUDE": ["../common/sklearn.json", "../common/hdbscan.json"], + "PARAMETERS_SETS": { + "hdbscan tree-friendly datasets": { + "data": [ + { + "dataset": "skin_segmentation", + "split_kwargs": { "train_size": 100000 }, + "preprocessing_kwargs": { "normalize": "standard" } + }, + { + "source": "make_blobs", + "generation_kwargs": { + "centers": 100, + "n_samples": 100000, + "n_features": 8, + "cluster_std": 1.0 + }, + "split_kwargs": { "ignore": true } + } + ] + }, + "hdbscan brute-friendly datasets": { + "data": [ + { + "dataset": "mnist", + "split_kwargs": { "train_size": 20000 }, + "preprocessing_kwargs": { "normalize": "standard" } + }, + { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": 25000, + "n_features": 4, + "cluster_std": 1.0 + }, + "split_kwargs": { "ignore": true } + } + ] + } + }, + "TEMPLATES": { + "hdbscan kd_tree": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan kd_tree method", + "hdbscan tree-friendly datasets" + ] + }, + "hdbscan brute force": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan brute method", + "hdbscan brute-friendly datasets" + ] + } + } +} diff --git a/configs/weekly/hdbscan.json b/configs/weekly/hdbscan.json new file mode 100644 index 00000000..a11d82a2 --- /dev/null +++ b/configs/weekly/hdbscan.json @@ -0,0 +1,68 @@ +{ + "INCLUDE": ["../common/sklearn.json", "../common/hdbscan.json"], + "PARAMETERS_SETS": { + "high-load hdbscan tree-friendly datasets": { + "data": [ + { + "dataset": ["road_network", "covtype"], + "split_kwargs": { "train_size": 100000 }, + "preprocessing_kwargs": { "normalize": "standard" } + }, + { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": 100000, + "n_features": [4, 16, 64], + "cluster_std": 2.0 + }, + "split_kwargs": { "ignore": true } + } + ] + }, + "high-load hdbscan brute-friendly datasets": { + "data": [ + { + "dataset": "cifar", + "split_kwargs": { "train_size": 15000 }, + "preprocessing_kwargs": { "normalize": "mean" } + }, + { + "dataset": "sensit", + "split_kwargs": { "train_size": 25000 }, + "preprocessing_kwargs": { "normalize": "standard" } + }, + { + "source": "make_blobs", + "generation_kwargs": { + "centers": 10, + "n_samples": 25000, + "n_features": [64, 256, 1024], + "cluster_std": 4.0 + }, + "split_kwargs": { "ignore": true } + } + ] + } + }, + "TEMPLATES": { + "hdbscan tree methods": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan tree methods", + "high-load hdbscan tree-friendly datasets" + ] + }, + "hdbscan brute force": { + "SETS": [ + "hdbscan sklearn-ex[cpu] implementations", + "common hdbscan parameters", + "sklearn hdbscan parameters", + "hdbscan brute method", + "high-load hdbscan brute-friendly datasets" + ] + } + } +} diff --git a/sklbench/benchmarks/common.py b/sklbench/benchmarks/common.py index 1df1e1a5..bd7b3376 100644 --- a/sklbench/benchmarks/common.py +++ b/sklbench/benchmarks/common.py @@ -16,6 +16,7 @@ import argparse import json +import re from typing import Dict from ..utils.bench_case import get_bench_case_value, get_data_name @@ -26,16 +27,23 @@ def enrich_result(result: Dict, bench_case: BenchCase) -> Dict: """Common function for all benchmarks to update the result with additional information""" + library = ( + get_bench_case_value(bench_case, "algorithm:library") + .replace( + # skipping emulators namespace for conciseness + "sklbench.emulators.", + "", + ) + .replace(".utils", "") + ) + # estimators available only from a preview namespace (like `sklearnex.preview.cluster`) + # are reported under their library so that they are comparable + # with the stock implementation in the report + library = re.sub(r"\.preview(\..+)?$", "", library) result.update( { "dataset": get_data_name(bench_case, shortened=True), - "library": get_bench_case_value(bench_case, "algorithm:library") - .replace( - # skipping emulators namespace for conciseness - "sklbench.emulators.", - "", - ) - .replace(".utils", ""), + "library": library, "device": get_bench_case_value(bench_case, "algorithm:device"), } ) diff --git a/sklbench/utils/measurement.py b/sklbench/utils/measurement.py index 4df6c57b..66e1f100 100644 --- a/sklbench/utils/measurement.py +++ b/sklbench/utils/measurement.py @@ -56,6 +56,11 @@ def box_filter(array, left=0.2, right=0.8): return array[0], 0.0 lower, upper = array[int(size * left)], array[int(size * right)] result = np.array([item for item in array if lower < item < upper]) + if result.size == 0: + # the box is empty for short series (2 measurements always are, and a + # `time_limit` early stop can leave exactly 2), which would make the + # aggregated time NaN - fall back to the whole series instead + result = np.array(array) return np.mean(result), np.std(result)