diff --git a/CHANGELOG.md b/CHANGELOG.md index f9c3f2ea..311b825e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ - Example: 10.2.1.4 is the 5th version that supports khiops 10.2.1. - Internals: Changes in *Internals* sections are unlikely to be of interest for data scientists. +## Unreleased + +### Changed +- (`sklearn`) The `fit` methods in estimators accept `max_cores` to limit the number of CPU cores allocated to the training +- (General) When `max_cores` is set, use also its value to limit the number of pre-allocated CPU cores (instead of using all the available ones). + ## 11.0.1.0 - 2026-07-02 ### Added diff --git a/khiops/core/api.py b/khiops/core/api.py index efdc2fc5..22fc32fe 100644 --- a/khiops/core/api.py +++ b/khiops/core/api.py @@ -193,7 +193,12 @@ def _preprocess_arguments(args): if arg == "max_cores": max_cores = args[arg] if max_cores is not None: + # This `max_cores` system setting will be used in the khiops scenario + # to limit the CPU cores to use for the training system_settings.max_cores = int(max_cores) + # An additional environment variable (local to this specific run) + # MUST also be set to avoid allocating all the available CPU cores. + # Thus, allocated CPU cores = max number of CPU cores used elif arg == "memory_limit_mb": memory_limit_mb = args[arg] if memory_limit_mb is not None: diff --git a/khiops/core/internals/runner.py b/khiops/core/internals/runner.py index e9d7cb71..b724f0fe 100644 --- a/khiops/core/internals/runner.py +++ b/khiops/core/internals/runner.py @@ -332,11 +332,18 @@ def _get_current_library_installer(): return "unknown" -def _build_khiops_process_environment(): +def _build_khiops_process_environment(system_settings=None): """Build a specific environment used for the execution of khiops in a process This environment can be modified freely without interfering with the global one. + + Parameters + ---------- + + system_settings: `SystemSettings` + Set of settings that must be taken into account + for this specific run """ khiops_env = os.environ.copy() @@ -344,6 +351,11 @@ def _build_khiops_process_environment(): # (using KHIOPS_MPI_HOME if it exists) if "HOME" not in khiops_env: khiops_env["HOME"] = khiops_env.get("KHIOPS_MPI_HOME", "") + if system_settings is not None and system_settings.max_cores is not None: + # An additional environment variable (local to this specific run) + # must also be set to avoid allocating all the available CPU cores. + # Thus, allocated CPU cores = max number of CPU cores used + khiops_env["KHIOPS_PROC_NUMBER"] = system_settings.max_cores return khiops_env @@ -690,6 +702,7 @@ class for more information. scenario_path, command_line_options, trace, + system_settings, ) # pylint: enable=assignment-from-no-return # Catch an OS level error if any @@ -895,6 +908,7 @@ def _run( scenario_path, command_line_options, trace, + system_settings, ): """Abstract run method to be implemented in child classes @@ -1449,7 +1463,14 @@ def _get_samples_dir(self): self._samples_dir_checked = True return self._samples_dir - def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False): + def raw_run( + self, + tool_name, + command_line_args=None, + use_mpi=True, + trace=False, + system_settings=None, + ): """Execute a Khiops tool with given command line arguments Parameters @@ -1462,6 +1483,9 @@ def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False): Whether to execute the application with MPI trace : bool, default False If ``True`` print the trace of the process. + system_settings: `SystemSettings` + Set of settings that must be taken into account + for this specific run Examples -------- @@ -1499,9 +1523,9 @@ def raw_run(self, tool_name, command_line_args=None, use_mpi=True, trace=False): print(f"Khiops execution call: {khiops_call}") # Build custom Khiops process environment - # which makes sure HOME is defined and set + # which makes sure for example HOME is defined and set # according to khiops_env's KHIOPS_MPI_HOME - khiops_env = _build_khiops_process_environment() + khiops_env = _build_khiops_process_environment(system_settings) # Execute the process with subprocess.Popen( @@ -1524,11 +1548,15 @@ def _run( scenario_path, command_line_options, trace, + system_settings, ): # Execute the tool khiops_args = command_line_options.build_command_line_options(scenario_path) stdout, stderr, return_code = self.raw_run( - tool_name, command_line_args=khiops_args, trace=trace + tool_name, + command_line_args=khiops_args, + trace=trace, + system_settings=system_settings, ) return return_code, stdout, stderr diff --git a/khiops/extras/docker.py b/khiops/extras/docker.py index da8bebaa..e0591e86 100644 --- a/khiops/extras/docker.py +++ b/khiops/extras/docker.py @@ -103,6 +103,7 @@ def _run( scenario_path, command_line_options, trace, + system_settings, ): # Check arguments if command_line_options.output_scenario_path: diff --git a/khiops/sklearn/estimators.py b/khiops/sklearn/estimators.py index 71990518..c3163106 100644 --- a/khiops/sklearn/estimators.py +++ b/khiops/sklearn/estimators.py @@ -348,6 +348,18 @@ def _cleanup_computation_dir(self, computation_dir): def fit(self, X, y=None, **kwargs): """Fit the estimator + Parameters + ---------- + X : :external:term:`array-like` of shape (n_samples, n_features_in) or dict + Training dataset. Either an :external:term:`array-like` or a ``dict`` + specification for multi-table datasets (see :doc:`/multi_table_primer`). + + y : :external:term:`array-like` of shape (n_samples,) + The target values. + + max_cores : int, optional + Maximum number of CPU cores allocated and used for the training. + Returns ------- self : `KhiopsEstimator` @@ -689,6 +701,8 @@ def fit(self, X, y=None, **kwargs): The column that contains the id of the instance. columns : list, optional The columns to be co-clustered. If not specified it uses all columns. + max_cores : int, optional + Maximum number of CPU cores allocated and used for the training. Returns ------- @@ -764,6 +778,7 @@ def _fit_train_model(self, ds, computation_dir, **kwargs): main_table_path, variables, coclustering_file_path, + max_cores=kwargs.get("max_cores"), log_file_path=train_log_file_path, trace=self.verbose, ) @@ -1179,7 +1194,25 @@ def _transform_prepare_deployment_for_predict(self, _): return self.model_.copy(), None def fit_predict(self, X, y=None, **kwargs): - """Performs clustering on X and returns result (instead of labels)""" + """Performs clustering on X and returns result (instead of labels) + + Parameters + ---------- + X : :external:term:`array-like` of shape (n_samples, n_features_in) or dict + Training dataset. Either an :external:term:`array-like` or a ``dict`` + specification for multi-table datasets (see :doc:`/multi_table_primer`). + + y : :external:term:`array-like` of shape (n_samples,) + The target values. + + max_cores : int, optional + Maximum number of CPU cores allocated and used for the training. + + Returns + ------- + results : `numpy.array` + """ + return self.fit(X, y, **kwargs).predict(X) @@ -1253,6 +1286,9 @@ def fit(self, X, y=None, **kwargs): y : :external:term:`array-like` of shape (n_samples,) The target values. + max_cores : int, optional + Maximum number of CPU cores allocated and used for the training. + Returns ------- self : `KhiopsSupervisedEstimator` @@ -1314,7 +1350,7 @@ def _fit_check_params(self, ds, **kwargs): def _fit_train_model(self, ds, computation_dir, **kwargs): # Train the model with Khiops train_args, train_kwargs = self._fit_prepare_training_function_inputs( - ds, computation_dir + ds, computation_dir, **kwargs ) report_file_path, model_kdic_file_path = self._fit_core_training_function( *train_args, **train_kwargs @@ -1335,7 +1371,7 @@ def _fit_train_model(self, ds, computation_dir, **kwargs): def _fit_core_training_function(self, *args, **kwargs): """A wrapper to the khiops.core training function for the estimator""" - def _fit_prepare_training_function_inputs(self, ds, computation_dir): + def _fit_prepare_training_function_inputs(self, ds, computation_dir, **fit_kwargs): # Set output path files output_dir = self._get_output_dir(computation_dir) report_file_path = fs.get_child_path( @@ -1374,7 +1410,8 @@ def _fit_prepare_training_function_inputs(self, ds, computation_dir): report_file_path, ] - # Build the optional parameters from a copy of the estimator parameters + # Build the optional parameters from a copy + # of the estimator initializer parameters kwargs = self.get_params() # Remove non core.api params @@ -1404,6 +1441,10 @@ def _fit_prepare_training_function_inputs(self, ds, computation_dir): kwargs["trace"] = kwargs["verbose"] del kwargs["verbose"] + # Set the technical parameters + if "max_cores" in fit_kwargs: + kwargs["max_cores"] = fit_kwargs["max_cores"] + return args, kwargs def _fit_training_post_process(self, ds): @@ -1584,10 +1625,10 @@ def predict(self, X): assert isinstance(y_pred, (str, pd.DataFrame)), "Expected str or DataFrame" return y_pred - def _fit_prepare_training_function_inputs(self, ds, computation_dir): + def _fit_prepare_training_function_inputs(self, ds, computation_dir, **fit_kwargs): # Call the parent method args, kwargs = super()._fit_prepare_training_function_inputs( - ds, computation_dir + ds, computation_dir, **fit_kwargs ) # Rename parameters to be compatible with khiops.core @@ -1855,10 +1896,10 @@ def _fit_check_params(self, ds, **kwargs): # Check the pair related parameters _check_pair_parameters(self) - def _fit_prepare_training_function_inputs(self, ds, computation_dir): + def _fit_prepare_training_function_inputs(self, ds, computation_dir, **fit_kwargs): # Call the parent method args, kwargs = super()._fit_prepare_training_function_inputs( - ds, computation_dir + ds, computation_dir, **fit_kwargs ) # Rename parameters to be compatible with khiops.core @@ -1878,6 +1919,9 @@ def fit(self, X, y, **kwargs): y : :external:term:`array-like` of shape (n_samples,) The target values. + max_cores : int, optional + Maximum number of CPU cores allocated and used for the training. + Returns ------- self : `KhiopsClassifier` @@ -2200,6 +2244,9 @@ def fit(self, X, y=None, **kwargs): y : :external:term:`array-like` of shape (n_samples,) The target values. + max_cores : int, optional + Maximum number of CPU cores allocated and used for the training. + Returns ------- self : `KhiopsRegressor` @@ -2606,6 +2653,9 @@ def fit(self, X, y=None, **kwargs): y : :external:term:`array-like` of shape (n_samples,) The target values. + max_cores : int, optional + Maximum number of CPU cores allocated and used for the training. + Returns ------- self : `KhiopsEncoder` @@ -2616,10 +2666,10 @@ def fit(self, X, y=None, **kwargs): # pylint: enable=useless-super-delegation - def _fit_prepare_training_function_inputs(self, ds, computation_dir): + def _fit_prepare_training_function_inputs(self, ds, computation_dir, **fit_kwargs): # Call the parent method args, kwargs = super()._fit_prepare_training_function_inputs( - ds, computation_dir + ds, computation_dir, **fit_kwargs ) # Rename encoder parameters, delete unused ones # to be compatible with khiops.core @@ -2725,6 +2775,9 @@ def fit_transform(self, X, y=None, **kwargs): y : :external:term:`array-like` of shape (n_samples,) The target values. + max_cores : int, optional + Maximum number of CPU cores allocated and used for the training. + Returns ------- self : `KhiopsEncoder` diff --git a/tests/test_core.py b/tests/test_core.py index 59e645aa..d6e06d3c 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -2639,9 +2639,11 @@ def run( self, task, task_args, - command_line_options, + command_line_options=None, trace=False, system_settings=None, + stdout_file_path="", + stderr_file_path="", force_ansi_scenario=False, **kwargs, ): @@ -2677,6 +2679,7 @@ def _run( scenario_path, command_line_options, trace, + system_settings, ): return 0, "", "" diff --git a/tests/test_sklearn.py b/tests/test_sklearn.py index 9eb2f117..f08894d3 100644 --- a/tests/test_sklearn.py +++ b/tests/test_sklearn.py @@ -709,7 +709,8 @@ def setUpClass(cls): ("khiops.core", "train_coclustering"): { "log_file_path": os.path.join( cls.output_dir, "khiops_train_cc.log" - ) + ), + "max_cores": 62, }, ("khiops.core", "simplify_coclustering"): { "max_part_numbers": {"SampleId": 2}, @@ -767,6 +768,7 @@ def setUpClass(cls): "group_target_value": False, "additional_data_tables": {}, "keep_selected_variables_only": False, + "max_cores": 63, } }, "predict": { @@ -797,6 +799,7 @@ def setUpClass(cls): "max_parts": 5, "keep_selected_variables_only": False, "additional_data_tables": {}, + "max_cores": 65, } }, "predict": { @@ -834,6 +837,7 @@ def setUpClass(cls): "numerical_recoding_method": "part Id", "pairs_recoding_method": "part Id", "additional_data_tables": {}, + "max_cores": 67, } }, "predict": { @@ -1419,14 +1423,29 @@ def _test_template( self.expected_kwargs.get(schema_type), source_type ) ) + # the original object is a generator not a list + expected_kwargs_list = list(expected_kwargs_list) + # add the custom_kwargs parameters to the expected ones + if ( + len(expected_kwargs_list) + and custom_kwargs is not None + and len(custom_kwargs) + ): + expected_kwargs_list[0].update(custom_kwargs) + special_kwarg_checkers = ( self.special_kwarg_checkers.get(estimator_type_key) .get(estimator_method) .get((module_name, function_name)) ) + union_of_initializer_params_and_method_params = kwargs + if custom_kwargs is not None and len(custom_kwargs): + union_of_initializer_params_and_method_params.update( + custom_kwargs + ) for expected_kwargs in expected_kwargs_list: self._check_kwargs( - kwargs, + union_of_initializer_params_and_method_params, expected_kwargs=expected_kwargs, special_checkers=special_kwarg_checkers, ) @@ -1453,6 +1472,9 @@ def test_parameter_transfer_classifier_fit_from_monotable_dataframe(self): "group_target_value": False, "keep_selected_variables_only": False, }, + custom_kwargs={ + "max_cores": 63, + }, ) def test_parameter_transfer_classifier_fit_from_monotable_dataframe_with_df_y( @@ -1479,6 +1501,9 @@ def test_parameter_transfer_classifier_fit_from_monotable_dataframe_with_df_y( "group_target_value": False, "keep_selected_variables_only": False, }, + custom_kwargs={ + "max_cores": 63, + }, ) def test_parameter_transfer_classifier_fit_from_multitable_dataframe(self): @@ -1546,6 +1571,9 @@ def test_parameter_transfer_encoder_fit_from_monotable_dataframe(self): "transform_type_numerical": "part_id", "transform_type_pairs": "part_id", }, + custom_kwargs={ + "max_cores": 67, + }, ) def test_parameter_transfer_encoder_fit_from_monotable_dataframe_with_df_y( @@ -1574,6 +1602,9 @@ def test_parameter_transfer_encoder_fit_from_monotable_dataframe_with_df_y( "transform_type_numerical": "part_id", "transform_type_pairs": "part_id", }, + custom_kwargs={ + "max_cores": 67, + }, ) def test_parameter_transfer_encoder_fit_from_multitable_dataframe(self): @@ -1637,6 +1668,9 @@ def test_parameter_transfer_regressor_fit_from_monotable_dataframe(self): "n_feature_parts": 5, "keep_selected_variables_only": False, }, + custom_kwargs={ + "max_cores": 65, + }, ) def test_parameter_transfer_regressor_fit_from_monotable_dataframe_with_df_y( @@ -1658,6 +1692,9 @@ def test_parameter_transfer_regressor_fit_from_monotable_dataframe_with_df_y( "n_feature_parts": 5, "keep_selected_variables_only": False, }, + custom_kwargs={ + "max_cores": 65, + }, ) def test_parameter_transfer_regressor_fit_from_multitable_dataframe(self): @@ -1709,6 +1746,7 @@ def test_parameter_transfer_coclustering_fit_from_dataframe(self): "columns": ("SampleId", "Pos", "Char"), "id_column": "SampleId", "max_part_numbers": {"SampleId": 2}, + "max_cores": 62, } }, )