From e8758cba179884ccfc8240b34fe0364ba35de1d8 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 19 Aug 2026 10:33:29 -0400 Subject: [PATCH 1/8] DAOS-19532 test: Add register_cleanup_method for fio Add a new RunRemoteCommand class which uses run_remote() to run a command with parameters. This class includes support for using the Test.register_cleanup() method to handle stopping the command when the test times out or fails prematurely with the command running in the background. Update FioCommand to inherit from RunRemoteCommand. Quick-functional: true Test-tag: FioPil4dfsSmall FioSmall EcodFioRebuild Ecodtruncate EcodFaultInjection ParallelIo Pil4dfsFio Signed-off-by: Phil Henderson --- src/tests/ftest/util/command_utils.py | 283 +++++++++++++++++++++++++- src/tests/ftest/util/fio_utils.py | 63 +----- 2 files changed, 283 insertions(+), 63 deletions(-) diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index f5d2885f94e..baf66903dc0 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -23,7 +23,7 @@ from file_utils import change_file_owner, create_directory, distribute_files from general_utils import (DaosTestError, check_file_exists, get_file_listing, get_job_manager_class, get_subprocess_stdout, run_command) -from run_utils import command_as_user, run_remote +from run_utils import command_as_user, run_remote, stop_processes from user_utils import get_primary_group from yaml_utils import get_yaml_data @@ -1511,3 +1511,282 @@ def __init__(self, run_user='root'): self.unit_command = BasicParameter(None, position=1) self.service = BasicParameter(None, position=2) + + +class RunRemoteCommand(CommandWithParameters): + """A class for command run with run_remote().""" + + def __init__(self, namespace, command, path="", check_results=None, run_user=None): + """Create a ExecutableCommand object. + + Uses Avocado's utils.process module to run a command str provided. + + Args: + namespace (str): yaml namespace (path to parameters) + command (str): string of the command to be executed. + path (str, optional): path to location of command binary file. Defaults to "". + check_results (list, optional): list of words used to mark the command as failed if + any are found in the command output. Defaults to None. + run_user (str, optional): user to run as. Defaults to None, which will run commands as + the current user. + """ + super().__init__(namespace, command, path) + self._hosts = None + self.timeout = None + self.exit_status_exception = True + self.register_cleanup_method = None + self.env = EnvironmentVariables() + self.verbose = True + self.result = None + + # User to run the command as. "root" is equivalent to sudo + self.run_user = run_user + + # List of CPU cores to pass to taskset + self.bind_cores = None + + # Define a list of executable names associated with the command. This + # list is used to generate the 'command_regex' property, which can be + # used to check on the progress or terminate the command. + self._exe_names = [self.command] + + # If set use the full command string when returning the 'command_regex' property + self.full_command_regex = False + + # Optional list of words used to to mark the command as failed if any of + # the words are found in the command output (self.result.stdout_text and + # self.result.stderr_text). Useful for detecting a command failure that + # may not be caught through the command exit status. + self.check_results_list = [] + if check_results: + self.check_results_list = list(check_results) + + def __str__(self): + """Return the command with all of its defined parameters as a string. + + Returns: + str: the command with all the defined parameters + + """ + return self.with_sudo + + @property + def hosts(self): + """Get the host(s) on which to remotely run the command via run(). + + Returns: + NodeSet: remote host(s) on which the command will run. + + """ + return self._hosts + + @hosts.setter + def hosts(self, value): + """Set the host(s) on which to remotely run the command via run(). + + If the specified host is None the command will run locally w/o ssh. + + Args: + value (NodeSet): remote host(s) on which to run the command + + Raises: + TypeError: if value is not a NodeSet + + """ + if not isinstance(value, NodeSet): + raise TypeError(f"Invalid {self.command} host NodeSet: {value} ({type(value)})") + self._hosts = value.copy() + + @property + def sudo(self): + """Get the sudo flag. + + Returns: + bool: whether to run as sudo/root + + """ + return self.run_user == 'root' + + @sudo.setter + def sudo(self, value): + """Set the sudo flag. + + Args: + value (bool): whether to run as sudo + + """ + self.run_user = 'root' if value else None + + @property + def command_regex(self): + """Get the regular expression to use to search for the command. + + Typical use would include combining with pgrep to verify a subprocess is running. + + Returns: + str: regular expression to use to search for the command + + """ + _exe_names = '|'.join(self._exe_names) + return f"'({_exe_names})'" + + @property + def with_bind(self): + """Get the command string with bind_cores. + + Returns: + str: the command string with bind_cores + + """ + command = super().__str__() + if self.bind_cores: + command = ' '.join(['taskset', '-c', self.bind_cores, command]) + return command + + @property + def with_sudo(self): + """Get the command string with bind_cores and sudo, but not env exports. + + Returns: + str: the command string with bind_cores and sudo + + """ + return command_as_user(self.with_bind, self.run_user) + + @property + def with_exports(self): + """Get the command string with bind_cores, sudo, and env exports. + + Returns: + str: the command string with bind_cores, sudo, and env exports + + """ + return command_as_user(self.with_bind, self.run_user, self.env) + + @contextlib.contextmanager + def no_exception(self): + """Temporarily disable raising exceptions for failed commands.""" + original_value = self.exit_status_exception + self.exit_status_exception = False + yield + self.exit_status_exception = original_value + + @contextlib.contextmanager + def as_user(self, user): + """Temporarily run commands as a different user. + + Args: + user (str): the user to temporarily run as + """ + original_value = self.run_user + self.run_user = user + yield + self.run_user = original_value + + def run(self, raise_exception=None): + """Run the command. + + Args: + raise_exception (bool, optional): whether or not to raise an exception if the command + fails. This overrides the self.exit_status_exception + setting if defined. Defaults to None. + + Raises: + CommandFailure: if there are no hosts specified + + Returns: + CommandResult: result from running the command + """ + if not self._hosts: + raise CommandFailure(f'Unable to run {self.command}: No hosts specified!') + + if raise_exception is None: + raise_exception = self.exit_status_exception + + if callable(self.register_cleanup_method): + # Stop any running processes started by this job manager when the test completes + # pylint: disable=not-callable + self.register_cleanup_method(self.stop) + + # Run fio remotely + self.result = None + result = run_remote( + self.log, self._hosts, self.with_exports, timeout=self.timeout, verbose=self.verbose) + self.result = result + if raise_exception and not result.passed: + raise CommandFailure(f"Error running fio on: {result.failed_hosts}") + return result + + def stop(self): + """Stop the command. + + Raises: + CommandFailure: if there are no hosts specified + """ + if not self._hosts: + raise CommandFailure(f'Unable to stop {self.command}: No hosts specified!') + + regex = self.command_regex + if self.full_command_regex: + regex = f"'{str(self)}'" + detected, running = stop_processes( + self.log, self._hosts, regex, full_command=self.job.full_command_regex) + if not detected: + self.log.info( + "No remote %s processes killed on %s (none found), done.", regex, self._hosts) + elif running: + self.log.info( + "***Unable to kill remote %s process on %s! Please investigate/report.***", + regex, running) + else: + self.log.info( + "***At least one remote %s process needed to be killed on %s! Please investigate/" + "report.***", regex, detected) + + def check_results(self): + """Check the command result for any bad keywords. + + Returns: + bool: True if either there were no items from self.check_result_list + to verify or if none of the items were found in the command + output; False if a item was found in the command output. + + """ + status = True + if self.result and self.check_results_list: + regex = fr"({'|'.join(self.check_results_list)})" + self.log.debug("Checking the %s output for any bad keywords: %s", self.command, regex) + for output in (self.result.joined_stdout, self.result.joined_stderr): + match = re.findall(regex, output) + if match: + self.log.info( + "The following error messages have been detected in " + "the %s output:", self.command) + for item in match: + self.log.info(" %s", item) + status = False + break + return status + + def get_params(self, test): + """Get values for all of the command params from the yaml file. + + Also gets env_vars from /run/client/* and self.namespace. + + Args: + test (Test): avocado Test object + + """ + super().get_params(test) + for namespace in ['/run/client/*', self.namespace]: + if namespace is not None: + self.env.update_from_list(test.params.get("env_vars", namespace, None) or []) + + def _get_new(self): + """Get a new object based upon this one. + + Returns: + RunRemoteCommand: a new RunRemoteCommand object + """ + return RunRemoteCommand( + self.namespace, self._command, self._path, self.check_results_list, self.run_user) diff --git a/src/tests/ftest/util/fio_utils.py b/src/tests/ftest/util/fio_utils.py index 2c4bd11c822..3ce8935c7c4 100644 --- a/src/tests/ftest/util/fio_utils.py +++ b/src/tests/ftest/util/fio_utils.py @@ -4,14 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from ClusterShell.NodeSet import NodeSet -from command_utils import ExecutableCommand +from command_utils import RunRemoteCommand from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter -from exception_utils import CommandFailure -from run_utils import run_remote -class FioCommand(ExecutableCommand): +class FioCommand(RunRemoteCommand): # pylint: disable=too-many-instance-attributes """Defines a object representing a fio command.""" @@ -65,36 +62,6 @@ def __init__(self, path=""): self.names = BasicParameter(None) self._jobs = {} - # List of hosts on which the fio command will run - self._hosts = None - - @property - def hosts(self): - """Get the host(s) on which to remotely run the fio command via run(). - - Returns: - NodeSet: remote host(s) on which the fio command will run. - - """ - return self._hosts - - @hosts.setter - def hosts(self, value): - """Set the host(s) on which to remotely run the fio command via run(). - - If the specified host is None the command will run locally w/o ssh. - - Args: - value (NodeSet): remote host(s) on which to run the fio command - - Raises: - TypeError: if value is not a NodeSet - - """ - if not isinstance(value, NodeSet): - raise TypeError("Invalid fio host NodeSet: {} ({})".format(value, type(value))) - self._hosts = value.copy() - def get_params(self, test): """Get values for all of the command params from the yaml file. @@ -169,32 +136,6 @@ def command_with_params(self): command.append(str(self._jobs[name])) return " ".join(command) - def _run_process(self, raise_exception=None): - """Run the command remotely as a foreground process. - - Args: - raise_exception (bool, optional): whether or not to raise an exception if the command - fails. This overrides the self.exit_status_exception - setting if defined. Defaults to None. - - Raises: - CommandFailure: if there is an error running the command - - Returns: - CommandResult: groups of command results from the same hosts with the same return status - """ - if not self._hosts: - raise CommandFailure('No hosts specified for fio command') - - if raise_exception is None: - raise_exception = self.exit_status_exception - - # Run fio remotely - result = run_remote(self.log, self._hosts, self.with_exports, timeout=None) - if raise_exception and not result.passed: - raise CommandFailure("Error running fio on: {}".format(result.failed_hosts)) - return result - class FioJob(CommandWithParameters): # pylint: disable=too-many-instance-attributes """Defines a object representing a fio job sub-command.""" From 706200ddcb40d4906d2349676604e7a9d221b102 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 19 Aug 2026 19:17:00 -0400 Subject: [PATCH 2/8] Updates. Quick-functional: true Test-tag: FioPil4dfsSmall FioSmall EcodFioRebuild Ecodtruncate EcodFaultInjection ParallelIo Pil4dfsFio Signed-off-by: Phil Henderson --- src/tests/ftest/util/command_utils.py | 32 ++++----------------------- src/tests/ftest/util/fio_utils.py | 2 +- src/tests/ftest/util/run_utils.py | 30 ++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index baf66903dc0..829ab298da0 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -1708,13 +1708,14 @@ def run(self, raise_exception=None): # pylint: disable=not-callable self.register_cleanup_method(self.stop) - # Run fio remotely + # Run the command on the remote hosts self.result = None result = run_remote( self.log, self._hosts, self.with_exports, timeout=self.timeout, verbose=self.verbose) self.result = result - if raise_exception and not result.passed: - raise CommandFailure(f"Error running fio on: {result.failed_hosts}") + if raise_exception and not result.passed and not result.search( + self.log, fr"({'|'.join(self.check_results_list)})"): + raise CommandFailure(f"Error running {self.command} on: {result.failed_hosts}") return result def stop(self): @@ -1743,31 +1744,6 @@ def stop(self): "***At least one remote %s process needed to be killed on %s! Please investigate/" "report.***", regex, detected) - def check_results(self): - """Check the command result for any bad keywords. - - Returns: - bool: True if either there were no items from self.check_result_list - to verify or if none of the items were found in the command - output; False if a item was found in the command output. - - """ - status = True - if self.result and self.check_results_list: - regex = fr"({'|'.join(self.check_results_list)})" - self.log.debug("Checking the %s output for any bad keywords: %s", self.command, regex) - for output in (self.result.joined_stdout, self.result.joined_stderr): - match = re.findall(regex, output) - if match: - self.log.info( - "The following error messages have been detected in " - "the %s output:", self.command) - for item in match: - self.log.info(" %s", item) - status = False - break - return status - def get_params(self, test): """Get values for all of the command params from the yaml file. diff --git a/src/tests/ftest/util/fio_utils.py b/src/tests/ftest/util/fio_utils.py index 3ce8935c7c4..e102c4240ed 100644 --- a/src/tests/ftest/util/fio_utils.py +++ b/src/tests/ftest/util/fio_utils.py @@ -152,7 +152,7 @@ def __init__(self, namespace, name): """ job_namespace = namespace.split("/") job_namespace.insert(-1, name) - super().__init__("/".join(job_namespace), "--name={}".format(name)) + super().__init__("/".join(job_namespace), f"--name={name}") # fio global/local job options self.description = FormattedParameter("--description={}") diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index 3a0510127c2..f7d475c6eac 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -330,6 +330,34 @@ def log_output(self, log): for data in self.output: log_result_data(log, data) + def search(self, log, regex): + """Check a CommandResult for any keywords in stdout/stderr. + + Args: + log (logging.Logger): Logger object for logging messages. + regex (str): The regular expression pattern to search for in the command output. + + Returns: + bool: True if the regular expression pattern was not found in the CommandResult; + False otherwise. + """ + if not self.output: + log.debug("No output to search for keywords: %s", regex) + return False + + status = True + log.debug("Searching the command output for any keywords: %s", regex) + for output in (self.joined_stdout, self.joined_stderr): + match = re.findall(regex, output) + if match: + log.info( + "The following error messages have been detected in the command output:") + for item in match: + log.info(" %s", item) + status = False + break + return status + def log_result_data(log, data): """Log a single command result data entry. From baf095f5257d01b25a2630e6bf423b6312ca1772 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 20 Aug 2026 14:29:32 -0400 Subject: [PATCH 3/8] Updates Quick-functional: true Test-tag: FioPil4dfsSmall FioSmall EcodFioRebuild Ecodtruncate EcodFaultInjection ParallelIo Pil4dfsFio Signed-off-by: Phil Henderson --- src/tests/ftest/dfuse/pil4dfs_fio.py | 3 + src/tests/ftest/util/command_utils.py | 230 ++++++++------------------ src/tests/ftest/util/fio_test_base.py | 2 + src/tests/ftest/util/fio_utils.py | 4 +- src/tests/ftest/util/run_utils.py | 8 +- 5 files changed, 78 insertions(+), 169 deletions(-) diff --git a/src/tests/ftest/dfuse/pil4dfs_fio.py b/src/tests/ftest/dfuse/pil4dfs_fio.py index a149f5610d8..1a57b1910b8 100644 --- a/src/tests/ftest/dfuse/pil4dfs_fio.py +++ b/src/tests/ftest/dfuse/pil4dfs_fio.py @@ -1,5 +1,6 @@ """ (C) Copyright 2019-2024 Intel Corporation. + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -105,6 +106,7 @@ def _run_fio_pil4dfs(self, ioengine): start_dfuse(self, dfuse, container.pool, container) fio_cmd = FioCommand() + fio_cmd.register_cleanup_method = self.register_cleanup fio_cmd.get_params(self) fio_cmd.update_directory(dfuse.mount_dir.value) fio_cmd.update("global", "ioengine", ioengine, f"fio --name=global --ioengine='{ioengine}'") @@ -146,6 +148,7 @@ def _run_fio_dfs(self): container = self._create_container() fio_cmd = FioCommand() + fio_cmd.register_cleanup_method = self.register_cleanup fio_cmd.get_params(self) fio_cmd.update("global", "ioengine", "dfs", "fio --name=global --ioengine='dfs'") fio_cmd.update( diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 829ab298da0..76bc2b88488 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -23,7 +23,7 @@ from file_utils import change_file_owner, create_directory, distribute_files from general_utils import (DaosTestError, check_file_exists, get_file_listing, get_job_manager_class, get_subprocess_stdout, run_command) -from run_utils import command_as_user, run_remote, stop_processes +from run_utils import command_as_user, run_local, run_remote, stop_processes from user_utils import get_primary_group from yaml_utils import get_yaml_data @@ -263,7 +263,7 @@ def check_results(self): if self.result and self.check_results_list: regex = r"({})".format("|".join(self.check_results_list)) self.log.debug("Checking the command output for any bad keywords: %s", regex) - for output in (self.result.stdout_text, self.result.stderr_text): + for output in (self._result_stdout(), self._result_stderr()): match = re.findall(regex, output) if match: self.log.info( @@ -275,6 +275,26 @@ def check_results(self): break return status + def _result_stdout(self): + """Get all the stdout from the command result. + + Returns: + str: the command result's stdout as a string + """ + if not self.result: + raise CommandFailure("No command result available to return stdout") + return self.result.stdout_text + + def _result_stderr(self): + """Get all the stderr from the command result. + + Returns: + str: the command result's stderr as a string + """ + if not self.result: + raise CommandFailure("No command result available to return stderr") + return self.result.stderr_text + def _run_subprocess(self): """Run the command as a sub process. @@ -447,7 +467,7 @@ def get_output(self, method_name, regex_method=None, **kwargs): # Parse the output and return if not regex_method: regex_method = method_name - return self.parse_output(result.stdout_text, regex_method) + return self.parse_output(self._result_stdout(), regex_method) def parse_output(self, stdout, regex_method): """Parse output using findall() with supplied 'regex_method' as pattern. @@ -464,7 +484,7 @@ def parse_output(self, stdout, regex_method): """ if regex_method not in self.METHOD_REGEX: - raise CommandFailure("No pattern regex defined for '{}()'".format(regex_method)) + raise CommandFailure(f"No pattern regex defined for '{regex_method}()'") return re.findall(self.METHOD_REGEX[regex_method], stdout) def get_params(self, test): @@ -1513,13 +1533,13 @@ def __init__(self, run_user='root'): self.service = BasicParameter(None, position=2) -class RunRemoteCommand(CommandWithParameters): - """A class for command run with run_remote().""" +class RunCommand(ExecutableCommand): + """A class for command run with run_remote()/run_local().""" def __init__(self, namespace, command, path="", check_results=None, run_user=None): - """Create a ExecutableCommand object. + """Create a RunCommand object. - Uses Avocado's utils.process module to run a command str provided. + Uses run_remote() to run a command str provided. Args: namespace (str): yaml namespace (path to parameters) @@ -1530,45 +1550,9 @@ def __init__(self, namespace, command, path="", check_results=None, run_user=Non run_user (str, optional): user to run as. Defaults to None, which will run commands as the current user. """ - super().__init__(namespace, command, path) + super().__init__(namespace, command, path, False, check_results, run_user) self._hosts = None - self.timeout = None - self.exit_status_exception = True self.register_cleanup_method = None - self.env = EnvironmentVariables() - self.verbose = True - self.result = None - - # User to run the command as. "root" is equivalent to sudo - self.run_user = run_user - - # List of CPU cores to pass to taskset - self.bind_cores = None - - # Define a list of executable names associated with the command. This - # list is used to generate the 'command_regex' property, which can be - # used to check on the progress or terminate the command. - self._exe_names = [self.command] - - # If set use the full command string when returning the 'command_regex' property - self.full_command_regex = False - - # Optional list of words used to to mark the command as failed if any of - # the words are found in the command output (self.result.stdout_text and - # self.result.stderr_text). Useful for detecting a command failure that - # may not be caught through the command exit status. - self.check_results_list = [] - if check_results: - self.check_results_list = list(check_results) - - def __str__(self): - """Return the command with all of its defined parameters as a string. - - Returns: - str: the command with all the defined parameters - - """ - return self.with_sudo @property def hosts(self): @@ -1597,94 +1581,8 @@ def hosts(self, value): raise TypeError(f"Invalid {self.command} host NodeSet: {value} ({type(value)})") self._hosts = value.copy() - @property - def sudo(self): - """Get the sudo flag. - - Returns: - bool: whether to run as sudo/root - - """ - return self.run_user == 'root' - - @sudo.setter - def sudo(self, value): - """Set the sudo flag. - - Args: - value (bool): whether to run as sudo - - """ - self.run_user = 'root' if value else None - - @property - def command_regex(self): - """Get the regular expression to use to search for the command. - - Typical use would include combining with pgrep to verify a subprocess is running. - - Returns: - str: regular expression to use to search for the command - - """ - _exe_names = '|'.join(self._exe_names) - return f"'({_exe_names})'" - - @property - def with_bind(self): - """Get the command string with bind_cores. - - Returns: - str: the command string with bind_cores - - """ - command = super().__str__() - if self.bind_cores: - command = ' '.join(['taskset', '-c', self.bind_cores, command]) - return command - - @property - def with_sudo(self): - """Get the command string with bind_cores and sudo, but not env exports. - - Returns: - str: the command string with bind_cores and sudo - - """ - return command_as_user(self.with_bind, self.run_user) - - @property - def with_exports(self): - """Get the command string with bind_cores, sudo, and env exports. - - Returns: - str: the command string with bind_cores, sudo, and env exports - - """ - return command_as_user(self.with_bind, self.run_user, self.env) - - @contextlib.contextmanager - def no_exception(self): - """Temporarily disable raising exceptions for failed commands.""" - original_value = self.exit_status_exception - self.exit_status_exception = False - yield - self.exit_status_exception = original_value - - @contextlib.contextmanager - def as_user(self, user): - """Temporarily run commands as a different user. - - Args: - user (str): the user to temporarily run as - """ - original_value = self.run_user - self.run_user = user - yield - self.run_user = original_value - - def run(self, raise_exception=None): - """Run the command. + def _run_process(self, raise_exception=None): + """Run the command as a foreground process. Args: raise_exception (bool, optional): whether or not to raise an exception if the command @@ -1692,17 +1590,11 @@ def run(self, raise_exception=None): setting if defined. Defaults to None. Raises: - CommandFailure: if there are no hosts specified + CommandFailure: if there is an error running the command Returns: CommandResult: result from running the command """ - if not self._hosts: - raise CommandFailure(f'Unable to run {self.command}: No hosts specified!') - - if raise_exception is None: - raise_exception = self.exit_status_exception - if callable(self.register_cleanup_method): # Stop any running processes started by this job manager when the test completes # pylint: disable=not-callable @@ -1710,23 +1602,44 @@ def run(self, raise_exception=None): # Run the command on the remote hosts self.result = None - result = run_remote( - self.log, self._hosts, self.with_exports, timeout=self.timeout, verbose=self.verbose) + if not self.hosts: + result = run_local(self.log, self.with_exports, self.verbose, self.timeout) + else: + result = run_remote(self.log, self.hosts, self.with_exports, self.verbose, self.timeout) self.result = result - if raise_exception and not result.passed and not result.search( - self.log, fr"({'|'.join(self.check_results_list)})"): - raise CommandFailure(f"Error running {self.command} on: {result.failed_hosts}") + if raise_exception or (raise_exception is None and self.exit_status_exception): + if not result.passed: + raise CommandFailure(f"Error running {self.command} on: {result.failed_hosts}") + if not result.search(self.log, fr"({'|'.join(self.check_results_list)})"): + raise CommandFailure(f"Error running {self.command}: check results failed") return result + def _result_stdout(self): + """Get all the stdout from the command result. + + Returns: + str: the command result's stdout as a string + """ + if not self.result: + raise CommandFailure("No command result available to return stdout") + return self.result.joined_stdout + + def _result_stderr(self): + """Get all the stderr from the command result. + + Returns: + str: the command result's stderr as a string + """ + if not self.result: + raise CommandFailure("No command result available to return stderr") + return self.result.joined_stderr + def stop(self): """Stop the command. Raises: CommandFailure: if there are no hosts specified """ - if not self._hosts: - raise CommandFailure(f'Unable to stop {self.command}: No hosts specified!') - regex = self.command_regex if self.full_command_regex: regex = f"'{str(self)}'" @@ -1734,7 +1647,8 @@ def stop(self): self.log, self._hosts, regex, full_command=self.job.full_command_regex) if not detected: self.log.info( - "No remote %s processes killed on %s (none found), done.", regex, self._hosts) + "No remote %s processes killed on %s (none found), done.", + regex, "local host" if not self.hosts else self.hosts) elif running: self.log.info( "***Unable to kill remote %s process on %s! Please investigate/report.***", @@ -1744,25 +1658,11 @@ def stop(self): "***At least one remote %s process needed to be killed on %s! Please investigate/" "report.***", regex, detected) - def get_params(self, test): - """Get values for all of the command params from the yaml file. - - Also gets env_vars from /run/client/* and self.namespace. - - Args: - test (Test): avocado Test object - - """ - super().get_params(test) - for namespace in ['/run/client/*', self.namespace]: - if namespace is not None: - self.env.update_from_list(test.params.get("env_vars", namespace, None) or []) - def _get_new(self): """Get a new object based upon this one. Returns: - RunRemoteCommand: a new RunRemoteCommand object + RunCommand: a new RunCommand object """ - return RunRemoteCommand( + return RunCommand( self.namespace, self._command, self._path, self.check_results_list, self.run_user) diff --git a/src/tests/ftest/util/fio_test_base.py b/src/tests/ftest/util/fio_test_base.py index 93c802718c3..86b20a48c71 100644 --- a/src/tests/ftest/util/fio_test_base.py +++ b/src/tests/ftest/util/fio_test_base.py @@ -1,5 +1,6 @@ """ (C) Copyright 2020-2024 Intel Corporation. + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -30,6 +31,7 @@ def setUp(self): # Get the parameters for Fio self.fio_cmd = FioCommand() + self.fio_cmd.register_cleanup_method = self.register_cleanup self.fio_cmd.get_params(self) self.processes = self.params.get("np", '/run/fio/client_processes/*') self.manager = self.params.get("manager", '/run/fio/*', "MPICH") diff --git a/src/tests/ftest/util/fio_utils.py b/src/tests/ftest/util/fio_utils.py index e102c4240ed..83ee8be3a6e 100644 --- a/src/tests/ftest/util/fio_utils.py +++ b/src/tests/ftest/util/fio_utils.py @@ -4,11 +4,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ -from command_utils import RunRemoteCommand +from command_utils import RunCommand from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter -class FioCommand(RunRemoteCommand): +class FioCommand(RunCommand): # pylint: disable=too-many-instance-attributes """Defines a object representing a fio command.""" diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index f7d475c6eac..aa4f2dff0c8 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -605,8 +605,12 @@ def stop_processes(log, hosts, pattern, verbose=True, timeout=60, exclude=None, search_command = f"/usr/bin/pgrep --list-full --full -x {pattern}" # Search for any active processes - log.debug("Searching for any processes on %s that match %s", hosts, pattern_match) - result = run_remote(log, hosts, search_command, verbose, timeout) + if not hosts: + log.debug("Searching for any local processes that match %s", pattern_match) + result = run_local(log, search_command, verbose, timeout) + else: + log.debug("Searching for any processes on %s that match %s", hosts, pattern_match) + result = run_remote(log, hosts, search_command, verbose, timeout) if not result.passed_hosts: log.debug("No processes found on %s that match %s", result.failed_hosts, pattern_match) return processes_detected, processes_running From 725937bf758557212dfb211fd303a01b49fb3e21 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 20 Aug 2026 14:36:41 -0400 Subject: [PATCH 4/8] Cleanup Quick-functional: true Test-tag: FioPil4dfsSmall FioSmall EcodFioRebuild Ecodtruncate EcodFaultInjection ParallelIo Pil4dfsFio Signed-off-by: Phil Henderson --- src/tests/ftest/util/command_utils.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 76bc2b88488..04fc177feed 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -1539,7 +1539,7 @@ class RunCommand(ExecutableCommand): def __init__(self, namespace, command, path="", check_results=None, run_user=None): """Create a RunCommand object. - Uses run_remote() to run a command str provided. + Uses run_remote()/run_local() to run a command str provided. Args: namespace (str): yaml namespace (path to parameters) @@ -1560,7 +1560,6 @@ def hosts(self): Returns: NodeSet: remote host(s) on which the command will run. - """ return self._hosts @@ -1575,7 +1574,6 @@ def hosts(self, value): Raises: TypeError: if value is not a NodeSet - """ if not isinstance(value, NodeSet): raise TypeError(f"Invalid {self.command} host NodeSet: {value} ({type(value)})") @@ -1590,7 +1588,8 @@ def _run_process(self, raise_exception=None): setting if defined. Defaults to None. Raises: - CommandFailure: if there is an error running the command + CommandFailure: if there is an error running the command with raise_exception or + self.exit_status_exception (when raise_exception is None) set to True. Returns: CommandResult: result from running the command @@ -1635,16 +1634,12 @@ def _result_stderr(self): return self.result.joined_stderr def stop(self): - """Stop the command. - - Raises: - CommandFailure: if there are no hosts specified - """ + """Stop the command.""" regex = self.command_regex if self.full_command_regex: regex = f"'{str(self)}'" detected, running = stop_processes( - self.log, self._hosts, regex, full_command=self.job.full_command_regex) + self.log, self.hosts, regex, full_command=self.full_command_regex) if not detected: self.log.info( "No remote %s processes killed on %s (none found), done.", From f008312b3e63245bfe827f92e5cc0581bd778090 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 20 Aug 2026 15:04:14 -0400 Subject: [PATCH 5/8] Add get_fio() method Quick-functional: true Test-tag: FioPil4dfsSmall FioSmall EcodFioRebuild Ecodtruncate EcodFaultInjection ParallelIo Pil4dfsFio Signed-off-by: Phil Henderson --- src/tests/ftest/dfuse/pil4dfs_fio.py | 12 +++--------- src/tests/ftest/io/parallel_io.py | 4 ++-- src/tests/ftest/util/fio_test_base.py | 7 ++----- src/tests/ftest/util/fio_utils.py | 24 ++++++++++++++++++++++-- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/tests/ftest/dfuse/pil4dfs_fio.py b/src/tests/ftest/dfuse/pil4dfs_fio.py index 1a57b1910b8..96feec4cf45 100644 --- a/src/tests/ftest/dfuse/pil4dfs_fio.py +++ b/src/tests/ftest/dfuse/pil4dfs_fio.py @@ -12,7 +12,7 @@ from ClusterShell.NodeSet import NodeSet from cpu_utils import CpuInfo from dfuse_utils import get_dfuse, start_dfuse -from fio_utils import FioCommand +from fio_utils import get_fio from general_utils import bytes_to_human, get_log_file, percent_change @@ -105,9 +105,7 @@ def _run_fio_pil4dfs(self, ioengine): dfuse = get_dfuse(self, self.hostlist_clients) start_dfuse(self, dfuse, container.pool, container) - fio_cmd = FioCommand() - fio_cmd.register_cleanup_method = self.register_cleanup - fio_cmd.get_params(self) + fio_cmd = get_fio(self, self.hostlist_clients) fio_cmd.update_directory(dfuse.mount_dir.value) fio_cmd.update("global", "ioengine", ioengine, f"fio --name=global --ioengine='{ioengine}'") fio_cmd.update( @@ -120,7 +118,6 @@ def _run_fio_pil4dfs(self, ioengine): fio_cmd.env['D_DYNAMIC_CTX'] = 1 fio_cmd.env["D_LOG_FILE"] = get_log_file(self.client_log) fio_cmd.env["D_LOG_MASK"] = 'INFO' - fio_cmd.hosts = self.hostlist_clients bws = {} for rw in Pil4dfsFio._FIO_RW_NAMES: @@ -147,9 +144,7 @@ def _run_fio_dfs(self): """ container = self._create_container() - fio_cmd = FioCommand() - fio_cmd.register_cleanup_method = self.register_cleanup - fio_cmd.get_params(self) + fio_cmd = get_fio(self, self.hostlist_clients) fio_cmd.update("global", "ioengine", "dfs", "fio --name=global --ioengine='dfs'") fio_cmd.update( "job", "numjobs", self.fio_numjobs, f"fio --name=job --numjobs={self.fio_numjobs}") @@ -163,7 +158,6 @@ def _run_fio_dfs(self): fio_cmd.env['D_DYNAMIC_CTX'] = 1 fio_cmd.env["D_LOG_FILE"] = get_log_file(self.client_log) fio_cmd.env["D_LOG_MASK"] = 'INFO' - fio_cmd.hosts = self.hostlist_clients bws = {} for rw in Pil4dfsFio._FIO_RW_NAMES: diff --git a/src/tests/ftest/io/parallel_io.py b/src/tests/ftest/io/parallel_io.py index 2925dc863a6..ba775b11044 100644 --- a/src/tests/ftest/io/parallel_io.py +++ b/src/tests/ftest/io/parallel_io.py @@ -154,7 +154,7 @@ def test_parallelio(self): cmd, result.failed_hosts)) # run fio on all containers self.fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, cont.uuid)) - thread = threading.Thread(target=self.execute_fio) + thread = threading.Thread(target=self.fio_cmd.run) threads.append(thread) thread.start() @@ -172,7 +172,7 @@ def test_parallelio(self): # try accessing destroyed container, it should fail try: self.fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, container_to_destroy)) - self.execute_fio() + self.fio_cmd.run() self.fail(f"Fio was able to access destroyed container: {self.container[0]}") except CommandFailure: self.log.info("fio failed as expected") diff --git a/src/tests/ftest/util/fio_test_base.py b/src/tests/ftest/util/fio_test_base.py index 86b20a48c71..2a10d02c462 100644 --- a/src/tests/ftest/util/fio_test_base.py +++ b/src/tests/ftest/util/fio_test_base.py @@ -5,7 +5,7 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ from apricot import TestWithServers -from fio_utils import FioCommand +from fio_utils import get_fio class FioBase(TestWithServers): @@ -30,13 +30,10 @@ def setUp(self): super().setUp() # Get the parameters for Fio - self.fio_cmd = FioCommand() - self.fio_cmd.register_cleanup_method = self.register_cleanup - self.fio_cmd.get_params(self) + self.fio_cmd = get_fio(self, self.hostlist_clients) self.processes = self.params.get("np", '/run/fio/client_processes/*') self.manager = self.params.get("manager", '/run/fio/*', "MPICH") def execute_fio(self): """Runner method for Fio.""" - self.fio_cmd.hosts = self.hostlist_clients self.fio_cmd.run() diff --git a/src/tests/ftest/util/fio_utils.py b/src/tests/ftest/util/fio_utils.py index 83ee8be3a6e..2dc657b2ce8 100644 --- a/src/tests/ftest/util/fio_utils.py +++ b/src/tests/ftest/util/fio_utils.py @@ -8,17 +8,37 @@ from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter +def get_fio(test, hosts, path="", namespace="/run/fio/*"): + """Get a FioCommand object with parameters from the test yaml file. + + Args: + test (Test): avocado Test object + hosts (NodeSet): hosts on which to run the ior command + path (str, optional): path to location of command binary file. Defaults to "". + namespace (str, optional): path to yaml parameters. Defaults to "/run/fio/*". + + Returns: + FioCommand: a FioCommand object with parameters from the test yaml file + """ + fio = FioCommand(path, namespace) + fio.hosts = hosts + fio.register_cleanup_method = test.register_cleanup + fio.get_params(test) + return fio + + class FioCommand(RunCommand): # pylint: disable=too-many-instance-attributes """Defines a object representing a fio command.""" - def __init__(self, path=""): + def __init__(self, path="", namespace="/run/fio/*"): """Create a FioCommand object. Args: path (str, optional): path to location of command binary file. Defaults to "". + namespace (str, optional): path to yaml parameters. Defaults to "/run/fio/*". """ - super().__init__("/run/fio/*", "fio", path) + super().__init__(namespace, "fio", path) # fio command-line options self.debug = FormattedParameter("--debug={}") From 2d44573b637ae1b961c8b257d7b89404acd83e30 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 20 Aug 2026 15:06:42 -0400 Subject: [PATCH 6/8] Revert a change. Quick-functional: true Test-tag: FioPil4dfsSmall FioSmall EcodFioRebuild Ecodtruncate EcodFaultInjection ParallelIo Pil4dfsFio Signed-off-by: Phil Henderson --- src/tests/ftest/io/parallel_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/io/parallel_io.py b/src/tests/ftest/io/parallel_io.py index ba775b11044..2925dc863a6 100644 --- a/src/tests/ftest/io/parallel_io.py +++ b/src/tests/ftest/io/parallel_io.py @@ -154,7 +154,7 @@ def test_parallelio(self): cmd, result.failed_hosts)) # run fio on all containers self.fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, cont.uuid)) - thread = threading.Thread(target=self.fio_cmd.run) + thread = threading.Thread(target=self.execute_fio) threads.append(thread) thread.start() @@ -172,7 +172,7 @@ def test_parallelio(self): # try accessing destroyed container, it should fail try: self.fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, container_to_destroy)) - self.fio_cmd.run() + self.execute_fio() self.fail(f"Fio was able to access destroyed container: {self.container[0]}") except CommandFailure: self.log.info("fio failed as expected") From daff60d11bfc130f669855ef5e47a39f6090088e Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 21 Aug 2026 00:11:20 -0400 Subject: [PATCH 7/8] Further cleanup. Quick-functional: true Signed-off-by: Phil Henderson --- src/tests/ftest/dfuse/fio_pil4dfs_small.py | 19 +++---- src/tests/ftest/dfuse/fio_small.py | 17 +++--- src/tests/ftest/erasurecode/rebuild_fio.py | 41 ++++++++------- src/tests/ftest/erasurecode/truncate.py | 21 ++++---- src/tests/ftest/fault_injection/ec.py | 9 ++-- src/tests/ftest/io/parallel_io.py | 14 ++--- src/tests/ftest/util/command_utils.py | 60 ++++++++++++---------- src/tests/ftest/util/fio_test_base.py | 39 -------------- src/tests/ftest/util/fio_utils.py | 36 +++++++------ 9 files changed, 120 insertions(+), 136 deletions(-) delete mode 100644 src/tests/ftest/util/fio_test_base.py diff --git a/src/tests/ftest/dfuse/fio_pil4dfs_small.py b/src/tests/ftest/dfuse/fio_pil4dfs_small.py index 205cfb21d66..717ee4a6b45 100644 --- a/src/tests/ftest/dfuse/fio_pil4dfs_small.py +++ b/src/tests/ftest/dfuse/fio_pil4dfs_small.py @@ -8,10 +8,10 @@ import os from dfuse_utils import get_dfuse, start_dfuse -from fio_test_base import FioBase +from fio_utils import TestFio -class FioPil4dfsSmall(FioBase): +class FioPil4dfsSmall(TestFio): """Test class Description: Runs Fio with in small config. :avocado: recursive @@ -28,7 +28,8 @@ def test_fio_pil4dfs_small(self): :avocado: tags=dfuse,fio,checksum,tx,pil4dfs :avocado: tags=FioPil4dfsSmall,test_fio_pil4dfs_small """ - self.fio_cmd.env['LD_PRELOAD'] = os.path.join(self.prefix, 'lib64', 'libpil4dfs.so') + fio_cmd = self.get_fio_command() + fio_cmd.env['LD_PRELOAD'] = os.path.join(self.prefix, 'lib64', 'libpil4dfs.so') self.log_step('Create a pool') pool = self.get_pool(connect=False) @@ -42,18 +43,18 @@ def test_fio_pil4dfs_small(self): self.log_step('Start dfuse') dfuse = get_dfuse(self, self.hostlist_clients) start_dfuse(self, dfuse, pool, container) - self.fio_cmd.update_directory(dfuse.mount_dir.value) + fio_cmd.update_directory(dfuse.mount_dir.value) # Run with various fio parameters for variant in self.params.get("variants", '/run/fio/global/*'): self.log_step( f'Run fio with direct={variant[0]}, blocksize={variant[1]}, ' f'size={variant[2]}, rw={variant[3]}') - self.fio_cmd.update('global', 'direct', variant[0], 'global.direct') - self.fio_cmd.update('global', 'blocksize', variant[1], 'global.blocksize') - self.fio_cmd.update('global', 'size', variant[2], 'global.size') - self.fio_cmd.update('global', 'rw', variant[3], 'global.rw') - self.execute_fio() + fio_cmd.update('global', 'direct', variant[0], 'global.direct') + fio_cmd.update('global', 'blocksize', variant[1], 'global.blocksize') + fio_cmd.update('global', 'size', variant[2], 'global.size') + fio_cmd.update('global', 'rw', variant[3], 'global.rw') + fio_cmd.run() self.log_step('Stop dfuse and destroy container') dfuse.stop() diff --git a/src/tests/ftest/dfuse/fio_small.py b/src/tests/ftest/dfuse/fio_small.py index 4337cbf3937..838abdd91b5 100644 --- a/src/tests/ftest/dfuse/fio_small.py +++ b/src/tests/ftest/dfuse/fio_small.py @@ -6,10 +6,10 @@ """ from dfuse_utils import get_dfuse, start_dfuse -from fio_test_base import FioBase +from fio_utils import TestFio -class FioSmall(FioBase): +class FioSmall(TestFio): """Test class Description: Runs Fio with in small config. :avocado: recursive @@ -38,18 +38,19 @@ def test_fio_small(self): self.log_step('Start dfuse') dfuse = get_dfuse(self, self.hostlist_clients) start_dfuse(self, dfuse, pool, container) - self.fio_cmd.update_directory(dfuse.mount_dir.value) + fio_cmd = self.get_fio_command() + fio_cmd.update_directory(dfuse.mount_dir.value) # Run with various fio parameters for variant in self.params.get("variants", '/run/fio/global/*'): self.log_step( f'Run fio with direct={variant[0]}, blocksize={variant[1]}, ' f'size={variant[2]}, rw={variant[3]}') - self.fio_cmd.update('global', 'direct', variant[0], 'global.direct') - self.fio_cmd.update('global', 'blocksize', variant[1], 'global.blocksize') - self.fio_cmd.update('global', 'size', variant[2], 'global.size') - self.fio_cmd.update('global', 'rw', variant[3], 'global.rw') - self.execute_fio() + fio_cmd.update('global', 'direct', variant[0], 'global.direct') + fio_cmd.update('global', 'blocksize', variant[1], 'global.blocksize') + fio_cmd.update('global', 'size', variant[2], 'global.size') + fio_cmd.update('global', 'rw', variant[3], 'global.rw') + fio_cmd.run() self.log_step('Stop dfuse and destroy container') dfuse.stop() diff --git a/src/tests/ftest/erasurecode/rebuild_fio.py b/src/tests/ftest/erasurecode/rebuild_fio.py index c62e4daec16..bb2a900bbbc 100644 --- a/src/tests/ftest/erasurecode/rebuild_fio.py +++ b/src/tests/ftest/erasurecode/rebuild_fio.py @@ -9,20 +9,21 @@ import time from dfuse_utils import get_dfuse, start_dfuse -from fio_test_base import FioBase +from fio_utils import TestFio -class EcodFioRebuild(FioBase): +class EcodFioRebuild(TestFio): """Test class Description: Runs Fio with EC object type over POSIX and verify on-line, off-line for rebuild and verify the data. :avocado: recursive """ - def execution(self, rebuild_mode): + def execution(self, fio_cmd, rebuild_mode): """Execute test. Args: + fio_cmd (FioCommand): Fio command object rebuild_mode (str): On-line or off-line rebuild mode """ aggregation_timeout = self.params.get("aggregation_timeout", "/run/pool/*") @@ -42,15 +43,15 @@ def execution(self, rebuild_mode): container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) dfuse = get_dfuse(self, self.hostlist_clients) start_dfuse(self, dfuse, pool, container) - self.fio_cmd.update_directory(dfuse.mount_dir.value) + fio_cmd.update_directory(dfuse.mount_dir.value) # Write the Fio data and kill the last server rank if rebuild_mode is on-line if 'on-line' in rebuild_mode: self.log_step(f"Start fio and stop the last server rank ({rank_to_kill})") - self.start_online_fio(dfuse.mount_dir.value, rank_to_kill) + self.start_online_fio(fio_cmd, dfuse.mount_dir.value, rank_to_kill) else: self.log_step("Start fio and leave all servers running") - self.start_online_fio(dfuse.mount_dir.value, None) + self.start_online_fio(fio_cmd, dfuse.mount_dir.value, None) # Get initial total free space (scm+nvme) self.log_step("Get initial total free space (scm+nvme)") @@ -87,36 +88,37 @@ def execution(self, rebuild_mode): # Adding unlink option for final read command self.log_step("Adding unlink option for final read command") if int(container.properties.value.split(":")[1]) == 1: - self.fio_cmd._jobs['test'].unlink.value = 1 # pylint: disable=protected-access + fio_cmd._jobs['test'].unlink.value = 1 # pylint: disable=protected-access # Read and verify the original data. self.log_step("Read and verify the original data.") - self.fio_cmd._jobs['test'].rw.value = read_option # pylint: disable=protected-access - self.fio_cmd.run() + fio_cmd._jobs['test'].rw.value = read_option # pylint: disable=protected-access + fio_cmd.run() # If RF is 2 kill one more server and validate the data is not corrupted. if int(container.properties.value.split(":")[1]) == 2: # Kill one more server rank rank_to_kill = num_ranks - 2 self.log_step(f"Kill one more server rank {rank_to_kill} when RF=2") - self.fio_cmd._jobs['test'].unlink.value = 1 # pylint: disable=protected-access + fio_cmd._jobs['test'].unlink.value = 1 # pylint: disable=protected-access self.server_managers[0].stop_ranks([rank_to_kill], force=True) # Read and verify the original data. self.log_step(f"Verify the data is not corrupted after stopping rank {rank_to_kill}.") - self.fio_cmd.run() + fio_cmd.run() # Pre-teardown: make sure rebuild is done before too-quickly trying to destroy container. pool.wait_for_rebuild_to_end() self.log.info("Test passed") - def start_online_fio(self, directory, rank_to_kill=None): + def start_online_fio(self, fio_cmd, directory, rank_to_kill=None): """Run Fio operation with thread in background. Trigger the server failure while Fio is running Args: + fio_cmd (FioCommand): Fio command object directory (str): directory to use with the fio command rank_to_kill (int, optional): the server rank to kill while IO operation is in progress. Set to None to leave all servers running during IO. Defaults to None. @@ -126,7 +128,7 @@ def start_online_fio(self, directory, rank_to_kill=None): # Create the Fio run thread job = threading.Thread( target=self.write_single_fio_dataset, - kwargs={"directory": directory, "results": results_queue}) + kwargs={"fio_cmd": fio_cmd, "directory": directory, "results": results_queue}) # Launch the Fio thread job.start() @@ -144,16 +146,17 @@ def start_online_fio(self, directory, rank_to_kill=None): if results_queue.get() == "FAIL": self.fail("Error running fio as a thread") - def write_single_fio_dataset(self, directory, results): + def write_single_fio_dataset(self, fio_cmd, directory, results): """Run Fio Benchmark. Args: + fio_cmd (FioCommand): Fio command object directory (str): directory to use with the fio command results (queue): queue for returning thread results """ try: - self.fio_cmd.update_directory(directory) - self.execute_fio() + fio_cmd.update_directory(directory) + fio_cmd.run() results.put("PASS") except Exception: # pylint: disable=broad-except results.put("FAIL") @@ -183,7 +186,8 @@ def test_ec_online_rebuild_fio(self): :avocado: tags=ec,ec_array,fio,ec_online_rebuild :avocado: tags=EcodFioRebuild,test_ec_online_rebuild_fio """ - self.execution('on-line') + fio_cmd = self.get_fio_command() + self.execution(fio_cmd, 'on-line') def test_ec_offline_rebuild_fio(self): """Jira ID: DAOS-7320. @@ -203,4 +207,5 @@ def test_ec_offline_rebuild_fio(self): :avocado: tags=ec,ec_array,fio,ec_offline_rebuild :avocado: tags=EcodFioRebuild,test_ec_offline_rebuild_fio """ - self.execution('off-line') + fio_cmd = self.get_fio_command() + self.execution(fio_cmd, 'off-line') diff --git a/src/tests/ftest/erasurecode/truncate.py b/src/tests/ftest/erasurecode/truncate.py index 6635e7d5008..f06c4265640 100644 --- a/src/tests/ftest/erasurecode/truncate.py +++ b/src/tests/ftest/erasurecode/truncate.py @@ -1,18 +1,18 @@ ''' (C) Copyright 2019-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent ''' import os from dfuse_utils import get_dfuse, start_dfuse -from fio_test_base import FioBase +from fio_utils import TestFio from general_utils import get_remote_file_size from run_utils import run_remote -class Ecodtruncate(FioBase): +class Ecodtruncate(TestFio): # pylint: disable=protected-access """Test class Description: Runs Fio with EC object type over POSIX and verify truncate file does not corrupt the data. @@ -48,16 +48,17 @@ def test_ec_truncate(self): container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) dfuse = get_dfuse(self, self.hostlist_clients) start_dfuse(self, dfuse, pool, container) - self.fio_cmd.update_directory(dfuse.mount_dir.value) - self.execute_fio() + fio_cmd = self.get_fio_command() + fio_cmd.update_directory(dfuse.mount_dir.value) + fio_cmd.run() # Get the fuse file name. testfile = "{}.0.0".format(os.path.join(dfuse.mount_dir.value, fname[0])) - original_fs = int(self.fio_cmd._jobs['test'].size.value) + original_fs = int(fio_cmd._jobs['test'].size.value) # Read and verify the original data. - self.fio_cmd._jobs['test'].rw = 'read' - self.fio_cmd.run() + fio_cmd._jobs['test'].rw = 'read' + fio_cmd.run() # Get the file stats and confirm size file_size = get_remote_file_size(self.hostlist_clients[0], testfile) @@ -75,7 +76,7 @@ def test_ec_truncate(self): self.assertEqual(truncate_size, file_size) # Read and verify the data after truncate. - self.fio_cmd.run() + fio_cmd.run() # Truncate the original file and shrink to original size. result = run_remote( @@ -89,4 +90,4 @@ def test_ec_truncate(self): original_fs, file_size, "file size after truncase is not equal to original") # Read and verify the data after truncate. - self.fio_cmd.run() + fio_cmd.run() diff --git a/src/tests/ftest/fault_injection/ec.py b/src/tests/ftest/fault_injection/ec.py index fc8728c5c1e..0f6435b3bd2 100644 --- a/src/tests/ftest/fault_injection/ec.py +++ b/src/tests/ftest/fault_injection/ec.py @@ -5,11 +5,11 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' from dfuse_utils import get_dfuse, start_dfuse -from fio_test_base import FioBase +from fio_utils import TestFio from ior_test_base import IorTestBase -class EcodFaultInjection(IorTestBase, FioBase): +class EcodFaultInjection(IorTestBase, TestFio): """EC Fault domains Test class. Test Class Description: To validate Erasure code object type classes with Fault injection. @@ -57,5 +57,6 @@ def test_ec_fio_fault(self): container.set_attr(attrs={'dfuse-direct-io-disable': 'on'}) dfuse = get_dfuse(self, self.hostlist_clients) start_dfuse(self, dfuse, pool, container) - self.fio_cmd.update_directory(dfuse.mount_dir.value) - self.execute_fio() + fio_cmd = self.get_fio_command() + fio_cmd.update_directory(dfuse.mount_dir.value) + fio_cmd.run() diff --git a/src/tests/ftest/io/parallel_io.py b/src/tests/ftest/io/parallel_io.py index 2925dc863a6..4e5943304a4 100644 --- a/src/tests/ftest/io/parallel_io.py +++ b/src/tests/ftest/io/parallel_io.py @@ -13,12 +13,12 @@ from dfuse_utils import get_dfuse, start_dfuse from exception_utils import CommandFailure -from fio_test_base import FioBase +from fio_utils import TestFio from ior_test_base import IorTestBase from run_utils import run_remote -class ParallelIo(FioBase, IorTestBase): +class ParallelIo(TestFio, IorTestBase): """Base Parallel IO test class. :avocado: recursive @@ -153,8 +153,9 @@ def test_parallelio(self): self.fail("Error running '{}' on the following hosts: {}".format( cmd, result.failed_hosts)) # run fio on all containers - self.fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, cont.uuid)) - thread = threading.Thread(target=self.execute_fio) + fio_cmd = self.get_fio_command() + fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, cont.uuid)) + thread = threading.Thread(target=fio_cmd.run) threads.append(thread) thread.start() @@ -171,8 +172,9 @@ def test_parallelio(self): # try accessing destroyed container, it should fail try: - self.fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, container_to_destroy)) - self.execute_fio() + fio_cmd = self.get_fio_command() + fio_cmd.update_directory(os.path.join(dfuse.mount_dir.value, container_to_destroy)) + fio_cmd.run() self.fail(f"Fio was able to access destroyed container: {self.container[0]}") except CommandFailure: self.log.info("fio failed as expected") diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 04fc177feed..079448994e0 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -95,6 +95,9 @@ def __init__(self, namespace, command, path="", subprocess=False, check_results= if check_results: self.check_results_list = list(check_results) + # Internal flag used to indicate if cleanup is required for the command. + self.__cleanup_needed = False + def __str__(self): """Return the command with all of its defined parameters as a string. @@ -206,6 +209,7 @@ def run(self, raise_exception=None): CommandFailure: if there is an error running the command """ + self.__cleanup_needed = True if self.run_as_subprocess: self._run_subprocess() return None @@ -392,6 +396,35 @@ def stop(self): self.log.info("%s stopped successfully", self.command) self._process = None + def cleanup_command(self): + """Cleanup the command.""" + if not self.__cleanup_needed: + self.log.info("No cleanup needed for %s", self.command) + return + + self.log.info("Cleaning up %s", self.command) + regex = self.command_regex + if self.full_command_regex: + regex = f"'{str(self)}'" + hosts = None + if hasattr(self, "hosts"): + hosts = self.hosts + detected, running = stop_processes( + self.log, hosts, regex, full_command=self.full_command_regex) + if not detected: + self.log.info( + "No remote %s processes killed on %s (none found), done.", + regex, "local host" if not hosts else hosts) + elif running: + self.log.info( + "***Unable to kill remote %s process on %s! Please investigate/report.***", + regex, running) + else: + self.log.info( + "***At least one remote %s process needed to be killed on %s! Please investigate/" + "report.***", regex, detected) + self.__cleanup_needed = False + def wait(self): """Wait for the sub process to complete. @@ -1552,7 +1585,6 @@ def __init__(self, namespace, command, path="", check_results=None, run_user=Non """ super().__init__(namespace, command, path, False, check_results, run_user) self._hosts = None - self.register_cleanup_method = None @property def hosts(self): @@ -1594,12 +1626,6 @@ def _run_process(self, raise_exception=None): Returns: CommandResult: result from running the command """ - if callable(self.register_cleanup_method): - # Stop any running processes started by this job manager when the test completes - # pylint: disable=not-callable - self.register_cleanup_method(self.stop) - - # Run the command on the remote hosts self.result = None if not self.hosts: result = run_local(self.log, self.with_exports, self.verbose, self.timeout) @@ -1633,26 +1659,6 @@ def _result_stderr(self): raise CommandFailure("No command result available to return stderr") return self.result.joined_stderr - def stop(self): - """Stop the command.""" - regex = self.command_regex - if self.full_command_regex: - regex = f"'{str(self)}'" - detected, running = stop_processes( - self.log, self.hosts, regex, full_command=self.full_command_regex) - if not detected: - self.log.info( - "No remote %s processes killed on %s (none found), done.", - regex, "local host" if not self.hosts else self.hosts) - elif running: - self.log.info( - "***Unable to kill remote %s process on %s! Please investigate/report.***", - regex, running) - else: - self.log.info( - "***At least one remote %s process needed to be killed on %s! Please investigate/" - "report.***", regex, detected) - def _get_new(self): """Get a new object based upon this one. diff --git a/src/tests/ftest/util/fio_test_base.py b/src/tests/ftest/util/fio_test_base.py deleted file mode 100644 index 2a10d02c462..00000000000 --- a/src/tests/ftest/util/fio_test_base.py +++ /dev/null @@ -1,39 +0,0 @@ -""" - (C) Copyright 2020-2024 Intel Corporation. - (C) Copyright 2026 Hewlett Packard Enterprise Development LP - - SPDX-License-Identifier: BSD-2-Clause-Patent -""" -from apricot import TestWithServers -from fio_utils import get_fio - - -class FioBase(TestWithServers): - """Base fio class. - - :avocado: recursive - """ - - def __init__(self, *args, **kwargs): - """Initialize a FioBase object.""" - super().__init__(*args, **kwargs) - self.fio_cmd = None - self.processes = None - self.manager = None - - def setUp(self): - """Set up each test case.""" - # obtain separate logs - self.update_log_file_names() - - # Start the servers and agents - super().setUp() - - # Get the parameters for Fio - self.fio_cmd = get_fio(self, self.hostlist_clients) - self.processes = self.params.get("np", '/run/fio/client_processes/*') - self.manager = self.params.get("manager", '/run/fio/*', "MPICH") - - def execute_fio(self): - """Runner method for Fio.""" - self.fio_cmd.run() diff --git a/src/tests/ftest/util/fio_utils.py b/src/tests/ftest/util/fio_utils.py index 2dc657b2ce8..0649cec997f 100644 --- a/src/tests/ftest/util/fio_utils.py +++ b/src/tests/ftest/util/fio_utils.py @@ -4,27 +4,33 @@ SPDX-License-Identifier: BSD-2-Clause-Patent """ +from apricot import TestWithServers from command_utils import RunCommand from command_utils_base import BasicParameter, CommandWithParameters, FormattedParameter -def get_fio(test, hosts, path="", namespace="/run/fio/*"): - """Get a FioCommand object with parameters from the test yaml file. +class TestFio(TestWithServers): + # pylint: disable=too-few-public-methods + """Base class for Fio tests. - Args: - test (Test): avocado Test object - hosts (NodeSet): hosts on which to run the ior command - path (str, optional): path to location of command binary file. Defaults to "". - namespace (str, optional): path to yaml parameters. Defaults to "/run/fio/*". - - Returns: - FioCommand: a FioCommand object with parameters from the test yaml file + :avocado: recursive """ - fio = FioCommand(path, namespace) - fio.hosts = hosts - fio.register_cleanup_method = test.register_cleanup - fio.get_params(test) - return fio + + def get_fio_command(self, path="", namespace="/run/fio/*"): + """Get a FioCommand object with parameters from the test yaml file. + + Args: + path (str, optional): path to location of command binary file. Defaults to "". + namespace (str, optional): path to yaml parameters. Defaults to "/run/fio/*". + + Returns: + FioCommand: a FioCommand object with parameters from the test yaml file + """ + fio = FioCommand(path, namespace) + self.register_cleanup(fio.cleanup_command) + fio.hosts = self.hostlist_clients + fio.get_params(self) + return fio class FioCommand(RunCommand): From db47f57541a092641d18a6469f5fd0319bf5fc11 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 21 Aug 2026 00:20:47 -0400 Subject: [PATCH 8/8] Missed one. Quick-functional: true Signed-off-by: Phil Henderson --- src/tests/ftest/dfuse/pil4dfs_fio.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/tests/ftest/dfuse/pil4dfs_fio.py b/src/tests/ftest/dfuse/pil4dfs_fio.py index 96feec4cf45..c61ddda0959 100644 --- a/src/tests/ftest/dfuse/pil4dfs_fio.py +++ b/src/tests/ftest/dfuse/pil4dfs_fio.py @@ -8,15 +8,14 @@ import json import os -from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet from cpu_utils import CpuInfo from dfuse_utils import get_dfuse, start_dfuse -from fio_utils import get_fio +from fio_utils import TestFio from general_utils import bytes_to_human, get_log_file, percent_change -class Pil4dfsFio(TestWithServers): +class Pil4dfsFio(TestFio): """Test class Description: Runs Fio with in small config. :avocado: recursive @@ -105,7 +104,7 @@ def _run_fio_pil4dfs(self, ioengine): dfuse = get_dfuse(self, self.hostlist_clients) start_dfuse(self, dfuse, container.pool, container) - fio_cmd = get_fio(self, self.hostlist_clients) + fio_cmd = self.get_fio_command() fio_cmd.update_directory(dfuse.mount_dir.value) fio_cmd.update("global", "ioengine", ioengine, f"fio --name=global --ioengine='{ioengine}'") fio_cmd.update( @@ -144,7 +143,7 @@ def _run_fio_dfs(self): """ container = self._create_container() - fio_cmd = get_fio(self, self.hostlist_clients) + fio_cmd = self.get_fio_command() fio_cmd.update("global", "ioengine", "dfs", "fio --name=global --ioengine='dfs'") fio_cmd.update( "job", "numjobs", self.fio_numjobs, f"fio --name=job --numjobs={self.fio_numjobs}")