diff --git a/doc/changes/DM-51000.misc.rst b/doc/changes/DM-51000.misc.rst new file mode 100644 index 0000000..016004d --- /dev/null +++ b/doc/changes/DM-51000.misc.rst @@ -0,0 +1 @@ +Updated the local submissions to handle the case when the given executable name is not an absolute path. It uses the submit-side location of the executable or prints a warning that the job may be held. This was triggered by ``ctrl_bps`` default yaml definitions no longer using ``*_DIR`` environment variables; instead they now rely on the PATH. diff --git a/python/lsst/ctrl/bps/htcondor/prepare_utils.py b/python/lsst/ctrl/bps/htcondor/prepare_utils.py index dd8b456..07fac74 100644 --- a/python/lsst/ctrl/bps/htcondor/prepare_utils.py +++ b/python/lsst/ctrl/bps/htcondor/prepare_utils.py @@ -30,6 +30,7 @@ import logging import os import re +import shutil from collections import defaultdict from pathlib import Path from typing import Any, cast @@ -283,6 +284,50 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob): return jobcmds +def _resolve_executable(executable: str) -> str: + """Resolve an executable name to its full path via PATH. + + HTCondor resolves a relative ``executable`` in the submit description + relative to the job's sandbox directory rather than searching ``PATH``. + So a bare or relative command name (e.g., ``pipetask``) must be turned + into a full path at submit time, otherwise the job is held with a + "No such file or directory" error on the compute node. + + Parameters + ---------- + executable : `str` + The executable value from the job description. + + Returns + ------- + executable : `str` + A full path if ``executable`` was a bare name found on ``PATH``, + otherwise the value unchanged (already absolute/relative paths and + values containing HTCondor env syntax are left as is). + """ + # Leave anything that already starts with a path separator or + # environment-variable syntax (e.g., $ENV(...), $(...)) untouched. + if executable.startswith(("/", "$")): + _LOG.debug("Executable started with / or $. Using as-is (%s)", executable) + return executable + + exec_path = shutil.which(executable) + if exec_path: + _LOG.debug("results of 'which %s' = %s", executable, exec_path) + # Still have to make sure it is not a relative path as + # shutil.which would skip checking PATH if executable + # contains '/' and then only check existence. + exec_path = os.path.abspath(exec_path) + return exec_path + + _LOG.warning( + "Could not resolve executable '%s' on PATH; HTCondor resolves an " + "executable name relative to the job sandbox, so the job may be held.", + executable, + ) + return executable + + def _translate_command_line( cached_vals: dict[str, Any], generic_workflow: GenericWorkflow, gwjob: GenericWorkflowJob ) -> dict[str, Any]: @@ -353,7 +398,7 @@ def _translate_command_line( jobcmds["transfer_executable"] = "True" jobcmds["executable"] = gwjob.executable.src_uri else: - jobcmds["executable"] = _fix_env_var_syntax(gwjob.executable.src_uri) + jobcmds["executable"] = _resolve_executable(_fix_env_var_syntax(gwjob.executable.src_uri)) if arguments: arguments = _fix_env_var_syntax(arguments) diff --git a/tests/test_prepare_utils.py b/tests/test_prepare_utils.py index d30a69f..567d400 100644 --- a/tests/test_prepare_utils.py +++ b/tests/test_prepare_utils.py @@ -29,6 +29,7 @@ import logging import os +import shutil import unittest from copy import deepcopy @@ -222,6 +223,26 @@ def testMakeCommandExecutableEnvVar(self): jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob) self.assertEqual(jobcmds["executable"], "$ENV(CTRL_BPS_DIR)/bin/pipetask") + def testMakeCommandBareExecutableResolvedViaPath(self): + # A bare executable name (no path, no env var) is resolved to its + # full path via PATH because HTCondor resolves the executable + # relative to the job sandbox, not via PATH. + resolved = shutil.which("sh") + self.assertIsNotNone(resolved, "test requires 'sh' on PATH") + gw_exec = GenericWorkflowExec("sh", "sh") + gw, gwjob = self._make_job(executable=gw_exec) + jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob) + self.assertEqual(jobcmds["executable"], resolved) + + def testMakeCommandBareExecutableNotOnPath(self): + # A bare executable name that cannot be found on PATH is left as is. + gw_exec = GenericWorkflowExec("nosuchcmd", "nosuchcmd_xyzzy") + gw, gwjob = self._make_job(executable=gw_exec) + with self.assertLogs(level="WARNING") as cm_log: + jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob) + self.assertRegex(cm_log.output[0], "Could not resolve executable 'nosuchcmd_xyzzy' on PATH.*") + self.assertEqual(jobcmds["executable"], "nosuchcmd_xyzzy") + def testMakeCommandArguments(self): # Arguments should have cmd, wms, file, and env placeholders replaced. gw, gwjob = self._make_job(arguments="run --attempt {opt}")