From 45b9ca8dbe9b232e7924aa8e6920ce221439ebea Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 18 Nov 2026 09:09:19 +0100 Subject: [PATCH] fix(operator): Wait for the tee processes of capture_shell_output on exit --- Cargo.lock | 1 + crates/stackable-operator/CHANGELOG.md | 9 + crates/stackable-operator/Cargo.toml | 1 + .../src/crd/git_sync/v1alpha1_impl.rs | 84 +++++- .../src/crd/git_sync/v1alpha2_impl.rs | 140 +++++++++- .../src/product_logging/framework.rs | 247 ++++++++++++++++-- 6 files changed, 446 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9571b3637..1b7072869 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3844,6 +3844,7 @@ dependencies = [ "stackable-versioned", "stackable-webhook", "strum", + "tempfile", "tokio", "tracing", "tracing-appender", diff --git a/crates/stackable-operator/CHANGELOG.md b/crates/stackable-operator/CHANGELOG.md index d5fafa12a..8510fd534 100644 --- a/crates/stackable-operator/CHANGELOG.md +++ b/crates/stackable-operator/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Fixed + +- `product_logging::framework::capture_shell_output` no longer loses the output of short-lived + containers such as init containers. When stdout or stderr go to the console and to a file, the + generated command now sets an EXIT trap that waits up to five seconds for its `tee` processes. + Callers must not set their own EXIT trap ([#1336]). + +[#1336]: https://github.com/stackabletech/operator-rs/pull/1336 + ## [0.122.0] - 2026-11-03 ### Added diff --git a/crates/stackable-operator/Cargo.toml b/crates/stackable-operator/Cargo.toml index 9e18ee666..a6ce89484 100644 --- a/crates/stackable-operator/Cargo.toml +++ b/crates/stackable-operator/Cargo.toml @@ -70,6 +70,7 @@ xml.workspace = true [dev-dependencies] indoc.workspace = true rstest.workspace = true +tempfile.workspace = true [lints] workspace = true diff --git a/crates/stackable-operator/src/crd/git_sync/v1alpha1_impl.rs b/crates/stackable-operator/src/crd/git_sync/v1alpha1_impl.rs index 2b575929c..5c0e54209 100644 --- a/crates/stackable-operator/src/crd/git_sync/v1alpha1_impl.rs +++ b/crates/stackable-operator/src/crd/git_sync/v1alpha1_impl.rs @@ -487,7 +487,19 @@ mod tests { assert_eq!( r#"args: - |- - mkdir --parents /stackable/log/git-sync-0 && exec > >(tee /stackable/log/git-sync-0/container.stdout.log) 2> >(tee /stackable/log/git-sync-0/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-0 && + exec > >(tee /stackable/log/git-sync-0/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-0/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT prepare_signal_handlers() { @@ -556,7 +568,19 @@ volumeMounts: assert_eq!( r#"args: - |- - mkdir --parents /stackable/log/git-sync-1 && exec > >(tee /stackable/log/git-sync-1/container.stdout.log) 2> >(tee /stackable/log/git-sync-1/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-1 && + exec > >(tee /stackable/log/git-sync-1/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-1/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT prepare_signal_handlers() { @@ -630,7 +654,19 @@ volumeMounts: assert_eq!( r#"args: - |- - mkdir --parents /stackable/log/git-sync-2 && exec > >(tee /stackable/log/git-sync-2/container.stdout.log) 2> >(tee /stackable/log/git-sync-2/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-2 && + exec > >(tee /stackable/log/git-sync-2/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-2/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT prepare_signal_handlers() { @@ -701,7 +737,19 @@ volumeMounts: assert_eq!( r"args: - |- - mkdir --parents /stackable/log/git-sync-0-init && exec > >(tee /stackable/log/git-sync-0-init/container.stdout.log) 2> >(tee /stackable/log/git-sync-0-init/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-0-init && + exec > >(tee /stackable/log/git-sync-0-init/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-0-init/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT /stackable/git-sync --depth=1 --git-config='safe.directory:/tmp/git' --link=current --one-time=true --period=20s --ref=main --repo=https://github.com/stackabletech/repo1 --root=/tmp/git command: - /bin/bash @@ -738,7 +786,19 @@ volumeMounts: assert_eq!( r"args: - |- - mkdir --parents /stackable/log/git-sync-1-init && exec > >(tee /stackable/log/git-sync-1-init/container.stdout.log) 2> >(tee /stackable/log/git-sync-1-init/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-1-init && + exec > >(tee /stackable/log/git-sync-1-init/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-1-init/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT /stackable/git-sync --depth=3 --git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt' --link=current --one-time=true --period=60s --ref=trunk --repo=https://github.com/stackabletech/repo2 --rev=HEAD --root=/tmp/git command: - /bin/bash @@ -780,7 +840,19 @@ volumeMounts: assert_eq!( r"args: - |- - mkdir --parents /stackable/log/git-sync-2-init && exec > >(tee /stackable/log/git-sync-2-init/container.stdout.log) 2> >(tee /stackable/log/git-sync-2-init/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-2-init && + exec > >(tee /stackable/log/git-sync-2-init/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-2-init/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT /stackable/git-sync --depth=1 --git-config='safe.directory:/tmp/git,key:value,safe.directory:/safe-dir' --link=current --one-time=true --period=20s --ref=feat/git-sync --repo=https://github.com/stackabletech/repo3 --root=/tmp/git command: - /bin/bash diff --git a/crates/stackable-operator/src/crd/git_sync/v1alpha2_impl.rs b/crates/stackable-operator/src/crd/git_sync/v1alpha2_impl.rs index 4fdd76e0e..889b2f545 100644 --- a/crates/stackable-operator/src/crd/git_sync/v1alpha2_impl.rs +++ b/crates/stackable-operator/src/crd/git_sync/v1alpha2_impl.rs @@ -618,7 +618,19 @@ mod tests { assert_eq!( r#"args: - |- - mkdir --parents /stackable/log/git-sync-0 && exec > >(tee /stackable/log/git-sync-0/container.stdout.log) 2> >(tee /stackable/log/git-sync-0/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-0 && + exec > >(tee /stackable/log/git-sync-0/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-0/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT prepare_signal_handlers() { @@ -687,7 +699,19 @@ volumeMounts: assert_eq!( r#"args: - |- - mkdir --parents /stackable/log/git-sync-1 && exec > >(tee /stackable/log/git-sync-1/container.stdout.log) 2> >(tee /stackable/log/git-sync-1/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-1 && + exec > >(tee /stackable/log/git-sync-1/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-1/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT prepare_signal_handlers() { @@ -761,7 +785,19 @@ volumeMounts: assert_eq!( r#"args: - |- - mkdir --parents /stackable/log/git-sync-2 && exec > >(tee /stackable/log/git-sync-2/container.stdout.log) 2> >(tee /stackable/log/git-sync-2/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-2 && + exec > >(tee /stackable/log/git-sync-2/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-2/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT prepare_signal_handlers() { @@ -832,7 +868,19 @@ volumeMounts: assert_eq!( r"args: - |- - mkdir --parents /stackable/log/git-sync-0-init && exec > >(tee /stackable/log/git-sync-0-init/container.stdout.log) 2> >(tee /stackable/log/git-sync-0-init/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-0-init && + exec > >(tee /stackable/log/git-sync-0-init/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-0-init/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT /stackable/git-sync --depth=1 --git-config='safe.directory:/tmp/git' --link=current --one-time=true --period=20s --ref=main --repo=https://github.com/stackabletech/repo1 --root=/tmp/git command: - /bin/bash @@ -869,7 +917,19 @@ volumeMounts: assert_eq!( r"args: - |- - mkdir --parents /stackable/log/git-sync-1-init && exec > >(tee /stackable/log/git-sync-1-init/container.stdout.log) 2> >(tee /stackable/log/git-sync-1-init/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-1-init && + exec > >(tee /stackable/log/git-sync-1-init/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-1-init/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT /stackable/git-sync --depth=3 --git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt' --link=current --one-time=true --period=60s --ref=trunk --repo=https://github.com/stackabletech/repo2 --rev=HEAD --root=/tmp/git command: - /bin/bash @@ -911,7 +971,19 @@ volumeMounts: assert_eq!( r"args: - |- - mkdir --parents /stackable/log/git-sync-2-init && exec > >(tee /stackable/log/git-sync-2-init/container.stdout.log) 2> >(tee /stackable/log/git-sync-2-init/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-2-init && + exec > >(tee /stackable/log/git-sync-2-init/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-2-init/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT /stackable/git-sync --depth=1 --git-config='safe.directory:/tmp/git,key:value,safe.directory:/safe-dir' --link=current --one-time=true --period=20s --ref=feat/git-sync --repo=https://github.com/stackabletech/repo3 --root=/tmp/git command: - /bin/bash @@ -1073,7 +1145,19 @@ name: content-from-git-2 assert_eq!( r#"args: - |- - mkdir --parents /stackable/log/git-sync-0 && exec > >(tee /stackable/log/git-sync-0/container.stdout.log) 2> >(tee /stackable/log/git-sync-0/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-0 && + exec > >(tee /stackable/log/git-sync-0/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-0/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT prepare_signal_handlers() { @@ -1148,7 +1232,19 @@ volumeMounts: assert_eq!( r"args: - |- - mkdir --parents /stackable/log/git-sync-0-init && exec > >(tee /stackable/log/git-sync-0-init/container.stdout.log) 2> >(tee /stackable/log/git-sync-0-init/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-0-init && + exec > >(tee /stackable/log/git-sync-0-init/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-0-init/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT /stackable/git-sync --depth=3 --git-config='safe.directory:/tmp/git,http.sslCAInfo:/tmp/ca-cert/ca.crt' --link=current --one-time=true --period=60s --ref=trunk --repo=ssh://git@github.com/stackabletech/repo.git --rev=HEAD --root=/tmp/git command: - /bin/bash @@ -1283,7 +1379,19 @@ secret: assert_eq!( r#"args: - |- - mkdir --parents /stackable/log/git-sync-0 && exec > >(tee /stackable/log/git-sync-0/container.stdout.log) 2> >(tee /stackable/log/git-sync-0/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-0 && + exec > >(tee /stackable/log/git-sync-0/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-0/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT prepare_signal_handlers() { @@ -1354,7 +1462,19 @@ volumeMounts: assert_eq!( r"args: - |- - mkdir --parents /stackable/log/git-sync-0-init && exec > >(tee /stackable/log/git-sync-0-init/container.stdout.log) 2> >(tee /stackable/log/git-sync-0-init/container.stderr.log >&2) + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /stackable/log/git-sync-0-init && + exec > >(tee /stackable/log/git-sync-0-init/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /stackable/log/git-sync-0-init/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT /stackable/git-sync --depth=3 --git-config='http.sslCAInfo:/stackable/gitca-0/ca.crt,safe.directory:/tmp/git' --link=current --one-time=true --period=60s --ref=trunk --repo=ssh://git@github.com/stackabletech/repo.git --rev=HEAD --root=/tmp/git command: - /bin/bash diff --git a/crates/stackable-operator/src/product_logging/framework.rs b/crates/stackable-operator/src/product_logging/framework.rs index 35c7bab02..8070ca8da 100644 --- a/crates/stackable-operator/src/product_logging/framework.rs +++ b/crates/stackable-operator/src/product_logging/framework.rs @@ -120,9 +120,41 @@ pub fn calculate_log_volume_size_limit(max_log_files_size: &[MemoryQuantity]) -> log_volume_size_limit.into() } +/// Waits for the `tee` processes in `stackable_tee_pids`, at most for five seconds. +/// +/// Redirecting to /dev/null closes the shell's ends of the pipes, so tee reads EOF once no child +/// process has them open any more. A background process that outlives the script keeps them open +/// forever, so the watchdog kills tee after a bounded time instead of letting the container hang. +/// `set +e` keeps a failing `wait` or `kill` from replacing the exit code of the script under +/// `set -e`. +const WAIT_FOR_TEE_FUNCTION: &str = "\ +stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! +}"; + /// Create a Bash command which filters stdout and stderr according to the given log configuration /// and additionally stores the output in log files /// +/// The command must run in the same shell as the commands whose output it captures, so join it +/// with them using `" && "` or a newline. +/// +/// When stdout or stderr go to the console and to a file at the same time, they are piped through +/// `tee` processes running in the background. When Bash is PID 1 of the container and exits, the +/// kernel kills these processes, possibly before they wrote the last output. To prevent this, the +/// command sets an EXIT trap which waits for the `tee` processes, at most for five seconds, so a +/// background process which still has stdout or stderr open delays the exit by that long. This +/// has the following consequences for the caller: +/// +/// - The caller must not set its own EXIT trap, because it would replace this one. +/// - The shell variable `stackable_tee_pids` and the function `stackable_wait_for_tee` are +/// reserved. +/// - A caller that ends with `exec` replaces Bash, so the trap does not run. That is fine for a +/// long-running process, but a short-lived one can lose the end of its output. +/// /// # Example /// /// ``` @@ -189,37 +221,60 @@ pub fn capture_shell_output( let log_file_dir = format!("{log_dir}/{container}"); - let stdout_redirect = match ( + let mut commands = Vec::new(); + let mut uses_tee = false; + + if file_log_level <= LogLevel::ERROR { + commands.push(format!("mkdir --parents {log_file_dir}")); + } + + match ( console_log_level <= LogLevel::INFO, file_log_level <= LogLevel::INFO, ) { - (true, true) => format!(" > >(tee {log_file_dir}/container.stdout.log)"), - (true, false) => String::new(), - (false, true) => format!(" > {log_file_dir}/container.stdout.log"), - (false, false) => " > /dev/null".into(), - }; + (true, true) => { + commands.push(format!("exec > >(tee {log_file_dir}/container.stdout.log)")); + commands.push("stackable_tee_pids+=($!)".into()); + uses_tee = true; + } + (true, false) => {} + (false, true) => commands.push(format!("exec > {log_file_dir}/container.stdout.log")), + (false, false) => commands.push("exec > /dev/null".into()), + } - let stderr_redirect = match ( + match ( console_log_level <= LogLevel::ERROR, file_log_level <= LogLevel::ERROR, ) { - (true, true) => format!(" 2> >(tee {log_file_dir}/container.stderr.log >&2)"), - (true, false) => String::new(), - (false, true) => format!(" 2> {log_file_dir}/container.stderr.log"), - (false, false) => " 2> /dev/null".into(), - }; + (true, true) => { + commands.push(format!( + "exec 2> >(tee {log_file_dir}/container.stderr.log >&2)" + )); + commands.push("stackable_tee_pids+=($!)".into()); + uses_tee = true; + } + (true, false) => {} + (false, true) => commands.push(format!("exec 2> {log_file_dir}/container.stderr.log")), + (false, false) => commands.push("exec 2> /dev/null".into()), + } - let mut args = Vec::new(); - if file_log_level <= LogLevel::ERROR { - args.push(format!("mkdir --parents {log_file_dir}")); + if uses_tee { + commands.push("trap stackable_wait_for_tee EXIT".into()); } - if stdout_redirect.is_empty() && stderr_redirect.is_empty() { - args.push(":".into()); - } else { - args.push(format!("exec{stdout_redirect}{stderr_redirect}")); + + if commands.is_empty() { + commands.push(":".into()); } - args.join(" && ") + // Bash continues a command list after a trailing `&&`, so the commands can go on separate + // lines without changing what they do. + let commands = commands.join(" &&\n "); + + if uses_tee { + format!("{WAIT_FOR_TEE_FUNCTION}\n{commands}") + } else { + commands + } } /// Create the content of a log4j properties file according to the given log configuration @@ -1510,6 +1565,7 @@ pub fn remove_vector_shutdown_file_command(stackable_log_dir: &str) -> String { mod tests { use std::collections::BTreeMap; + use indoc::indoc; use rstest::rstest; use super::*; @@ -1536,6 +1592,157 @@ mod tests { ); } + fn shell_log_config( + console_level: LogLevel, + file_level: LogLevel, + ) -> AutomaticContainerLogConfig { + AutomaticContainerLogConfig { + loggers: BTreeMap::from([( + AutomaticContainerLogConfig::ROOT_LOGGER.to_string(), + LoggerConfig { + level: LogLevel::TRACE, + }, + )]), + console: Some(AppenderConfig { + level: Some(console_level), + }), + file: Some(AppenderConfig { + level: Some(file_level), + }), + } + } + + #[rstest] + #[case::console_and_file( + LogLevel::INFO, + LogLevel::INFO, + indoc! {" + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /log/test && + exec > >(tee /log/test/container.stdout.log) && + stackable_tee_pids+=($!) && + exec 2> >(tee /log/test/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT"} + )] + #[case::console_and_file_only_for_errors( + LogLevel::INFO, + LogLevel::ERROR, + indoc! {" + stackable_wait_for_tee() { + exec > /dev/null 2>&1 + set +e + { sleep 5; kill ${stackable_tee_pids[*]}; } & + wait ${stackable_tee_pids[*]} + kill $! + } + mkdir --parents /log/test && + exec 2> >(tee /log/test/container.stderr.log >&2) && + stackable_tee_pids+=($!) && + trap stackable_wait_for_tee EXIT"} + )] + #[case::file_only( + LogLevel::NONE, + LogLevel::INFO, + indoc! {" + mkdir --parents /log/test && + exec > /log/test/container.stdout.log && + exec 2> /log/test/container.stderr.log"} + )] + #[case::console_only(LogLevel::INFO, LogLevel::NONE, ":")] + #[case::neither( + LogLevel::NONE, + LogLevel::NONE, + "exec > /dev/null &&\n exec 2> /dev/null" + )] + fn capture_shell_output_command( + #[case] console_level: LogLevel, + #[case] file_level: LogLevel, + #[case] expected_command: &str, + ) { + let command = + capture_shell_output("/log", "test", &shell_log_config(console_level, file_level)); + + assert_eq!(expected_command, command); + } + + struct CapturedRun { + status: std::process::ExitStatus, + duration: std::time::Duration, + stdout_log: String, + stderr_log: String, + } + + /// Runs `script` behind the output capturing in a new Bash process and returns the log files + /// as they are at the moment Bash has exited. + fn run_captured(script: &str) -> CapturedRun { + let log_dir = tempfile::tempdir().expect("a temporary directory can be created"); + let log_dir_path = log_dir + .path() + .to_str() + .expect("the temporary directory path is valid UTF-8"); + let capture = capture_shell_output( + log_dir_path, + "test", + &shell_log_config(LogLevel::INFO, LogLevel::INFO), + ); + + let start = std::time::Instant::now(); + // The console output is discarded, so that a background process of the script cannot + // keep this process waiting for EOF. + let status = std::process::Command::new("bash") + .args(["-c", &format!("{capture}\n{script}")]) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .expect("bash can be started"); + let duration = start.elapsed(); + + let read_log = |name: &str| { + std::fs::read_to_string(log_dir.path().join("test").join(name)).unwrap_or_default() + }; + + CapturedRun { + status, + duration, + stdout_log: read_log("container.stdout.log"), + stderr_log: read_log("container.stderr.log"), + } + } + + /// A script this short ends before tee has started, so the log files are only complete if Bash + /// waits for tee. + #[rstest] + #[case::success("true", Some(0))] + #[case::explicit_exit("exit 3", Some(3))] + #[case::errexit("set -e\nfalse\necho unreachable", Some(1))] + fn captured_output_is_complete_and_keeps_exit_code( + #[case] script: &str, + #[case] expected_code: Option, + ) { + let run = run_captured(&format!("echo output\necho error >&2\n{script}")); + + assert_eq!(expected_code, run.status.code()); + assert_eq!("output\n", run.stdout_log); + assert_eq!("error\n", run.stderr_log); + } + + #[test] + fn captured_output_does_not_wait_forever_for_background_processes() { + let run = run_captured("sleep 20 &\necho output"); + + assert!(run.status.success()); + assert!(run.duration < std::time::Duration::from_secs(15)); + assert_eq!("output\n", run.stdout_log); + } + #[test] fn log4j2_config() { let log_config = AutomaticContainerLogConfig {