diff --git a/scripts/test_databases.sh b/scripts/test_databases.sh index a0cae393d..e8dd1d1c2 100755 --- a/scripts/test_databases.sh +++ b/scripts/test_databases.sh @@ -3,14 +3,14 @@ set -euo pipefail if [[ $# -lt 1 || $# -gt 2 ]]; then - printf 'Usage: %s BUILD_DIR [local|redis|mysql|redis-awaitable|mysql-awaitable|public-redis|public-mysql|public-services|all]\n' "$0" >&2 + printf 'Usage: %s BUILD_DIR [local|redis|mysql|redis-awaitable|mysql-awaitable|public-redis|public-mysql|public-services|public-redis-failures|public-mysql-failures|all]\n' "$0" >&2 exit 2 fi database_test_dir=$(cd -- "$1" && pwd)/tests/database database_test_mode=${2:-local} case "$database_test_mode" in - local|redis|mysql|redis-awaitable|mysql-awaitable|public-redis|public-mysql|public-services|all) ;; + local|redis|mysql|redis-awaitable|mysql-awaitable|public-redis|public-mysql|public-services|public-redis-failures|public-mysql-failures|all) ;; *) printf 'Unknown database test mode: %s\n' "$database_test_mode" >&2; exit 2 ;; esac @@ -56,3 +56,9 @@ fi if [[ "$database_test_mode" == public-mysql || "$database_test_mode" == public-services || "$database_test_mode" == all ]]; then run_database_test ark_database_public_service_tests mysql fi +if [[ "$database_test_mode" == public-redis-failures ]]; then + run_database_test ark_database_public_failure_service_tests redis +fi +if [[ "$database_test_mode" == public-mysql-failures ]]; then + run_database_test ark_database_public_failure_service_tests mysql +fi diff --git a/tests/database/CMakeLists.txt b/tests/database/CMakeLists.txt index c29f11d48..74e0350be 100644 --- a/tests/database/CMakeLists.txt +++ b/tests/database/CMakeLists.txt @@ -36,10 +36,13 @@ target_link_libraries(ark_database_redis_stop_tests PRIVATE arknet::arknet) ark_database_test(ark_database_public_redis_tests main.cpp public_redis_module.cpp "${ROOT_DIR}/src/plugins/redis/redis_module_impl.cpp") target_link_libraries(ark_database_public_redis_tests PRIVATE ark::core arknet::arknet) -# Explicit real-service acceptance; deliberately excluded from check-database. +# Real-service programs are compiled by check-database, but never run there. ark_database_test(ark_database_public_service_tests public_service.cpp "${ROOT_DIR}/src/plugins/redis/redis_module_impl.cpp") target_link_libraries(ark_database_public_service_tests PRIVATE ark::core arknet::arknet) +ark_database_test(ark_database_public_failure_service_tests public_failure_service.cpp + "${ROOT_DIR}/src/plugins/redis/redis_module_impl.cpp") +target_link_libraries(ark_database_public_failure_service_tests PRIVATE ark::core arknet::arknet) add_library(ark_database_redis_plugin SHARED "${ROOT_DIR}/src/plugins/redis/redis_plugin.cpp" "${ROOT_DIR}/src/plugins/redis/redis_module_impl.cpp") target_include_directories(ark_database_redis_plugin PRIVATE "${ROOT_DIR}/src/plugins") @@ -72,9 +75,14 @@ add_custom_target(check-database USES_TERMINAL) # Compile real-service acceptance in ordinary CI without running any server work. -add_dependencies(check-database ark_database_redis_sample_check ark_database_public_service_tests) +add_dependencies(check-database ark_database_redis_sample_check ark_database_public_service_tests + ark_database_public_failure_service_tests) if(ARKNET_ENABLE_SSL) + target_sources(ark_database_public_failure_service_tests PRIVATE + "${ROOT_DIR}/src/plugins/mysql/mysql_module_impl.cpp") + target_compile_definitions(ark_database_public_failure_service_tests PRIVATE + ARK_DATABASE_UNTRUSTED_CA="${ARKNET_SOURCE_DIR}/tests/certs/wrong.pem") target_sources(ark_database_public_service_tests PRIVATE "${ROOT_DIR}/src/plugins/mysql/mysql_module_impl.cpp") target_compile_definitions(ark_database_public_redis_tests PRIVATE diff --git a/tests/database/README.md b/tests/database/README.md index 6a0044fef..e5cd6de9b 100644 --- a/tests/database/README.md +++ b/tests/database/README.md @@ -238,3 +238,51 @@ configuration or failure, and does not print credentials or arbitrary native err messages. It never starts or stops servers. `all` also includes both public real-service modes and therefore requires an SSL-enabled build and both endpoint configurations. + +## Public real-service failure checks + +The separate `ark_database_public_failure_service_tests` executable checks wrong +Redis/MySQL passwords, an untrusted MySQL CA, and real slow-command deadlines. +`check-database` compiles it without running it. Choose a failure mode explicitly; +these modes are not included in `all`. + +```sh +cmake --build database-build --target ark_database_public_failure_service_tests --parallel 2 +bash scripts/test_databases.sh database-build public-redis-failures +bash scripts/test_databases.sh database-build public-mysql-failures +# Or invoke the executable with redis or mysql. +``` + +Use the same isolated endpoint variables as the public service checks above. +The Redis failure mode additionally requires a nonempty +`ARK_DATABASE_REDIS_PASSWORD` and permission for `CLIENT LIST`. MySQL requires SSL, +a trusted `ARK_DATABASE_MYSQL_CA` and access to the current account's sessions in +`information_schema.processlist`; no PROCESS privilege or extra schema is needed. +The unrelated CA fixture is `ARKNET_SOURCE_DIR/tests/certs/wrong.pem`, already used +by the local TLS regressions. No new environment variable or service management is +introduced. Missing configuration, observer permission errors, or a requested +protocol unavailable in the build fail with nonzero status. + +Credential/TLS failures must retain their native error and known-unsent business +command state. Subsequent explicit requests use the valid configuration. The +warmed primary then runs BLPOP on an absent unique `ark_failure_...` key (30-second +server block) or a tagged `SELECT SLEEP(10)`, with a two-second client deadline. +BLPOP never receives producer data; the test first checks that its key is absent. +Both deadlines must be delivered within five seconds. + +An independently configured connection observes Redis's named blocked client via +`CLIENT LIST`, or MySQL's tagged sleeping session via `processlist`. Only this +server-visible observation establishes that the request was sent; polling timers +only throttle observations. The timeout must retain its unknown-execution flag. +An explicit new PING/SELECT must recover through the same public connection name. +The observer rejects a second slow client/session, then checks for 500 ms after +the original disappears. This bounds the no-replay evidence to these tagged +slow requests during the observed recovery window; it does not prove write-replay +behavior or absence of later activity outside the window. + +All coroutines and task completions must stay on the owner thread/shared manual +runtime. Accepted slow/observer tasks finish during natural manager drain before +native retirement and idempotent stop. These cases do not test cancellation, +server termination, dynamic-library unloading or capacity. Slow-query results +remain owned after module destruction. Native/server error text and credentials +are not printed. diff --git a/tests/database/README_CN.md b/tests/database/README_CN.md index 6a774b42e..518e75a17 100644 --- a/tests/database/README_CN.md +++ b/tests/database/README_CN.md @@ -190,3 +190,40 @@ MySQL 要求设置 `ARK_DATABASE_MYSQL_HOST`、`ARK_DATABASE_MYSQL_PORT`、 程序必须显式配置端点；配置缺失、无效或验收失败均返回非零状态，不打印凭据或任意原生错误文本， 也不会启动或停止服务。`all` 也运行这两种公开模块真实服务模式，因此要求启用 SSL， 并配置两种服务的端点。 + +## 公开模块真实服务负面验收 + +独立的 `ark_database_public_failure_service_tests` 验证 Redis/MySQL 错误密码、 +MySQL 不可信 CA，以及真实慢命令的截止时间。`check-database` 仅编译该程序，不运行。 +需显式选择负面验收模式；`all` 不包含这些模式。 + +```sh +cmake --build database-build --target ark_database_public_failure_service_tests --parallel 2 +bash scripts/test_databases.sh database-build public-redis-failures +bash scripts/test_databases.sh database-build public-mysql-failures +# 也可以直接运行程序，参数为 redis 或 mysql。 +``` + +沿用上方公开服务验收的隔离端点环境变量。Redis 负面模式额外要求非空的 +`ARK_DATABASE_REDIS_PASSWORD`，账号需有 `CLIENT LIST` 权限。MySQL 要求开启 SSL、 +使用可信的 `ARK_DATABASE_MYSQL_CA`，且账号能通过 `information_schema.processlist` +查看自身会话；不需要 PROCESS 权限或额外数据库。不可信 CA 使用本地 TLS 回归已有的 +`ARKNET_SOURCE_DIR/tests/certs/wrong.pem`，不引入新的环境变量或服务管理操作。 +配置缺失、观察连接权限不足或构建中没有请求的协议时，程序返回非零状态。 + +认证/TLS 失败必须保留原生错误，以及业务命令尚未发送的已知状态。随后使用正确配置显式发起 +新请求。预热后的主连接对不存在的唯一 `ark_failure_...` key 执行 BLPOP（服务端阻塞 +30 秒），或执行带唯一注释的 `SELECT SLEEP(10)`，客户端截止时间均为 2 秒。 +测试先检查 Redis key 不存在，且不会向该 key 写入数据。两种超时结果均须在 5 秒内交付。 + +独立配置的观察连接通过 Redis `CLIENT LIST` 确认具名客户端正在阻塞，或通过 MySQL +`processlist` 确认带唯一注释的会话正在休眠。只有服务端可见状态用于确认请求已经发送； +轮询定时器仅控制观察频率。超时结果必须保留执行状态未知标志。随后通过同一公开连接名显式 +执行 PING/SELECT，并验证恢复成功。观察连接拒绝出现第二个慢客户端或会话，且在原请求消失后 +继续检查 500 毫秒。该证据仅覆盖观察窗口内的慢查询恢复，不证明写操作的重放行为，也不证明 +窗口以外没有后续活动。 + +所有协程及任务完成回调均须在拥有线程及共享手动运行时上执行。管理器进入自然排空状态后， +已受理的慢请求和观察任务继续执行，完成后才退出原生资源，并验证停止操作可重复调用。 +这些用例不测试取消、服务端终止、动态库卸载或容量。模块销毁后，慢查询结果仍须有效。 +程序不打印凭据、原生错误文本或服务端错误文本。 diff --git a/tests/database/public_failure_service.cpp b/tests/database/public_failure_service.cpp new file mode 100644 index 000000000..c8f7d0817 --- /dev/null +++ b/tests/database/public_failure_service.cpp @@ -0,0 +1,522 @@ +// Copyright (c) 2026 OpenArkStudio. Licensed under Apache-2.0. See LICENSE. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ark/redis/redis_module_impl.hpp" + +#if defined(ARKNET_ENABLE_SSL) +#include "ark/mysql/mysql_module_impl.hpp" +#endif + +namespace { + +using namespace std::chrono_literals; +unsigned checks{}; + +class acceptance_failure : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +void require(bool condition, const std::string& message) { + ++checks; + if (!condition) + { + throw acceptance_failure(message); + } +} + +std::string environment(const char* name, bool allow_empty = false) { + const auto* value = std::getenv(name); + require(value && (allow_empty || *value), std::string("Required test environment variable: ") + name); + return value; +} + +std::uint16_t port(const char* name) { + const auto text = environment(name); + unsigned number{}; + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), number); + require(parsed.ec == std::errc{} && parsed.ptr == text.data() + text.size() && number > 0 && number <= 65535, + std::string(name) + " must be an integer between 1 and 65535"); + return static_cast(number); +} + +// This timer throttles observer polling; server-visible state, never a guessed +// delay, establishes whether the slow command reached the server. +ark::task<> observer_interval() { + asio::steady_timer timer(co_await asio::this_coro::executor); + timer.expires_after(20ms); + co_await timer.async_wait(asio::use_awaitable); +} + +template +struct module_owner +{ + ark::plugin_manager manager; + module_type database; + ark::module consumer; + ark::task_group* tasks{}; + std::shared_ptr runtime; + const std::thread::id thread{std::this_thread::get_id()}; + std::exception_ptr failure; + unsigned completions{}; + bool completions_on_owner{true}; + bool stopped{}; + + module_owner() { + database.set_plugin_manager(&manager); + database.set_name("database"); + require(manager.add_module("database", &database) && manager.add_module("consumer", &consumer), + "Cannot install public database failure test modules"); + require(manager.add_update_module(&database), "Cannot install public database update hook"); + runtime = manager.get_task_runtime(); + tasks = &manager.create_task_group(consumer); + require(database.init(), "Cannot initialize public database module"); + } + + ~module_owner() { + if (!stopped) + { + try + { finish(); } + catch (...) + { + std::cerr << "Public database failure test could not drain\n" << std::flush; + std::_Exit(1); + } + } + } + + void check_executor() const { + require(std::this_thread::get_id() == thread && runtime->get_executor().running_in_this_thread(), + "Failure coroutine escaped its owner thread/shared manual runtime"); + } + + void start(ark::task<> workflow) { + require(tasks->start(std::move(workflow), + [this](std::exception_ptr error) { + completions_on_owner = completions_on_owner && std::this_thread::get_id() == thread && + runtime->get_executor().running_in_this_thread(); + if (error && !failure) + { + failure = std::move(error); + } + ++completions; + }) != 0, + "Failure workflow was not accepted"); + } + + void check_failure() const { + if (failure) + { + std::rethrow_exception(failure); + } + } + + void preflight(ark::task<> workflow) { + start(std::move(workflow)); + const auto deadline = std::chrono::steady_clock::now() + 20s; + while (tasks->pending()) + { + require(std::chrono::steady_clock::now() < deadline, "Failure preflight did not complete"); + require(manager.update(0), "Failure preflight update failed"); + std::this_thread::sleep_for(1ms); + } + check_failure(); + require(completions == 1 && completions_on_owner, "Preflight completion escaped its owner or repeated"); + } + + void finish() { + const auto deadline = std::chrono::steady_clock::now() + 30s; + while (!(stopped = manager.stop(0))) + { + require(manager.get_shutdown_state() != ark::plugin_manager::shutdown_state::failed && + std::chrono::steady_clock::now() < deadline, + "Failure workflows did not naturally drain/retire native resources"); + require(manager.update(0), "Failure workflow update failed during drain"); + std::this_thread::sleep_for(1ms); + } + } + + void verify() { + require(tasks->pending() == 2, "Slow request and independent observer were not both admitted"); + require(!manager.stop(0), "Manager stopped before failure workflows drained"); + require(manager.get_shutdown_state() == ark::plugin_manager::shutdown_state::draining, + "Manager did not begin natural drain"); + finish(); + check_failure(); + require(completions == 3 && completions_on_owner, "Failure completions repeated or escaped their owner"); + require(tasks->closed() && tasks->pending() == 0, "Manager stopped with pending failure tasks"); + require(manager.stop(0), "Manager stop was not idempotent after failures"); + } +}; + +using redis_owner = module_owner; + +void redis_success(const arknet::redis::result& result) { + require(bool(result) && result.replies.size() == 1, "Explicit Redis recovery/observer request failed"); +} + +ark::task<> redis_preflight(redis_owner& owner, std::string key) { + ark::redis_module& database = owner.database; + arknet::redis::command rejected_ping{"PING"}; + auto rejected = co_await database.command("wrong-password", std::move(rejected_ping), 5s); + owner.check_executor(); + require(rejected.error == arknet::redis::client_errc::handshake_failed && !rejected.command_may_have_run, + "Wrong Redis password lost its known-unsent handshake failure"); + require(!rejected.replies.empty() && rejected.replies[0].type == arknet::redis::value_type::error, + "Wrong Redis password lost the native server error reply"); + arknet::redis::command recovery_ping{"PING"}; + auto recovered = co_await database.command("primary", std::move(recovery_ping), 5s); + owner.check_executor(); + redis_success(recovered); + require(recovered.replies[0].text == "PONG", "Explicit Redis request did not recover with valid credentials"); + arknet::redis::command check_key{"EXISTS", std::move(key)}; + auto absent = co_await database.command("primary", std::move(check_key), 5s); + owner.check_executor(); + redis_success(absent); + require(absent.replies[0].type == arknet::redis::value_type::integer && absent.replies[0].integer == 0, + "Redis slow-test key already exists; refusing a destructive pop"); + arknet::redis::command observer_ping{"PING"}; + auto observer = co_await database.command("observer", std::move(observer_ping), 5s); + owner.check_executor(); + redis_success(observer); + require(observer.replies[0].text == "PONG", "Redis observer could not authenticate"); +} + +struct redis_slow_state +{ + std::optional result; + bool observed{}; + bool recovered{}; +}; + +ark::task<> redis_slow(redis_owner& owner, redis_slow_state& state, std::string key) { + // An absent unique key and no producer keep the server blocked for 30 seconds. + // The independently warmed client deadline is two seconds. + const auto started = std::chrono::steady_clock::now(); + arknet::redis::command block{"BLPOP", std::move(key), "30"}; + state.result = co_await owner.database.command("primary", std::move(block), 2s); + owner.check_executor(); + require(std::chrono::steady_clock::now() - started < 5s, + "Redis client deadline was not delivered well before its server block timeout"); +} + +std::string client_field(std::string_view line, std::string_view name) { + const auto marker = std::string(name) + "="; + for (std::size_t position{}; position < line.size();) + { + const auto end = line.find(' ', position); + const auto field = line.substr(position, end == std::string_view::npos ? end : end - position); + if (field.starts_with(marker)) + { + return std::string(field.substr(marker.size())); + } + if (end == std::string_view::npos) + { + break; + } + position = end + 1; + } + return {}; +} + +std::optional blocked_redis_client(const arknet::redis::result& listing, const std::string& tag) { + redis_success(listing); + const auto& reply = listing.replies[0]; + require(reply.type == arknet::redis::value_type::bulk_string || + reply.type == arknet::redis::value_type::verbatim_string, + "Redis CLIENT LIST did not return a bulk/verbatim string"); + std::string_view remaining = reply.text; + if (reply.type == arknet::redis::value_type::verbatim_string) + { + // Native RESP3 results retain their format tag. Parse a view without + // altering the owning result or accepting unrelated format encodings. + require(remaining.starts_with("txt:"), "Redis CLIENT LIST returned a non-text verbatim string"); + remaining.remove_prefix(4); + } + std::optional blocked; + while (!remaining.empty()) + { + const auto newline = remaining.find('\n'); + const auto line = remaining.substr(0, newline); + if (client_field(line, "name") == tag && client_field(line, "cmd") == "blpop" && + client_field(line, "flags").find('b') != std::string::npos) + { + require(!blocked, "Redis observer saw concurrent replay of the uniquely named BLPOP"); + blocked = client_field(line, "id"); + require(!blocked->empty(), "Redis observer could not identify its blocked client"); + } + if (newline == std::string_view::npos) + { + break; + } + remaining.remove_prefix(newline + 1); + } + return blocked; +} + +ark::task<> redis_observe(redis_owner& owner, redis_slow_state& state, std::string tag) { + std::optional original; + std::optional quiet_until; + const auto deadline = std::chrono::steady_clock::now() + 12s; + for (;;) + { + owner.check_failure(); + require(std::chrono::steady_clock::now() < deadline, "Redis observer did not see deadline and recovery"); + arknet::redis::command clients{"CLIENT", "LIST"}; + auto listing = co_await owner.database.command("observer", std::move(clients), 3s); + owner.check_executor(); + const auto blocked = blocked_redis_client(listing, tag); + if (blocked) + { + require(!quiet_until, "Redis observer saw a new blocked command after the original drained"); + require(!original || *original == *blocked, "Redis observer saw BLPOP replayed on a new connection"); + original = *blocked; + state.observed = true; + } + if (state.result && !state.recovered) + { + require(state.observed, "Redis deadline completed before server-visible BLPOP observation"); + require(state.result->error == arknet::redis::client_errc::timeout && state.result->command_may_have_run, + "Sent Redis deadline lost its timeout/unknown execution state"); + arknet::redis::command recovery_ping{"PING"}; + auto recovered = co_await owner.database.command("primary", std::move(recovery_ping), 5s); + owner.check_executor(); + redis_success(recovered); + require(recovered.replies[0].text == "PONG", "Explicit new Redis request did not recover after deadline"); + state.recovered = true; + } + if (!blocked && state.recovered) + { + if (!quiet_until) + { + quiet_until = std::chrono::steady_clock::now() + 500ms; + } + if (std::chrono::steady_clock::now() >= *quiet_until) + { + break; + } + } + co_await observer_interval(); + owner.check_executor(); + } +} + +void check_redis() { + ark::redis_options options; + options.pool.connection.host = environment("ARK_DATABASE_REDIS_HOST"); + options.pool.connection.port = std::to_string(port("ARK_DATABASE_REDIS_PORT")); + options.pool.connection.password = environment("ARK_DATABASE_REDIS_PASSWORD"); + if (const auto* user = std::getenv("ARK_DATABASE_REDIS_USER")) + { + options.pool.connection.username = user; + } + options.pool.max_connections = 1; + options.pool.client_configuration.connect_timeout = 3s; + options.pool.client_configuration.drain_timeout = 100ms; + const auto tag = "ark_failure_" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); + options.pool.connection.client_name = tag; + redis_slow_state state; + { + redis_owner owner; + require(owner.database.configure("primary", options), "Cannot configure Redis primary"); + auto wrong_password = options; + wrong_password.pool.connection.password += "-ark-rejected-" + tag; + require(owner.database.configure("wrong-password", std::move(wrong_password)), + "Cannot configure rejected Redis credentials"); + options.pool.connection.client_name = tag + "_observer"; + require(owner.database.configure("observer", options), "Cannot configure Redis observer"); + owner.preflight(redis_preflight(owner, tag)); + owner.start(redis_slow(owner, state, tag)); + owner.start(redis_observe(owner, state, tag)); + owner.verify(); + } + require(state.observed && state.recovered && state.result && state.result->command_may_have_run, + "Redis deadline result/lifecycle evidence did not survive module destruction"); +} + +#if defined(ARKNET_ENABLE_SSL) +using mysql_owner = module_owner; + +void mysql_success(const ark::mysql_reply& reply) { + require(bool(reply), "Explicit MySQL recovery/observer request failed (code " + + std::to_string(static_cast(reply.result.status.code)) + ", server code " + + std::to_string(reply.result.status.server_code) + ")"); +} + +ark::task<> mysql_preflight(mysql_owner& owner) { + ark::mysql_module& database = owner.database; + auto rejected = co_await database.query("wrong-password", "SELECT 1", 5s); + owner.check_executor(); + require(rejected.result.status.code == arknet::mysql::connection_errc::authentication && + rejected.result.status.server_code == 1045 && rejected.result.status.sql_state == "28000" && + !rejected.result.status.outcome_unknown && rejected.result.rows.empty(), + "Wrong MySQL password lost its known-unsent authentication error"); + auto recovered = co_await database.query("primary", "SELECT 7", 5s); + owner.check_executor(); + mysql_success(recovered); + require(recovered.result.rows.size() == 1 && std::get(recovered.result.rows[0][0]) == 7, + "Explicit MySQL request did not recover with valid credentials"); + auto untrusted = co_await database.query("untrusted-ca", "SELECT 1", 5s); + owner.check_executor(); + require(untrusted.result.status.code == arknet::mysql::connection_errc::tls && + !untrusted.result.status.outcome_unknown && untrusted.result.rows.empty(), + "Untrusted MySQL CA lost its known-unsent TLS error"); + auto verified = co_await database.query("primary", "SELECT 8", 5s); + owner.check_executor(); + mysql_success(verified); + require(verified.result.rows.size() == 1 && std::get(verified.result.rows[0][0]) == 8, + "Explicit MySQL request did not recover with a trusted CA"); + auto observer = co_await database.query("observer", "SELECT 1", 5s); + owner.check_executor(); + mysql_success(observer); +} + +struct mysql_slow_state +{ + std::optional result; + bool observed{}; + bool recovered{}; +}; + +ark::task<> mysql_slow(mysql_owner& owner, mysql_slow_state& state, std::string tag) { + const auto started = std::chrono::steady_clock::now(); + state.result = co_await owner.database.query("primary", "SELECT SLEEP(10) /*" + tag + "*/", 2s); + owner.check_executor(); + require(std::chrono::steady_clock::now() - started < 5s, + "MySQL client deadline was not delivered well before its server sleep ended"); +} + +ark::task<> mysql_observe(mysql_owner& owner, mysql_slow_state& state, std::string tag) { + std::optional original; + std::optional quiet_until; + const auto deadline = std::chrono::steady_clock::now() + 18s; + const auto sql = "SELECT CAST(ID AS SIGNED) FROM information_schema.processlist " + "WHERE ID <> CONNECTION_ID() AND STATE = 'User sleep' AND INFO LIKE '%" + + tag + "%'"; + for (;;) + { + owner.check_failure(); + require(std::chrono::steady_clock::now() < deadline, "MySQL observer did not see deadline and recovery"); + auto listing = co_await owner.database.query("observer", sql, 3s); + owner.check_executor(); + mysql_success(listing); + require(listing.result.rows.size() <= 1, "MySQL observer saw concurrent replay of its tagged SLEEP"); + if (!listing.result.rows.empty()) + { + require(!quiet_until, "MySQL observer saw a new SLEEP after the original drained"); + const auto id = std::get(listing.result.rows[0][0]); + require(!original || *original == id, "MySQL observer saw SLEEP replayed on a new connection"); + original = id; + state.observed = true; + } + if (state.result && !state.recovered) + { + require(state.observed, "MySQL deadline completed before server-visible SLEEP observation"); + require(state.result->result.status.code == arknet::mysql::connection_errc::timeout && + state.result->result.status.outcome_unknown, + "Sent MySQL deadline lost its timeout/unknown execution state"); + auto recovered = co_await owner.database.query("primary", "SELECT 9", 5s); + owner.check_executor(); + mysql_success(recovered); + require(recovered.result.rows.size() == 1 && std::get(recovered.result.rows[0][0]) == 9, + "Explicit new MySQL request did not recover after deadline"); + state.recovered = true; + } + if (listing.result.rows.empty() && state.recovered) + { + if (!quiet_until) + { + quiet_until = std::chrono::steady_clock::now() + 500ms; + } + if (std::chrono::steady_clock::now() >= *quiet_until) + { + break; + } + } + co_await observer_interval(); + owner.check_executor(); + } +} + +void check_mysql() { + ark::mysql_options options; + options.pool.connection.host = environment("ARK_DATABASE_MYSQL_HOST"); + options.pool.connection.port = port("ARK_DATABASE_MYSQL_PORT"); + options.pool.connection.username = environment("ARK_DATABASE_MYSQL_USER"); + options.pool.connection.password = environment("ARK_DATABASE_MYSQL_PASSWORD", true); + options.pool.connection.database = environment("ARK_DATABASE_MYSQL_DATABASE"); + options.pool.connection.ca_file = environment("ARK_DATABASE_MYSQL_CA"); + options.pool.connection.tls = arknet::mysql::tls_mode::required; + options.pool.connection.timeout = 5s; + options.pool.max_connections = 1; + const auto tag = "ark_failure_" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); + mysql_slow_state state; + { + mysql_owner owner; + require(owner.database.configure("primary", options), "Cannot configure MySQL primary"); + auto wrong_password = options; + wrong_password.pool.connection.password += "-ark-rejected-" + tag; + require(owner.database.configure("wrong-password", std::move(wrong_password)), + "Cannot configure rejected MySQL credentials"); + auto untrusted = options; + untrusted.pool.connection.ca_file = ARK_DATABASE_UNTRUSTED_CA; + require(owner.database.configure("untrusted-ca", std::move(untrusted)), "Cannot configure untrusted MySQL CA"); + require(owner.database.configure("observer", options), "Cannot configure MySQL observer"); + owner.preflight(mysql_preflight(owner)); + owner.start(mysql_slow(owner, state, tag)); + owner.start(mysql_observe(owner, state, tag)); + owner.verify(); + } + require(state.observed && state.recovered && state.result && state.result->result.status.outcome_unknown, + "MySQL deadline result/lifecycle evidence did not survive module destruction"); +} +#endif + +} // namespace + +int main(int argc, char** argv) { + try + { + require(argc == 2, "Usage: ark_database_public_failure_service_tests redis|mysql"); + const std::string_view protocol(argv[1]); + if (protocol == "redis") + { + check_redis(); + } +#if defined(ARKNET_ENABLE_SSL) + else if (protocol == "mysql") + { + check_mysql(); + } +#endif + else + { + throw acceptance_failure("Requested failure-test protocol is not enabled"); + } + std::cout << protocol << " public module real-service failure checks passed: " << checks << '\n'; + return 0; + } + catch (const acceptance_failure& error) + { std::cerr << error.what() << '\n'; } + catch (...) + { + // Never emit authentication configuration or server-controlled messages. + std::cerr << "Public database failure acceptance raised an unexpected exception\n"; + } + return 1; +}