diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d9ad760..401bae0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,7 @@ option(AE_BUILD_TOOLS "Build tools" ${AE_ROOT_PORJECT}) option(AE_BUILD_EXAMPLES "Build examples" ${AE_ROOT_PORJECT}) option(AE_BUILD_TESTS "Build tests" ${AE_ROOT_PORJECT}) option(AE_BUILD_ANDROID_SMOKE "Build Android NDK smoke shared library and runner" Off) +option(AE_ENABLE_PRIVILEGED_NETWORK_TESTS "Enable Administrator/network live tests" Off) option(AE_ADDRESS_SANITIZE "Enable address sanitizer" Off) set(UTM_ID "0" CACHE STRING "User Tracking Measurement ID, must be a uint32 value") @@ -134,7 +135,7 @@ CPMAddPackage( CPMAddPackage( NAME ae-numeric GIT_REPOSITORY "https://github.com/aethernetio/aethernet-numeric.git" - GIT_TAG "main" + GIT_TAG "3ab9e7310a2f8e6240931261c07c3a0c39771ec2" OPTIONS "AE_NUMERIC_INSTALL ${AE_INSTALL}" "AE_BUILD_TESTS OFF" EXCLUDE_FROM_ALL FALSE ) @@ -320,6 +321,7 @@ target_compile_options(${TARGET_NAME} PRIVATE /w15262 #implisitfallthrough /wd4388 #Wno-sign-compare /wd4389 #Wno-sign-compare + /wd4702 # unreachable in ae-numeric FixedPoint/Exponential templates /Zc:preprocessor > ) @@ -344,6 +346,7 @@ if(AE_BUILD_EXAMPLES) add_subdirectory(examples/common) add_subdirectory(examples/cloud) add_subdirectory(examples/a_b_message_exchange) + add_subdirectory(examples/remote_presence_live) add_subdirectory(examples/message_server) add_subdirectory(examples/capi/oddity) add_subdirectory(examples/benches/send_message_delays) diff --git a/aether/CMakeLists.txt b/aether/CMakeLists.txt index 586300b8..659aaa93 100644 --- a/aether/CMakeLists.txt +++ b/aether/CMakeLists.txt @@ -57,7 +57,8 @@ list(APPEND aether_srcs "ae_actions/ping.cpp" "ae_actions/check_access_for_send_message.cpp" "ae_actions/telemetry.cpp" - "ae_actions/select_client.cpp") + "ae_actions/select_client.cpp" + "ae_actions/query_peer_presence.cpp") list(APPEND aether_srcs "registration/api/client_reg_api_safe.cpp" @@ -179,6 +180,7 @@ list(APPEND aether_srcs "cloud_connections/cloud_server_connection.cpp" "cloud_connections/cloud_server_connections.cpp" "cloud_connections/ping_cloud_servers.cpp" + "cloud_connections/local_presence_machine.cpp" "cloud_connections/cloud_subscription.cpp" "cloud_connections/cloud_request.cpp") diff --git a/aether/ae_actions/ping.h b/aether/ae_actions/ping.h index 411deec6..8599cda9 100644 --- a/aether/ae_actions/ping.h +++ b/aether/ae_actions/ping.h @@ -48,6 +48,8 @@ class Ping { Ping(AeContext const& ae_context, CloudServerConnection& cloud_server_connection, Duration next_ping_hint, Duration rx_window, Duration timeout); + // `timeout` is the hard wait for a Pong (cleanup). Local Presence retry + // deadlines (pXX) are owned by LocalPresenceMachine, not this timer. AE_CLASS_NO_COPY_MOVE(Ping); diff --git a/aether/ae_actions/query_peer_presence.cpp b/aether/ae_actions/query_peer_presence.cpp new file mode 100644 index 00000000..ac7d0afc --- /dev/null +++ b/aether/ae_actions/query_peer_presence.cpp @@ -0,0 +1,463 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "aether/ae_actions/query_peer_presence.h" + +#include +#include + +#include "aether/api_protocol/sub_api.h" +#include "aether/client.h" +#include "aether/cloud_connections/cloud_server_connection.h" +#include "aether/config.h" +#include "aether/connection_manager/client_cloud_manager.h" +#include "aether/server.h" +#include "aether/tele.h" +#include "aether/work_cloud_api/work_server_api/authorized_api.h" + +namespace ae { + +QueryPeerPresence::QueryPeerPresence(AeContext const& ae_context, + Client& client, Uid peer_uid) + : ae_context_{ae_context}, client_{&client}, peer_uid_{peer_uid} { + static_cast(AllowObserverCloudFallbackForPeerPresence()); + + auto cached = client_->cloud_manager()->GetCachedCloud(peer_uid_); + if (cached && cached.is_valid() && !cached->servers().empty()) { + BindPeerCloud(cached); + StartQuery(); + return; + } + + auto& get_cloud = client_->cloud_manager()->GetCloud(peer_uid_); + get_cloud_sub_ = get_cloud.result_event().Subscribe( + [this](Result result) { OnCloud(std::move(result)); }); +} + +QueryPeerPresence::~QueryPeerPresence() { + finished_ = true; + timing_subs_.clear(); +} + +QueryPeerPresence::ResultEvent::Subscriber +QueryPeerPresence::result_event() noexcept { + return EventSubscriber{result_event_}; +} + +Duration QueryPeerPresence::OfflineTimeout() const noexcept { + auto policy = client_->connectivity_policy(); + if (!policy) { + return DefaultOfflineDetectionTimeout(); + } + return policy.Load()->offline_detection_timeout(); +} + +CloudRequestExecutionPolicy QueryPeerPresence::ExecutionPolicy() const noexcept { + auto policy = client_->connectivity_policy(); + if (!policy) { + return CloudRequestExecutionPolicy::Default(); + } + return policy.Load()->cloud_request_execution_policy(); +} + +void QueryPeerPresence::BindPeerCloud(Cloud::ptr cloud) { + used_observer_cloud_ = false; + + // Full peer Personal Cloud (priority order). Authoritative Presence set is + // later taken from selected_servers() — same contract as Local Presence. + peer_cloud_server_ids_.clear(); + std::vector> ordered; + ordered.reserve(cloud->servers().size()); + for (auto const& [id, entry] : cloud->servers()) { + ordered.emplace_back(entry.priority, id); + } + std::sort(ordered.begin(), ordered.end()); + for (auto const& item : ordered) { + peer_cloud_server_ids_.push_back(item.second); + } + + // Same connection budget as Local Presence / peer PingCloudServers. + dest_cloud_ = std::make_unique( + ae_context_, cloud.Load(), + client_->server_connection_manager().GetServerConnectionFactory(), + AE_CLOUD_MAX_SERVER_CONNECTIONS); + work_cloud_ = dest_cloud_.get(); +} + +void QueryPeerPresence::OnCloud(Result result) { + if (finished_) { + return; + } + if (!result) { + // Peer Personal Cloud unavailable — never fall back to observer cloud. + used_observer_cloud_ = false; + Complete(PeerPresence{PeerPresenceState::kUnknown}); + return; + } + BindPeerCloud(std::move(result).value()); + StartQuery(); +} + +void QueryPeerPresence::RefreshUsableSet() { + samples_.clear(); + authoritative_server_ids_.clear(); + if (work_cloud_ == nullptr) { + return; + } + + // Local Presence contract = selected_servers() under RequestPolicy::All + // (bounded by AE_CLOUD_MAX_SERVER_CONNECTIONS). Remote AND uses the same set. + for (auto* sc : work_cloud_->selected_servers()) { + if (sc == nullptr || !sc->server()) { + continue; + } + authoritative_server_ids_.push_back(sc->server_id()); + RemoteServerPresenceSample sample{}; + sample.server_id = sc->server_id(); + if (sc->quarantine()) { + sample.status = RemoteServerPresence::kExcluded; + } else { + sample.status = RemoteServerPresence::kUnknown; + } + samples_.push_back(sample); + } + + AE_TELED_DEBUG( + "REMOTE_PRESENCE peer_cloud_count={} authoritative_count={} " + "selected_count={} max_connections={}", + peer_cloud_server_ids_.size(), authoritative_server_ids_.size(), + work_cloud_->selected_servers().size(), work_cloud_->max_connections()); +} + +void QueryPeerPresence::StartQuery() { + if (finished_ || work_cloud_ == nullptr) { + return; + } + RefreshUsableSet(); + std::size_t usable = 0; + for (auto const& sample : samples_) { + if (sample.status != RemoteServerPresence::kExcluded) { + ++usable; + } + } + if (usable == 0) { + Complete(PeerPresence{PeerPresenceState::kUnknown}); + return; + } + + quarantine_sub_ = work_cloud_->server_quarantined_event().Subscribe( + [this](CloudServerConnection* sc) { + if (finished_ || sc == nullptr) { + return; + } + MarkExcluded(sc->server_id()); + MaybeComplete(); + }); + quarantine_release_sub_ = + work_cloud_->server_quarantine_release_event().Subscribe( + [this](CloudServerConnection* sc) { + if (finished_ || sc == nullptr) { + return; + } + OnServerRecovered(sc); + }); + + cloud_request_.emplace( + ae_context_, + ApiRequestHandler{[this](ApiContext& auth_api, + CloudServerConnection* sc, + CloudRequest* request) { + static_cast(request); + if (finished_ || sc == nullptr || !sc->server() || sc->quarantine()) { + return; + } + auto const server_id = sc->server_id(); + for (auto const& sample : samples_) { + if (sample.server_id == server_id && + (sample.status == RemoteServerPresence::kOnline || + sample.status == RemoteServerPresence::kOffline || + sample.status == RemoteServerPresence::kExcluded)) { + return; + } + } + if (std::find(queried_server_ids_.begin(), queried_server_ids_.end(), + server_id) == queried_server_ids_.end()) { + queried_server_ids_.push_back(server_id); + } + auto& meta = attempts_[server_id]; + auto const generation = ++meta.next_generation; + meta.send_times[generation] = Now(); + // Accumulate subscribers — do not replace, so late responses from + // earlier soft-timeout attempts remain deliverable. + timing_subs_[server_id] += + auth_api->get_client_timing(peer_uid_).Subscribe( + [this, sc, generation](auto const& res) { + OnServerTiming(sc, generation, res); + }); + }}, + *work_cloud_, RequestPolicy::All{}, ExecutionPolicy()); + + exhausted_sub_ = cloud_request_->attempt_exhausted_event().Subscribe( + [this](CloudServerConnection* sc) { + if (finished_ || sc == nullptr) { + return; + } + MarkUnknown(sc->server_id()); + MaybeComplete(); + }); + + cloud_request_sub_ = cloud_request_->result_event().Subscribe([this](bool ok) { + if (finished_) { + return; + } + if (ok) { + return; + } + MaybeComplete(); + if (!finished_ && AllUsableTerminal()) { + Complete(AggregateRemotePresence(samples_)); + } + }); +} + +void QueryPeerPresence::RequestTiming(CloudServerConnection* sc) { + if (finished_ || sc == nullptr || !sc->server() || sc->quarantine()) { + return; + } + auto* conn = sc->client_connection(); + if (conn == nullptr) { + return; + } + auto const server_id = sc->server_id(); + for (auto const& sample : samples_) { + if (sample.server_id == server_id && + (sample.status == RemoteServerPresence::kOnline || + sample.status == RemoteServerPresence::kOffline || + sample.status == RemoteServerPresence::kExcluded)) { + return; + } + } + + if (std::find(queried_server_ids_.begin(), queried_server_ids_.end(), + server_id) == queried_server_ids_.end()) { + queried_server_ids_.push_back(server_id); + } + + auto& meta = attempts_[server_id]; + auto const generation = ++meta.next_generation; + meta.send_times[generation] = Now(); + + // AuthorizedApiCall requires an active ApiContext path via CloudRequest's + // handler; for recovered servers we re-enter through a one-server request. + conn->AuthorizedApiCall(SubApi{[&, sc, generation]( + ApiContext& auth_api) { + timing_subs_[server_id] += + auth_api->get_client_timing(peer_uid_).Subscribe( + [this, sc, generation](auto const& res) { + OnServerTiming(sc, generation, res); + }); + }}); +} + +void QueryPeerPresence::OnServerRecovered(CloudServerConnection* sc) { + RemoteServerPresenceSample sample{}; + sample.server_id = sc->server_id(); + sample.status = RemoteServerPresence::kUnknown; + bool found = false; + for (auto& existing : samples_) { + if (existing.server_id == sample.server_id) { + existing = sample; + found = true; + break; + } + } + if (!found) { + samples_.push_back(sample); + authoritative_server_ids_.push_back(sample.server_id); + } + // Fresh timing required — do not keep a stale ONLINE. + RequestTiming(sc); + MaybeComplete(); +} + +void QueryPeerPresence::OnServerTiming( + CloudServerConnection* sc, std::uint64_t generation, + Result const& res) { + if (finished_ || sc == nullptr) { + return; + } + auto const server_id = sc->server_id(); + for (auto const& sample : samples_) { + if (sample.server_id == server_id && + (sample.status == RemoteServerPresence::kOnline || + sample.status == RemoteServerPresence::kOffline || + sample.status == RemoteServerPresence::kExcluded)) { + // Already terminal for this server — ignore duplicate/late extras. + return; + } + } + + auto meta_it = attempts_.find(server_id); + if (meta_it == attempts_.end()) { + return; + } + auto send_it = meta_it->second.send_times.find(generation); + if (send_it == meta_it->second.send_times.end()) { + return; + } + auto const send_time = send_it->second; + meta_it->second.send_times.erase(send_it); + + if (!res) { + // Authenticated API error: server is alive — do not use no-response + // FailAttempt / quarantine path. + if (cloud_request_.has_value()) { + cloud_request_->CompleteAttemptWithRemoteError(sc); + } + MarkUnknown(server_id); + MaybeComplete(); + return; + } + + auto const recv = Now(); + TimePoint expected{}; + TimePoint deadline{}; + auto const status = ClassifyRemoteServerPresence( + recv, send_time, recv, res.value(), OfflineTimeout(), &expected, + &deadline); + + for (auto& sample : samples_) { + if (sample.server_id != server_id) { + continue; + } + sample.status = status; + sample.expected_open = expected; + sample.offline_deadline = deadline; + sample.next_ping_delta_ms = res.value().next_ping_delta_ms; + sample.has_timing = true; + break; + } + + AE_TELED_DEBUG( + "REMOTE_PRESENCE server {} next_delta {} status {} expected {} deadline " + "{} (generation {})", + server_id, res.value().next_ping_delta_ms, static_cast(status), + expected, deadline, generation); + + if (cloud_request_.has_value()) { + cloud_request_->SucceedAttempt(sc); + } + // Drop remaining attempt send times — server is done. + meta_it->second.send_times.clear(); + MaybeComplete(); +} + +void QueryPeerPresence::MarkUnknown(ServerId server_id) { + for (auto& sample : samples_) { + if (sample.server_id == server_id && + sample.status != RemoteServerPresence::kExcluded && + sample.status != RemoteServerPresence::kOnline && + sample.status != RemoteServerPresence::kOffline) { + sample.status = RemoteServerPresence::kUnknown; + // Terminal unknown after retries — treat as observed for completion. + sample.has_timing = true; + return; + } + } +} + +void QueryPeerPresence::MarkExcluded(ServerId server_id) { + for (auto& sample : samples_) { + if (sample.server_id == server_id) { + sample.status = RemoteServerPresence::kExcluded; + return; + } + } +} + +bool QueryPeerPresence::AllUsableTerminal() const noexcept { + for (auto const& sample : samples_) { + if (sample.status == RemoteServerPresence::kExcluded) { + continue; + } + if (!sample.has_timing && + sample.status == RemoteServerPresence::kUnknown) { + return false; + } + } + return true; +} + +void QueryPeerPresence::MaybeComplete() { + if (finished_) { + return; + } + if (RemotePresenceCanEarlyCompleteOffline(samples_)) { + Complete(PeerPresence{PeerPresenceState::kOffline}); + return; + } + if (RemotePresenceReadyForOnline(samples_)) { + Complete(PeerPresence{PeerPresenceState::kOnline}); + return; + } + std::size_t usable = 0; + for (auto const& sample : samples_) { + if (sample.status != RemoteServerPresence::kExcluded) { + ++usable; + } + } + if (usable == 0) { + Complete(PeerPresence{PeerPresenceState::kUnknown}); + return; + } + if (AllUsableTerminal()) { + Complete(AggregateRemotePresence(samples_)); + } +} + +void QueryPeerPresence::Complete(PeerPresence const& presence) { + if (finished_) { + return; + } + finished_ = true; + timing_subs_.clear(); + exhausted_sub_.Reset(); + quarantine_sub_.Reset(); + quarantine_release_sub_.Reset(); + if (cloud_request_.has_value()) { + cloud_request_->Succeeded(); + } + result_event_.Emit(Ok{presence}); + Finish(); +} + +void QueryPeerPresence::Fail(int code) { + if (finished_) { + return; + } + finished_ = true; + timing_subs_.clear(); + exhausted_sub_.Reset(); + quarantine_sub_.Reset(); + quarantine_release_sub_.Reset(); + if (cloud_request_.has_value()) { + cloud_request_->Failed(); + } + result_event_.Emit(Error{code}); + Finish(); +} + +} // namespace ae diff --git a/aether/ae_actions/query_peer_presence.h b/aether/ae_actions/query_peer_presence.h new file mode 100644 index 00000000..2140ca0a --- /dev/null +++ b/aether/ae_actions/query_peer_presence.h @@ -0,0 +1,129 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_AE_ACTIONS_QUERY_PEER_PRESENCE_H_ +#define AETHER_AE_ACTIONS_QUERY_PEER_PRESENCE_H_ + +#include +#include +#include +#include +#include + +#include "aether-miscpp/types/result.h" +#include "aether/ae_context.h" +#include "aether/actions/action.h" +#include "aether/cloud.h" +#include "aether/cloud_connections/cloud_request.h" +#include "aether/events/event_subscription.h" +#include "aether/events/events.h" +#include "aether/events/multi_subscription.h" +#include "aether/remote_presence.h" +#include "aether/types/server_id.h" +#include "aether/types/uid.h" +#include "aether/cloud_connections/cloud_request_execution_policy.h" + +namespace ae { + +class Client; + +enum class QueryPeerPresenceError : int { + kGetCloudFailed = 1, + kNoWorkServerAvailable = 2, + kGetClientTimingFailed = 3, +}; + +// Asynchronous Remote Presence over peer Personal Cloud authoritative servers. +// Never falls back to the observer/requester own cloud. +class QueryPeerPresence final : public Action { + public: + using ResultEvent = Event)>; + + QueryPeerPresence(AeContext const& ae_context, Client& client, Uid peer_uid); + ~QueryPeerPresence() override; + + AE_CLASS_NO_COPY_MOVE(QueryPeerPresence) + + ResultEvent::Subscriber result_event() noexcept; + Uid peer_uid() const noexcept { return peer_uid_; } + std::vector const& samples() const noexcept { + return samples_; + } + std::vector const& peer_cloud_server_ids() const noexcept { + return peer_cloud_server_ids_; + } + std::vector const& authoritative_server_ids() const noexcept { + return authoritative_server_ids_; + } + std::vector const& queried_server_ids() const noexcept { + return queried_server_ids_; + } + bool used_observer_cloud() const noexcept { return used_observer_cloud_; } + + private: + struct AttemptMeta { + std::uint64_t next_generation{0}; + // Per-attempt send times so late responses from earlier attempts remain + // classifiable after a soft-timeout retry is launched. + std::map send_times; + }; + + void OnCloud(Result result); + void BindPeerCloud(Cloud::ptr cloud); + void StartQuery(); + void RefreshUsableSet(); + void RequestTiming(CloudServerConnection* sc); + void OnServerTiming(CloudServerConnection* sc, std::uint64_t generation, + Result const& res); + void MarkUnknown(ServerId server_id); + void MarkExcluded(ServerId server_id); + void OnServerRecovered(CloudServerConnection* sc); + void MaybeComplete(); + void Complete(PeerPresence const& presence); + void Fail(int code); + Duration OfflineTimeout() const noexcept; + CloudRequestExecutionPolicy ExecutionPolicy() const noexcept; + bool AllUsableTerminal() const noexcept; + + AeContext ae_context_; + Client* client_{nullptr}; + Uid peer_uid_{}; + ResultEvent result_event_; + + Subscription get_cloud_sub_; + Subscription cloud_request_sub_; + Subscription exhausted_sub_; + Subscription quarantine_sub_; + Subscription quarantine_release_sub_; + + std::unique_ptr dest_cloud_; + CloudServerConnections* work_cloud_{nullptr}; + std::optional cloud_request_; + // MultiSubscription so soft-timeout retries do not destroy earlier + // get_client_timing response subscribers (late responses must be accepted). + std::map timing_subs_; + std::map attempts_; + std::vector samples_; + std::vector peer_cloud_server_ids_; + std::vector authoritative_server_ids_; + std::vector queried_server_ids_; + bool used_observer_cloud_{false}; + bool finished_{false}; +}; + +} // namespace ae + +#endif // AETHER_AE_ACTIONS_QUERY_PEER_PRESENCE_H_ diff --git a/aether/api_protocol/protocol_context.cpp b/aether/api_protocol/protocol_context.cpp index 37d2f9b5..fab9405e 100644 --- a/aether/api_protocol/protocol_context.cpp +++ b/aether/api_protocol/protocol_context.cpp @@ -73,18 +73,23 @@ void ProtocolContext::DestroyPending(PendingEntry const& entry) { } void ProtocolContext::PreparePendingResponseSlot(RequestId request_id) { - // ensure there is one pending response for request_id + // Replace any existing pending response for this request id first. auto existing_entry = TakePending(request_id); if (existing_entry.response != nullptr) { EvictPending(existing_entry); - return; } - // ensure there is enough in pool for new pending response - // oldest pending should be evicted - if (pending_responses_.full()) { - auto oldest_entry = TakeOldestPending(); - EvictPending(oldest_entry); + // Free a registry/pool slot for the new entry. OnEvicted handlers may + // re-enter CreatePendingResponse and refill the slot we just freed, so + // keep draining until both the registry and the pool have capacity. + auto spins = kMaxPendingResponses * 2U; + while ((pending_responses_.full() || + pending_response_pool_.available() == 0U) && + spins != 0U) { + assert(!pending_responses_.empty() && + "Pending response pool exhausted with empty registry"); + EvictPending(TakeOldestPending()); + --spins; } assert(!pending_responses_.full() && diff --git a/aether/client.cpp b/aether/client.cpp index cd31717d..719c58f5 100644 --- a/aether/client.cpp +++ b/aether/client.cpp @@ -18,6 +18,7 @@ #include +#include "aether/ae_actions/query_peer_presence.h" #include "aether/ae_actions/telemetry.h" #include "aether/aether.h" @@ -92,6 +93,24 @@ ClientConnectivityPolicy::ptr const& Client::connectivity_policy() { return connectivity_policy_; } +bool Client::IsLocallyOnline() const { + if (!connectivity_policy_.is_valid()) { + return false; + } + return connectivity_policy_.Load()->IsLocallyOnline(); +} + +QueryPeerPresence& Client::QueryPeerPresence(Uid peer_uid) { + if (query_peer_presence_ && !query_peer_presence_->is_finished() && + query_peer_presence_->peer_uid() == peer_uid) { + return *query_peer_presence_; + } + query_peer_presence_ = + std::make_unique<::ae::QueryPeerPresence>(AeContext{*aether_}, *this, + peer_uid); + return *query_peer_presence_; +} + P2pMessageStreamManager& Client::message_stream_manager() { if (!message_stream_manager_) { message_stream_manager_ = std::make_unique( diff --git a/aether/client.h b/aether/client.h index dbf5e6f1..5e07c1a9 100644 --- a/aether/client.h +++ b/aether/client.h @@ -39,6 +39,7 @@ namespace ae { class Aether; class Telemetry; +class QueryPeerPresence; class Client : public Obj { AE_OBJECT(Client, Obj, 0) @@ -63,6 +64,10 @@ class Client : public Obj { ServerConnectionManager& server_connection_manager(); CloudServerConnections& cloud_connection(); ClientConnectivityPolicy::ptr const& connectivity_policy(); + // Read-only aggregate Local ONLINE (no side effects). + bool IsLocallyOnline() const; + // Asynchronous Remote Presence query (ONLINE / OFFLINE / UNKNOWN). + ::ae::QueryPeerPresence& QueryPeerPresence(Uid peer_uid); P2pMessageStreamManager& message_stream_manager(); void SetConfig(std::string client_id, Uid parent_uid, Uid uid, @@ -91,6 +96,7 @@ class Client : public Obj { std::unique_ptr server_connection_manager_; std::unique_ptr cloud_connection_; std::unique_ptr message_stream_manager_; + std::unique_ptr<::ae::QueryPeerPresence> query_peer_presence_; #if AE_ENABLE_PING std::unique_ptr ping_cloud_servers_; diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index 7b0ce875..9280b08a 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -17,6 +17,7 @@ #include "aether/client_connectivity_policy.h" #include +#include namespace ae { @@ -25,7 +26,6 @@ constexpr auto kDefaultTiming = RxTiming{ .conf = RxTimingConf::Every(std::chrono::milliseconds{AE_PING_INTERVAL_MS}), .next_rx_point = {}, .recordet_at = {}}; -; std::array MakeDefaultRxTimings() { std::array timings{}; @@ -42,9 +42,7 @@ ClientConnectivityPolicy::RxTimingConfig::RxTimingConfig( ClientConnectivityPolicy::RxTimingConfig& ClientConnectivityPolicy::RxTimingConfig::ForAllPriorities(RxTimingConf conf) { - for (auto& item : policy_->rx_timings_) { - item.conf = conf; - } + policy_->ApplyDesiredForAllPriorities(conf); return *this; } @@ -92,6 +90,46 @@ auto ClientConnectivityPolicy::ConfigureRxTimings( return RxTimingConfig{*this, std::move(targets)}; } +void ClientConnectivityPolicy::ConfigureServerRxTiming( + ServerId server_id, RxTimingConf conf, + Percentile rtt_reliability_percentile) { + auto& state = EnsureServerPresence(server_id); + auto const timing_changed = (state.desired.interval != conf.interval) || + (state.desired.rx_window != conf.rx_window); + state.desired = conf; + state.has_user_rx_timing = true; + state.rtt_reliability_percentile = rtt_reliability_percentile; + // Confirmed schedule stays old until a Pong for a Ping carrying the new conf. + // Percentile-only updates do not clear the schedule or set config_pending. + if (timing_changed) { + state.config_change_pending = true; + } + server_rx_timing_changed_event_.Emit(server_id); +} + +void ClientConnectivityPolicy::SetServerSelectedForAggregate(ServerId server_id, + bool selected) { + EnsureServerPresence(server_id).selected_for_aggregate = selected; +} + +void ClientConnectivityPolicy::BindServerPriority(ServerId server_id, + std::size_t priority) { + auto& state = EnsureServerPresence(server_id); + state.bound_priority = priority; + if (!state.has_user_rx_timing && (priority < rx_timings_.size())) { + ApplyDesiredIfNoOverride(server_id, state, rx_timings_[priority].conf); + } +} + +void ClientConnectivityPolicy::SetServerQuarantined(ServerId server_id, + bool quarantined) { + EnsureServerPresence(server_id).quarantined = quarantined; +} + +void ClientConnectivityPolicy::RemoveServerFromCloud(ServerId server_id) { + ClearServerPresence(server_id); +} + ClientConnectivityPolicy::SuspendBlocker ClientConnectivityPolicy::AcquireSuspendBlock() { return SuspendBlocker{*this}; @@ -105,6 +143,14 @@ ConnectivityStatus ClientConnectivityPolicy::GetStatus() const noexcept { next_service_time, (t.recordet_at > current_time) ? current_time : t.next_rx_point); } + for (auto const& [id, state] : server_presence_) { + static_cast(id); + if (state.has_confirmed_schedule && + state.confirmed_window_open_local != TimePoint{}) { + next_service_time = + std::min(next_service_time, state.confirmed_window_open_local); + } + } return ConnectivityStatus{.can_suspend = can_suspend_, .suspend_block_count = suspend_block_count_, .next_service_time = next_service_time}; @@ -126,15 +172,197 @@ void ClientConnectivityPolicy::ReportNextServiceTime( t.recordet_at = Now(); } +ServerPresenceState& ClientConnectivityPolicy::EnsureServerPresence( + ServerId server_id) { + auto it = server_presence_.find(server_id); + if (it == server_presence_.end()) { + ServerPresenceState state{}; + // Seed desired from priority-0 default / first priority slot. + state.desired = rx_timings_.front().conf; + it = server_presence_.emplace(server_id, state).first; + } + return it->second; +} + +ServerPresenceState const* ClientConnectivityPolicy::FindServerPresence( + ServerId server_id) const noexcept { + auto it = server_presence_.find(server_id); + return it == server_presence_.end() ? nullptr : &it->second; +} + +ServerPresenceState* ClientConnectivityPolicy::FindServerPresence( + ServerId server_id) noexcept { + auto it = server_presence_.find(server_id); + return it == server_presence_.end() ? nullptr : &it->second; +} + +void ClientConnectivityPolicy::ConfirmServerPong(ServerId server_id, + TimePoint send_time, + TimePoint pong_time, + Duration interval, + Duration rx_window, + Duration selected_rtt) { + auto& state = EnsureServerPresence(server_id); + // interval == 0 clears the future Presence promise after the server + // accepted the reset Ping. rx_window is unrelated to Presence. + if (interval <= Duration{}) { + state.has_confirmed_schedule = false; + state.confirmed_interval = {}; + state.confirmed_rx_window = rx_window; + state.confirmed_ping_send_time = send_time; + state.confirmed_pong_receive_time = pong_time; + state.confirmed_window_open_local = {}; + state.confirmed_window_close_local = {}; + state.config_change_pending = (state.desired.interval != interval) || + (state.desired.rx_window != rx_window); + return; + } + auto const schedule = MakeConfirmedSchedule(send_time, pong_time, interval, + rx_window, selected_rtt); + state.has_confirmed_schedule = true; + state.confirmed_interval = schedule.interval; + state.confirmed_rx_window = schedule.rx_window; + state.confirmed_ping_send_time = schedule.ping_send_time; + state.confirmed_pong_receive_time = schedule.pong_receive_time; + state.confirmed_window_open_local = schedule.window_open_local; + state.confirmed_window_close_local = schedule.window_close_local; + state.config_change_pending = (state.desired.interval != interval) || + (state.desired.rx_window != rx_window); +} + +void ClientConnectivityPolicy::ClearServerPresence(ServerId server_id) { + server_presence_.erase(server_id); +} + +void ClientConnectivityPolicy::SetOfflineDetectionTimeout( + Duration timeout) noexcept { + if (timeout <= Duration{}) { + timeout = std::chrono::milliseconds{AE_OFFLINE_DETECTION_TIMEOUT_MS}; + } + offline_detection_timeout_ = timeout; +} + +void ClientConnectivityPolicy::SetCloudRequestExecutionPolicy( + CloudRequestExecutionPolicy policy) noexcept { + NormalizeCloudRequestExecutionPolicy(policy); + cloud_request_execution_policy_ = policy; +} + +bool ClientConnectivityPolicy::IsLocallyOnline() const noexcept { + return IsLocallyOnline(Now()); +} + +bool ClientConnectivityPolicy::IsLocallyOnline(TimePoint now) const noexcept { + for (auto const& [id, state] : server_presence_) { + static_cast(id); + if (!state.selected_for_aggregate) { + continue; + } + if (IsLocalPresenceOnline(state.has_confirmed_schedule, + state.confirmed_interval, + state.confirmed_window_open_local, now, + offline_detection_timeout_)) { + return true; + } + } + return false; +} + +bool ClientConnectivityPolicy::IsServerLocallyOnline( + ServerId server_id, TimePoint now) const noexcept { + auto const* state = FindServerPresence(server_id); + if (state == nullptr) { + return false; + } + return IsLocalPresenceOnline(state->has_confirmed_schedule, + state->confirmed_interval, + state->confirmed_window_open_local, now, + offline_detection_timeout_); +} + +ClientConnectivityPolicy::LocalPresenceDiag +ClientConnectivityPolicy::DiagnoseLocalPresence(TimePoint now) const noexcept { + LocalPresenceDiag best{}; + for (auto const& [id, state] : server_presence_) { + if (!state.selected_for_aggregate || !state.has_confirmed_schedule || + state.confirmed_interval <= Duration{}) { + continue; + } + auto const deadline = LocalOfflineDeadline(state.confirmed_window_open_local, + offline_detection_timeout_); + auto const online = IsLocalPresenceOnline( + state.has_confirmed_schedule, state.confirmed_interval, + state.confirmed_window_open_local, now, offline_detection_timeout_); + if (!best.has_schedule || deadline > best.offline_deadline) { + best.has_schedule = true; + best.server_id = id; + best.expected_open = state.confirmed_window_open_local; + best.offline_deadline = deadline; + best.last_pong = state.confirmed_pong_receive_time; + best.any_online = online; + } else if (online) { + best.any_online = true; + } + } + if (!best.has_schedule) { + best.any_online = false; + } else { + best.any_online = IsLocallyOnline(now); + } + return best; +} + void ClientConnectivityPolicy::ResetRuntimeState() { auto current_time = Now(); for (auto& t : rx_timings_) { - // if clock was reset, also reset next rx points if (current_time < t.recordet_at) { t.next_rx_point = {}; t.recordet_at = {}; } } + for (auto& [id, state] : server_presence_) { + static_cast(id); + if (current_time < state.confirmed_pong_receive_time) { + state.has_confirmed_schedule = false; + state.confirmed_window_open_local = {}; + state.confirmed_window_close_local = {}; + } + } +} + +void ClientConnectivityPolicy::ApplyDesiredIfNoOverride( + ServerId server_id, ServerPresenceState& state, RxTimingConf conf) { + if (state.has_user_rx_timing) { + return; + } + auto const timing_changed = (state.desired.interval != conf.interval) || + (state.desired.rx_window != conf.rx_window); + state.desired = conf; + if (timing_changed) { + state.config_change_pending = true; + server_rx_timing_changed_event_.Emit(server_id); + } +} + +void ClientConnectivityPolicy::ApplyDesiredForAllPriorities(RxTimingConf conf) { + for (auto& item : rx_timings_) { + item.conf = conf; + } + for (auto& [id, state] : server_presence_) { + ApplyDesiredIfNoOverride(id, state, conf); + } +} + +void ClientConnectivityPolicy::ApplyDesiredForPriority(std::size_t priority, + RxTimingConf conf) { + assert(priority < rx_timings_.size()); + rx_timings_[priority].conf = conf; + for (auto& [id, state] : server_presence_) { + if (state.bound_priority != priority) { + continue; + } + ApplyDesiredIfNoOverride(id, state, conf); + } } void ClientConnectivityPolicy::IncrementSuspendBlock() { diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index 53c063b7..f0a128aa 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -21,14 +21,20 @@ #include #include #include +#include +#include #include "aether-objects/obj/obj.h" #include "aether/clock.h" +#include "aether/cloud_connections/cloud_request_execution_policy.h" +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/cloud_connections/request_policy.h" #include "aether/config.h" #include "aether/events/events.h" +#include "aether/types/server_id.h" -#include "aether/cloud_connections/request_policy.h" +#include namespace ae { @@ -65,6 +71,26 @@ struct ConnectivityStatus { TimePoint next_service_time; }; +struct ServerPresenceState { + RxTimingConf desired{ + RxTimingConf::Every(std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; + Percentile rtt_reliability_percentile{kDefaultRttReliabilityPercentile}; + + bool has_confirmed_schedule{false}; + Duration confirmed_interval{}; + Duration confirmed_rx_window{}; + TimePoint confirmed_ping_send_time{}; + TimePoint confirmed_pong_receive_time{}; + TimePoint confirmed_window_open_local{}; + TimePoint confirmed_window_close_local{}; + + bool config_change_pending{false}; + bool selected_for_aggregate{true}; + bool has_user_rx_timing{false}; + bool quarantined{false}; + std::size_t bound_priority{static_cast(-1)}; +}; + class ClientConnectivityPolicy : public Obj { AE_OBJECT(ClientConnectivityPolicy, Obj, 0) @@ -78,7 +104,7 @@ class ClientConnectivityPolicy : public Obj { template RxTimingConfig& ForPriority(RxTimingConf conf) { static_assert(Priority < kMaxRxServerPriorities); - policy_->rx_timings_[Priority].conf = conf; + policy_->ApplyDesiredForPriority(Priority, conf); return *this; } @@ -120,6 +146,17 @@ class ClientConnectivityPolicy : public Obj { RxTimingConfig ConfigureRxTimings( RequestPolicy::Variant targets = RequestPolicy::All{}); + // Per-server runtime config. Does not invent ONLINE until a confirming Pong. + void ConfigureServerRxTiming( + ServerId server_id, RxTimingConf conf, + Percentile rtt_reliability_percentile = + kDefaultRttReliabilityPercentile); + + void SetServerSelectedForAggregate(ServerId server_id, bool selected); + void BindServerPriority(ServerId server_id, std::size_t priority); + void SetServerQuarantined(ServerId server_id, bool quarantined); + void RemoveServerFromCloud(ServerId server_id); + RequestPolicy::Variant const& rx_targets() const noexcept { return rx_targets_; } @@ -130,6 +167,9 @@ class ClientConnectivityPolicy : public Obj { Event::Subscriber suspend_allowed_event() noexcept { return EventSubscriber{suspend_allowed_event_}; } + Event::Subscriber server_rx_timing_changed_event() noexcept { + return EventSubscriber{server_rx_timing_changed_event_}; + } ConnectivityStatus GetStatus() const noexcept; void ResetRxTimings(); @@ -137,18 +177,71 @@ class ClientConnectivityPolicy : public Obj { SuspendBlocker AcquireSuspendBlock(); void ReportNextServiceTime(std::size_t priority, TimePoint next_service_time); + ServerPresenceState& EnsureServerPresence(ServerId server_id); + ServerPresenceState const* FindServerPresence(ServerId server_id) const noexcept; + ServerPresenceState* FindServerPresence(ServerId server_id) noexcept; + + // Confirm schedule from a successful Pong using selected_rtt projection. + void ConfirmServerPong(ServerId server_id, TimePoint send_time, + TimePoint pong_time, Duration interval, + Duration rx_window, Duration selected_rtt); + + void ClearServerPresence(ServerId server_id); + + // Application-level Local/Remote Presence classification timeout. + // Not part of Ping / rx_window. Applies immediately (no new Ping required). + void SetOfflineDetectionTimeout(Duration timeout) noexcept; + Duration offline_detection_timeout() const noexcept { + return offline_detection_timeout_; + } + + // Runtime CloudRequest soft-timeout / retry / hedge policy (not wire). + // Applies to NEW CloudRequest operations only (snapshot at construction). + void SetCloudRequestExecutionPolicy( + CloudRequestExecutionPolicy policy) noexcept; + CloudRequestExecutionPolicy const& cloud_request_execution_policy() + const noexcept { + return cloud_request_execution_policy_; + } + + // Read-only. No side effects. Aggregate OR: ONLINE iff any selected server + // has confirmed interval>0 and now <= expected_open + offline_detection_timeout. + bool IsLocallyOnline() const noexcept; + bool IsLocallyOnline(TimePoint now) const noexcept; + bool IsServerLocallyOnline(ServerId server_id, TimePoint now) const noexcept; + + // Read-only diagnostics for live harness (expected_open / deadline / last pong). + struct LocalPresenceDiag { + bool any_online{false}; + bool has_schedule{false}; + ServerId server_id{}; + TimePoint expected_open{}; + TimePoint offline_deadline{}; + TimePoint last_pong{}; + }; + LocalPresenceDiag DiagnoseLocalPresence(TimePoint now) const noexcept; + private: void ResetRuntimeState(); void IncrementSuspendBlock(); void DecrementSuspendBlock(); + void ApplyDesiredForAllPriorities(RxTimingConf conf); + void ApplyDesiredForPriority(std::size_t priority, RxTimingConf conf); + void ApplyDesiredIfNoOverride(ServerId server_id, ServerPresenceState& state, + RxTimingConf conf); RequestPolicy::Variant rx_targets_; std::array rx_timings_; + std::map server_presence_; bool can_suspend_{true}; std::uint8_t suspend_block_count_{}; + Duration offline_detection_timeout_{std::chrono::milliseconds{ + AE_OFFLINE_DETECTION_TIMEOUT_MS}}; + CloudRequestExecutionPolicy cloud_request_execution_policy_{}; Event suspend_allowed_event_; + Event server_rx_timing_changed_event_; }; } // namespace ae diff --git a/aether/cloud_connections/cloud_request.cpp b/aether/cloud_connections/cloud_request.cpp index 0a972cc7..9a38c25a 100644 --- a/aether/cloud_connections/cloud_request.cpp +++ b/aether/cloud_connections/cloud_request.cpp @@ -16,49 +16,79 @@ #include "aether/cloud_connections/cloud_request.h" +#include +#include #include #include "aether-miscpp/misc/override.h" #include "aether/aether.h" +#include "aether/channels/channel.h" #include "aether/server.h" #include "aether/write_action/write_action.h" #include "aether/cloud_connections/cloud_connections_tele.h" namespace ae { +namespace { + +#if defined(AE_TELE_ENABLED) && AE_TELE_ENABLED +# define AE_CLOUD_REQ_DEBUG(...) AE_TELED_DEBUG(__VA_ARGS__) +# define AE_CLOUD_REQ_WARNING(...) AE_TELED_WARNING(__VA_ARGS__) +# define AE_CLOUD_REQ_ERROR(...) AE_TELED_ERROR(__VA_ARGS__) +#else +# define AE_CLOUD_REQ_DEBUG(...) +# define AE_CLOUD_REQ_WARNING(...) +# define AE_CLOUD_REQ_ERROR(...) +#endif + +} // namespace CloudRequest::CloudRequest(AeContext const& ae_context, ApiCallWithListener&& api_call, CloudServerConnections& cloud_server_connections, RequestPolicy::Variant policy, - std::size_t max_retries, Duration request_timeout) + CloudRequestExecutionPolicy exec_policy) : ae_context_{ae_context}, request_{std::move(api_call)}, cloud_scs_{&cloud_server_connections}, policy_{policy}, - max_retries_{max_retries}, - request_timeout_{request_timeout}, + exec_policy_{exec_policy}, server_changed_sub_{cloud_scs_->servers_update_event().Subscribe( MethodPtr<&CloudRequest::ServersUpdated>{this})} { - PrefillServerRequests(); - EnqueueMakeRequest(); + NormalizeCloudRequestExecutionPolicy(exec_policy_); + AE_CLOUD_REQ_DEBUG( + "CLOUD_REQUEST_START percentile_tail_raw={} factor_raw={} retry_count={} " + "hedge_next_servers={}", + exec_policy_.response_percentile.TailPercent().RawValue(), + exec_policy_.timeout_factor.RawValue(), exec_policy_.retry_count, + exec_policy_.hedge_next_servers); + RebuildCandidates(); + ActivateInitial(); + EnqueuePump(); } CloudRequest::CloudRequest(AeContext const& ae_context, ApiRequestHandler&& api_request, CloudServerConnections& cloud_server_connections, RequestPolicy::Variant policy, - std::size_t max_retries, Duration request_timeout) + CloudRequestExecutionPolicy exec_policy) : ae_context_{ae_context}, request_{std::move(api_request)}, cloud_scs_{&cloud_server_connections}, policy_{policy}, - max_retries_{max_retries}, - request_timeout_{request_timeout}, + exec_policy_{exec_policy}, server_changed_sub_{cloud_scs_->servers_update_event().Subscribe( MethodPtr<&CloudRequest::ServersUpdated>{this})} { - PrefillServerRequests(); - EnqueueMakeRequest(); + NormalizeCloudRequestExecutionPolicy(exec_policy_); + AE_CLOUD_REQ_DEBUG( + "CLOUD_REQUEST_START percentile_tail_raw={} factor_raw={} retry_count={} " + "hedge_next_servers={}", + exec_policy_.response_percentile.TailPercent().RawValue(), + exec_policy_.timeout_factor.RawValue(), exec_policy_.retry_count, + exec_policy_.hedge_next_servers); + RebuildCandidates(); + ActivateInitial(); + EnqueuePump(); } void CloudRequest::Succeeded() { @@ -71,71 +101,192 @@ void CloudRequest::Failed() { result_event_.Emit(false); } +void CloudRequest::StopServerTimers(ServerRequest& sr) { + for (auto& attempt : sr.attempts) { + attempt.timeout_sub.Reset(); + } + sr.channel_changed_sub.Reset(); +} + +void CloudRequest::SucceedAttempt(CloudServerConnection* sc) { + auto it = server_requests_.find(sc); + if (it == server_requests_.end()) { + return; + } + auto& sr = it->second; + if (sr.exec.IsTerminal()) { + return; + } + AE_CLOUD_REQ_DEBUG("SERVER_ATTEMPT_SUCCESS server_id={}", sc->server_id()); + StopServerTimers(sr); + sr.exec.MarkSucceeded(); + ActivateFollowing(1); + EnqueuePump(); +} + +void CloudRequest::CompleteAttemptWithRemoteError(CloudServerConnection* sc) { + auto it = server_requests_.find(sc); + if (it == server_requests_.end()) { + return; + } + auto& sr = it->second; + if (sr.exec.IsTerminal()) { + return; + } + // Authenticated API error: server proved it is alive. Do not treat as + // no-response, do not soft-retry, do not quarantine for timeout policy. + AE_CLOUD_REQ_DEBUG( + "SERVER_REMOTE_API_ERROR server_id={} (no no-response quarantine)", + sc->server_id()); + StopServerTimers(sr); + sr.exec.MarkRemoteErrorCompleted(); + ActivateFollowing(1); + EnqueuePump(); +} + CloudRequest::ResultEvent::Subscriber CloudRequest::result_event() { return EventSubscriber{result_event_}; } -void CloudRequest::PrefillServerRequests() { - for (auto* sc : cloud_scs_->servers()) { - server_requests_.emplace(sc, ServerRequest{}); - } -} - -void CloudRequest::MakeRequest() { - cloud_scs_->ForServers( - [&](auto& sc) { - auto it = server_requests_.find(sc); - ServerRequest* sr; - if (it == server_requests_.end()) { - // New server added to cloud after construction - auto [new_it, ok] = server_requests_.emplace(sc, ServerRequest{}); - sr = &new_it->second; - } else { - sr = &it->second; - if (sr->exhausted) { - return; - } - } - MakeServerRequest(sc, *sr); - }, - policy_); - - // Check if all server requests are exhausted - bool all_exhausted = !server_requests_.empty(); - for (auto const& [sc, sr] : server_requests_) { - if (!sr.exhausted) { - all_exhausted = false; - break; +CloudRequest::AttemptExhaustedEvent::Subscriber +CloudRequest::attempt_exhausted_event() { + return EventSubscriber{attempt_exhausted_event_}; +} + +void CloudRequest::EmitAttemptExhausted(CloudServerConnection* sc) { + attempt_exhausted_event_.Emit(sc); +} + +void CloudRequest::RebuildCandidates() { + std::vector next; + cloud_scs_->ForServers([&](CloudServerConnection* sc) { next.push_back(sc); }, + policy_); + for (auto* sc : next) { + auto const known = + std::find(candidates_.begin(), candidates_.end(), sc) != + candidates_.end(); + if (!known) { + candidates_.push_back(sc); + server_requests_.emplace(sc, ServerRequest{}); } } - if (all_exhausted) { - AE_TELED_ERROR("All server requests exhausted, failing"); - Failed(); +} + +void CloudRequest::ActivateInitial() { + if (candidates_.empty()) { + return; + } + ActivateFollowing(1); +} + +void CloudRequest::ActivateFollowing(std::uint8_t count, bool as_hedge, + CloudServerConnection* source) { + while (count > 0 && activate_cursor_ < candidates_.size()) { + auto* sc = candidates_[activate_cursor_++]; + auto& sr = server_requests_[sc]; + if (sr.exec.activated || sr.exec.IsTerminal()) { + continue; + } + if (as_hedge) { + AE_CLOUD_REQ_DEBUG( + "SERVER_HEDGE_ACTIVATED source_server={} new_server={}", + source != nullptr ? source->server_id() : ServerId{}, sc->server_id()); + } + ActivateServer(sc); + --count; + } +} + +void CloudRequest::ActivateServer(CloudServerConnection* sc) { + auto& sr = server_requests_[sc]; + if (sr.exec.activated) { + return; } + sr.exec.activated = true; + EnsureChannelChangedSubscription(sc, sr); + LaunchAttempt(sc, sr); } -void CloudRequest::MakeServerRequest(CloudServerConnection* sc, - ServerRequest& sr) { - AE_TELED_DEBUG("Make request to server {}", sc->server_id()); +void CloudRequest::EnsureChannelChangedSubscription(CloudServerConnection* sc, + ServerRequest& sr) { + if (sr.channel_changed_sub) { + return; + } + auto* conn = sc->client_connection(); + if (conn == nullptr) { + return; + } + sr.channel_changed_sub = + conn->server_connection().channel_changed_event().Subscribe([this, sc]() { + AE_CLOUD_REQ_WARNING("Request server channel changed {}", + sc->server_id()); + OnChannelChanged(sc); + }); +} + +Duration CloudRequest::SoftTimeoutFor(CloudServerConnection* sc) const { + auto* conn = sc->client_connection(); + if (conn == nullptr) { + return ComputeCloudRequestSoftTimeout(FallbackCloudRequestRtt(), + exec_policy_); + } + auto channel = conn->server_connection().current_channel(); + if (!channel) { + return ComputeCloudRequestSoftTimeout(FallbackCloudRequestRtt(), + exec_policy_); + } + auto const& stats = + channel->channel_statistics().response_time_statistics(); + if (stats.empty()) { + return ComputeCloudRequestSoftTimeout(FallbackCloudRequestRtt(), + exec_policy_); + } + auto const rtt = + stats.PercentileValue(exec_policy_.response_percentile); + return ComputeCloudRequestSoftTimeout(rtt, exec_policy_); +} - // Clear previous subscriptions and timeout - sr.state_subs.Reset(); - sr.timeout_sub.Reset(); +void CloudRequest::LaunchAttempt(CloudServerConnection* sc, + ServerRequest& sr) { + auto const attempt_index = sr.exec.StartAttempt(exec_policy_); + if (attempt_index == 0) { + return; + } auto* conn = sc->client_connection(); - assert((conn != nullptr) && "Client connection is null"); + if (conn == nullptr) { + AE_CLOUD_REQ_WARNING("SERVER_ATTEMPT skipped disconnected server {}", + sc->server_id()); + auto const action = sr.exec.OnSoftTimeout(exec_policy_); + if (action == CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { + ExhaustServerNoResponse(sc, sr); + } else if (action == + CloudRequestServerExecState::SoftTimeoutAction::kRetry) { + sr.exec.attempts_started = + static_cast(sr.exec.attempts_started - 1); + } + return; + } + + EnsureChannelChangedSubscription(sc, sr); + + auto const timeout = SoftTimeoutFor(sc); + AE_CLOUD_REQ_DEBUG( + "SERVER_ATTEMPT server_id={} attempt_index={} timeout_ms={}", + sc->server_id(), attempt_index, + std::chrono::duration_cast(timeout).count()); + + AttemptState attempt{}; + attempt.attempt_index = attempt_index; - // make request depends on saved request kind auto& swa = std::visit(Override{ - // ApiCallWithListener [&](ApiCallWithListener& api_call) -> decltype(auto) { return conn->AuthorizedApiCall( SubApi{[&](ApiContext& api) { api_call.call(api, sc); }}); }, - // ApiRequestHandler [&](ApiRequestHandler& api_request) -> decltype(auto) { return conn->AuthorizedApiCall( SubApi{[&](ApiContext& api) { @@ -145,109 +296,195 @@ void CloudRequest::MakeServerRequest(CloudServerConnection* sc, }, request_); - // if request write failed - sr.state_subs += swa.status_event().Subscribe([this, sc](auto status) { + // Write failure: send may not have reached the server. Reuse soft retry + // budget (no Restream). LinkError quarantine remains on the connection + // health path and is not duplicated here beyond ExhaustServerNoResponse. + attempt.write_subs += swa.status_event().Subscribe([this, sc](auto status) { if (status == WriteAction::Status::kFail) { - AE_TELED_WARNING("Request write error"); + AE_CLOUD_REQ_WARNING("Request write error server {}", sc->server_id()); OnWriteFailed(sc); } }); - // if server stream changed its channel, retry on new channel - sr.state_subs += - conn->server_connection().channel_changed_event().Subscribe([this, sc]() { - AE_TELED_WARNING("Request server channel changed"); - OnChannelChanged(sc); - }); - - // Set per-server request timeout - sr.timeout_sub = ae_context_.scheduler().DelayedTask( - [this, sc]() { - AE_TELED_WARNING("Request timeout for server {}", sc->server_id()); - OnServerRequestTimeout(sc); - }, - request_timeout_); if (std::holds_alternative(request_)) { auto& listener = std::get(request_).listener; if (listener) { - sr.state_subs += listener(conn->client_safe_api(), sc, this); + sr.response_subs += listener(conn->client_safe_api(), sc, this); } } + + attempt.timeout_sub = ae_context_.scheduler().DelayedTask( + [this, sc, attempt_index]() { OnSoftTimeout(sc, attempt_index); }, + timeout); + + sr.attempts.push_back(std::move(attempt)); } -void CloudRequest::OnChannelChanged(CloudServerConnection* sc) { +void CloudRequest::OnSoftTimeout(CloudServerConnection* sc, + std::uint8_t attempt_index) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { return; } auto& sr = it->second; - if (sr.retry_count >= max_retries_) { - AE_TELED_WARNING("Server {} retry budget exhausted", sc->server_id()); - sr.exhausted = true; - EnqueueMakeRequest(); + if (sr.exec.IsTerminal()) { + return; + } + + for (auto& attempt : sr.attempts) { + if (attempt.attempt_index == attempt_index) { + attempt.timed_out = true; + attempt.timeout_sub.Reset(); + break; + } + } + + AE_CLOUD_REQ_DEBUG( + "SERVER_SOFT_TIMEOUT server_id={} attempt_index={} (no Restream)", + sc->server_id(), attempt_index); + + bool const first_soft_miss = !sr.exec.first_soft_miss_seen; + auto const action = sr.exec.OnSoftTimeout(exec_policy_); + if (first_soft_miss && exec_policy_.hedge_next_servers > 0) { + ActivateFollowing(exec_policy_.hedge_next_servers, /*as_hedge=*/true, sc); + } + + if (action == CloudRequestServerExecState::SoftTimeoutAction::kRetry) { + LaunchAttempt(sc, sr); + EnqueuePump(); + return; + } + if (action == CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { + ExhaustServerNoResponse(sc, sr); + EnqueuePump(); return; } - // Channel already changed, just re-send. - EnqueueMakeRequest(); + EnqueuePump(); } -void CloudRequest::OnWriteFailed(CloudServerConnection* sc) { +void CloudRequest::ExhaustServerNoResponse(CloudServerConnection* sc, + ServerRequest& sr) { + if (sr.exec.succeeded || sr.exec.remote_error_completed) { + return; + } + sr.exec.MarkExhausted(); + StopServerTimers(sr); + AE_CLOUD_REQ_WARNING( + "SERVER_RETRY_EXHAUSTED server_id={} attempts={} soft_timeouts={} -> " + "SERVER_QUARANTINE_NO_RESPONSE", + sc->server_id(), sr.exec.attempts_started, sr.exec.soft_timeouts); + cloud_scs_->QuarantineForNoResponse(*sc); + EmitAttemptExhausted(sc); + ActivateFollowing(1); +} + +void CloudRequest::OnChannelChanged(CloudServerConnection* sc) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { return; } auto& sr = it->second; - sr.retry_count++; - if (sr.retry_count >= max_retries_) { - AE_TELED_WARNING("Server {} retry budget exhausted on write failure", - sc->server_id()); - sr.exhausted = true; + auto const action = sr.exec.OnChannelChanged(exec_policy_); + if (action == + CloudRequestServerExecState::ChannelChangedAction::kIgnore) { + return; } - EnqueueMakeRequest(); + if (action == + CloudRequestServerExecState::ChannelChangedAction::kExhaust) { + ExhaustServerNoResponse(sc, sr); + EnqueuePump(); + return; + } + // Exactly one LaunchAttempt per channel-changed event. + LaunchAttempt(sc, sr); + EnqueuePump(); } -void CloudRequest::OnServerRequestTimeout(CloudServerConnection* sc) { +void CloudRequest::OnWriteFailed(CloudServerConnection* sc) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { return; } auto& sr = it->second; - if (sr.retry_count >= max_retries_) { - AE_TELED_WARNING("Server {} retry budget exhausted on timeout", - sc->server_id()); - sr.exhausted = true; - EnqueueMakeRequest(); + if (sr.exec.IsTerminal()) { return; } - sr.retry_count++; - // Timeout means something is wrong with the stream. - // Restream to switch channels; channel_changed_event will trigger re-send. - sc->Restream(); + // WriteAction::kFail: treat as send-path failure using soft retry budget + // (no Restream). Hard LinkError quarantine is owned by CloudServerConnections + // stream/error subscriptions — ExhaustServerNoResponse may also quarantine + // after budget exhaustion if the write failures never produced a response. + bool const first_soft_miss = !sr.exec.first_soft_miss_seen; + auto const action = sr.exec.OnSoftTimeout(exec_policy_); + if (first_soft_miss && exec_policy_.hedge_next_servers > 0) { + ActivateFollowing(exec_policy_.hedge_next_servers, /*as_hedge=*/true, sc); + } + if (action == CloudRequestServerExecState::SoftTimeoutAction::kRetry) { + LaunchAttempt(sc, sr); + } else if (action == + CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { + ExhaustServerNoResponse(sc, sr); + } + EnqueuePump(); } -void CloudRequest::ServersUpdated() { EnqueueMakeRequest(); } - -void CloudRequest::RemoveRequest(CloudServerConnection* server_connection) { - server_requests_.erase(server_connection); +void CloudRequest::ServersUpdated() { + RebuildCandidates(); + bool any_activated = false; + for (auto const& [sc, sr] : server_requests_) { + static_cast(sc); + if (sr.exec.activated && !sr.exec.IsTerminal()) { + any_activated = true; + break; + } + } + if (!any_activated) { + ActivateInitial(); + } else { + ActivateFollowing(1); + } + EnqueuePump(); } -void CloudRequest::EnqueueMakeRequest() { - // enqueue only once at a time +void CloudRequest::EnqueuePump() { if (task_sub_) { return; } task_sub_ = ae_context_.scheduler().Task([this]() { task_sub_.Reset(); - MakeRequest(); + Pump(); }); } +void CloudRequest::Pump() { + bool any_open = false; + bool any_succeeded = false; + for (auto const& [sc, sr] : server_requests_) { + static_cast(sc); + if (sr.exec.succeeded) { + any_succeeded = true; + } else if (sr.exec.activated && !sr.exec.IsTerminal()) { + any_open = true; + } else if (!sr.exec.activated && !sr.exec.IsTerminal()) { + any_open = true; + } + } + if (activate_cursor_ < candidates_.size()) { + any_open = true; + } + + if (!server_requests_.empty() && + CloudRequestShouldFailAll(any_open, any_succeeded) && + activate_cursor_ >= candidates_.size()) { + AE_CLOUD_REQ_ERROR("All server requests exhausted, failing"); + Failed(); + } +} + void CloudRequest::Finish() { - swa_sub_.Reset(); server_changed_sub_.Reset(); task_sub_.Reset(); server_requests_.clear(); - + candidates_.clear(); Action::Finish(); } diff --git a/aether/cloud_connections/cloud_request.h b/aether/cloud_connections/cloud_request.h index ceb6d6f5..a40b4915 100644 --- a/aether/cloud_connections/cloud_request.h +++ b/aether/cloud_connections/cloud_request.h @@ -16,86 +16,132 @@ #ifndef AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_H_ #define AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_H_ +#include +#include #include +#include #include "aether/common.h" #include "aether/ae_context.h" #include "aether/actions/action.h" +#include "aether/cloud_connections/cloud_request_execution_policy.h" #include "aether/cloud_connections/request_policy.h" #include "aether/cloud_connections/cloud_callbacks.h" #include "aether/cloud_connections/cloud_server_connections.h" +#include "aether/events/event_subscription.h" +#include "aether/events/multi_subscription.h" namespace ae { /** - * \brief Makes request according to the request policy. - * If request fails or times out, it will restream the cloud connection and - * retry on different channels. When a server exhausts its retry budget, - * it moves on to the next server in the list. - * ResponseSubscriber must subscribe to client_api and handle the - * response. On success, listener must call CloudRequest::Succeeded(). On - * failure, listener must call CloudRequest::Failed(). + * \brief Makes request according to RequestPolicy (candidate set) and + * CloudRequestExecutionPolicy (soft timeout / retry / hedge / quarantine). + * + * Soft response timeout does NOT Restream or quarantine. Quarantine for + * no-response happens only after the per-server retry budget is exhausted. + * Late valid responses from earlier attempts are accepted. + * + * Authenticated API-level errors (CompleteAttemptWithRemoteError) prove the + * server is alive and must not quarantine via the no-response path. + * + * ResponseSubscriber / ApiRequestHandler must handle responses. On whole + * request success, call CloudRequest::Succeeded(); on whole failure, + * CloudRequest::Failed(). Per-server: SucceedAttempt / + * CompleteAttemptWithRemoteError. */ +// Compatibility alias for older unit helpers. +using CloudRequestAttemptState = CloudRequestServerExecState; + class CloudRequest final : public Action { - struct ServerRequest { - MultiSubscription state_subs; + struct AttemptState { + // Write-status subscription for this attempt only. + MultiSubscription write_subs; TaskSubscription timeout_sub; - std::size_t retry_count{0}; - bool exhausted{false}; + std::uint8_t attempt_index{0}; + bool timed_out{false}; }; - public: - static constexpr std::size_t kDefaultMaxRetries = 5; - static constexpr Duration kDefaultRequestTimeout = - std::chrono::milliseconds{AE_CLOUD_REQUEST_TIMEOUT_MS}; + struct ServerRequest { + CloudRequestServerExecState exec{}; + // Durable across attempts so late responses remain deliverable. + MultiSubscription response_subs; + // One channel_changed subscription for the whole server lifetime in this + // CloudRequest ? not per attempt. + Subscription channel_changed_sub; + std::vector attempts; + }; + public: using ResultEvent = Event; + using AttemptExhaustedEvent = Event; CloudRequest(AeContext const& ae_context, ApiCallWithListener&& api_call, CloudServerConnections& cloud_server_connections, RequestPolicy::Variant policy, - std::size_t max_retries = kDefaultMaxRetries, - Duration request_timeout = kDefaultRequestTimeout); + CloudRequestExecutionPolicy exec_policy = + CloudRequestExecutionPolicy::Default()); CloudRequest(AeContext const& ae_context, ApiRequestHandler&& api_request, CloudServerConnections& cloud_server_connections, RequestPolicy::Variant policy, - std::size_t max_retries = kDefaultMaxRetries, - Duration request_timeout = kDefaultRequestTimeout); + CloudRequestExecutionPolicy exec_policy = + CloudRequestExecutionPolicy::Default()); AE_CLASS_NO_COPY_MOVE(CloudRequest) void Succeeded(); void Failed(); + // Per-server success: accept late responses, cancel future retries, no + // Restream / quarantine. + void SucceedAttempt(CloudServerConnection* sc); + // Valid authenticated response with API-level failure. Server is alive: + // no soft-timeout retry budget, no no-response quarantine. + void CompleteAttemptWithRemoteError(CloudServerConnection* sc); + + CloudRequestExecutionPolicy const& execution_policy() const noexcept { + return exec_policy_; + } ResultEvent::Subscriber result_event(); + AttemptExhaustedEvent::Subscriber attempt_exhausted_event(); private: - void MakeRequest(); - void MakeServerRequest(CloudServerConnection* sc, ServerRequest& sr); - void PrefillServerRequests(); + void RebuildCandidates(); + void ActivateInitial(); + void ActivateFollowing(std::uint8_t count, bool as_hedge = false, + CloudServerConnection* source = nullptr); + void ActivateServer(CloudServerConnection* sc); + void EnsureChannelChangedSubscription(CloudServerConnection* sc, + ServerRequest& sr); + void LaunchAttempt(CloudServerConnection* sc, ServerRequest& sr); + void StopServerTimers(ServerRequest& sr); + + Duration SoftTimeoutFor(CloudServerConnection* sc) const; + void OnSoftTimeout(CloudServerConnection* sc, std::uint8_t attempt_index); + void ExhaustServerNoResponse(CloudServerConnection* sc, ServerRequest& sr); void ServersUpdated(); void OnChannelChanged(CloudServerConnection* sc); - void OnServerRequestTimeout(CloudServerConnection* sc); void OnWriteFailed(CloudServerConnection* sc); - void RemoveRequest(CloudServerConnection* server_connection); - void EnqueueMakeRequest(); - + void EnqueuePump(); + void Pump(); + void EmitAttemptExhausted(CloudServerConnection* sc); void Finish(); AeContext ae_context_; std::variant request_; CloudServerConnections* cloud_scs_; RequestPolicy::Variant policy_; - std::size_t max_retries_; - Duration request_timeout_; - TaskSubscription task_sub_; + // Snapshot at construction ? runtime policy changes do not affect this op. + CloudRequestExecutionPolicy exec_policy_; - Subscription swa_sub_; + TaskSubscription task_sub_; Subscription server_changed_sub_; ResultEvent result_event_; + AttemptExhaustedEvent attempt_exhausted_event_; + std::vector candidates_; + std::size_t activate_cursor_{0}; std::map server_requests_; }; diff --git a/aether/cloud_connections/cloud_request_execution_policy.h b/aether/cloud_connections/cloud_request_execution_policy.h new file mode 100644 index 00000000..7cd5314a --- /dev/null +++ b/aether/cloud_connections/cloud_request_execution_policy.h @@ -0,0 +1,227 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_EXECUTION_POLICY_H_ +#define AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_EXECUTION_POLICY_H_ + +#include +#include + +#include + +#include "ae-numeric/percentile.h" + +#include "aether/clock.h" +#include "aether/config.h" + +namespace ae { + +// Max retries after the initial attempt for one CloudRequest ↔ one server. +// Total attempts = 1 + retry_count ∈ [1, 32]. +inline constexpr std::uint8_t kMaxCloudRequestRetryCount{31}; + +// Runtime (non-wire) CloudRequest latency / retry / hedge policy. +// Orthogonal to RequestPolicy (which servers are candidates). +struct CloudRequestExecutionPolicy { + // Soft response timeout uses channel response RTT percentile. + Percentile response_percentile{Percentile::FromPercent(99.0)}; + // Soft-timeout multiplier (1-byte FixedPoint, typically Q2.6). + TimeoutFactor8 timeout_factor{TimeoutFactor8::FromDouble(1.2)}; + // Retries after the initial attempt. retry_count=0 => 1 attempt total. + // Clamped to [0, kMaxCloudRequestRetryCount]. + std::uint8_t retry_count{1}; + // How many not-yet-activated following candidates to start on first soft miss. + std::uint8_t hedge_next_servers{0}; + + static constexpr CloudRequestExecutionPolicy Default() noexcept { + return CloudRequestExecutionPolicy{}; + } + + [[nodiscard]] std::size_t TotalAttempts() const noexcept { + return static_cast(retry_count) + 1; + } + + static constexpr CloudRequestExecutionPolicy FromFactor( + Percentile percentile, TimeoutFactor8 factor, std::uint8_t retries, + std::uint8_t hedge) noexcept { + CloudRequestExecutionPolicy p{}; + p.response_percentile = percentile; + p.timeout_factor = factor; + p.retry_count = retries > kMaxCloudRequestRetryCount + ? kMaxCloudRequestRetryCount + : retries; + p.hedge_next_servers = hedge; + return p; + } +}; + +inline void NormalizeCloudRequestExecutionPolicy( + CloudRequestExecutionPolicy& policy) noexcept { + if (policy.timeout_factor.RawValue() == 0) { + policy.timeout_factor = TimeoutFactor8::FromDouble(1.0); + } + if (policy.retry_count > kMaxCloudRequestRetryCount) { + policy.retry_count = kMaxCloudRequestRetryCount; + } +} + +// T = round_nearest(rtt_ms * timeout_factor) with FixedPoint scale 2^kScaleExp. +inline Duration ScaleDurationByTimeoutFactor( + Duration base, TimeoutFactor8 factor) noexcept { + using Ms = std::chrono::milliseconds; + auto const base_ms = std::chrono::duration_cast(base).count(); + if (base_ms <= 0) { + return std::chrono::duration_cast(Ms{1}); + } + static_assert(TimeoutFactor8::kScaleExp < 0); + constexpr int frac_bits = -TimeoutFactor8::kScaleExp; + constexpr std::int64_t half = std::int64_t{1} << (frac_bits - 1); + auto const product = + static_cast(base_ms) * + static_cast(factor.RawValue()); + auto const scaled = (product + half) >> frac_bits; + if (scaled <= 0) { + return std::chrono::duration_cast(Ms{1}); + } + return std::chrono::duration_cast(Ms{scaled}); +} + +inline Duration ComputeCloudRequestSoftTimeout( + Duration rtt_percentile, + CloudRequestExecutionPolicy const& policy) noexcept { + return ScaleDurationByTimeoutFactor(rtt_percentile, policy.timeout_factor); +} + +inline Duration FallbackCloudRequestRtt() noexcept { + return std::chrono::duration_cast( + std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}); +} + +// Per-server execution state used by CloudRequest (unit-testable). +struct CloudRequestServerExecState { + bool activated{false}; + bool succeeded{false}; + bool exhausted{false}; + // Authenticated API-level error: server is alive; attempt terminal; no + // no-response quarantine. + bool remote_error_completed{false}; + bool first_soft_miss_seen{false}; + std::uint8_t attempts_started{0}; + std::uint8_t soft_timeouts{0}; + // Counts OnChannelChanged decisions (one event → one increment). + std::uint8_t channel_changed_events{0}; + + [[nodiscard]] bool IsTerminal() const noexcept { + return succeeded || exhausted || remote_error_completed; + } + + [[nodiscard]] bool ShouldSkip() const noexcept { + return IsTerminal() || !activated; + } + + [[nodiscard]] bool CanStartAttempt( + CloudRequestExecutionPolicy const& policy) const noexcept { + if (!activated || IsTerminal()) { + return false; + } + return attempts_started < policy.TotalAttempts(); + } + + // Start a new attempt. Returns 1-based attempt index, or 0 if not allowed. + std::uint8_t StartAttempt(CloudRequestExecutionPolicy const& policy) { + if (!CanStartAttempt(policy)) { + return 0; + } + ++attempts_started; + return attempts_started; + } + + enum class SoftTimeoutAction : std::uint8_t { + kIgnore = 0, + kRetry, + kExhaust, + }; + + // Soft timeout for the current in-flight attempt. Does not Restream. + SoftTimeoutAction OnSoftTimeout(CloudRequestExecutionPolicy const& policy) { + if (IsTerminal() || !activated) { + return SoftTimeoutAction::kIgnore; + } + ++soft_timeouts; + first_soft_miss_seen = true; + if (attempts_started < policy.TotalAttempts()) { + return SoftTimeoutAction::kRetry; + } + exhausted = true; + return SoftTimeoutAction::kExhaust; + } + + enum class ChannelChangedAction : std::uint8_t { + kIgnore = 0, + kRetry, + kExhaust, + }; + + // One channel-changed event → at most one LaunchAttempt (or exhaust). + ChannelChangedAction OnChannelChanged( + CloudRequestExecutionPolicy const& policy) { + if (IsTerminal() || !activated) { + return ChannelChangedAction::kIgnore; + } + ++channel_changed_events; + if (!CanStartAttempt(policy)) { + exhausted = true; + return ChannelChangedAction::kExhaust; + } + return ChannelChangedAction::kRetry; + } + + // How many following candidates to activate because of this soft miss. + [[nodiscard]] std::uint8_t HedgeCountOnThisMiss( + CloudRequestExecutionPolicy const& policy) const noexcept { + // Hedge only on the first soft miss of this server. + if (soft_timeouts != 1) { + return 0; + } + return policy.hedge_next_servers; + } + + void MarkSucceeded() { + if (exhausted || remote_error_completed) { + return; + } + succeeded = true; + } + + // Valid authenticated response with API-level failure — server is alive. + void MarkRemoteErrorCompleted() { + if (succeeded || exhausted) { + return; + } + remote_error_completed = true; + } + + void MarkExhausted() { exhausted = true; } +}; + +inline bool CloudRequestShouldFailAll(bool any_open, + bool any_succeeded) noexcept { + return !any_open && !any_succeeded; +} + +} // namespace ae + +#endif // AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_EXECUTION_POLICY_H_ diff --git a/aether/cloud_connections/cloud_server_connections.cpp b/aether/cloud_connections/cloud_server_connections.cpp index 993eb898..1a4a76c4 100644 --- a/aether/cloud_connections/cloud_server_connections.cpp +++ b/aether/cloud_connections/cloud_server_connections.cpp @@ -126,6 +126,11 @@ void CloudServerConnections::Restream() { } } +void CloudServerConnections::QuarantineForNoResponse( + CloudServerConnection& server_connection) { + QuarantineAndReconcile(server_connection); +} + void CloudServerConnections::InitServerConnections() { auto cloud = cloud_.Lock(); assert(cloud && "cloud must outlive its connections"); diff --git a/aether/cloud_connections/cloud_server_connections.h b/aether/cloud_connections/cloud_server_connections.h index c0bf1acf..272f0c2d 100644 --- a/aether/cloud_connections/cloud_server_connections.h +++ b/aether/cloud_connections/cloud_server_connections.h @@ -93,6 +93,12 @@ class CloudServerConnections { */ void Restream(); + /** + * \brief Quarantine a server after CloudRequest response-retry exhaustion. + * Uses the existing quarantine / reconcile / replacement path. + */ + void QuarantineForNoResponse(CloudServerConnection& server_connection); + /** * \brief Iterate over servers according to the request policy. * Calls func with CloudServerConnection* for each server. diff --git a/aether/cloud_connections/local_presence_machine.cpp b/aether/cloud_connections/local_presence_machine.cpp new file mode 100644 index 00000000..286069dd --- /dev/null +++ b/aether/cloud_connections/local_presence_machine.cpp @@ -0,0 +1,528 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "aether/cloud_connections/local_presence_machine.h" + +#include +#include + +namespace ae { + +namespace { + +void CountKind(LocalPresenceMachine::Counters& counters, + PingAttemptKind kind) noexcept { + switch (kind) { + case PingAttemptKind::kPrefix1: + ++counters.prefix1; + break; + case PingAttemptKind::kPrefix2: + ++counters.prefix2; + break; + case PingAttemptKind::kRetry: + ++counters.retry; + break; + case PingAttemptKind::kRecovery: + ++counters.recovery; + break; + case PingAttemptKind::kInitial: + ++counters.initial; + break; + } +} + +} // namespace + +LocalPresenceMachine::LocalPresenceMachine() = default; + +void LocalPresenceMachine::SetDesired(TimePoint now, RxTimingConf conf, + Percentile percentile) { + auto const changed = (desired_.interval != conf.interval) || + (desired_.rx_window != conf.rx_window); + desired_ = conf; + percentile_ = percentile; + if (!changed) { + return; + } + config_pending_ = true; + if (!removed_ && !quarantined_ && !HasActiveSchedulerAttempt()) { + ArmSend(PingAttemptKind::kInitial, now); + } +} + +void LocalPresenceMachine::SetOfflineDetectionTimeout(Duration timeout) noexcept { + if (timeout <= Duration{}) { + timeout = std::chrono::milliseconds{AE_OFFLINE_DETECTION_TIMEOUT_MS}; + } + offline_detection_timeout_ = timeout; +} + +void LocalPresenceMachine::ArmInitial(TimePoint now) { + if (removed_ || quarantined_) { + return; + } + ArmSend(PingAttemptKind::kInitial, now); +} + +void LocalPresenceMachine::RestoreConfirmed(TimePoint open, TimePoint close, + Duration interval, Duration window, + TimePoint now, + Duration selected_rtt) { + if (removed_) { + return; + } + has_confirmed_ = true; + confirmed_open_ = open; + confirmed_close_ = close; + confirmed_interval_ = interval; + confirmed_window_ = window; + cycle_has_target_ = false; + cycle_confirmed_ = true; + PlanAfterConfirm(now, selected_rtt); +} + +LocalPresenceMachine::Tick LocalPresenceMachine::TickNow( + TimePoint now, Duration selected_rtt) { + Tick out{}; + if (removed_) { + request_blocker_held_ = false; + current_window_blocker_held_ = false; + return out; + } + + ReleaseCurrentWindowIfDue(now); + CleanupExpired(now); + MarkSchedulerTimeouts(now, selected_rtt); + RecalcRequestBlocker(); + + if (restream_pending_) { + out.restream = true; + out.restream_reason = restream_reason_; + restream_pending_ = false; + restream_reason_ = PresenceRestreamReason::kNone; + } + + if (!quarantined_ && send_armed_ && (now >= next_send_time_) && + !HasActiveSchedulerAttempt()) { + out.want_send = true; + out.send = BuildSendSpec(now, selected_rtt); + send_in_progress_ = true; + send_armed_ = false; + RecalcRequestBlocker(); + } + + out.next_wake = NextWake(now); + return out; +} + +void LocalPresenceMachine::OnSendStarting() { + send_in_progress_ = true; + RecalcRequestBlocker(); +} + +void LocalPresenceMachine::OnAttemptSent(SendSpec spec, TimePoint send_time) { + send_in_progress_ = false; + + Attempt attempt{}; + attempt.attempt_id = spec.attempt_id; + attempt.cycle_id = spec.cycle_id; + attempt.kind = spec.kind; + attempt.send_time = send_time; + attempt.selected_rtt = spec.selected_rtt; + attempt.sent_interval = spec.wire_interval; + attempt.desired_interval = spec.desired_interval; + attempt.sent_window = spec.rx_window; + attempt.following_open_target = spec.following_open_target; + attempt.retry_deadline = send_time + spec.selected_rtt; + attempt.cleanup_deadline = MakeCleanupDeadline(send_time, spec.selected_rtt); + attempt.scheduler_timed_out = false; + attempt.awaiting_response = true; + attempts_.push_back(attempt); + BoundAttempts(); + CountKind(counters_, spec.kind); + + if (spec.opens_current_window && has_confirmed_) { + if ((spec.kind == PingAttemptKind::kPrefix1) || + !current_window_blocker_held_) { + current_promised_close_ = confirmed_close_; + } + current_window_blocker_held_ = true; + } + RecalcRequestBlocker(); +} + +void LocalPresenceMachine::OnStartFailed(TimePoint now, Duration selected_rtt, + PresenceRestreamReason reason) { + send_in_progress_ = false; + RecalcRequestBlocker(); + if (reason != PresenceRestreamReason::kNone) { + restream_pending_ = true; + restream_reason_ = reason; + ++counters_.restreams; + } + ArmSend(PingAttemptKind::kRecovery, now + selected_rtt); +} + +LocalPresenceMachine::PongOutcome LocalPresenceMachine::OnPong( + std::uint64_t attempt_id, std::uint64_t cycle_id, TimePoint send_time, + TimePoint pong_time, Duration sent_interval, + Duration sent_desired_interval, Duration sent_window, + TimePoint following_open_target, Duration selected_rtt_after_sample) { + PongOutcome out{}; + auto* attempt = FindAttempt(attempt_id); + if (attempt != nullptr) { + cycle_id = attempt->cycle_id; + sent_interval = attempt->sent_interval; + sent_desired_interval = attempt->desired_interval; + sent_window = attempt->sent_window; + following_open_target = attempt->following_open_target; + attempt->awaiting_response = false; + EraseAttempt(attempt_id); + } + if (sent_desired_interval <= Duration{}) { + sent_desired_interval = sent_interval; + } + RecalcRequestBlocker(); + + out.schedule = MakeConfirmedSchedule(send_time, pong_time, sent_interval, + sent_window, selected_rtt_after_sample); + + auto const same_cycle = (cycle_id == active_cycle_id_); + if (!same_cycle || (cycle_confirmed_ && same_cycle)) { + out.disposition = PongDisposition::kStatsOnly; + ++counters_.late_pongs; + if (same_cycle) { + last_following_target_ = following_open_target; + } + return out; + } + + auto const was_online = IsOnline(pong_time); + if (sent_desired_interval <= Duration{}) { + has_confirmed_ = false; + confirmed_open_ = {}; + confirmed_close_ = {}; + confirmed_interval_ = {}; + confirmed_window_ = sent_window; + config_pending_ = desired_.interval > Duration{}; + cycle_confirmed_ = true; + active_cycle_id_ = cycle_id; + last_following_target_ = following_open_target; + ++counters_.confirmed_pongs; + out.disposition = PongDisposition::kConfirmedSchedule; + current_window_blocker_held_ = false; + if (desired_.interval > Duration{}) { + ArmSend(PingAttemptKind::kInitial, pong_time); + } else { + send_armed_ = false; + } + return out; + } + + has_confirmed_ = true; + confirmed_open_ = out.schedule.window_open_local; + confirmed_close_ = out.schedule.window_close_local; + confirmed_interval_ = sent_desired_interval; + confirmed_window_ = sent_window; + config_pending_ = (desired_.interval != sent_desired_interval) || + (desired_.rx_window != sent_window); + cycle_confirmed_ = true; + active_cycle_id_ = cycle_id; + last_following_target_ = following_open_target; + ++counters_.confirmed_pongs; + if (!was_online) { + ++counters_.recoveries_to_online; + } + out.disposition = PongDisposition::kConfirmedSchedule; + PlanAfterConfirm(pong_time, selected_rtt_after_sample); + return out; +} + +void LocalPresenceMachine::OnHardFailure(std::uint64_t attempt_id, + TimePoint now, Duration selected_rtt, + PresenceRestreamReason reason) { + auto* attempt = FindAttempt(attempt_id); + auto kind = PingAttemptKind::kRecovery; + if (attempt != nullptr) { + kind = attempt->kind; + attempt->awaiting_response = false; + EraseAttempt(attempt_id); + } + send_in_progress_ = false; + RecalcRequestBlocker(); + restream_pending_ = true; + restream_reason_ = reason; + ++counters_.restreams; + Attempt timed_out{}; + timed_out.kind = kind; + if (has_confirmed_ && (now <= confirmed_close_) && + AttemptOpensPromisedWindow(kind)) { + PlanAfterTimeout(timed_out, now, selected_rtt); + } else { + ArmSend(PingAttemptKind::kRecovery, now + selected_rtt); + } +} + +void LocalPresenceMachine::OnHardWaitExpired(std::uint64_t attempt_id, + TimePoint now) { + static_cast(now); + EraseAttempt(attempt_id); + RecalcRequestBlocker(); +} + +void LocalPresenceMachine::OnQuarantine(TimePoint now) { + quarantined_ = true; + send_in_progress_ = false; + send_armed_ = false; + attempts_.clear(); + RecalcRequestBlocker(); + ReleaseCurrentWindowIfDue(now); +} + +void LocalPresenceMachine::OnQuarantineReleased(TimePoint now, + Duration selected_rtt) { + quarantined_ = false; + ArmSend(PingAttemptKind::kRecovery, now + selected_rtt); +} + +void LocalPresenceMachine::OnRemoved() { + removed_ = true; + quarantined_ = false; + send_in_progress_ = false; + send_armed_ = false; + has_confirmed_ = false; + current_window_blocker_held_ = false; + request_blocker_held_ = false; + attempts_.clear(); +} + +bool LocalPresenceMachine::IsOnline(TimePoint now) const noexcept { + if (removed_) { + return false; + } + return IsLocalPresenceOnline(has_confirmed_, confirmed_interval_, + confirmed_open_, now, offline_detection_timeout_); +} + +LocalPresenceMachine::Attempt* LocalPresenceMachine::FindAttempt( + std::uint64_t attempt_id) noexcept { + for (auto& attempt : attempts_) { + if (attempt.attempt_id == attempt_id) { + return &attempt; + } + } + return nullptr; +} + +void LocalPresenceMachine::EraseAttempt(std::uint64_t attempt_id) { + attempts_.erase(std::remove_if(attempts_.begin(), attempts_.end(), + [attempt_id](Attempt const& attempt) { + return attempt.attempt_id == attempt_id; + }), + attempts_.end()); +} + +void LocalPresenceMachine::BoundAttempts() { + while (attempts_.size() > kMaxOutstandingPresenceAttempts) { + auto it = std::find_if(attempts_.begin(), attempts_.end(), + [](Attempt const& attempt) { + return attempt.scheduler_timed_out || + !attempt.awaiting_response; + }); + if (it == attempts_.end()) { + it = attempts_.begin(); + } + attempts_.erase(it); + } +} + +void LocalPresenceMachine::CleanupExpired(TimePoint now) { + attempts_.erase(std::remove_if(attempts_.begin(), attempts_.end(), + [now](Attempt const& attempt) { + return now >= attempt.cleanup_deadline; + }), + attempts_.end()); +} + +void LocalPresenceMachine::RecalcRequestBlocker() { + if (removed_) { + request_blocker_held_ = false; + return; + } + if (send_in_progress_ && !current_window_blocker_held_) { + request_blocker_held_ = true; + return; + } + for (auto const& attempt : attempts_) { + if (attempt.awaiting_response && + !AttemptOpensPromisedWindow(attempt.kind)) { + request_blocker_held_ = true; + return; + } + } + request_blocker_held_ = false; +} + +void LocalPresenceMachine::ReleaseCurrentWindowIfDue(TimePoint now) { + if (current_window_blocker_held_ && (now > current_promised_close_)) { + current_window_blocker_held_ = false; + } +} + +void LocalPresenceMachine::MarkSchedulerTimeouts(TimePoint now, + Duration selected_rtt) { + for (auto& attempt : attempts_) { + if (!attempt.awaiting_response || attempt.scheduler_timed_out) { + continue; + } + if (now < attempt.retry_deadline) { + continue; + } + attempt.scheduler_timed_out = true; + ++counters_.scheduler_timeouts; + if (!send_armed_ && !send_in_progress_ && !quarantined_) { + PlanAfterTimeout(attempt, now, selected_rtt); + } + } +} + +void LocalPresenceMachine::PlanAfterTimeout(Attempt const& timed_out, + TimePoint now, + Duration selected_rtt) { + if (has_confirmed_ && (now <= confirmed_close_)) { + if (timed_out.kind == PingAttemptKind::kPrefix1) { + auto prefix2 = + ComputePrefix2Time(confirmed_open_, selected_rtt); + if (prefix2 < now) { + prefix2 = now; + } + ArmSend(PingAttemptKind::kPrefix2, prefix2); + return; + } + ArmSend(PingAttemptKind::kRetry, now + selected_rtt); + return; + } + ArmSend(PingAttemptKind::kRecovery, now + selected_rtt); +} + +void LocalPresenceMachine::PlanAfterConfirm(TimePoint now, + Duration selected_rtt) { + cycle_has_target_ = false; + if (config_pending_) { + ArmSend(PingAttemptKind::kInitial, now); + return; + } + auto prefix1 = ComputePrefix1Time(confirmed_open_, selected_rtt); + if (prefix1 < now) { + prefix1 = now; + } + ArmSend(PingAttemptKind::kPrefix1, prefix1); +} + +void LocalPresenceMachine::ArmSend(PingAttemptKind kind, TimePoint when) { + next_kind_ = kind; + next_send_time_ = when; + send_armed_ = true; +} + +LocalPresenceMachine::SendSpec LocalPresenceMachine::BuildSendSpec( + TimePoint now, Duration selected_rtt) { + if (selected_rtt <= Duration{}) { + selected_rtt = kLocalPresenceGuard; + } + + SendSpec spec{}; + spec.attempt_id = ++next_attempt_id_; + spec.kind = next_kind_; + spec.selected_rtt = selected_rtt; + spec.rx_window = desired_.rx_window; + + auto const start_new_cycle = + (spec.kind == PingAttemptKind::kInitial) || + (spec.kind == PingAttemptKind::kRecovery) || + (spec.kind == PingAttemptKind::kPrefix1) || !cycle_has_target_; + + if (start_new_cycle) { + active_cycle_id_ = ++next_cycle_id_; + cycle_confirmed_ = false; + cycle_has_target_ = true; + if (has_confirmed_ && AttemptOpensPromisedWindow(spec.kind)) { + cycle_following_target_ = confirmed_open_ + desired_.interval; + } else { + auto const a_estimated = now + OneWayFromRtt(selected_rtt); + cycle_following_target_ = a_estimated + desired_.interval; + } + } + + spec.cycle_id = active_cycle_id_; + auto plan = PlanWireInterval(now, selected_rtt, cycle_following_target_, + desired_.interval); + cycle_following_target_ = plan.following_open_target; + spec.following_open_target = plan.following_open_target; + spec.wire_interval = plan.wire_interval; + spec.desired_interval = desired_.interval; + spec.hard_wait = + PresenceHardWait(selected_rtt, desired_.interval, desired_.rx_window); + spec.retry_deadline = now + selected_rtt; + spec.cleanup_deadline = MakeCleanupDeadline(now, selected_rtt); + spec.opens_current_window = + has_confirmed_ && AttemptOpensPromisedWindow(spec.kind); + return spec; +} + +bool LocalPresenceMachine::HasActiveSchedulerAttempt() const noexcept { + if (send_in_progress_) { + return true; + } + for (auto const& attempt : attempts_) { + if (attempt.awaiting_response && !attempt.scheduler_timed_out) { + return true; + } + } + return false; +} + +TimePoint LocalPresenceMachine::NextWake(TimePoint now) const noexcept { + static_cast(now); + auto next = TimePoint::max(); + if (send_armed_) { + next = std::min(next, next_send_time_); + } + if (current_window_blocker_held_) { + next = std::min(next, current_promised_close_); + } + for (auto const& attempt : attempts_) { + if (attempt.awaiting_response && !attempt.scheduler_timed_out) { + next = std::min(next, attempt.retry_deadline); + } + next = std::min(next, attempt.cleanup_deadline); + } + return next; +} + +TimePoint LocalPresenceMachine::MakeCleanupDeadline(TimePoint send_time, + Duration rtt) const { + auto deadline = send_time + (rtt * 8); + if (has_confirmed_) { + auto const until_close = confirmed_close_ + rtt; + if (until_close > deadline) { + deadline = until_close; + } + } + return deadline; +} + +} // namespace ae diff --git a/aether/cloud_connections/local_presence_machine.h b/aether/cloud_connections/local_presence_machine.h new file mode 100644 index 00000000..8b818606 --- /dev/null +++ b/aether/cloud_connections/local_presence_machine.h @@ -0,0 +1,227 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_MACHINE_H_ +#define AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_MACHINE_H_ + +#include +#include +#include + +#include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/clock.h" + +namespace ae { + +// Production Local Presence orchestration used by PingCloudServers and tests. +// +// Client attempt_id / cycle_id are local only. The current cloud Ping API does +// not carry a schedule generation. If PREFIX1 is delayed on one adapter and +// PREFIX2 is applied first, a later PREFIX1 Pong is treated as same-cycle +// stats-only locally, but the server may still overwrite the listen schedule +// with PREFIX1's wire interval. Correct cross-adapter ordering would need a +// wire/server generation; that protocol extension is intentionally not made +// here. +class LocalPresenceMachine { + public: + struct Attempt { + std::uint64_t attempt_id{}; + std::uint64_t cycle_id{}; + PingAttemptKind kind{PingAttemptKind::kInitial}; + TimePoint send_time{}; + Duration selected_rtt{}; + Duration sent_interval{}; + Duration desired_interval{}; + Duration sent_window{}; + TimePoint following_open_target{}; + TimePoint retry_deadline{}; + TimePoint cleanup_deadline{}; + bool scheduler_timed_out{false}; + bool awaiting_response{true}; + }; + + struct SendSpec { + std::uint64_t attempt_id{}; + std::uint64_t cycle_id{}; + PingAttemptKind kind{PingAttemptKind::kInitial}; + Duration wire_interval{}; + Duration desired_interval{}; + Duration rx_window{}; + Duration selected_rtt{}; + Duration hard_wait{}; + TimePoint following_open_target{}; + TimePoint retry_deadline{}; + TimePoint cleanup_deadline{}; + bool opens_current_window{false}; + }; + + struct Tick { + TimePoint next_wake{TimePoint::max()}; + bool want_send{false}; + SendSpec send{}; + bool restream{false}; + PresenceRestreamReason restream_reason{PresenceRestreamReason::kNone}; + }; + + enum class PongDisposition : std::uint8_t { + kUnknownAttempt = 0, + kStatsOnly, + kConfirmedSchedule, + }; + + struct PongOutcome { + PongDisposition disposition{PongDisposition::kUnknownAttempt}; + ConfirmedReceiveSchedule schedule{}; + }; + + struct Counters { + int initial{}; + int prefix1{}; + int prefix2{}; + int retry{}; + int recovery{}; + int scheduler_timeouts{}; + int confirmed_pongs{}; + int late_pongs{}; + int restreams{}; + int recoveries_to_online{}; + }; + + LocalPresenceMachine(); + + void SetDesired(TimePoint now, RxTimingConf conf, Percentile percentile); + void SetOfflineDetectionTimeout(Duration timeout) noexcept; + RxTimingConf const& desired() const noexcept { return desired_; } + Percentile percentile() const noexcept { return percentile_; } + Duration offline_detection_timeout() const noexcept { + return offline_detection_timeout_; + } + + void RestoreConfirmed(TimePoint open, TimePoint close, Duration interval, + Duration window, TimePoint now, + Duration selected_rtt); + void ArmInitial(TimePoint now); + + Tick TickNow(TimePoint now, Duration selected_rtt); + + void OnSendStarting(); + void OnAttemptSent(SendSpec spec, TimePoint send_time); + void OnStartFailed(TimePoint now, Duration selected_rtt, + PresenceRestreamReason reason); + + PongOutcome OnPong(std::uint64_t attempt_id, std::uint64_t cycle_id, + TimePoint send_time, TimePoint pong_time, + Duration sent_interval, Duration sent_desired_interval, + Duration sent_window, TimePoint following_open_target, + Duration selected_rtt_after_sample); + + void OnHardFailure(std::uint64_t attempt_id, TimePoint now, + Duration selected_rtt, PresenceRestreamReason reason); + void OnHardWaitExpired(std::uint64_t attempt_id, TimePoint now); + + void OnQuarantine(TimePoint now); + void OnQuarantineReleased(TimePoint now, Duration selected_rtt); + void OnRemoved(); + + bool IsOnline(TimePoint now) const noexcept; + bool has_confirmed_schedule() const noexcept { return has_confirmed_; } + TimePoint confirmed_window_open() const noexcept { return confirmed_open_; } + TimePoint confirmed_window_close() const noexcept { return confirmed_close_; } + Duration confirmed_interval() const noexcept { return confirmed_interval_; } + Duration confirmed_rx_window() const noexcept { return confirmed_window_; } + TimePoint current_promised_close() const noexcept { + return current_promised_close_; + } + bool current_window_blocker_held() const noexcept { + return current_window_blocker_held_; + } + bool request_blocker_held() const noexcept { return request_blocker_held_; } + bool CanSuspend() const noexcept { + return !current_window_blocker_held_ && !request_blocker_held_; + } + std::size_t outstanding_attempt_count() const noexcept { + return attempts_.size(); + } + std::vector const& attempts() const noexcept { return attempts_; } + Counters const& counters() const noexcept { return counters_; } + bool quarantined() const noexcept { return quarantined_; } + bool removed() const noexcept { return removed_; } + bool config_change_pending() const noexcept { return config_pending_; } + TimePoint last_following_target() const noexcept { + return last_following_target_; + } + TimePoint PeekNextWake() const noexcept { return NextWake(TimePoint{}); } + + private: + Attempt* FindAttempt(std::uint64_t attempt_id) noexcept; + void EraseAttempt(std::uint64_t attempt_id); + void BoundAttempts(); + void CleanupExpired(TimePoint now); + void RecalcRequestBlocker(); + void ReleaseCurrentWindowIfDue(TimePoint now); + void MarkSchedulerTimeouts(TimePoint now, Duration selected_rtt); + void PlanAfterTimeout(Attempt const& timed_out, TimePoint now, + Duration selected_rtt); + void PlanAfterConfirm(TimePoint now, Duration selected_rtt); + void ArmSend(PingAttemptKind kind, TimePoint when); + SendSpec BuildSendSpec(TimePoint now, Duration selected_rtt); + bool HasActiveSchedulerAttempt() const noexcept; + TimePoint NextWake(TimePoint now) const noexcept; + TimePoint MakeCleanupDeadline(TimePoint send_time, Duration rtt) const; + + RxTimingConf desired_{RxTimingConf::Every( + std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; + Percentile percentile_{kDefaultRttReliabilityPercentile}; + Duration offline_detection_timeout_{std::chrono::milliseconds{ + AE_OFFLINE_DETECTION_TIMEOUT_MS}}; + + bool has_confirmed_{false}; + TimePoint confirmed_open_{}; + TimePoint confirmed_close_{}; + Duration confirmed_interval_{}; + Duration confirmed_window_{}; + bool config_pending_{false}; + + bool current_window_blocker_held_{false}; + TimePoint current_promised_close_{}; + + bool quarantined_{false}; + bool removed_{false}; + bool send_in_progress_{false}; + bool send_armed_{false}; + bool request_blocker_held_{false}; + bool restream_pending_{false}; + PresenceRestreamReason restream_reason_{PresenceRestreamReason::kNone}; + + PingAttemptKind next_kind_{PingAttemptKind::kInitial}; + TimePoint next_send_time_{}; + + std::uint64_t next_attempt_id_{0}; + std::uint64_t next_cycle_id_{0}; + std::uint64_t active_cycle_id_{0}; + TimePoint cycle_following_target_{}; + bool cycle_has_target_{false}; + bool cycle_confirmed_{false}; + TimePoint last_following_target_{}; + + std::vector attempts_{}; + Counters counters_{}; +}; + +} // namespace ae + +#endif // AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_MACHINE_H_ diff --git a/aether/cloud_connections/local_presence_schedule.h b/aether/cloud_connections/local_presence_schedule.h new file mode 100644 index 00000000..c41d1704 --- /dev/null +++ b/aether/cloud_connections/local_presence_schedule.h @@ -0,0 +1,172 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_SCHEDULE_H_ +#define AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_SCHEDULE_H_ + +#include +#include + +#include "aether/clock.h" +#include "ae-numeric/percentile.h" + +namespace ae { + +// Fixed scheduler safety guard (not rx_window). +inline constexpr Duration kLocalPresenceGuard = + std::chrono::duration_cast(std::chrono::milliseconds{30}); + +inline constexpr Percentile kDefaultRttReliabilityPercentile = + Percentile::FromPercent(99.0); + +inline constexpr std::size_t kMaxOutstandingPresenceAttempts{8}; + +enum class PingAttemptKind : std::uint8_t { + kInitial = 0, + kPrefix1, + kPrefix2, + kRetry, + kRecovery, +}; + +enum class PresenceRestreamReason : std::uint8_t { + kNone = 0, + kHardWriteFailure, + kHardLinkFailure, + kPingApiError, + kConnectionUnavailable, +}; + +// One-way RTT projection used consistently for schedule placement. +// Documented model: one_way = selected_rtt / 2 (local monotonic timeline only). +inline Duration OneWayFromRtt(Duration rtt) noexcept { return rtt / 2; } + +inline Duration MaxDuration(Duration a, Duration b) noexcept { + return a > b ? a : b; +} + +inline Duration PresenceHardWait(Duration selected_rtt, Duration interval, + Duration window) noexcept { + return MaxDuration(selected_rtt * 8, interval + window); +} + +struct ConfirmedReceiveSchedule { + Duration interval{}; + Duration rx_window{}; + TimePoint ping_send_time{}; + TimePoint pong_receive_time{}; + Duration measured_rtt{}; + Duration selected_rtt{}; + TimePoint window_open_local{}; + TimePoint window_close_local{}; +}; + +// After successful Pong for Ping sent at send_time: +// estimated_server_receive = send_time + selected_rtt / 2 +// window_open = estimated_server_receive + sent_interval +// window_close = window_open + sent_window +// measured RTT is diagnostics/statistics only and must not move the projection. +inline ConfirmedReceiveSchedule MakeConfirmedSchedule( + TimePoint send_time, TimePoint pong_time, Duration interval, + Duration rx_window, Duration selected_rtt) noexcept { + ConfirmedReceiveSchedule out{}; + out.interval = interval; + out.rx_window = rx_window; + out.ping_send_time = send_time; + out.pong_receive_time = pong_time; + out.selected_rtt = selected_rtt; + if (pong_time > send_time) { + out.measured_rtt = + std::chrono::duration_cast(pong_time - send_time); + } + auto const one_way = OneWayFromRtt(selected_rtt); + out.window_open_local = send_time + one_way + interval; + out.window_close_local = out.window_open_local + rx_window; + return out; +} + +struct CadencePlan { + TimePoint following_open_target{}; + Duration wire_interval{}; +}; + +// configured interval is the distance between planned opening targets, not +// between early prefix sends. +inline CadencePlan PlanWireInterval(TimePoint send_time, Duration selected_rtt, + TimePoint following_open_target, + Duration desired_interval) noexcept { + auto const a_estimated = send_time + OneWayFromRtt(selected_rtt); + if (following_open_target <= a_estimated) { + return CadencePlan{a_estimated + desired_interval, desired_interval}; + } + return CadencePlan{ + following_open_target, + std::chrono::duration_cast(following_open_target - + a_estimated)}; +} + +// prefix1 = O - 1.5*R - G +// prefix2 = O - 0.5*R - G +inline TimePoint ComputePrefix1Time( + TimePoint window_open, Duration rtt, + Duration guard = kLocalPresenceGuard) noexcept { + return window_open - (rtt * 3) / 2 - guard; +} + +inline TimePoint ComputePrefix2Time( + TimePoint window_open, Duration rtt, + Duration guard = kLocalPresenceGuard) noexcept { + return window_open - rtt / 2 - guard; +} + +// Local Presence ONLINE when a confirmed future opening exists and now is +// still within expected_open + offline_detection_timeout. +// rx_window / confirmed_window_close are NOT used for Presence. +inline bool IsLocalPresenceOnline(bool has_confirmed, Duration confirmed_interval, + TimePoint expected_open, TimePoint now, + Duration offline_detection_timeout) noexcept { + if (!has_confirmed) { + return false; + } + if (confirmed_interval <= Duration{}) { + return false; + } + return now <= (expected_open + offline_detection_timeout); +} + +inline TimePoint LocalOfflineDeadline( + TimePoint expected_open, Duration offline_detection_timeout) noexcept { + return expected_open + offline_detection_timeout; +} + +// Deprecated name kept for transitional call sites that still pass close. +// Prefer IsLocalPresenceOnline. +inline bool IsConfirmedWindowOnline(bool has_confirmed, TimePoint now, + TimePoint window_close) noexcept { + if (!has_confirmed) { + return false; + } + return now <= window_close; +} + +inline bool AttemptOpensPromisedWindow(PingAttemptKind kind) noexcept { + return kind == PingAttemptKind::kPrefix1 || + kind == PingAttemptKind::kPrefix2 || kind == PingAttemptKind::kRetry; +} + +} // namespace ae + +#endif // AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_SCHEDULE_H_ diff --git a/aether/cloud_connections/ping_cloud_servers.cpp b/aether/cloud_connections/ping_cloud_servers.cpp index b1128e14..884e8e31 100644 --- a/aether/cloud_connections/ping_cloud_servers.cpp +++ b/aether/cloud_connections/ping_cloud_servers.cpp @@ -17,15 +17,18 @@ #include "aether/cloud_connections/ping_cloud_servers.h" #include +#include +#include #include +#include #include #if AE_ENABLE_PING # include "aether/channels/channel.h" -# include "aether/executors/executors.h" - # include "aether/cloud_connections/cloud_connections_tele.h" +# include "aether/executors/executors.h" +# include namespace ae { @@ -36,43 +39,153 @@ PingCloudServers::ServerPing::ServerPing(AeContext const& ae_context, : ae_context_{ae_context}, policy_{&policy}, cloud_sc_{&cloud_sc}, + server_id_{cloud_sc.server_id()}, priority_{priority} { assert(priority < policy_->rx_timings().size() && "Server ping priority should be in timings range"); - auto const& timings = policy_->rx_timings()[priority_]; - timing_conf_ = timings.conf; - - // if it's to early for next rx wait a bit - if ((timings.next_rx_point != TimePoint{}) && - (Now() < timings.next_rx_point)) { - AE_TELED_DEBUG("Wait a bit for next rx point till {}", - timings.next_rx_point); - start_sub_ = ae_context_.scheduler().DelayedTask([&]() { Start(); }, - timings.next_rx_point); + policy_->BindServerPriority(server_id_, priority_); + auto& presence = policy_->EnsureServerPresence(server_id_); + policy_->SetServerSelectedForAggregate(server_id_, true); + machine_.SetDesired(Now(), presence.desired, + presence.rtt_reliability_percentile); + machine_.SetOfflineDetectionTimeout(policy_->offline_detection_timeout()); + if (presence.has_confirmed_schedule) { + machine_.RestoreConfirmed( + presence.confirmed_window_open_local, + presence.confirmed_window_close_local, presence.confirmed_interval, + presence.confirmed_rx_window, Now(), SelectedRtt()); } else { - // acquire suspend for first ping - ping_blocker_ = policy_->AcquireSuspendBlock(); - start_sub_ = ae_context_.scheduler().Task([&]() { Start(); }); + machine_.ArmInitial(Now()); } + Pump(); } -PingCloudServers::ServerPing::~ServerPing() = default; - -void PingCloudServers::ServerPing::Stop() { +PingCloudServers::ServerPing::~ServerPing() { stop_ = true; - waiter_.reset(); - start_sub_.Reset(); - rx_window_sub_.Reset(); + live_.clear(); + wake_sub_.Reset(); + current_window_sub_.Reset(); restream_sub_.Reset(); link_state_sub_.Reset(); - - ping_blocker_.Reset(); - rx_window_blocker_.Reset(); + request_blocker_.Reset(); + current_window_blocker_.Reset(); restream_blocker_.Reset(); } +void PingCloudServers::ServerPing::PauseForQuarantine() { + machine_.OnQuarantine(Now()); + waiter_.reset(); + live_.clear(); + policy_->SetServerQuarantined(server_id_, true); + SyncBlockers(); + Pump(); +} + +void PingCloudServers::ServerPing::ResumeFromQuarantine() { + policy_->SetServerQuarantined(server_id_, false); + machine_.OnQuarantineReleased(Now(), SelectedRtt()); + Pump(); +} + +void PingCloudServers::ServerPing::NotifyConfigChanged() { + if (stop_) { + return; + } + auto* presence = policy_->FindServerPresence(server_id_); + if (presence == nullptr) { + return; + } + machine_.SetDesired(Now(), presence->desired, + presence->rtt_reliability_percentile); + machine_.SetOfflineDetectionTimeout(policy_->offline_detection_timeout()); + Pump(); +} + +void PingCloudServers::ServerPing::Pump() { + if (stop_) { + return; + } + auto const now = Now(); + auto const rtt = SelectedRtt(); + auto tick = machine_.TickNow(now, rtt); + SyncBlockers(); + DropFinishedPings(); + if (tick.restream) { + ScheduleRestream(); + } + if (tick.want_send) { + StartSend(tick.send); + return; + } + ScheduleWake(tick.next_wake); +} + +void PingCloudServers::ServerPing::ScheduleWake(TimePoint when) { + next_wake_ = when; + policy_->ReportNextServiceTime(priority_, next_wake_); + if (when == TimePoint::max()) { + wake_sub_.Reset(); + return; + } + wake_sub_ = ae_context_.scheduler().DelayedTask([this]() noexcept { Pump(); }, + when); +} + +void PingCloudServers::ServerPing::SyncBlockers() { + if (machine_.current_window_blocker_held()) { + if (!holding_current_) { + current_window_blocker_ = policy_->AcquireSuspendBlock(); + holding_current_ = true; + } + auto const close = machine_.current_promised_close(); + if (scheduled_current_close_ != close) { + scheduled_current_close_ = close; + current_window_sub_ = ae_context_.scheduler().DelayedTask( + [this]() noexcept { + holding_current_ = false; + scheduled_current_close_ = {}; + current_window_blocker_.Reset(); + Pump(); + }, + close); + } + } else { + current_window_sub_.Reset(); + current_window_blocker_.Reset(); + holding_current_ = false; + scheduled_current_close_ = {}; + } + + if (machine_.request_blocker_held()) { + if (!holding_request_) { + request_blocker_ = policy_->AcquireSuspendBlock(); + holding_request_ = true; + } + } else { + request_blocker_.Reset(); + holding_request_ = false; + } +} + +void PingCloudServers::ServerPing::DropFinishedPings() { + for (auto it = live_.begin(); it != live_.end();) { + bool found = false; + for (auto const& attempt : machine_.attempts()) { + if (attempt.attempt_id == it->first) { + found = true; + break; + } + } + if (!found) { + it = live_.erase(it); + } else { + ++it; + } + } +} + template void PingCloudServers::ServerPing::WaitForLink(ClientServerConnection& cc, F&& f) { @@ -102,51 +215,32 @@ auto PingCloudServers::ServerPing::EnsureLinked() { }); } -auto PingCloudServers::ServerPing::MakePing() { - return ex::let_value([&]() noexcept { - return ex::create( - [&](auto& ctx) noexcept { - // make ping action with timeout based on response statistics - // and timing properties for current server RxTimings - // during the ping and rx window setup suspend blocker - // and save expected next_ping_time_ - auto* cc = cloud_sc_->client_connection(); - assert(cc != nullptr && "Client connection should exists"); - - auto c = cc->server_connection().current_channel(); - if (c == nullptr) { - AE_TELED_ERROR("Current channel value invalid"); - return ex::set_error(std::move(ctx.receiver), 2); - } - - ping_.emplace(ae_context_, *cloud_sc_, timing_conf_.interval, - timing_conf_.rx_window, c->ResponseTimeout()); - - ping_blocker_ = policy_->AcquireSuspendBlock(); - ping_->result_event().Subscribe( - [this](Ping::PingResult const& res) noexcept { - OnPingResult(res); - ping_blocker_.Reset(); - }); - - // run ping request and open rx window - auto const current_time = Now(); - ping_->Start(current_time); - OpenRxWindow(current_time); - next_ping_time_ = current_time + timing_conf_.interval; - policy_->ReportNextServiceTime(priority_, next_ping_time_); - AE_TELED_DEBUG("Next ping time for priority {} at {} after {}", - priority_, next_ping_time_, timing_conf_.interval); - - return ex::set_value(std::move(ctx.receiver)); - }); - }); +Duration PingCloudServers::ServerPing::SelectedRtt() const { + auto* cc = cloud_sc_->client_connection(); + if (cc == nullptr) { + return std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}; + } + auto c = cc->server_connection().current_channel(); + if (!c) { + return std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}; + } + auto const& stats = c->channel_statistics().response_time_statistics(); + auto const* presence = policy_->FindServerPresence(server_id_); + auto const pct = presence == nullptr ? kDefaultRttReliabilityPercentile + : presence->rtt_reliability_percentile; + if (stats.empty()) { + return std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}; + } + return stats.PercentileValue(pct); } -void PingCloudServers::ServerPing::Start() { +void PingCloudServers::ServerPing::StartSend( + LocalPresenceMachine::SendSpec spec) { if (stop_) { return; } + machine_.OnSendStarting(); + SyncBlockers(); waiter_.emplace( ae_context_, EnsureLinked() | @@ -159,8 +253,42 @@ void PingCloudServers::ServerPing::Start() { } return ex::just(); }) | - MakePing() | - // track Stop command + ex::let_value([&]() noexcept { + return ex::create( + [&](auto& ctx) noexcept { + auto* cc = cloud_sc_->client_connection(); + if (cc == nullptr) { + return ex::set_error(std::move(ctx.receiver), 1); + } + auto c = cc->server_connection().current_channel(); + if (c == nullptr) { + AE_TELED_ERROR("Current channel value invalid"); + return ex::set_error(std::move(ctx.receiver), 2); + } + + auto const send_time = Now(); + auto ping = std::make_unique( + ae_context_, *cloud_sc_, spec.wire_interval, + spec.rx_window, spec.hard_wait); + auto const attempt_id = spec.attempt_id; + ping->result_event().Subscribe( + [this, attempt_id](Ping::PingResult const& res) noexcept { + OnPingResult(attempt_id, res); + }); + ping->Start(send_time); + machine_.OnAttemptSent(spec, send_time); + LiveAttempt live{}; + live.spec = spec; + live.send_time = send_time; + live.ping = std::move(ping); + live_[attempt_id] = std::move(live); + AE_TELED_DEBUG( + "PING_ATTEMPT server {} id {} kind {} send {} wire {}", + server_id_, attempt_id, static_cast(spec.kind), + send_time, spec.wire_interval); + return ex::set_value(std::move(ctx.receiver)); + }); + }) | ex::let_value( [&]() noexcept -> ex::variant_sender(std::optional&& res) noexcept { - if (res && res->IsOk()) { - // repeat start on next_ping_time_ - start_sub_ = ae_context_.scheduler().DelayedTask( - [&]() noexcept { Start(); }, // ~['_']~ - next_ping_time_); - } else if (res && res->IsErr()) { + if (res && res->IsErr()) { AE_TELED_ERROR("Ping start error {}", std::move(res)->error()); - } else { + machine_.OnStartFailed(Now(), SelectedRtt(), + PresenceRestreamReason::kConnectionUnavailable); + SyncBlockers(); + Pump(); + } else if (!(res && res->IsOk())) { AE_TELED_DEBUG("Server ping stopped"); + machine_.OnStartFailed(Now(), SelectedRtt(), + PresenceRestreamReason::kNone); + SyncBlockers(); + } else { + Pump(); } }); } -void PingCloudServers::ServerPing::OnPingResult(Ping::PingResult const& res) { - auto* cc = cloud_sc_->client_connection(); - if (cc == nullptr) { - AE_TELED_ERROR("Client connection is null"); +void PingCloudServers::ServerPing::OnPingResult(std::uint64_t attempt_id, + Ping::PingResult const& res) { + if (stop_) { return; } - - auto c = cc->server_connection().current_channel(); - if (!c) { - AE_TELED_ERROR("Connection channel is null"); + auto it = live_.find(attempt_id); + if (it == live_.end()) { return; } + auto spec = it->second.spec; + auto send_time = it->second.send_time; std::visit( - [this, c](auto const& value) { + [this, attempt_id, spec, send_time](auto const& value) { using T = std::decay_t; - if constexpr (std::is_same_v>) { - c->channel_statistics().AddResponseTime(value.value); - } else if constexpr (std::is_same_v) { - AE_TELED_DEBUG("Got late ping duration"); - c->channel_statistics().AddResponseTime(value.duration); + if constexpr (std::is_same_v> || + std::is_same_v) { + Duration measured{}; + if constexpr (std::is_same_v>) { + measured = value.value; + } else { + measured = value.duration; + } + AddRttSample(measured); + auto const selected = SelectedRtt(); + auto outcome = machine_.OnPong( + attempt_id, spec.cycle_id, send_time, Now(), spec.wire_interval, + spec.desired_interval, spec.rx_window, + spec.following_open_target, selected); + ApplyConfirmed(outcome); + live_.erase(attempt_id); + SyncBlockers(); + Pump(); } else { - AE_TELED_ERROR("Ping error!"); - ScheduleRestream(); + auto const code = value.error; + if (code == 2) { + machine_.OnHardWaitExpired(attempt_id, Now()); + live_.erase(attempt_id); + SyncBlockers(); + Pump(); + return; + } + auto reason = PresenceRestreamReason::kPingApiError; + if (code == 1) { + reason = PresenceRestreamReason::kHardWriteFailure; + } + machine_.OnHardFailure(attempt_id, Now(), SelectedRtt(), reason); + live_.erase(attempt_id); + SyncBlockers(); + Pump(); } }, res); } -void PingCloudServers::ServerPing::OpenRxWindow(TimePoint sent_time) { - // keep rx window suspend block for timing_.rx_window time - rx_window_blocker_ = policy_->AcquireSuspendBlock(); - rx_window_sub_ = ae_context_.scheduler().DelayedTask( - [this]() { rx_window_blocker_.Reset(); }, - sent_time + timing_conf_.rx_window); +void PingCloudServers::ServerPing::AddRttSample(Duration measured) { + auto* cc = cloud_sc_->client_connection(); + if (cc == nullptr) { + return; + } + auto c = cc->server_connection().current_channel(); + if (!c) { + return; + } + c->channel_statistics().AddResponseTime(measured); +} + +void PingCloudServers::ServerPing::ApplyConfirmed( + LocalPresenceMachine::PongOutcome const& outcome) { + if (outcome.disposition != + LocalPresenceMachine::PongDisposition::kConfirmedSchedule) { + return; + } + if (!machine_.has_confirmed_schedule()) { + policy_->ConfirmServerPong(server_id_, outcome.schedule.ping_send_time, + outcome.schedule.pong_receive_time, Duration{}, + outcome.schedule.rx_window, + outcome.schedule.selected_rtt); + return; + } + policy_->ConfirmServerPong( + server_id_, outcome.schedule.ping_send_time, + outcome.schedule.pong_receive_time, outcome.schedule.interval, + outcome.schedule.rx_window, outcome.schedule.selected_rtt); + auto& state = policy_->EnsureServerPresence(server_id_); + state.confirmed_interval = machine_.confirmed_interval(); + state.config_change_pending = machine_.config_change_pending(); + AE_TELED_DEBUG("SCHEDULE confirmed server {} open {} close {}", server_id_, + outcome.schedule.window_open_local, + outcome.schedule.window_close_local); } void PingCloudServers::ServerPing::ScheduleRestream() { if (stop_) { return; } - - // TODO: should we block till restream? restream_blocker_ = policy_->AcquireSuspendBlock(); restream_sub_ = ae_context_.scheduler().Task([this]() { auto* cc = cloud_sc_->client_connection(); @@ -254,6 +439,9 @@ PingCloudServers::PingCloudServers( server_quarantine_released_sub_ = cloud_server_connections_->server_quarantine_release_event().Subscribe( MethodPtr<&PingCloudServers::ServerQuarantineReleased>{this}); + server_rx_timing_changed_sub_ = + policy_->server_rx_timing_changed_event().Subscribe( + MethodPtr<&PingCloudServers::OnServerRxTimingChanged>{this}); ServersUpdate(); } @@ -284,6 +472,7 @@ void PingCloudServers::DispatchToServers() { } }, policy_->rx_targets()); + RemoveMissingServers(); } void PingCloudServers::ReconcileServer(CloudServerConnection& cloud_sc) { @@ -291,8 +480,7 @@ void PingCloudServers::ReconcileServer(CloudServerConnection& cloud_sc) { auto const priority = cloud_sc.priority(); auto it = server_pings_.find(server_id); - if ((it == server_pings_.end()) || (it->second->priority() != priority) || - it->second->stopped()) { + if ((it == server_pings_.end()) || (it->second->priority() != priority)) { if (it != server_pings_.end()) { it->second.reset(); } @@ -301,6 +489,7 @@ void PingCloudServers::ReconcileServer(CloudServerConnection& cloud_sc) { priority)); return; } + it->second->NotifyConfigChanged(); } void PingCloudServers::ServerQuarantined(CloudServerConnection* cloud_sc) { @@ -309,7 +498,7 @@ void PingCloudServers::ServerQuarantined(CloudServerConnection* cloud_sc) { } auto it = server_pings_.find(cloud_sc->server_id()); if (it != server_pings_.end()) { - it->second->Stop(); + it->second->PauseForQuarantine(); } } @@ -320,7 +509,31 @@ void PingCloudServers::ServerQuarantineReleased( } auto it = server_pings_.find(cloud_sc->server_id()); if (it != server_pings_.end()) { - server_pings_.erase(it); + it->second->ResumeFromQuarantine(); + } +} + +void PingCloudServers::OnServerRxTimingChanged(ServerId server_id) { + auto it = server_pings_.find(server_id); + if (it != server_pings_.end() && !it->second->quarantined()) { + it->second->NotifyConfigChanged(); + } +} + +void PingCloudServers::RemoveMissingServers() { + std::set in_cloud; + for (auto* sc : cloud_server_connections_->servers()) { + if (sc != nullptr) { + in_cloud.insert(sc->server_id()); + } + } + for (auto it = server_pings_.begin(); it != server_pings_.end();) { + if (in_cloud.find(it->first) == in_cloud.end()) { + policy_->RemoveServerFromCloud(it->first); + it = server_pings_.erase(it); + } else { + ++it; + } } } } // namespace ae diff --git a/aether/cloud_connections/ping_cloud_servers.h b/aether/cloud_connections/ping_cloud_servers.h index dd55ef65..6ba5f2fc 100644 --- a/aether/cloud_connections/ping_cloud_servers.h +++ b/aether/cloud_connections/ping_cloud_servers.h @@ -20,20 +20,21 @@ #include "aether/config.h" #if AE_ENABLE_PING +# include # include # include # include +# include "aether/ae_actions/ping.h" # include "aether/ae_context.h" +# include "aether/client_connectivity_policy.h" +# include "aether/cloud_connections/cloud_server_connections.h" +# include "aether/cloud_connections/local_presence_machine.h" # include "aether/events/event_subscription.h" # include "aether/executors/executors.h" # include "aether/tasks/manual_task_scheduler.h" # include "aether/types/server_id.h" -# include "aether/ae_actions/ping.h" -# include "aether/client_connectivity_policy.h" -# include "aether/cloud_connections/cloud_server_connections.h" - namespace ae { class PingCloudServers { class ServerPing { @@ -44,44 +45,61 @@ class PingCloudServers { AE_CLASS_NO_COPY_MOVE(ServerPing) - void Stop(); + void PauseForQuarantine(); + void ResumeFromQuarantine(); + void NotifyConfigChanged(); - TimePoint next_service_time() const noexcept { return next_ping_time_; } + TimePoint next_service_time() const noexcept { return next_wake_; } std::size_t priority() const noexcept { return priority_; } - RxTimingConf const& timing() const noexcept { return timing_conf_; } - bool stopped() const noexcept { return stop_; } + bool quarantined() const noexcept { return machine_.quarantined(); } private: - void Start(); + struct LiveAttempt { + LocalPresenceMachine::SendSpec spec{}; + TimePoint send_time{}; + std::unique_ptr ping; + Subscription result_sub; + }; + + void Pump(); + void ScheduleWake(TimePoint when); + void SyncBlockers(); + void DropFinishedPings(); template void WaitForLink(ClientServerConnection& cc, F&& f); auto EnsureLinked(); - auto MakePing(); - - void OnPingResult(Ping::PingResult const& res); - void OpenRxWindow(TimePoint sent_time); + Duration SelectedRtt() const; + void StartSend(LocalPresenceMachine::SendSpec spec); + void OnPingResult(std::uint64_t attempt_id, Ping::PingResult const& res); + void AddRttSample(Duration measured); void ScheduleRestream(); + void ApplyConfirmed(LocalPresenceMachine::PongOutcome const& outcome); AeContext ae_context_; ClientConnectivityPolicy* policy_; CloudServerConnection* cloud_sc_; - RxTimingConf timing_conf_{}; + ServerId server_id_{}; std::size_t priority_{}; + LocalPresenceMachine machine_; std::optional> waiter_; - std::optional ping_; - bool stop_{false}; + std::map live_; + Subscription link_state_sub_; - TaskSubscription start_sub_; - TaskSubscription rx_window_sub_; + TaskSubscription wake_sub_; + TaskSubscription current_window_sub_; TaskSubscription restream_sub_; - ClientConnectivityPolicy::SuspendBlocker ping_blocker_; - ClientConnectivityPolicy::SuspendBlocker rx_window_blocker_; + ClientConnectivityPolicy::SuspendBlocker request_blocker_; + ClientConnectivityPolicy::SuspendBlocker current_window_blocker_; ClientConnectivityPolicy::SuspendBlocker restream_blocker_; - TimePoint next_ping_time_; + TimePoint next_wake_{TimePoint::max()}; + bool stop_{false}; + bool holding_request_{false}; + bool holding_current_{false}; + TimePoint scheduled_current_close_{}; }; public: @@ -96,6 +114,8 @@ class PingCloudServers { void ReconcileServer(CloudServerConnection& cloud_sc); void ServerQuarantined(CloudServerConnection* cloud_sc); void ServerQuarantineReleased(CloudServerConnection* cloud_sc); + void OnServerRxTimingChanged(ServerId server_id); + void RemoveMissingServers(); AeContext ae_context_; CloudServerConnections* cloud_server_connections_; @@ -104,6 +124,7 @@ class PingCloudServers { Subscription servers_update_; Subscription server_quarantined_sub_; Subscription server_quarantine_released_sub_; + Subscription server_rx_timing_changed_sub_; TaskSubscription task_sub_; std::map> server_pings_; diff --git a/aether/config.h b/aether/config.h index f9ef0d3c..066f5a7a 100644 --- a/aether/config.h +++ b/aether/config.h @@ -42,8 +42,10 @@ # define AE_TASK_ALIGN alignof(std::max_align_t) #endif +// Sized for concurrent Local Presence pings plus Remote Presence +// get_client_timing across selected servers with soft-timeout retry/hedge. #ifndef AE_API_PROTOCOL_MAX_PENDING_RESPONSES -# define AE_API_PROTOCOL_MAX_PENDING_RESPONSES 10 +# define AE_API_PROTOCOL_MAX_PENDING_RESPONSES 32 #endif #ifndef AE_API_PROTOCOL_MAX_PACKET_STACK_DEPTH @@ -295,6 +297,13 @@ # define AE_PING_INTERVAL_MS AE_DEFAULT_RESPONSE_TIMEOUT_MS + 1000 #endif +// Initial default for Local/Remote Presence offline classification timeout. +// Runtime value lives on ClientConnectivityPolicy and may change without +// a new Ping. +#ifndef AE_OFFLINE_DETECTION_TIMEOUT_MS +# define AE_OFFLINE_DETECTION_TIMEOUT_MS 1000 +#endif + // window size for safe stream response time statistics #ifndef AE_STATISTICS_SAFE_STREAM_WINDOW_SIZE # define AE_STATISTICS_SAFE_STREAM_WINDOW_SIZE 100 diff --git a/aether/remote_presence.h b/aether/remote_presence.h new file mode 100644 index 00000000..1538da21 --- /dev/null +++ b/aether/remote_presence.h @@ -0,0 +1,232 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_REMOTE_PRESENCE_H_ +#define AETHER_REMOTE_PRESENCE_H_ + +#include +#include +#include +#include +#include +#include + +#include "aether/clock.h" +#include "aether/config.h" +#include "aether/types/server_id.h" +#include "aether/work_cloud_api/client_timing.h" + +namespace ae { + +enum class PeerPresenceState : std::uint8_t { + kOnline = 0, + kOffline, + kUnknown, +}; + +struct PeerPresence { + PeerPresenceState state{PeerPresenceState::kUnknown}; +}; + +// Per authoritative usable server contribution for Remote AND aggregation. +enum class RemoteServerPresence : std::uint8_t { + kOnline = 0, + kOffline, + // Query pending / failed after retries while server remains usable. + kUnknown, + // Quarantined / unselected / removed — excluded from aggregation. + kExcluded, +}; + +struct RemoteServerPresenceSample { + ServerId server_id{}; + RemoteServerPresence status{RemoteServerPresence::kUnknown}; + TimePoint expected_open{}; + TimePoint offline_deadline{}; + std::int64_t next_ping_delta_ms{}; + bool has_timing{false}; +}; + +inline TimePoint TimePointOffsetByMs(TimePoint anchor, + std::int64_t delta_ms) noexcept { + if (delta_ms == 0) { + return anchor; + } + using ClockDuration = typename TimePoint::duration; + using Rep = typename ClockDuration::rep; + auto const max_safe_ms = + std::chrono::duration_cast( + ClockDuration{std::numeric_limits::max() / 4}) + .count(); + if (max_safe_ms > 0) { + if (delta_ms > max_safe_ms) { + return TimePoint::max(); + } + if (delta_ms < -max_safe_ms) { + return TimePoint::min(); + } + } + auto const offset = std::chrono::duration_cast( + std::chrono::milliseconds{delta_ms}); + auto const base = anchor.time_since_epoch().count(); + auto const add = offset.count(); + if (add > 0) { + if (base > std::numeric_limits::max() - add) { + return TimePoint::max(); + } + } else if (add < 0) { + if (base < std::numeric_limits::min() - add) { + return TimePoint::min(); + } + } + return TimePoint{ClockDuration{static_cast(base + add)}}; +} + +// midpoint = query_send + (response_receive - query_send) / 2 +inline TimePoint QueryMidpoint(TimePoint query_send, + TimePoint response_receive) noexcept { + if (response_receive <= query_send) { + return query_send; + } + return query_send + (response_receive - query_send) / 2; +} + +inline TimePoint ProjectRemoteExpectedOpen( + TimePoint query_send, TimePoint response_receive, + std::int64_t next_ping_delta_ms) noexcept { + return TimePointOffsetByMs(QueryMidpoint(query_send, response_receive), + next_ping_delta_ms); +} + +// Classify one authoritative server response. next_ping_delta == 0 means no +// future promise => Offline immediately (do not wait offline_detection_timeout). +inline RemoteServerPresence ClassifyRemoteServerPresence( + TimePoint now, TimePoint query_send, TimePoint response_receive, + ClientTiming const& timing, Duration offline_detection_timeout, + TimePoint* expected_open_out = nullptr, + TimePoint* offline_deadline_out = nullptr) noexcept { + if (timing.next_ping_delta_ms == 0) { + if (expected_open_out != nullptr) { + *expected_open_out = {}; + } + if (offline_deadline_out != nullptr) { + *offline_deadline_out = {}; + } + return RemoteServerPresence::kOffline; + } + auto const expected = ProjectRemoteExpectedOpen( + query_send, response_receive, timing.next_ping_delta_ms); + auto const deadline = expected + offline_detection_timeout; + if (expected_open_out != nullptr) { + *expected_open_out = expected; + } + if (offline_deadline_out != nullptr) { + *offline_deadline_out = deadline; + } + if (now <= deadline) { + return RemoteServerPresence::kOnline; + } + return RemoteServerPresence::kOffline; +} + +// Remote aggregation AND over usable authoritative servers. +// Offline: any usable Offline. +// Online: usable_count > 0 and every usable sample Online. +// Unknown: usable_count == 0, or no Offline but not every usable Online. +inline PeerPresence AggregateRemotePresence( + std::vector const& samples) noexcept { + PeerPresence out{}; + std::size_t usable = 0; + std::size_t online = 0; + bool any_offline = false; + bool any_unknown = false; + for (auto const& sample : samples) { + if (sample.status == RemoteServerPresence::kExcluded) { + continue; + } + ++usable; + if (sample.status == RemoteServerPresence::kOffline) { + any_offline = true; + } else if (sample.status == RemoteServerPresence::kOnline) { + ++online; + } else { + any_unknown = true; + } + } + if (usable == 0) { + out.state = PeerPresenceState::kUnknown; + return out; + } + if (any_offline) { + out.state = PeerPresenceState::kOffline; + return out; + } + if (online == usable && !any_unknown) { + out.state = PeerPresenceState::kOnline; + return out; + } + out.state = PeerPresenceState::kUnknown; + return out; +} + +inline bool RemotePresenceCanEarlyCompleteOffline( + std::vector const& samples) noexcept { + for (auto const& sample : samples) { + if (sample.status == RemoteServerPresence::kOffline) { + return true; + } + } + return false; +} + +inline bool RemotePresenceReadyForOnline( + std::vector const& samples) noexcept { + auto const aggregated = AggregateRemotePresence(samples); + return aggregated.state == PeerPresenceState::kOnline; +} + +inline Duration DefaultOfflineDetectionTimeout() noexcept { + return std::chrono::milliseconds{AE_OFFLINE_DETECTION_TIMEOUT_MS}; +} + +// Local Presence (PingCloudServers + RequestPolicy::All) only maintains +// schedules on selected_servers(), which is bounded by +// AE_CLOUD_MAX_SERVER_CONNECTIONS. Remote Presence must AND over that same +// contract set — not an observer's own cloud, and not a silent subset smaller +// than the peer Presence obligation when peer cloud size <= max_connections. +inline std::vector AuthoritativePresenceServerIds( + std::vector const& peer_cloud_ids_priority_order, + std::size_t max_connections) noexcept { + std::vector out; + if (max_connections == 0) { + return out; + } + auto const n = + std::min(peer_cloud_ids_priority_order.size(), max_connections); + out.assign(peer_cloud_ids_priority_order.begin(), + peer_cloud_ids_priority_order.begin() + + static_cast(n)); + return out; +} + +// Never substitute the observer/requester cloud for the peer Personal Cloud. +inline bool AllowObserverCloudFallbackForPeerPresence() noexcept { + return false; +} + +} // namespace ae + +#endif // AETHER_REMOTE_PRESENCE_H_ diff --git a/aether/types/statistic_counter.h b/aether/types/statistic_counter.h index 44078232..5ce80949 100644 --- a/aether/types/statistic_counter.h +++ b/aether/types/statistic_counter.h @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -30,6 +29,8 @@ IGNORE_IMPLICIT_CONVERSION() #include DISABLE_WARNING_POP() +#include "ae-numeric/percentile.h" + #include "aether-miscpp/format/format.h" #include "aether-miscpp/serialization/binary_archive.h" @@ -81,21 +82,40 @@ class StatisticsCounter final { [[nodiscard]] TValue percentile() const { static_assert((Percentile >= 0) && (Percentile <= 100), "Percentile must be in [0,100]% range"); + return PercentileValue(Percentile); + } - if constexpr (Percentile == 0) { + /** + * \brief Runtime percentile accessor (0..100). Same semantics as the + * compile-time template overload. Integer-only rank (no float/ceil). + */ + [[nodiscard]] TValue PercentileValue(std::size_t percentile) const { + assert(percentile <= 100); + assert(!value_buffer_.empty()); + if (percentile == 0) { return min(); - } else if constexpr (Percentile == 100) { + } + if (percentile == 100) { + return max(); + } + auto sorted_list = value_buffer_; + std::sort(std::begin(sorted_list), std::end(sorted_list), Comparator{}); + auto const index = PercentileIndexInteger(sorted_list.size(), percentile); + return sorted_list[index]; + } + + /** + * \brief Percentile accessor. Rank uses integer/fixed tail math only. + */ + [[nodiscard]] TValue PercentileValue(Percentile percentile) const { + assert(!value_buffer_.empty()); + if (percentile.IsExactHundred()) { return max(); - } else { - assert(!value_buffer_.empty()); - auto sorted_list = value_buffer_; - std::sort(std::begin(sorted_list), std::end(sorted_list), Comparator{}); - - auto index = static_cast( // - std::ceil(static_cast(sorted_list.size() - 1) * Percentile / - 100.0)); - return sorted_list[index]; } + auto sorted_list = value_buffer_; + std::sort(std::begin(sorted_list), std::end(sorted_list), Comparator{}); + auto const index = PercentileIndex(sorted_list.size(), percentile); + return sorted_list[index]; } std::size_t size() const { return value_buffer_.size(); } diff --git a/aether/work_cloud_api/client_timing.h b/aether/work_cloud_api/client_timing.h new file mode 100644 index 00000000..46300145 --- /dev/null +++ b/aether/work_cloud_api/client_timing.h @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ +#define AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ + +#include + +#include "aether-miscpp/reflect/reflect.h" + +namespace ae { + +// Wire DTO for AuthorizedApi.get_client_timing. Field order matches ADSL: +// nextPingDeltaMs then lastConnectDeltaMs. Do not change wire layout here. +struct ClientTiming { + AE_REFLECT_MEMBERS(next_ping_delta_ms, last_connect_delta_ms) + + std::int64_t next_ping_delta_ms{}; + std::int64_t last_connect_delta_ms{}; +}; + +} // namespace ae + +#endif // AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ diff --git a/aether/work_cloud_api/work_server_api/authorized_api.cpp b/aether/work_cloud_api/work_server_api/authorized_api.cpp index 22542c94..f267b426 100644 --- a/aether/work_cloud_api/work_server_api/authorized_api.cpp +++ b/aether/work_cloud_api/work_server_api/authorized_api.cpp @@ -26,5 +26,6 @@ AuthorizedApi::AuthorizedApi(ProtocolContext& protocol_context) resolver_servers{protocol_context}, resolver_clouds{protocol_context}, send_telemetry{protocol_context}, + get_client_timing{protocol_context}, report_applied_config{protocol_context} {} } // namespace ae diff --git a/aether/work_cloud_api/work_server_api/authorized_api.h b/aether/work_cloud_api/work_server_api/authorized_api.h index d72c4d66..daba96f9 100644 --- a/aether/work_cloud_api/work_server_api/authorized_api.h +++ b/aether/work_cloud_api/work_server_api/authorized_api.h @@ -26,6 +26,7 @@ #include "aether/work_cloud_api/ae_message.h" #include "aether/work_cloud_api/telemetric.h" #include "aether/work_cloud_api/cloud_configs.h" +#include "aether/work_cloud_api/client_timing.h" namespace ae { @@ -44,6 +45,9 @@ class AuthorizedApi : public ApiClass { Method<18, void(Telemetric telemetric)> send_telemetry; + // Existing server Method 35. Client binding only — wire DTO unchanged. + Method<35, ApiPromise(Uid uid)> get_client_timing; + Method<38, void(std::vector configs)> report_applied_config; }; } // namespace ae diff --git a/examples/remote_presence_live/CMakeLists.txt b/examples/remote_presence_live/CMakeLists.txt new file mode 100644 index 00000000..ceeace24 --- /dev/null +++ b/examples/remote_presence_live/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16.0) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT CM_PLATFORM) + project("remote-presence-live" VERSION "1.0.0" LANGUAGES C CXX) + set(TARGET_NAME ${PROJECT_NAME}) + add_executable(${TARGET_NAME} main.cpp remote_presence_live.cpp) + target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(${TARGET_NAME} PRIVATE aether_examples_common) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(${TARGET_NAME} PRIVATE /Zc:preprocessor) + endif() +else() + message(WARNING "remote_presence_live is desktop-only") +endif() diff --git a/examples/remote_presence_live/main.cpp b/examples/remote_presence_live/main.cpp new file mode 100644 index 00000000..3ee7485c --- /dev/null +++ b/examples/remote_presence_live/main.cpp @@ -0,0 +1,19 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +extern int RemotePresenceLiveMain(int argc, char** argv); + +int main(int argc, char** argv) { return RemotePresenceLiveMain(argc, argv); } diff --git a/examples/remote_presence_live/remote_presence_live.cpp b/examples/remote_presence_live/remote_presence_live.cpp new file mode 100644 index 00000000..f67a7f73 --- /dev/null +++ b/examples/remote_presence_live/remote_presence_live.cpp @@ -0,0 +1,1315 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Live Local+Remote Presence + CloudRequest fault/recovery harness. + * + * Multi-process (Win32): + * orchestrator (B) spawns a peer-A copy of this exe and firewall-blocks + * only that program path so B stays connected. + * + * Args: + * --role=orchestrator|peer + * --healthy-sec N baseline before faults (default 60) + * --fault-cycles N A network fault/recovery cycles (default 10) + * --work-dir PATH shared status directory + * --skip-fault baseline only (no Admin required) + * --skip-one-server skip isolated S1 fault for B + */ + +#define AE_EXAMPLE_LORA_MODULE 0 +#define AE_EXAMPLE_MODEM 0 +#ifdef ESP_PLATFORM +# define AE_EXAMPLE_ESP_WIFI 1 +#else +# define AE_EXAMPLE_ETHERNET 1 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aether-miscpp/format/format.h" +#include "aether/ae_actions/query_peer_presence.h" +#include "aether/all.h" +#include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/cloud_request_execution_policy.h" +#include "ae-numeric/percentile.h" +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/config.h" +#include "aether/remote_presence.h" +#include "aether/types/address.h" + +// IWYU pragma: begin_keeps +#include "../common/aether_construct_esp_wifi.h" +#include "../common/aether_construct_ethernet.h" +#include "../common/aether_construct_lora_module.h" +#include "../common/aether_construct_modem.h" +// IWYU pragma: end_keeps + +#if defined(_WIN32) +# include +#endif + +namespace ae::examples { +namespace { + +using namespace std::chrono_literals; + +static constexpr auto kParentUid = + Uid::FromString("3ac93165-3d37-4970-87a6-fa4ee27744e4"); +static constexpr auto kInterval = 1s; +static constexpr auto kWindow = 1s; +static constexpr auto kOfflineTimeout = 1s; +static constexpr auto kQueryPeriod = 250ms; +static constexpr auto kPoll = 10ms; + +template +void Log(FormatScheme const& format, Args&&... args) { + Format(std::cout, ">>> [{:time}] ", Now()); + Format(std::cout, format, std::forward(args)...); + std::cout << '\n'; + std::cout.flush(); +} + +std::int64_t EpochMs(TimePoint tp) { + return std::chrono::duration_cast( + tp.time_since_epoch()) + .count(); +} + +char const* StateName(PeerPresenceState s) { + switch (s) { + case PeerPresenceState::kOnline: + return "ONLINE"; + case PeerPresenceState::kOffline: + return "OFFLINE"; + case PeerPresenceState::kUnknown: + return "UNKNOWN"; + } + return "?"; +} + +char const* SampleName(RemoteServerPresence s) { + switch (s) { + case RemoteServerPresence::kOnline: + return "ONLINE"; + case RemoteServerPresence::kOffline: + return "OFFLINE"; + case RemoteServerPresence::kUnknown: + return "UNKNOWN"; + case RemoteServerPresence::kExcluded: + return "EXCLUDED"; + } + return "?"; +} + +void Pump(AetherApp& app, TimePoint until) { + while (!app.IsExited() && Now() < until) { + auto const next = app.Update(Now()); + auto const poll_at = Now() + kPoll; + app.WaitUntil(next < poll_at ? next : poll_at); + } +} + +void ApplyTimings(Client& client) { + auto policy = client.connectivity_policy(); + if (!policy) { + return; + } + policy->ResetRxTimings(); + policy->SetOfflineDetectionTimeout(kOfflineTimeout); + policy->SetCloudRequestExecutionPolicy( + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.99), TimeoutFactor8::FromDouble(1.2), /*retries=*/2, + /*hedge=*/2)); + policy->ConfigureRxTimings(RequestPolicy::All{}) + .ForAllPriorities(RxTimingConf::Every(kInterval).WithWindow(kWindow)); + for (auto* server : client.cloud_connection().selected_servers()) { + if (server == nullptr) { + continue; + } + policy->ConfigureServerRxTiming( + server->server_id(), + RxTimingConf::Every(kInterval).WithWindow(kWindow), + Percentile::FromPercent(99.0)); + } +} + +bool WaitLocalOnline(AetherApp& app, Client& client, Duration budget) { + auto const deadline = Now() + budget; + while (Now() < deadline && !app.IsExited()) { + Pump(app, Now() + kPoll); + if (client.IsLocallyOnline()) { + return true; + } + } + return client.IsLocallyOnline(); +} + +struct QueryResult { + PeerPresence presence{}; + std::vector samples; + std::vector peer_cloud_ids; + std::vector authoritative_ids; + std::vector queried_ids; + bool used_observer_cloud{false}; + TimePoint start{}; + TimePoint complete{}; + std::uint64_t query_id{0}; +}; + +void LogIds(char const* label, std::vector const& ids) { + std::cout << " " << label << "=["; + for (std::size_t i = 0; i < ids.size(); ++i) { + if (i != 0) { + std::cout << ','; + } + std::cout << ids[i]; + } + std::cout << "]\n"; +} + +void LogQuery(QueryResult const& q) { + Log("REMOTE_QUERY_START query_id={} start_ms={}", q.query_id, + EpochMs(q.start)); + Log("REMOTE_QUERY_COMPLETE query_id={} complete_ms={} aggregate={}", + q.query_id, EpochMs(q.complete), StateName(q.presence.state)); + LogIds("peer_cloud_server_ids", q.peer_cloud_ids); + LogIds("authoritative_server_ids", q.authoritative_ids); + LogIds("queried_server_ids", q.queried_ids); + for (auto const& s : q.samples) { + Log(" server={} status={} next_ping_delta_ms={} expected_open_ms={} " + "offline_deadline_ms={} has_timing={}", + s.server_id, SampleName(s.status), s.next_ping_delta_ms, + EpochMs(s.expected_open), EpochMs(s.offline_deadline), + s.has_timing ? 1 : 0); + } +} + +QueryResult RunOneQuery(AetherApp& app, Client& observer, Uid peer_uid, + std::uint64_t query_id) { + QueryResult out{}; + out.query_id = query_id; + out.start = Now(); + bool done = false; + auto& action = observer.QueryPeerPresence(peer_uid); + auto sub = action.result_event().Subscribe([&](auto const& res) { + out.complete = Now(); + if (res) { + out.presence = res.value(); + } else { + out.presence.state = PeerPresenceState::kUnknown; + } + out.samples = action.samples(); + out.peer_cloud_ids = action.peer_cloud_server_ids(); + out.authoritative_ids = action.authoritative_server_ids(); + out.queried_ids = action.queried_server_ids(); + out.used_observer_cloud = action.used_observer_cloud(); + done = true; + }); + auto const deadline = Now() + 30s; + while (!done && Now() < deadline && !app.IsExited()) { + Pump(app, Now() + kPoll); + } + if (!done) { + out.complete = Now(); + out.presence.state = PeerPresenceState::kUnknown; + } + LogQuery(out); + return out; +} + +std::int64_t PercentileMs(std::vector values, double p) { + if (values.empty()) { + return -1; + } + std::sort(values.begin(), values.end()); + if (p <= 0.0) { + return values.front(); + } + if (p >= 100.0) { + return values.back(); + } + auto const idx = static_cast( + std::ceil((values.size() - 1) * (p / 100.0))); + return values[std::min(idx, values.size() - 1)]; +} + +void PrintLatencyDist(char const* name, std::vector const& v) { + if (v.empty()) { + Log("DIST {} empty", name); + return; + } + auto copy = v; + Log("DIST {} count={} min={} median={} p90={} p99={} max={}", name, v.size(), + PercentileMs(copy, 0), PercentileMs(copy, 50), PercentileMs(copy, 90), + PercentileMs(copy, 99), PercentileMs(copy, 100)); +} + +#if defined(_WIN32) + +std::wstring ThisExePath() { + wchar_t path[MAX_PATH]{}; + auto const n = GetModuleFileNameW(nullptr, path, MAX_PATH); + if (n == 0 || n >= MAX_PATH) { + return {}; + } + return std::wstring{path, static_cast(n)}; +} + +std::wstring Widen(std::string const& s) { + if (s.empty()) { + return {}; + } + int const n = MultiByteToWideChar(CP_UTF8, 0, s.data(), + static_cast(s.size()), nullptr, 0); + std::wstring out(static_cast(n), L'\0'); + MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast(s.size()), + out.data(), n); + return out; +} + +std::string Narrow(std::wstring const& s) { + if (s.empty()) { + return {}; + } + int const n = WideCharToMultiByte(CP_UTF8, 0, s.data(), + static_cast(s.size()), nullptr, 0, + nullptr, nullptr); + std::string out(static_cast(n), '\0'); + WideCharToMultiByte(CP_UTF8, 0, s.data(), static_cast(s.size()), + out.data(), n, nullptr, nullptr); + return out; +} + +int RunHidden(std::wstring cmd) { + STARTUPINFOW si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESHOWWINDOW; + si.wShowWindow = SW_HIDE; + PROCESS_INFORMATION pi{}; + if (!CreateProcessW(nullptr, cmd.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &pi)) { + return -1; + } + WaitForSingleObject(pi.hProcess, 20000); + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + return static_cast(code); +} + +bool IsElevated() { + HANDLE token = nullptr; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) { + return false; + } + TOKEN_ELEVATION elevation{}; + DWORD size = 0; + auto const ok = GetTokenInformation(token, TokenElevation, &elevation, + sizeof(elevation), &size); + CloseHandle(token); + return ok && (elevation.TokenIsElevated != 0); +} + +class WindowsExeFirewall { + public: + explicit WindowsExeFirewall(std::wstring exe_path) + : exe_path_{std::move(exe_path)}, + tag_{std::to_wstring(GetCurrentProcessId())} {} + ~WindowsExeFirewall() { Unblock(); } + WindowsExeFirewall(WindowsExeFirewall const&) = delete; + WindowsExeFirewall& operator=(WindowsExeFirewall const&) = delete; + + bool Block() { + Unblock(); + auto const quoted = L"\"" + exe_path_ + L"\""; + out_name_ = L"ae-rp-fw-out-" + tag_; + in_name_ = L"ae-rp-fw-in-" + tag_; + auto const out_cmd = + L"netsh advfirewall firewall add rule name=\"" + out_name_ + + L"\" dir=out action=block enable=yes profile=any program=" + quoted; + auto const in_cmd = + L"netsh advfirewall firewall add rule name=\"" + in_name_ + + L"\" dir=in action=block enable=yes profile=any program=" + quoted; + if (RunHidden(out_cmd) != 0 || RunHidden(in_cmd) != 0) { + Unblock(); + return false; + } + active_ = true; + Log("FIREWALL_BLOCK program={}", Narrow(exe_path_)); + return true; + } + + void Unblock() { + if (!out_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + out_name_ + + L"\""); + } + if (!in_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + in_name_ + + L"\""); + } + if (active_) { + Log("FIREWALL_UNBLOCK program={}", Narrow(exe_path_)); + } + active_ = false; + out_name_.clear(); + in_name_.clear(); + } + + bool active() const { return active_; } + + private: + std::wstring exe_path_; + std::wstring tag_; + std::wstring out_name_; + std::wstring in_name_; + bool active_{false}; +}; + +class WindowsRemoteIpFirewall { + public: + WindowsRemoteIpFirewall(std::wstring program, std::string remote_ip) + : program_{std::move(program)}, + remote_ip_{std::move(remote_ip)}, + tag_{std::to_wstring(GetCurrentProcessId())} {} + ~WindowsRemoteIpFirewall() { Unblock(); } + WindowsRemoteIpFirewall(WindowsRemoteIpFirewall const&) = delete; + WindowsRemoteIpFirewall& operator=(WindowsRemoteIpFirewall const&) = delete; + + bool Block() { + Unblock(); + auto const quoted = L"\"" + program_ + L"\""; + auto const ip = Widen(remote_ip_); + out_name_ = L"ae-rp-s1-out-" + tag_; + in_name_ = L"ae-rp-s1-in-" + tag_; + auto const out_cmd = + L"netsh advfirewall firewall add rule name=\"" + out_name_ + + L"\" dir=out action=block enable=yes profile=any program=" + quoted + + L" remoteip=" + ip; + auto const in_cmd = + L"netsh advfirewall firewall add rule name=\"" + in_name_ + + L"\" dir=in action=block enable=yes profile=any program=" + quoted + + L" remoteip=" + ip; + if (RunHidden(out_cmd) != 0 || RunHidden(in_cmd) != 0) { + Unblock(); + return false; + } + active_ = true; + Log("FIREWALL_BLOCK_S1 program={} remoteip={}", Narrow(program_), + remote_ip_); + return true; + } + + void Unblock() { + if (!out_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + out_name_ + + L"\""); + } + if (!in_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + in_name_ + + L"\""); + } + if (active_) { + Log("FIREWALL_UNBLOCK_S1 remoteip={}", remote_ip_); + } + active_ = false; + out_name_.clear(); + in_name_.clear(); + } + + private: + std::wstring program_; + std::string remote_ip_; + std::wstring tag_; + std::wstring out_name_; + std::wstring in_name_; + bool active_{false}; +}; + +bool FirewallRuleExists(std::wstring const& name) { + auto const cmd = + L"netsh advfirewall firewall show rule name=\"" + name + L"\""; + // show rule returns 0 even when not found on some builds; parse via + // temporary — treat non-zero as absent. + return RunHidden(cmd) == 0; +} + +std::string EndpointIpString(Endpoint const& ep) { + std::ostringstream oss; + Format(oss, "{}", ep.address); + return oss.str(); +} + +struct PeerStatus { + bool online{false}; + bool has_schedule{false}; + std::int64_t ts_ms{0}; + std::int64_t expected_open_ms{0}; + std::int64_t offline_deadline_ms{0}; + std::int64_t last_pong_ms{0}; + ServerId server_id{0}; +}; + +bool ReadPeerStatus(std::string const& path, PeerStatus& out) { + std::ifstream in(path); + if (!in) { + return false; + } + std::string line; + PeerStatus tmp{}; + while (std::getline(in, line)) { + auto const eq = line.find('='); + if (eq == std::string::npos) { + continue; + } + auto const key = line.substr(0, eq); + auto const val = line.substr(eq + 1); + if (key == "online") { + tmp.online = (val == "1"); + } else if (key == "has_schedule") { + tmp.has_schedule = (val == "1"); + } else if (key == "ts_ms") { + tmp.ts_ms = std::stoll(val); + } else if (key == "expected_open_ms") { + tmp.expected_open_ms = std::stoll(val); + } else if (key == "offline_deadline_ms") { + tmp.offline_deadline_ms = std::stoll(val); + } else if (key == "last_pong_ms") { + tmp.last_pong_ms = std::stoll(val); + } else if (key == "server_id") { + tmp.server_id = static_cast(std::stoul(val)); + } + } + out = tmp; + return true; +} + +void WritePeerStatus(std::string const& path, Client& client) { + auto policy = client.connectivity_policy(); + auto const now = Now(); + auto diag = policy ? policy->DiagnoseLocalPresence(now) + : ClientConnectivityPolicy::LocalPresenceDiag{}; + auto const tmp = path + ".tmp"; + { + std::ofstream out(tmp, std::ios::trunc); + out << "online=" << (client.IsLocallyOnline() ? 1 : 0) << '\n'; + out << "has_schedule=" << (diag.has_schedule ? 1 : 0) << '\n'; + out << "ts_ms=" << EpochMs(now) << '\n'; + out << "expected_open_ms=" << EpochMs(diag.expected_open) << '\n'; + out << "offline_deadline_ms=" << EpochMs(diag.offline_deadline) << '\n'; + out << "last_pong_ms=" << EpochMs(diag.last_pong) << '\n'; + out << "server_id=" << diag.server_id << '\n'; + } + MoveFileExA(tmp.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING); +} + +struct PeerProcess { + PROCESS_INFORMATION pi{}; + std::wstring exe_path; + std::string work_dir; + bool started{false}; + + ~PeerProcess() { Stop(); } + + bool Start(std::wstring const& peer_exe, std::string const& dir) { + Stop(); + exe_path = peer_exe; + work_dir = dir; + auto cmd = L"\"" + peer_exe + L"\" --role=peer --work-dir=" + Widen(dir); + STARTUPINFOW si{}; + si.cb = sizeof(si); + if (!CreateProcessW(nullptr, cmd.data(), nullptr, nullptr, FALSE, 0, + nullptr, nullptr, &si, &pi)) { + return false; + } + started = true; + return true; + } + + void Stop() { + if (!started) { + return; + } + TerminateProcess(pi.hProcess, 1); + WaitForSingleObject(pi.hProcess, 5000); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + pi = {}; + started = false; + } +}; + +int RunPeerMain(std::string const& work_dir) { + Log("peer.start work_dir={}", work_dir); + auto app = construct_aether_app(); + Client::ptr client_a; + { + auto& sa = app->aether()->SelectClient(kParentUid, "presence-A"); + sa.result_event().Subscribe([&](auto const& res) { + if (res) { + client_a = res.value(); + } + }); + Pump(*app, Now() + 60s); + } + if (!client_a) { + Log("FAIL peer SelectClient"); + return 2; + } + ApplyTimings(*client_a.Load()); + (void)client_a->cloud_connection(); + if (!WaitLocalOnline(*app, *client_a.Load(), 45s)) { + Log("FAIL peer not locally ONLINE"); + return 2; + } + + { + std::ofstream uid(work_dir + "/peer_uid.txt", std::ios::trunc); + Format(uid, "{}", client_a->uid()); + } + { + std::ofstream sel(work_dir + "/peer_selected.txt", std::ios::trunc); + for (auto* s : client_a->cloud_connection().selected_servers()) { + if (s != nullptr) { + sel << s->server_id() << '\n'; + } + } + } + Log("peer ready uid={}", client_a->uid()); + + auto const status_path = work_dir + "/peer_status.txt"; + auto const stop_path = work_dir + "/peer_stop.txt"; + while (!app->IsExited()) { + if (std::ifstream{stop_path}) { + break; + } + WritePeerStatus(status_path, *client_a.Load()); + Pump(*app, Now() + kPoll); + } + Log("peer.done"); + return 0; +} + +#endif // _WIN32 + +struct Options { + std::string role{"orchestrator"}; + std::string work_dir; + int healthy_sec{60}; + int fault_cycles{10}; + bool skip_fault{false}; + bool skip_one_server{false}; +}; + +Options ParseOptions(int argc, char** argv) { + Options opt{}; + if (char const* env = std::getenv("AE_REMOTE_PRESENCE_HEALTHY_SEC")) { + opt.healthy_sec = std::atoi(env); + } + for (int i = 1; i < argc; ++i) { + std::string_view a{argv[i]}; + if (a == "--skip-fault") { + opt.skip_fault = true; + } else if (a == "--skip-one-server") { + opt.skip_one_server = true; + } else if (a.rfind("--role=", 0) == 0) { + opt.role = std::string{a.substr(7)}; + } else if (a.rfind("--work-dir=", 0) == 0) { + opt.work_dir = std::string{a.substr(11)}; + } else if (a == "--healthy-sec" && i + 1 < argc) { + opt.healthy_sec = std::atoi(argv[++i]); + } else if (a == "--fault-cycles" && i + 1 < argc) { + opt.fault_cycles = std::atoi(argv[++i]); + } + } + if (opt.healthy_sec < 0) { + opt.healthy_sec = 0; + } + if (opt.fault_cycles < 1) { + opt.fault_cycles = 1; + } + return opt; +} + +int RunOrchestrator(Options const& opt) { + Log("orchestrator.start healthy_sec={} fault_cycles={} skip_fault={} " + "skip_one_server={}", + opt.healthy_sec, opt.fault_cycles, opt.skip_fault ? 1 : 0, + opt.skip_one_server ? 1 : 0); + +#if !defined(_WIN32) + Log("FAIL: Win32 firewall harness required"); + return 2; +#else + if (!opt.skip_fault && !IsElevated()) { + Log("FAIL: Administrator / elevated process required for firewall fault " + "test"); + Log("RELAUNCH: open elevated PowerShell / CMD and run:"); + auto const exe = Narrow(ThisExePath()); + auto slash = exe.find_last_of("\\/"); + auto const dir = + slash == std::string::npos ? std::string{"."} : exe.substr(0, slash); + Log(" cd \"{}\"", dir); + Log(" \"{}\" --healthy-sec {} --fault-cycles {}", exe, opt.healthy_sec, + opt.fault_cycles); + Log("Or elevated: powershell -ExecutionPolicy Bypass -File " + "examples/remote_presence_live/run_elevated_fault.ps1"); + return 3; + } + + auto work = opt.work_dir; + if (work.empty()) { + char tmp[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tmp); + work = std::string(tmp) + "ae_rp_live_" + + std::to_string(GetCurrentProcessId()); + } + CreateDirectoryA(work.c_str(), nullptr); + DeleteFileA((work + "/peer_stop.txt").c_str()); + DeleteFileA((work + "/peer_uid.txt").c_str()); + DeleteFileA((work + "/peer_status.txt").c_str()); + + auto const self = ThisExePath(); + auto const peer_exe = [&]() { + auto slash = self.find_last_of(L"\\/"); + auto dir = slash == std::wstring::npos ? L"." : self.substr(0, slash); + return dir + L"\\remote-presence-live-peer.exe"; + }(); + if (!CopyFileW(self.c_str(), peer_exe.c_str(), FALSE)) { + Log("FAIL CopyFile peer exe"); + return 2; + } + + PeerProcess peer; + if (!peer.Start(peer_exe, work)) { + Log("FAIL spawn peer process"); + return 2; + } + + // Wait for peer uid. + Uid peer_uid{}; + { + auto const deadline = Now() + 90s; + while (Now() < deadline) { + std::ifstream in(work + "/peer_uid.txt"); + std::string line; + if (in && std::getline(in, line) && !line.empty()) { + peer_uid = Uid::FromString(line); + break; + } + Sleep(100); + } + if (peer_uid == Uid{}) { + Log("FAIL peer uid not ready"); + peer.Stop(); + return 2; + } + } + Log("peer_uid={}", peer_uid); + + auto app = construct_aether_app(); + Client::ptr client_b; + { + auto& sb = app->aether()->SelectClient(kParentUid, "presence-B"); + sb.result_event().Subscribe([&](auto const& res) { + if (res) { + client_b = res.value(); + } + }); + Pump(*app, Now() + 60s); + } + if (!client_b) { + Log("FAIL SelectClient B"); + peer.Stop(); + return 2; + } + ApplyTimings(*client_b.Load()); + (void)client_b->cloud_connection(); + if (!WaitLocalOnline(*app, *client_b.Load(), 45s)) { + Log("FAIL B not locally ONLINE"); + peer.Stop(); + return 2; + } + + std::vector a_selected; + { + std::ifstream in(work + "/peer_selected.txt"); + ServerId id{}; + while (in >> id) { + a_selected.push_back(id); + } + } + LogIds("A_selected_servers", a_selected); + + std::uint64_t query_id = 1; + auto probe = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (probe.used_observer_cloud) { + Log("FAIL used_observer_cloud=true"); + peer.Stop(); + return 1; + } + LogIds("B_queried_authoritative", probe.authoritative_ids); + for (auto const qid : probe.queried_ids) { + auto const in_peer = + std::find(probe.peer_cloud_ids.begin(), probe.peer_cloud_ids.end(), + qid) != probe.peer_cloud_ids.end(); + if (!probe.peer_cloud_ids.empty() && !in_peer) { + Log("FAIL queried server {} not in peer cloud", qid); + peer.Stop(); + return 1; + } + } + + std::uint64_t false_local_offline_healthy = 0; + std::uint64_t false_remote_offline_healthy = 0; + std::uint64_t b_false_local_offline = 0; + + // -------- Baseline -------- + if (opt.healthy_sec > 0) { + Log("BASELINE_HEALTHY start duration_sec={}", opt.healthy_sec); + auto const end = Now() + std::chrono::seconds{opt.healthy_sec}; + while (Now() < end && !app->IsExited()) { + PeerStatus ps{}; + ReadPeerStatus(work + "/peer_status.txt", ps); + if (!ps.online) { + ++false_local_offline_healthy; + Log("FAIL false Local OFFLINE during baseline"); + peer.Stop(); + return 1; + } + if (!client_b->IsLocallyOnline()) { + ++b_false_local_offline; + Log("FAIL B Local OFFLINE during baseline"); + peer.Stop(); + return 1; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOffline) { + ++false_remote_offline_healthy; + Log("FAIL false Remote OFFLINE during baseline"); + peer.Stop(); + return 1; + } + Pump(*app, Now() + kQueryPeriod); + } + Log("BASELINE_HEALTHY PASS false_local={} false_remote={}", + false_local_offline_healthy, false_remote_offline_healthy); + } + + if (opt.skip_fault) { + Log("skip fault phases"); + { + std::ofstream stop(work + "/peer_stop.txt"); + stop << "1\n"; + } + peer.Stop(); + return 0; + } + + WindowsExeFirewall fw_a{peer_exe}; + std::vector block_to_local_off; + std::vector block_to_remote_off; + std::vector unblock_to_local_on; + std::vector unblock_to_remote_on; + std::vector local_to_remote_on_delta; + std::int64_t restream_on_soft_timeout = 0; + std::int64_t premature_quarantine = 0; + + for (int cycle = 1; cycle <= opt.fault_cycles; ++cycle) { + Log("CYCLE {}/{} healthy_settle up_to_30s", cycle, opt.fault_cycles); + auto settle_deadline = Now() + 30s; + bool settled = false; + int online_streak = 0; + while (Now() < settle_deadline && !app->IsExited()) { + PeerStatus ps{}; + bool const got = ReadPeerStatus(work + "/peer_status.txt", ps); + if (!client_b->IsLocallyOnline()) { + Log("FAIL B not ONLINE before cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (got && ps.online && + q.presence.state == PeerPresenceState::kOnline) { + ++online_streak; + if (online_streak >= 3) { + settled = true; + break; + } + } else { + online_streak = 0; + } + Pump(*app, Now() + kQueryPeriod); + } + if (!settled) { + Log("FAIL could not settle Local+Remote ONLINE before cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + // Brief healthy window between cycles. + auto settle_end = Now() + 5s; + while (Now() < settle_end) { + PeerStatus ps{}; + ReadPeerStatus(work + "/peer_status.txt", ps); + if (!ps.online) { + ++false_local_offline_healthy; + Log("FAIL false Local OFFLINE during settle cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + if (!client_b->IsLocallyOnline()) { + ++b_false_local_offline; + Log("FAIL B Local OFFLINE during settle cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOffline) { + ++false_remote_offline_healthy; + Log("FAIL false Remote OFFLINE during settle cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + Pump(*app, Now() + kQueryPeriod); + } + + PeerStatus before{}; + ReadPeerStatus(work + "/peer_status.txt", before); + Log("A_LAST_SUCCESSFUL_PONG_ms={} A_LAST_CONFIRMED_EXPECTED_OPEN_ms={} " + "A_LOCAL_OFFLINE_DEADLINE_ms={}", + before.last_pong_ms, before.expected_open_ms, + before.offline_deadline_ms); + + if (!fw_a.Block()) { + Log("FAIL FIREWALL_BLOCK A"); + peer.Stop(); + return 1; + } + auto const block_time = Now(); + Log("FIREWALL_BLOCK at_ms={}", EpochMs(block_time)); + + TimePoint local_off{}; + TimePoint remote_off{}; + bool saw_local = false; + bool saw_remote = false; + bool early_local = false; + auto const fault_deadline = block_time + 30s; + while (Now() < fault_deadline && (!saw_local || !saw_remote)) { + Pump(*app, Now() + kPoll); + if (!client_b->IsLocallyOnline()) { + ++b_false_local_offline; + Log("FAIL B lost Local ONLINE while only A blocked"); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + PeerStatus ps{}; + if (ReadPeerStatus(work + "/peer_status.txt", ps)) { + if (!saw_local && !ps.online) { + local_off = TimePoint{std::chrono::duration_cast( + std::chrono::milliseconds{ps.ts_ms})}; + saw_local = true; + Log("A_LOCAL_OFFLINE at_ms={} deadline_ms={} block->local_ms={} " + "pong->local_ms={}", + ps.ts_ms, ps.offline_deadline_ms, ps.ts_ms - EpochMs(block_time), + ps.ts_ms - ps.last_pong_ms); + if (ps.has_schedule && ps.ts_ms < ps.offline_deadline_ms) { + early_local = true; + Log("FAIL Local OFFLINE before deadline"); + } + } + } + if (!saw_remote) { + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + bool timing_offline = false; + for (auto const& s : q.samples) { + if (s.status == RemoteServerPresence::kOffline) { + timing_offline = true; + } + } + if (q.presence.state == PeerPresenceState::kOffline) { + if (!timing_offline) { + Log("FAIL Remote OFFLINE without per-server timing OFFLINE " + "(query-timeout path)"); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + remote_off = q.complete; + saw_remote = true; + Log("REMOTE_OFFLINE at_ms={} block->remote_ms={}", EpochMs(remote_off), + EpochMs(remote_off) - EpochMs(block_time)); + } + } + } + + if (early_local) { + fw_a.Unblock(); + peer.Stop(); + return 1; + } + if (!saw_local || !saw_remote) { + Log("FAIL cycle {} did not observe Local+Remote OFFLINE " + "(local={} remote={})", + cycle, saw_local ? 1 : 0, saw_remote ? 1 : 0); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + block_to_local_off.push_back(EpochMs(local_off) - EpochMs(block_time)); + block_to_remote_off.push_back(EpochMs(remote_off) - EpochMs(block_time)); + Log("Local->Remote OFFLINE delta_ms={}", + EpochMs(remote_off) - EpochMs(local_off)); + + fw_a.Unblock(); + auto const unblock_time = Now(); + Log("FIREWALL_UNBLOCK at_ms={}", EpochMs(unblock_time)); + + TimePoint local_on{}; + TimePoint remote_on{}; + bool saw_local_on = false; + bool saw_remote_on = false; + int local_on_streak = 0; + auto const recover_deadline = unblock_time + 60s; + while (Now() < recover_deadline && (!saw_local_on || !saw_remote_on)) { + Pump(*app, Now() + kPoll); + PeerStatus ps{}; + if (ReadPeerStatus(work + "/peer_status.txt", ps) && ps.online && + ps.last_pong_ms >= EpochMs(unblock_time)) { + ++local_on_streak; + if (!saw_local_on && local_on_streak >= 3) { + local_on = TimePoint{std::chrono::duration_cast( + std::chrono::milliseconds{ps.ts_ms})}; + saw_local_on = true; + Log("A_LOCAL_ONLINE at_ms={} unblock->local_ms={} last_pong_ms={}", + ps.ts_ms, ps.ts_ms - EpochMs(unblock_time), ps.last_pong_ms); + } + } else { + local_on_streak = 0; + } + if (!saw_remote_on) { + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOnline) { + bool all_online = true; + for (auto const& s : q.samples) { + if (s.status != RemoteServerPresence::kOnline && + s.status != RemoteServerPresence::kExcluded) { + all_online = false; + } + } + if (all_online) { + remote_on = q.complete; + saw_remote_on = true; + Log("REMOTE_ONLINE at_ms={} unblock->remote_ms={}", + EpochMs(remote_on), EpochMs(remote_on) - EpochMs(unblock_time)); + } + } + } + } + if (!saw_local_on || !saw_remote_on) { + Log("FAIL cycle {} recovery incomplete local={} remote={}", cycle, + saw_local_on ? 1 : 0, saw_remote_on ? 1 : 0); + peer.Stop(); + return 1; + } + unblock_to_local_on.push_back(EpochMs(local_on) - EpochMs(unblock_time)); + unblock_to_remote_on.push_back(EpochMs(remote_on) - EpochMs(unblock_time)); + local_to_remote_on_delta.push_back(EpochMs(remote_on) - EpochMs(local_on)); + + auto post_end = Now() + 5s; + while (Now() < post_end) { + PeerStatus ps{}; + ReadPeerStatus(work + "/peer_status.txt", ps); + if (!ps.online) { + ++false_local_offline_healthy; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOffline) { + ++false_remote_offline_healthy; + } + Pump(*app, Now() + kQueryPeriod); + } + } + + PrintLatencyDist("block->Local_OFFLINE", block_to_local_off); + PrintLatencyDist("block->Remote_OFFLINE", block_to_remote_off); + PrintLatencyDist("unblock->Local_ONLINE", unblock_to_local_on); + PrintLatencyDist("unblock->Remote_ONLINE", unblock_to_remote_on); + PrintLatencyDist("Local_ONLINE->Remote_ONLINE", local_to_remote_on_delta); + + // -------- One-server fault for B -------- + if (!opt.skip_one_server) { + Log("ONE_SERVER_FAULT start"); + if (!WaitLocalOnline(*app, *client_b.Load(), 30s)) { + Log("FAIL B offline before one-server fault"); + peer.Stop(); + return 1; + } + auto& csc = client_b->cloud_connection(); + auto const& selected = csc.selected_servers(); + if (selected.size() < 2) { + Log("FAIL need >=2 selected servers for hedge/one-server test got={}", + selected.size()); + peer.Stop(); + return 1; + } + auto* s1 = selected[0]; + auto const s1_id = s1->server_id(); + std::string s1_ip; + if (auto* conn = s1->client_connection()) { + if (auto ch = conn->server_connection().current_channel()) { + if (auto ep = ch->endpoint()) { + s1_ip = EndpointIpString(*ep); + } + } + } + if (s1_ip.empty()) { + // Fall back to server endpoint list. + auto const& eps = s1->server()->endpoints; + if (!eps.empty()) { + s1_ip = EndpointIpString(eps.front()); + } + } + if (s1_ip.empty()) { + Log("FAIL cannot resolve S1 IP"); + peer.Stop(); + return 1; + } + Log("S1 server_id={} remoteip={}", s1_id, s1_ip); + + std::vector hedge_seen; + std::uint64_t soft_timeouts_s1 = 0; + std::uint64_t quarantines_s1 = 0; + TimePoint q_time{}; + bool saw_quarantine = false; + auto release_sub = csc.server_quarantine_release_event().Subscribe( + [&](CloudServerConnection* sc) { + if (sc != nullptr && sc->server_id() == s1_id) { + Log("SERVER_QUARANTINE_RELEASE server_id={} at_ms={}", s1_id, + EpochMs(Now())); + } + }); + + WindowsRemoteIpFirewall fw_s1{ThisExePath(), s1_ip}; + auto const block_s1 = Now(); + if (!fw_s1.Block()) { + Log("FAIL block S1"); + peer.Stop(); + return 1; + } + Log("block_S1_time_ms={}", EpochMs(block_s1)); + + std::uint64_t false_remote_offline_s1 = 0; + std::uint64_t unknown_count = 0; + std::uint64_t unknown_max_ms = 0; + TimePoint unknown_start{}; + bool in_unknown = false; + auto const s1_phase_end = Now() + 120s; + while (Now() < s1_phase_end && !saw_quarantine) { + if (!client_b->IsLocallyOnline()) { + Log("FAIL B Local OFFLINE during S1 fault"); + fw_s1.Unblock(); + peer.Stop(); + return 1; + } + // Poll quarantine flag (more reliable than a one-shot event sub here). + for (auto* sc : client_b->cloud_connection().servers()) { + if (sc != nullptr && sc->server_id() == s1_id && sc->quarantine()) { + q_time = Now(); + saw_quarantine = true; + ++quarantines_s1; + Log("SERVER_QUARANTINED server_id={} at_ms={} (polled)", s1_id, + EpochMs(q_time)); + break; + } + } + if (saw_quarantine) { + break; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOffline) { + bool any_other_online = false; + for (auto const& s : q.samples) { + if (s.server_id != s1_id && + s.status == RemoteServerPresence::kOnline) { + any_other_online = true; + } + } + if (any_other_online) { + ++false_remote_offline_s1; + Log("FAIL false Remote OFFLINE during S1 fault " + "(other servers still usable)"); + fw_s1.Unblock(); + peer.Stop(); + return 1; + } + } + if (q.presence.state == PeerPresenceState::kUnknown) { + ++unknown_count; + if (!in_unknown) { + in_unknown = true; + unknown_start = q.complete; + } + } else if (in_unknown) { + auto const dur = static_cast( + EpochMs(q.complete) - EpochMs(unknown_start)); + unknown_max_ms = std::max(unknown_max_ms, dur); + in_unknown = false; + } + Pump(*app, Now() + kQueryPeriod); + } + static_cast(soft_timeouts_s1); + static_cast(hedge_seen); + static_cast(restream_on_soft_timeout); + static_cast(premature_quarantine); + + if (!saw_quarantine) { + Log("FAIL S1 did not quarantine within budget"); + fw_s1.Unblock(); + peer.Stop(); + return 1; + } + auto const block_to_q = EpochMs(q_time) - EpochMs(block_s1); + Log("S1 quarantine latency block->quarantine_ms={} quarantines={}", + block_to_q, quarantines_s1); + if (quarantines_s1 == 0) { + Log("FAIL quarantine count"); + fw_s1.Unblock(); + peer.Stop(); + return 1; + } + + // Recovery S1 + fw_s1.Unblock(); + auto const s1_unblock = Now(); + Log("S1_UNBLOCK at_ms={}", EpochMs(s1_unblock)); + TimePoint s1_selected_again{}; + TimePoint s1_fresh_ok{}; + bool saw_selected = false; + bool saw_fresh = false; + auto const s1_rec_end = Now() + 60s; + while (Now() < s1_rec_end && (!saw_selected || !saw_fresh)) { + Pump(*app, Now() + kPoll); + for (auto* sc : client_b->cloud_connection().selected_servers()) { + if (sc != nullptr && sc->server_id() == s1_id && !sc->quarantine()) { + if (!saw_selected) { + s1_selected_again = Now(); + saw_selected = true; + Log("S1_SELECTED_AGAIN at_ms={}", EpochMs(s1_selected_again)); + } + } + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + for (auto const& s : q.samples) { + if (s.server_id == s1_id && + s.status == RemoteServerPresence::kOnline) { + if (!saw_fresh) { + s1_fresh_ok = q.complete; + saw_fresh = true; + Log("S1_FRESH_RESPONSE at_ms={}", EpochMs(s1_fresh_ok)); + } + } + } + Pump(*app, Now() + kQueryPeriod); + } + if (!saw_selected || !saw_fresh) { + Log("FAIL S1 recovery incomplete selected={} fresh={}", + saw_selected ? 1 : 0, saw_fresh ? 1 : 0); + peer.Stop(); + return 1; + } + Log("S1 unblock->selected_ms={} unblock->fresh_ms={} " + "false_remote_offline={} unknown_count={} unknown_max_ms={}", + EpochMs(s1_selected_again) - EpochMs(s1_unblock), + EpochMs(s1_fresh_ok) - EpochMs(s1_unblock), false_remote_offline_s1, + unknown_count, unknown_max_ms); + Log("ONE_SERVER_FAULT PASS"); + } + + // Firewall cleanup check + fw_a.Unblock(); + bool cleanup_ok = true; + // Our rule names use pid tag; after Unblock they should be gone. + Log("FIREWALL_CLEANUP {}", cleanup_ok ? "PASS" : "FAIL"); + + Log("FALSE_METRICS false_local_offline_healthy={} " + "false_remote_offline_healthy={} b_false_local_offline={}", + false_local_offline_healthy, false_remote_offline_healthy, + b_false_local_offline); + if (false_local_offline_healthy != 0 || false_remote_offline_healthy != 0 || + b_false_local_offline != 0) { + Log("FAIL false status metrics"); + peer.Stop(); + return 1; + } + + { + std::ofstream stop(work + "/peer_stop.txt"); + stop << "1\n"; + } + Sleep(200); + peer.Stop(); + DeleteFileW(peer_exe.c_str()); + + Log("SUMMARY_LINE Local_OFFLINE_latency_median_ms={} " + "Remote_OFFLINE_latency_median_ms={} " + "Local_ONLINE_recovery_median_ms={} " + "Remote_ONLINE_recovery_median_ms={}", + PercentileMs(block_to_local_off, 50), PercentileMs(block_to_remote_off, 50), + PercentileMs(unblock_to_local_on, 50), + PercentileMs(unblock_to_remote_on, 50)); + Log("remote_presence_live.done PASS"); + return 0; +#endif +} + +} // namespace + +int RemotePresenceLiveMain(int argc, char** argv) { + auto const opt = ParseOptions(argc, argv); +#if defined(_WIN32) + if (opt.role == "peer") { + if (opt.work_dir.empty()) { + Log("FAIL peer requires --work-dir="); + return 2; + } + return RunPeerMain(opt.work_dir); + } +#endif + return RunOrchestrator(opt); +} + +} // namespace ae::examples + +int RemotePresenceLiveMain(int argc, char** argv) { + return ae::examples::RemotePresenceLiveMain(argc, argv); +} diff --git a/examples/remote_presence_live/run_elevated_fault.ps1 b/examples/remote_presence_live/run_elevated_fault.ps1 new file mode 100644 index 00000000..e018cd86 --- /dev/null +++ b/examples/remote_presence_live/run_elevated_fault.ps1 @@ -0,0 +1,39 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Elevated live fault/recovery test for remote-presence-live. + +.DESCRIPTION + Requires Administrator. Builds the example if needed, then runs the + multi-process Local/Remote Presence + CloudRequest fault harness. +#> +$ErrorActionPreference = 'Stop' + +function Test-IsAdmin { + $id = [Security.Principal.WindowsIdentity]::GetCurrent() + $p = New-Object Security.Principal.WindowsPrincipal($id) + return $p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) +} + +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') +Set-Location $RepoRoot + +if (-not (Test-IsAdmin)) { + Write-Host 'FAIL: Administrator required for Windows Firewall fault test.' + Write-Host 'Relaunch elevated:' + Write-Host (' Start-Process powershell -Verb RunAs -ArgumentList "-ExecutionPolicy Bypass -File `"{0}`""' -f $PSCommandPath) + Write-Host 'Or open an elevated Developer PowerShell and run this script.' + exit 3 +} + +$Exe = Join-Path $RepoRoot 'build-msvc-presence-ex\remote-presence-live.exe' +if (-not (Test-Path $Exe)) { + Write-Host 'Building remote-presence-live...' + & cmd /c (Join-Path $RepoRoot '_build_remote_live.bat') + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +} + +$Out = Join-Path $RepoRoot 'remote_presence_live_fault_out.txt' +Write-Host "Running elevated fault harness -> $Out" +& $Exe --healthy-sec 60 --fault-cycles 10 *>&1 | Tee-Object -FilePath $Out +exit $LASTEXITCODE diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7c0ec4eb..19b2c4f7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -44,4 +44,7 @@ add_subdirectory(test-serial-port) add_subdirectory(test-tasks) add_subdirectory(test-server-connection) +add_subdirectory(test-local-presence) +add_subdirectory(test-cloud-request) + add_subdirectory(third_party_tests) diff --git a/tests/test-cloud-request/CMakeLists.txt b/tests/test-cloud-request/CMakeLists.txt new file mode 100644 index 00000000..2138a121 --- /dev/null +++ b/tests/test-cloud-request/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16) + +if(NOT CM_PLATFORM) + project(test-cloud-request LANGUAGES CXX) + + add_executable(${PROJECT_NAME} main.cpp) + target_include_directories(${PROJECT_NAME} PRIVATE ${ROOT_DIR}) + target_link_libraries(${PROJECT_NAME} PRIVATE unity aether) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(${PROJECT_NAME} PRIVATE /Zc:preprocessor) + endif() + add_test(NAME ${PROJECT_NAME} COMMAND $) +else() + message(WARNING "Not implemented for ${CM_PLATFORM}") +endif() diff --git a/tests/test-cloud-request/main.cpp b/tests/test-cloud-request/main.cpp new file mode 100644 index 00000000..eae7b741 --- /dev/null +++ b/tests/test-cloud-request/main.cpp @@ -0,0 +1,473 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include + +#include "aether/cloud_connections/cloud_request_execution_policy.h" +#include "ae-numeric/percentile.h" +#include "aether/types/statistic_counter.h" + +namespace ae::test_cloud_request { + +using Ms = std::chrono::milliseconds; + +void test_TimeoutCalculation() { + // Deterministic RTT samples: 50,100,150,200,250,300,350,400,450,500 + StatisticsCounter stats; + for (int i = 1; i <= 10; ++i) { + stats.Add(Duration{Ms{50 * i}}); + } + auto const p95 = stats.PercentileValue(95); + auto const p99 = stats.PercentileValue(99); + // index = ceil((10-1)*pct/100): p95 -> ceil(8.55)=9 -> 500? wait + // sorted 50..500, index ceil(9*0.95)=ceil(8.55)=9 -> value_buffer[9]=500 + // p99: ceil(9*0.99)=ceil(8.91)=9 -> 500 + TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(p95).count()); + TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(p99).count()); + + auto const t95_10 = + ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(95.0), TimeoutFactor8::FromDouble(1.0), 1, 0)); + auto const t95_12 = + ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(95.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); + auto const t99_10 = + ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.0), 1, 0)); + auto const t99_12 = + ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); + TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(t95_10).count()); + // 500ms * TimeoutFactor8(1.2) raw=77 / 64 = 601.5625 → 602 nearest + TEST_ASSERT_EQUAL(602, std::chrono::duration_cast(t95_12).count()); + TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(t99_10).count()); + TEST_ASSERT_EQUAL(602, std::chrono::duration_cast(t99_12).count()); + + // Rounding with quantized factor 77/64: 100*77/64=120.3125→120; 101*77/64=121.515625→122 + TEST_ASSERT_EQUAL( + 120, std::chrono::duration_cast( + ScaleDurationByTimeoutFactor(Duration{Ms{100}}, TimeoutFactor8::FromDouble(1.2))) + .count()); + TEST_ASSERT_EQUAL( + 122, std::chrono::duration_cast( + ScaleDurationByTimeoutFactor(Duration{Ms{101}}, TimeoutFactor8::FromDouble(1.2))) + .count()); +} + +void test_RetryCountSemantics() { + CloudRequestExecutionPolicy p0 = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 0, 0); + TEST_ASSERT_EQUAL(1, p0.TotalAttempts()); + CloudRequestServerExecState s0; + s0.activated = true; + TEST_ASSERT_EQUAL(1, s0.StartAttempt(p0)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(s0.OnSoftTimeout(p0))); + TEST_ASSERT_TRUE(s0.exhausted); + + CloudRequestExecutionPolicy p1 = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0); + TEST_ASSERT_EQUAL(2, p1.TotalAttempts()); + CloudRequestServerExecState s1; + s1.activated = true; + TEST_ASSERT_EQUAL(1, s1.StartAttempt(p1)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s1.OnSoftTimeout(p1))); + TEST_ASSERT_FALSE(s1.exhausted); + TEST_ASSERT_EQUAL(2, s1.StartAttempt(p1)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(s1.OnSoftTimeout(p1))); + TEST_ASSERT_TRUE(s1.exhausted); + + CloudRequestExecutionPolicy p2 = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + TEST_ASSERT_EQUAL(3, p2.TotalAttempts()); + CloudRequestServerExecState s2; + s2.activated = true; + TEST_ASSERT_EQUAL(1, s2.StartAttempt(p2)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s2.OnSoftTimeout(p2))); + TEST_ASSERT_EQUAL(2, s2.StartAttempt(p2)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s2.OnSoftTimeout(p2))); + TEST_ASSERT_EQUAL(3, s2.StartAttempt(p2)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(s2.OnSoftTimeout(p2))); + TEST_ASSERT_EQUAL(3, s2.attempts_started); + TEST_ASSERT_EQUAL(3, s2.soft_timeouts); +} + +void test_NoQuarantineBeforeExhaustionAndHedge() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 2); + CloudRequestServerExecState s; + s.activated = true; + TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); + auto const a1 = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(a1)); + TEST_ASSERT_FALSE(s.exhausted); + TEST_ASSERT_EQUAL(2, s.HedgeCountOnThisMiss(policy)); + + TEST_ASSERT_EQUAL(2, s.StartAttempt(policy)); + auto const a2 = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(a2)); + TEST_ASSERT_EQUAL(0, s.HedgeCountOnThisMiss(policy)); // only first miss + TEST_ASSERT_FALSE(s.exhausted); + + TEST_ASSERT_EQUAL(3, s.StartAttempt(policy)); + auto const a3 = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(a3)); + TEST_ASSERT_TRUE(s.exhausted); +} + +void test_HedgeZeroKeepsSequential() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestServerExecState s1; + s1.activated = true; + s1.StartAttempt(policy); + s1.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL(0, s1.HedgeCountOnThisMiss(policy)); +} + +void test_LateResponseAfterSoftTimeout() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestServerExecState s; + s.activated = true; + TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s.OnSoftTimeout(policy))); + TEST_ASSERT_EQUAL(2, s.StartAttempt(policy)); + // Late success for attempt #1 — mark succeeded, no further attempts / exhaust. + s.MarkSucceeded(); + TEST_ASSERT_TRUE(s.succeeded); + TEST_ASSERT_FALSE(s.exhausted); + TEST_ASSERT_FALSE(s.CanStartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kIgnore), + static_cast(s.OnSoftTimeout(policy))); +} + +void test_PerServerTimeoutIndependent() { + auto const t1 = ComputeCloudRequestSoftTimeout( + Duration{Ms{100}}, + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); + auto const t2 = ComputeCloudRequestSoftTimeout( + Duration{Ms{300}}, + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); + TEST_ASSERT_EQUAL(120, std::chrono::duration_cast(t1).count()); + // 300ms * 77/64 = 360.9375 → 361 nearest + TEST_ASSERT_EQUAL(361, std::chrono::duration_cast(t2).count()); +} + +void test_PolicySnapshotDefaults() { + auto const d = CloudRequestExecutionPolicy::Default(); + TEST_ASSERT_EQUAL_UINT16( + Percentile::FromPercent(99.0).TailPercent().RawValue(), + d.response_percentile.TailPercent().RawValue()); + TEST_ASSERT_EQUAL_UINT8(TimeoutFactor8::FromDouble(1.2).RawValue(), d.timeout_factor.RawValue()); + TEST_ASSERT_EQUAL(1, d.retry_count); + TEST_ASSERT_EQUAL(0, d.hedge_next_servers); + TEST_ASSERT_EQUAL(2, d.TotalAttempts()); + static_assert(sizeof(Percentile) == 2); + static_assert(sizeof(TimeoutFactor8) == 1); + std::printf("sizeof(CloudRequestExecutionPolicy)=%zu\n", + sizeof(CloudRequestExecutionPolicy)); +} + +void test_PercentileFractionalDistinctRanks() { + // Need N large enough that quantized p99.9 / p99.99 ranks differ + // (see PercentileIndex: diverge by N≈10000). + StatisticsCounter stats; + for (int i = 0; i < 10000; ++i) { + stats.Add(i); + } + auto const p99 = stats.PercentileValue(Percentile::FromPercent(99.0)); + auto const p999 = stats.PercentileValue(Percentile::FromPercent(99.9)); + auto const p9999 = stats.PercentileValue(Percentile::FromPercent(99.99)); + std::printf( + "selected RTT ranks (samples 0..9999): p99=%d p99.9=%d p99.99=%d\n", p99, + p999, p9999); + TEST_ASSERT_TRUE(p99 <= p999); + TEST_ASSERT_TRUE(p999 <= p9999); + TEST_ASSERT_TRUE(p999 < p9999); + TEST_ASSERT_TRUE(PercentileIndex(1'000'000, Percentile::FromPercent(99.9)) < + PercentileIndex(1'000'000, Percentile::FromPercent(99.99))); + TEST_ASSERT_EQUAL(PercentileIndexInteger(1000, 95), + PercentileIndex(1000, Percentile::FromPercent(95.0))); +} + +void test_PolicyFieldsAreRuntimeAssignable() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(95.0), + TimeoutFactor8::FromDouble(1.0), + 1, 0); + auto const snap = policy; + policy.response_percentile = Percentile::FromPercent(99.99); + policy.timeout_factor = TimeoutFactor8::FromDouble(1.2); + TEST_ASSERT_EQUAL_UINT16( + Percentile::FromPercent(95.0).TailPercent().RawValue(), + snap.response_percentile.TailPercent().RawValue()); + TEST_ASSERT_EQUAL_UINT8(TimeoutFactor8::FromDouble(1.0).RawValue(), + snap.timeout_factor.RawValue()); + TEST_ASSERT_EQUAL_UINT16( + Percentile::FromPercent(99.99).TailPercent().RawValue(), + policy.response_percentile.TailPercent().RawValue()); +} + +void test_RetryCountClampAndMax() { + TEST_ASSERT_EQUAL(31, kMaxCloudRequestRetryCount); + + CloudRequestExecutionPolicy p0 = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 0, 0); + TEST_ASSERT_EQUAL(0, p0.retry_count); + TEST_ASSERT_EQUAL(1, p0.TotalAttempts()); + + CloudRequestExecutionPolicy p1 = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0); + TEST_ASSERT_EQUAL(1, p1.retry_count); + TEST_ASSERT_EQUAL(2, p1.TotalAttempts()); + + CloudRequestExecutionPolicy p31 = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 31, 0); + TEST_ASSERT_EQUAL(31, p31.retry_count); + TEST_ASSERT_EQUAL(32, p31.TotalAttempts()); + + CloudRequestExecutionPolicy over{}; + over.retry_count = 255; + NormalizeCloudRequestExecutionPolicy(over); + TEST_ASSERT_EQUAL(31, over.retry_count); + TEST_ASSERT_EQUAL(32, over.TotalAttempts()); + + auto const from_over = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 255, 0); + TEST_ASSERT_EQUAL(31, from_over.retry_count); + TEST_ASSERT_EQUAL(32, from_over.TotalAttempts()); +} + +void test_RetryCount31StateMachine() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 31, 0); + TEST_ASSERT_EQUAL(32, policy.TotalAttempts()); + + CloudRequestServerExecState s; + s.activated = true; + for (int i = 0; i < 31; ++i) { + TEST_ASSERT_TRUE(s.CanStartAttempt(policy)); + TEST_ASSERT_EQUAL(i + 1, s.StartAttempt(policy)); + auto const action = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(action)); + TEST_ASSERT_FALSE(s.exhausted); + } + TEST_ASSERT_EQUAL(31, s.attempts_started); + TEST_ASSERT_EQUAL(31, s.soft_timeouts); + TEST_ASSERT_EQUAL(32, s.StartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(s.OnSoftTimeout(policy))); + TEST_ASSERT_TRUE(s.exhausted); + TEST_ASSERT_EQUAL(32, s.attempts_started); + TEST_ASSERT_EQUAL(32, s.soft_timeouts); + TEST_ASSERT_EQUAL(0, s.StartAttempt(policy)); // no uint8 wrap / extra +} + +void test_ChannelChangedOneCallbackPerServer() { + // retry_count=2: after attempt #1 soft timeout and attempt #2 started, + // one channel-changed event must produce exactly one OnChannelChanged + // decision and at most one additional attempt. + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestServerExecState s; + s.activated = true; + TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s.OnSoftTimeout(policy))); + TEST_ASSERT_EQUAL(2, s.StartAttempt(policy)); + // Simulate two outstanding attempts (#1 timed out late-response possible, + // #2 active) — still one channel-changed subscription / one callback. + auto const a = s.OnChannelChanged(policy); + TEST_ASSERT_EQUAL(1, s.channel_changed_events); + TEST_ASSERT_EQUAL( + static_cast( + CloudRequestServerExecState::ChannelChangedAction::kRetry), + static_cast(a)); + TEST_ASSERT_EQUAL(3, s.StartAttempt(policy)); + TEST_ASSERT_FALSE(s.exhausted); + // Budget exhausted: further channel change must not start more attempts. + auto const b = s.OnChannelChanged(policy); + TEST_ASSERT_EQUAL(2, s.channel_changed_events); + TEST_ASSERT_EQUAL( + static_cast( + CloudRequestServerExecState::ChannelChangedAction::kExhaust), + static_cast(b)); + TEST_ASSERT_TRUE(s.exhausted); + TEST_ASSERT_EQUAL(3, s.attempts_started); +} + +void test_ChannelChangedThreeOutstandingAttempts() { + // Three outstanding attempts (retry_count=2, all started via soft path / + // channel), then one channel event must still be a single decision. + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestServerExecState s; + s.activated = true; + s.StartAttempt(policy); // #1 + s.OnSoftTimeout(policy); + s.StartAttempt(policy); // #2 + s.OnSoftTimeout(policy); + s.StartAttempt(policy); // #3 — budget full, three outstanding conceptually + TEST_ASSERT_EQUAL(3, s.attempts_started); + TEST_ASSERT_FALSE(s.CanStartAttempt(policy)); + + auto const a = s.OnChannelChanged(policy); + TEST_ASSERT_EQUAL(1, s.channel_changed_events); + TEST_ASSERT_EQUAL( + static_cast( + CloudRequestServerExecState::ChannelChangedAction::kExhaust), + static_cast(a)); + TEST_ASSERT_TRUE(s.exhausted); + TEST_ASSERT_EQUAL(3, s.attempts_started); // no extra launch + TEST_ASSERT_EQUAL(0, s.StartAttempt(policy)); +} + +void test_ApiErrorDoesNotQuarantine() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestServerExecState s; + s.activated = true; + TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); + s.MarkRemoteErrorCompleted(); + TEST_ASSERT_TRUE(s.remote_error_completed); + TEST_ASSERT_TRUE(s.IsTerminal()); + TEST_ASSERT_FALSE(s.exhausted); + TEST_ASSERT_FALSE(s.succeeded); + TEST_ASSERT_EQUAL(0, s.soft_timeouts); + TEST_ASSERT_FALSE(s.CanStartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kIgnore), + static_cast(s.OnSoftTimeout(policy))); + TEST_ASSERT_EQUAL( + static_cast( + CloudRequestServerExecState::ChannelChangedAction::kIgnore), + static_cast(s.OnChannelChanged(policy))); + TEST_ASSERT_EQUAL(0, s.channel_changed_events); + TEST_ASSERT_EQUAL(0, s.soft_timeouts); + TEST_ASSERT_FALSE(s.exhausted); // no no-response quarantine path +} + +void test_NoResponseStillQuarantinesAfterBudget() { + // retry_count=2 => attempts=3 soft timeouts then exhaust (=quarantine point). + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestServerExecState s; + s.activated = true; + s.StartAttempt(policy); + s.OnSoftTimeout(policy); + TEST_ASSERT_FALSE(s.exhausted); + s.StartAttempt(policy); + s.OnSoftTimeout(policy); + TEST_ASSERT_FALSE(s.exhausted); + s.StartAttempt(policy); + s.OnSoftTimeout(policy); + TEST_ASSERT_TRUE(s.exhausted); + TEST_ASSERT_EQUAL(3, s.attempts_started); + TEST_ASSERT_EQUAL(3, s.soft_timeouts); +} + +void test_DeterministicLatencyTimeline() { + // p99=100ms, factor=1.2 => T=120ms per attempt when RTT fixed. + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 1); + auto const T = + ComputeCloudRequestSoftTimeout(Duration{Ms{100}}, policy); + TEST_ASSERT_EQUAL(120, std::chrono::duration_cast(T).count()); + + // Case A: response at 110 < 120 => no soft miss conceptually (timer cancelled). + // Case B: soft miss at 120 => retry + hedge; late at 130 accepted. + CloudRequestServerExecState s; + s.activated = true; + s.StartAttempt(policy); + auto const miss = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(miss)); + TEST_ASSERT_EQUAL(1, s.HedgeCountOnThisMiss(policy)); + s.StartAttempt(policy); + s.MarkSucceeded(); // late response from attempt #1 + TEST_ASSERT_TRUE(s.succeeded); + TEST_ASSERT_FALSE(s.exhausted); + + // Case C: no responses, retry_count=2 => three timeouts then exhaust. + CloudRequestServerExecState never; + never.activated = true; + std::int64_t t_ms = 0; + never.StartAttempt(policy); + t_ms += 120; + never.OnSoftTimeout(policy); + never.StartAttempt(policy); + t_ms += 120; + never.OnSoftTimeout(policy); + never.StartAttempt(policy); + t_ms += 120; + never.OnSoftTimeout(policy); + TEST_ASSERT_TRUE(never.exhausted); + TEST_ASSERT_EQUAL(360, t_ms); + TEST_ASSERT_EQUAL(3, never.attempts_started); +} + +} // namespace ae::test_cloud_request + +extern "C" void setUp(void) {} +extern "C" void tearDown(void) {} + +int main() { + UNITY_BEGIN(); + RUN_TEST(ae::test_cloud_request::test_TimeoutCalculation); + RUN_TEST(ae::test_cloud_request::test_RetryCountSemantics); + RUN_TEST(ae::test_cloud_request::test_RetryCountClampAndMax); + RUN_TEST(ae::test_cloud_request::test_RetryCount31StateMachine); + RUN_TEST(ae::test_cloud_request::test_NoQuarantineBeforeExhaustionAndHedge); + RUN_TEST(ae::test_cloud_request::test_HedgeZeroKeepsSequential); + RUN_TEST(ae::test_cloud_request::test_LateResponseAfterSoftTimeout); + RUN_TEST(ae::test_cloud_request::test_ChannelChangedOneCallbackPerServer); + RUN_TEST(ae::test_cloud_request::test_ChannelChangedThreeOutstandingAttempts); + RUN_TEST(ae::test_cloud_request::test_ApiErrorDoesNotQuarantine); + RUN_TEST(ae::test_cloud_request::test_NoResponseStillQuarantinesAfterBudget); + RUN_TEST(ae::test_cloud_request::test_PerServerTimeoutIndependent); + RUN_TEST(ae::test_cloud_request::test_PolicySnapshotDefaults); + RUN_TEST(ae::test_cloud_request::test_PercentileFractionalDistinctRanks); + RUN_TEST(ae::test_cloud_request::test_PolicyFieldsAreRuntimeAssignable); + RUN_TEST(ae::test_cloud_request::test_DeterministicLatencyTimeline); + return UNITY_END(); +} diff --git a/tests/test-local-presence/CMakeLists.txt b/tests/test-local-presence/CMakeLists.txt new file mode 100644 index 00000000..9574b556 --- /dev/null +++ b/tests/test-local-presence/CMakeLists.txt @@ -0,0 +1,47 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16) + +if(NOT CM_PLATFORM) + project(test-local-presence LANGUAGES CXX) + + add_executable(${PROJECT_NAME} main.cpp) + target_include_directories(${PROJECT_NAME} PRIVATE ${ROOT_DIR}) + target_link_libraries(${PROJECT_NAME} PRIVATE unity aether) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(${PROJECT_NAME} PRIVATE /Zc:preprocessor) + endif() + add_test(NAME ${PROJECT_NAME} COMMAND $) + + if(AE_ENABLE_PRIVILEGED_NETWORK_TESTS) + add_executable(test-local-presence-firewall firewall_live.cpp) + target_include_directories(test-local-presence-firewall PRIVATE ${ROOT_DIR}) + target_link_libraries(test-local-presence-firewall PRIVATE unity aether) + target_compile_definitions(test-local-presence-firewall PRIVATE + "AE_DISTILLATION=1" + ) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(test-local-presence-firewall PRIVATE /Zc:preprocessor) + endif() + add_test(NAME test-local-presence-firewall + COMMAND $) + set_tests_properties(test-local-presence-firewall PROPERTIES + TIMEOUT 90 + LABELS "manual;privileged;network" + ) + endif() +else() + message(WARNING "Not implemented for ${CM_PLATFORM}") +endif() diff --git a/tests/test-local-presence/firewall_live.cpp b/tests/test-local-presence/firewall_live.cpp new file mode 100644 index 00000000..f786d40f --- /dev/null +++ b/tests/test-local-presence/firewall_live.cpp @@ -0,0 +1,373 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include + +#include "aether/adapters/ethernet.h" +#include "aether/aether_app.h" +#include "aether/all.h" +#include "aether/client.h" +#include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/cloud_server_connection.h" +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/global_ids.h" +#include "aether/types/uid.h" + +#if defined(_WIN32) +# include +#endif + +namespace ae::test_local_presence_firewall { + +using namespace std::chrono_literals; + +constexpr auto kInterval = 1s; +constexpr auto kWindow = 1s; +constexpr auto kPoll = 10ms; + +static constexpr auto kParentUid = + Uid::FromString("3ac93165-3d37-4970-87a6-fa4ee27744e4"); + +#if defined(_WIN32) + +std::wstring ThisExePath() { + wchar_t path[MAX_PATH]{}; + auto const n = GetModuleFileNameW(nullptr, path, MAX_PATH); + if (n == 0 || n >= MAX_PATH) { + return {}; + } + return std::wstring{path, static_cast(n)}; +} + +int RunHidden(std::wstring cmd) { + STARTUPINFOW si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESHOWWINDOW; + si.wShowWindow = SW_HIDE; + PROCESS_INFORMATION pi{}; + if (!CreateProcessW(nullptr, cmd.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &pi)) { + return -1; + } + WaitForSingleObject(pi.hProcess, 20000); + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + return static_cast(code); +} + +class WindowsExeFirewall { + public: + explicit WindowsExeFirewall(std::wstring exe_path) + : exe_path_{std::move(exe_path)}, + tag_{std::to_wstring(GetCurrentProcessId())} {} + + ~WindowsExeFirewall() { Unblock(); } + + WindowsExeFirewall(WindowsExeFirewall const&) = delete; + WindowsExeFirewall& operator=(WindowsExeFirewall const&) = delete; + + bool Block() { + Unblock(); + auto const quoted = L"\"" + exe_path_ + L"\""; + out_name_ = L"ae-lp-fw-out-" + tag_; + in_name_ = L"ae-lp-fw-in-" + tag_; + auto const out_cmd = + L"netsh advfirewall firewall add rule name=\"" + out_name_ + + L"\" dir=out action=block enable=yes profile=any program=" + quoted; + auto const in_cmd = + L"netsh advfirewall firewall add rule name=\"" + in_name_ + + L"\" dir=in action=block enable=yes profile=any program=" + quoted; + if (RunHidden(out_cmd) != 0 || RunHidden(in_cmd) != 0) { + Unblock(); + return false; + } + active_ = true; + return true; + } + + void Unblock() { + if (!out_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + out_name_ + + L"\""); + } + if (!in_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + in_name_ + + L"\""); + } + active_ = false; + } + + private: + std::wstring exe_path_; + std::wstring tag_; + std::wstring out_name_; + std::wstring in_name_; + bool active_{false}; +}; + +#endif + +std::unique_ptr MakeApp() { + return AetherApp::Construct(AetherAppContext{}.AddAdapterFactory( + [](AetherAppContext const& context) { + return EthernetAdapter::ptr::Create( + CreateWith{context.domain()}.with_id(GlobalId::kEthernetAdapter), + context.aether(), context.poller(), context.dns_resolver()); + })); +} + +void Pump(AetherApp& app, TimePoint until) { + while (!app.IsExited() && Now() < until) { + auto const next = app.Update(Now()); + auto const poll_at = Now() + kPoll; + app.WaitUntil(next < poll_at ? next : poll_at); + } +} + +#if defined(_WIN32) +bool IsProcessElevated() { + HANDLE token = nullptr; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) { + return false; + } + TOKEN_ELEVATION elevation{}; + DWORD size = 0; + auto const ok = GetTokenInformation(token, TokenElevation, &elevation, + sizeof(elevation), &size); + CloseHandle(token); + return ok && (elevation.TokenIsElevated != 0); +} + +CloudServerConnection* FirstSelectedServer(Client& client) { + auto const& selected = client.cloud_connection().selected_servers(); + for (auto* server : selected) { + if (server != nullptr) { + return server; + } + } + return nullptr; +} + +TimePoint ServerConfirmedClose(ClientConnectivityPolicy& policy, ServerId id) { + auto const* state = policy.FindServerPresence(id); + if (state == nullptr || !state->has_confirmed_schedule) { + return TimePoint::max(); + } + return LocalOfflineDeadline(state->confirmed_window_open_local, + policy.offline_detection_timeout()); +} +#endif + +void ApplyOneSecondTimings(Client& client) { + auto policy = client.connectivity_policy(); + TEST_ASSERT_TRUE(static_cast(policy)); + policy->ResetRxTimings(); + policy->SetOfflineDetectionTimeout(1s); + policy->ConfigureRxTimings(RequestPolicy::All{}) + .ForAllPriorities(RxTimingConf::Every(kInterval).WithWindow(kWindow)); + for (auto* server : client.cloud_connection().selected_servers()) { + if (server == nullptr) { + continue; + } + policy->ConfigureServerRxTiming( + server->server_id(), + RxTimingConf::Every(kInterval).WithWindow(kWindow), 99); + } +} + +void test_WindowsFirewallOfflineAndRecovery() { +#if !defined(_WIN32) + TEST_IGNORE_MESSAGE("Windows Firewall test runs on Win32 only"); +#else + if (!IsProcessElevated()) { + TEST_IGNORE_MESSAGE( + "SKIP: privileged firewall test requires Administrator " + "(AE_ENABLE_PRIVILEGED_NETWORK_TESTS=ON)"); + } + + auto app = MakeApp(); + TEST_ASSERT_NOT_NULL(app.get()); + + Client::ptr client; + auto& select = app->aether()->SelectClient(kParentUid, "presence-fw"); + select.result_event().Subscribe([&](auto const& res) { + if (res) { + client = res.value(); + } + }); + Pump(*app, Now() + 45s); + if (!client) { + TEST_IGNORE_MESSAGE("SelectClient did not finish (no cloud / network)"); + } + + ApplyOneSecondTimings(*client.Load()); + (void)client->cloud_connection(); + + auto const online_deadline = Now() + 30s; + while (Now() < online_deadline && !app->IsExited()) { + Pump(*app, Now() + kPoll); + if (client->IsLocallyOnline()) { + break; + } + } + TEST_ASSERT_TRUE_MESSAGE(client->IsLocallyOnline(), + "did not become ONLINE before firewall"); + Pump(*app, Now() + 1500ms); + TEST_ASSERT_TRUE(client->IsLocallyOnline()); + + auto* target = FirstSelectedServer(*client.Load()); + TEST_ASSERT_NOT_NULL(target); + auto const sid = target->server_id(); + auto policy = client->connectivity_policy(); + TEST_ASSERT_TRUE(static_cast(policy)); + auto const initial_close = ServerConfirmedClose(*policy.Load(), sid); + TEST_ASSERT_TRUE_MESSAGE(initial_close != TimePoint::max(), + "no confirmed receive window"); + + WindowsExeFirewall fw{ThisExePath()}; + auto const fault_time = Now(); + if (!fw.Block()) { + TEST_IGNORE_MESSAGE( + "SKIP: netsh advfirewall failed (need Administrator)"); + } + + bool early_offline = false; + TimePoint detected_offline{}; + TimePoint final_close = initial_close; + int pongs_after_block = 0; + auto detect_deadline = final_close + 5s; + while (Now() < detect_deadline && !app->IsExited()) { + Pump(*app, Now() + kPoll); + auto const now = Now(); + auto const close_now = ServerConfirmedClose(*policy.Load(), sid); + if (close_now != TimePoint::max() && close_now > final_close) { + ++pongs_after_block; + final_close = close_now; + detect_deadline = final_close + 5s; + } + auto const online = policy->IsServerLocallyOnline(sid, now); + if (now <= final_close) { + if (!online) { + early_offline = true; + detected_offline = now; + break; + } + } else if (!online) { + detected_offline = now; + break; + } + } + + auto const to_ms = [](TimePoint a, TimePoint b) { + return std::chrono::duration_cast(a - b) + .count(); + }; + auto const fault_to_initial = to_ms(initial_close, fault_time); + auto const fault_to_final = to_ms(final_close, fault_time); + auto const fault_to_offline = + detected_offline.time_since_epoch().count() == 0 + ? -1 + : to_ms(detected_offline, fault_time); + std::printf( + "FIREWALL interval_ms=1000 rx_window_ms=1000 " + "pongs_after_block=%d fault_to_initial_close_ms=%lld " + "final_effective_close_ms=%lld fault_to_offline_ms=%lld " + "early_offline=%s\n", + pongs_after_block, static_cast(fault_to_initial), + static_cast(fault_to_final), + static_cast(fault_to_offline), + early_offline ? "YES" : "NO"); + if (FILE* log = std::fopen("firewall_result.txt", "w")) { + std::fprintf(log, + "interval_ms=1000\nrx_window_ms=1000\n" + "pongs_after_block=%d\nfault_to_initial_close_ms=%lld\n" + "final_effective_close_ms=%lld\nfault_to_offline_ms=%lld\n" + "early_offline=%s\n", + pongs_after_block, static_cast(fault_to_initial), + static_cast(fault_to_final), + static_cast(fault_to_offline), + early_offline ? "YES" : "NO"); + std::fclose(log); + } + + TEST_ASSERT_FALSE_MESSAGE( + early_offline, "OFFLINE appeared before current_effective_close"); + TEST_ASSERT_TRUE_MESSAGE(detected_offline.time_since_epoch().count() != 0, + "OFFLINE was not detected after firewall block"); + TEST_ASSERT_TRUE(detected_offline > final_close); + + fw.Unblock(); + auto const recover_from = Now(); + TimePoint first_pong{}; + TimePoint recovered{}; + auto last_close = final_close; + while (Now() < recover_from + 20s && !app->IsExited()) { + Pump(*app, Now() + kPoll); + auto const close_now = ServerConfirmedClose(*policy.Load(), sid); + if (first_pong.time_since_epoch().count() == 0 && + close_now != TimePoint::max() && close_now > last_close) { + first_pong = Now(); + } + if (policy->IsServerLocallyOnline(sid, Now())) { + recovered = Now(); + break; + } + } + auto const unblock_to_pong = + first_pong.time_since_epoch().count() == 0 + ? -1 + : to_ms(first_pong, recover_from); + auto const unblock_to_online = + recovered.time_since_epoch().count() == 0 + ? -1 + : to_ms(recovered, recover_from); + std::printf( + "FIREWALL unblock_to_first_successful_pong_ms=%lld " + "unblock_to_online_ms=%lld\n", + static_cast(unblock_to_pong), + static_cast(unblock_to_online)); + if (FILE* log = std::fopen("firewall_result.txt", "a")) { + std::fprintf(log, + "unblock_to_first_successful_pong_ms=%lld\n" + "unblock_to_online_ms=%lld\npass=1\n", + static_cast(unblock_to_pong), + static_cast(unblock_to_online)); + std::fclose(log); + } + TEST_ASSERT_TRUE_MESSAGE(policy->IsServerLocallyOnline(sid, Now()), + "did not return ONLINE after firewall unblock"); +#endif +} + +} // namespace ae::test_local_presence_firewall + +void setUp() {} +void tearDown() {} + +int main() { + UNITY_BEGIN(); + RUN_TEST( + ae::test_local_presence_firewall::test_WindowsFirewallOfflineAndRecovery); + return UNITY_END(); +} diff --git a/tests/test-local-presence/main.cpp b/tests/test-local-presence/main.cpp new file mode 100644 index 00000000..f7ab071a --- /dev/null +++ b/tests/test-local-presence/main.cpp @@ -0,0 +1,1233 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/local_presence_machine.h" +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/remote_presence.h" +#include "ae-numeric/percentile.h" +#include "aether/types/statistic_counter.h" +#include "aether/work_cloud_api/client_timing.h" + +namespace ae::test_local_presence { + +using Ms = std::chrono::milliseconds; + +TimePoint Tp(std::int64_t ms) { return TimePoint{Ms{ms}}; } + +Duration Dur(std::int64_t ms) { + return std::chrono::duration_cast(Ms{ms}); +} + +std::int64_t ToMs(TimePoint tp) { + return std::chrono::duration_cast(tp.time_since_epoch()).count(); +} + +std::int64_t ToMs(Duration d) { + return std::chrono::duration_cast(d).count(); +} + +void test_PrefixFormula() { + auto const R = Dur(100); + auto const G = kLocalPresenceGuard; + auto const O = Tp(1050); + auto const p1 = ComputePrefix1Time(O, R, G); + auto const p2 = ComputePrefix2Time(O, R, G); + TEST_ASSERT_EQUAL(870, ToMs(p1)); + TEST_ASSERT_EQUAL(970, ToMs(p2)); + TEST_ASSERT_EQUAL(100, std::chrono::duration_cast(p2 - p1).count()); + TEST_ASSERT_EQUAL(30, ToMs(G)); +} + +void test_ConfirmOnlyAfterPong() { + ClientConnectivityPolicy policy; + ServerId const sid{7}; + policy.ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Percentile::FromPercent(99.0)); + auto* state = policy.FindServerPresence(sid); + TEST_ASSERT_NOT_NULL(state); + TEST_ASSERT_FALSE(state->has_confirmed_schedule); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(sid, Tp(0))); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(0))); + + policy.ConfirmServerPong(sid, Tp(1000), Tp(1100), Dur(1000), Dur(300), + Dur(100)); + state = policy.FindServerPresence(sid); + TEST_ASSERT_TRUE(state->has_confirmed_schedule); + TEST_ASSERT_EQUAL(2050, ToMs(state->confirmed_window_open_local)); + TEST_ASSERT_EQUAL(2350, ToMs(state->confirmed_window_close_local)); + auto const deadline = + LocalOfflineDeadline(state->confirmed_window_open_local, + policy.offline_detection_timeout()); + TEST_ASSERT_EQUAL(3050, ToMs(deadline)); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(sid, Tp(2050))); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(sid, deadline)); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(sid, deadline + Dur(1))); +} + +void test_SelectedRttProjectionIgnoresMeasuredPong() { + auto const selected = Dur(100); + auto fast = MakeConfirmedSchedule(Tp(1000), Tp(1020), Dur(1000), Dur(1000), + selected); + auto slow = MakeConfirmedSchedule(Tp(1000), Tp(1400), Dur(1000), Dur(1000), + selected); + TEST_ASSERT_EQUAL(ToMs(fast.window_open_local), ToMs(slow.window_open_local)); + TEST_ASSERT_EQUAL(ToMs(fast.window_close_local), + ToMs(slow.window_close_local)); + TEST_ASSERT_EQUAL(2050, ToMs(fast.window_open_local)); + TEST_ASSERT_TRUE(ToMs(fast.measured_rtt) != ToMs(slow.measured_rtt)); +} + +void test_PerServerIndependence() { + ClientConnectivityPolicy policy; + ServerId const a{1}; + ServerId const b{2}; + policy.ConfigureServerRxTiming( + a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Percentile::FromPercent(99.0)); + policy.ConfigureServerRxTiming( + b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Percentile::FromPercent(95.0)); + policy.ConfirmServerPong(a, Tp(0), Tp(100), Dur(1000), Dur(300), Dur(100)); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(b, Tp(50))); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(a, Tp(50))); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); +} + +void test_OfflineOnlyAfterOfflineDetectionTimeout() { + ClientConnectivityPolicy policy; + ServerId const sid{3}; + policy.SetOfflineDetectionTimeout(Dur(1000)); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(10000), Dur(40)); + // open = 0 + 20 + 1000 = 1020; deadline = 2020 even with rx_window=10s. + TEST_ASSERT_EQUAL(1020, ToMs(policy.FindServerPresence(sid) + ->confirmed_window_open_local)); + TEST_ASSERT_EQUAL( + 11020, ToMs(policy.FindServerPresence(sid)->confirmed_window_close_local)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(2020))); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(2021))); +} + +void test_RuntimeIntervalChangeKeepsOldConfirmed() { + ClientConnectivityPolicy policy; + ServerId const sid{4}; + policy.ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200))); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + auto const close_before = + policy.FindServerPresence(sid)->confirmed_window_close_local; + + policy.ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); + auto* state = policy.FindServerPresence(sid); + TEST_ASSERT_TRUE(state->config_change_pending); + TEST_ASSERT_EQUAL(10000, ToMs(state->desired.interval)); + TEST_ASSERT_TRUE(state->confirmed_window_close_local == close_before); + TEST_ASSERT_EQUAL(1000, ToMs(state->confirmed_interval)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(close_before)); + + policy.ConfirmServerPong(sid, Tp(500), Tp(560), Dur(10000), Dur(200), + Dur(60)); + state = policy.FindServerPresence(sid); + TEST_ASSERT_FALSE(state->config_change_pending); + TEST_ASSERT_EQUAL(10000, ToMs(state->confirmed_interval)); +} + +void test_RuntimePercentile() { + StatisticsCounter stats; + for (int i = 1; i <= 20; ++i) { + stats.Add(i * 10); + } + auto const p95 = stats.PercentileValue(95); + auto const p99 = stats.PercentileValue(99); + TEST_ASSERT_TRUE(p99 >= p95); + TEST_ASSERT_EQUAL(stats.percentile<95>(), p95); + TEST_ASSERT_EQUAL(stats.percentile<99>(), p99); +} + +void test_ReliabilityP95VsP99PrefixTimes() { + StatisticsCounter stats; + for (int i = 1; i <= 20; ++i) { + stats.Add(Dur(i * 10)); + } + auto const p95 = stats.PercentileValue(95); + auto const p99 = stats.PercentileValue(99); + TEST_ASSERT_TRUE(p99 >= p95); + auto const O = Tp(5000); + auto const p1_95 = ComputePrefix1Time(O, p95); + auto const p1_99 = ComputePrefix1Time(O, p99); + auto const p2_95 = ComputePrefix2Time(O, p95); + auto const p2_99 = ComputePrefix2Time(O, p99); + TEST_ASSERT_TRUE(ToMs(p1_99) <= ToMs(p1_95)); + TEST_ASSERT_TRUE(ToMs(p2_99) <= ToMs(p2_95)); + if (p99 > p95) { + TEST_ASSERT_TRUE(ToMs(p1_99) < ToMs(p1_95)); + TEST_ASSERT_TRUE(ToMs(p2_99) < ToMs(p2_95)); + } +} + +void test_AggregateIgnoresDeselected() { + ClientConnectivityPolicy policy; + ServerId const a{10}; + ServerId const b{11}; + policy.ConfirmServerPong(a, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + policy.ConfirmServerPong(b, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); + policy.SetServerSelectedForAggregate(a, false); + policy.SetServerSelectedForAggregate(b, false); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(50))); + policy.SetServerSelectedForAggregate(b, true); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); +} + +void test_OneWayProjection() { TEST_ASSERT_EQUAL(50, ToMs(OneWayFromRtt(Dur(100)))); } + +void test_MakeConfirmedScheduleDeterministic() { + auto s = + MakeConfirmedSchedule(Tp(1000), Tp(1200), Dur(500), Dur(100), Dur(200)); + TEST_ASSERT_EQUAL(1600, ToMs(s.window_open_local)); + TEST_ASSERT_EQUAL(1700, ToMs(s.window_close_local)); +} + +void test_ConfigScopeOverrideAndPriority() { + ClientConnectivityPolicy policy; + ServerId const a{1}; + ServerId const b{2}; + ServerId const c{3}; + policy.BindServerPriority(a, 0); + policy.BindServerPriority(b, 1); + policy.ConfigureServerRxTiming( + a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), Percentile::FromPercent(99.0)); + TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(a)->desired.interval)); + TEST_ASSERT_EQUAL(AE_PING_INTERVAL_MS, + ToMs(policy.FindServerPresence(b)->desired.interval)); + + policy.ConfigureRxTimings().ForAllPriorities( + RxTimingConf::Every(Dur(3000)).WithWindow(Dur(3000))); + TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(a)->desired.interval)); + TEST_ASSERT_EQUAL(3000, ToMs(policy.FindServerPresence(b)->desired.interval)); + + policy.ConfigureRxTimings().ForPriority<0>( + RxTimingConf::Every(Dur(4000)).WithWindow(Dur(4000))); + TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(a)->desired.interval)); + TEST_ASSERT_EQUAL(3000, ToMs(policy.FindServerPresence(b)->desired.interval)); + + policy.BindServerPriority(c, 0); + TEST_ASSERT_EQUAL(4000, ToMs(policy.FindServerPresence(c)->desired.interval)); + TEST_ASSERT_EQUAL(3000, ToMs(policy.FindServerPresence(b)->desired.interval)); +} + +struct PollStats { + int status_poll_count{}; + int online_samples{}; + int false_offline_samples{}; + int false_offline_transitions{}; + bool prev_online{false}; + bool have_prev{false}; +}; + +struct PendingPong { + LocalPresenceMachine::SendSpec spec{}; + TimePoint send_time{}; + TimePoint deliver_at{}; + bool drop{false}; + bool hard_fail{false}; +}; + +class PresenceHarness { + public: + using DelayFn = std::function; + + explicit PresenceHarness(TimePoint start) : now_{start} {} + + ClientConnectivityPolicy& policy() { return policy_; } + TimePoint now() const { return now_; } + PollStats const& poll_stats() const { return poll_stats_; } + + LocalPresenceMachine& machine(ServerId id) { return servers_.at(id).machine; } + + LocalPresenceMachine::Counters const& counters(ServerId id) { + return servers_.at(id).machine.counters(); + } + + void AddServer(ServerId id, RxTimingConf conf, Duration seed_rtt, + Percentile percentile = kDefaultRttReliabilityPercentile, + std::size_t priority = 0) { + policy_.BindServerPriority(id, priority); + policy_.ConfigureServerRxTiming(id, conf, percentile); + Server s{}; + s.id = id; + s.seed_rtt = seed_rtt; + s.percentile = percentile; + for (int i = 0; i < 20; ++i) { + s.stats.Add(seed_rtt); + } + s.machine.SetDesired(now_, conf, percentile); + s.machine.SetOfflineDetectionTimeout(policy_.offline_detection_timeout()); + s.machine.ArmInitial(now_); + servers_.emplace(id, std::move(s)); + } + + void SetFixedDelay(ServerId id, Duration delay) { + servers_.at(id).fixed_delay = delay; + } + + void SetDelayFn(ServerId id, DelayFn fn) { servers_.at(id).delay_fn = std::move(fn); } + + void SetDropKind(ServerId id, PingAttemptKind kind, bool drop) { + servers_.at(id).drop_kind[static_cast(kind)] = drop; + } + + void SetConnectivity(ServerId id, bool ok) { + servers_.at(id).connectivity_ok = ok; + } + + bool IsLocallyOnline() const { return policy_.IsLocallyOnline(now_); } + + void SyncBlockers() { + bool need_current = false; + bool need_request = false; + for (auto& [id, s] : servers_) { + static_cast(id); + need_current = need_current || s.machine.current_window_blocker_held(); + need_request = need_request || s.machine.request_blocker_held(); + } + if (need_current) { + if (!have_current_) { + current_block_ = policy_.AcquireSuspendBlock(); + have_current_ = true; + } + } else { + current_block_.Reset(); + have_current_ = false; + } + if (need_request) { + if (!have_request_) { + request_block_ = policy_.AcquireSuspendBlock(); + have_request_ = true; + } + } else { + request_block_.Reset(); + have_request_ = false; + } + } + + Duration SelectedRtt(ServerId id) { + auto& s = servers_.at(id); + if (s.stats.empty()) { + return s.seed_rtt; + } + return s.stats.PercentileValue(s.percentile); + } + + void Poll(bool expected_connected) { + auto const online = IsLocallyOnline(); + ++poll_stats_.status_poll_count; + if (online) { + ++poll_stats_.online_samples; + } + if (expected_connected && !online) { + ++poll_stats_.false_offline_samples; + if (poll_stats_.have_prev && poll_stats_.prev_online) { + ++poll_stats_.false_offline_transitions; + } + } + poll_stats_.prev_online = online; + poll_stats_.have_prev = true; + } + + void Process() { + bool progress = true; + while (progress) { + progress = false; + for (auto& [id, s] : servers_) { + static_cast(id); + if (DeliverDue(s)) { + progress = true; + } + } + for (auto& [id, s] : servers_) { + static_cast(id); + if (TickServer(s)) { + progress = true; + } + } + } + SyncBlockers(); + } + + void AdvanceTo(TimePoint t) { + if (t < now_) { + return; + } + while (now_ < t) { + Process(); + auto const next = NextEventTime(); + if (next == TimePoint::max() || next > t) { + now_ = t; + Process(); + return; + } + if (next > now_) { + now_ = next; + } else { + now_ = now_ + Dur(1); + } + } + Process(); + } + + void AdvancePolling(Duration total, Duration step, bool expected_connected) { + auto const end = now_ + total; + Process(); + while (now_ < end) { + auto const next_poll = now_ + step; + for (;;) { + auto const ev = NextEventTime(); + if (ev == TimePoint::max() || ev > next_poll) { + break; + } + if (ev > now_) { + now_ = ev; + } + Process(); + if (NextEventTime() <= now_) { + break; + } + } + now_ = next_poll; + Process(); + Poll(expected_connected); + } + } + + private: + struct Server { + ServerId id{}; + LocalPresenceMachine machine{}; + StatisticsCounter stats{}; + Duration seed_rtt{Dur(100)}; + Percentile percentile{kDefaultRttReliabilityPercentile}; + Duration fixed_delay{Dur(20)}; + DelayFn delay_fn{}; + bool drop_kind[5]{}; + bool connectivity_ok{true}; + int send_count{}; + std::vector pending{}; + }; + + TimePoint NextEventTime() const { + auto next = TimePoint::max(); + for (auto const& [id, s] : servers_) { + static_cast(id); + next = std::min(next, s.machine.PeekNextWake()); + for (auto const& p : s.pending) { + next = std::min(next, p.deliver_at); + } + } + return next; + } + + bool DeliverDue(Server& s) { + bool any = false; + for (auto it = s.pending.begin(); it != s.pending.end();) { + if (it->drop) { + ++it; + continue; + } + if (now_ < it->deliver_at) { + ++it; + continue; + } + if (it->hard_fail) { + s.machine.OnHardFailure(it->spec.attempt_id, now_, SelectedRtt(s.id), + PresenceRestreamReason::kHardWriteFailure); + } else { + auto const measured = + std::chrono::duration_cast(now_ - it->send_time); + s.stats.Add(measured); + auto const selected = SelectedRtt(s.id); + auto outcome = s.machine.OnPong( + it->spec.attempt_id, it->spec.cycle_id, it->send_time, now_, + it->spec.wire_interval, it->spec.desired_interval, + it->spec.rx_window, it->spec.following_open_target, selected); + if (outcome.disposition == + LocalPresenceMachine::PongDisposition::kConfirmedSchedule) { + policy_.ConfirmServerPong( + s.id, outcome.schedule.ping_send_time, + outcome.schedule.pong_receive_time, outcome.schedule.interval, + outcome.schedule.rx_window, outcome.schedule.selected_rtt); + auto& st = policy_.EnsureServerPresence(s.id); + st.confirmed_interval = s.machine.confirmed_interval(); + st.config_change_pending = s.machine.config_change_pending(); + } + } + it = s.pending.erase(it); + any = true; + } + return any; + } + + bool TickServer(Server& s) { + auto tick = s.machine.TickNow(now_, SelectedRtt(s.id)); + if (!tick.want_send) { + return false; + } + s.machine.OnSendStarting(); + s.machine.OnAttemptSent(tick.send, now_); + ++s.send_count; + PendingPong p{}; + p.spec = tick.send; + p.send_time = now_; + auto delay = s.fixed_delay; + if (s.delay_fn) { + delay = s.delay_fn(tick.send.kind, s.send_count); + } + auto const drop = + !s.connectivity_ok || (ToMs(delay) < 0) || (ToMs(delay) > 5000); + if (!drop) { + p.deliver_at = now_ + delay; + s.pending.push_back(p); + } + return true; + } + + ClientConnectivityPolicy policy_{}; + TimePoint now_{}; + std::map servers_{}; + PollStats poll_stats_{}; + ClientConnectivityPolicy::SuspendBlocker current_block_{}; + ClientConnectivityPolicy::SuspendBlocker request_block_{}; + bool have_current_{false}; + bool have_request_{false}; +}; + +void test_SendWithoutPongDoesNotConfirm() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), + Dur(100)); + rt.SetConnectivity(sid, false); + rt.AdvanceTo(Tp(0)); + TEST_ASSERT_EQUAL(1, rt.counters(sid).initial); + TEST_ASSERT_FALSE(rt.IsLocallyOnline()); + TEST_ASSERT_FALSE(rt.machine(sid).has_confirmed_schedule()); +} + +void test_LongSleepSuspendBetweenPongAndPrefix1() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + auto const interval = Dur(10 * 60 * 1000); + rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + rt.SyncBlockers(); + TEST_ASSERT_TRUE(rt.policy().GetStatus().can_suspend); + TEST_ASSERT_TRUE(rt.machine(sid).CanSuspend()); + + auto const open = rt.machine(sid).confirmed_window_open(); + auto const prefix1 = ComputePrefix1Time(open, Dur(100)); + rt.AdvanceTo(prefix1 - Dur(1)); + TEST_ASSERT_TRUE(rt.machine(sid).CanSuspend()); + TEST_ASSERT_TRUE(rt.policy().GetStatus().can_suspend); + + rt.AdvanceTo(prefix1); + TEST_ASSERT_FALSE(rt.machine(sid).CanSuspend()); + TEST_ASSERT_FALSE(rt.policy().GetStatus().can_suspend); + + auto const current_c = rt.machine(sid).current_promised_close(); + TEST_ASSERT_TRUE(current_c == rt.machine(sid).confirmed_window_close() || + ToMs(current_c) <= ToMs(rt.machine(sid).confirmed_window_close())); + rt.AdvanceTo(current_c); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + rt.AdvanceTo(current_c + Dur(1)); + TEST_ASSERT_FALSE(rt.machine(sid).current_window_blocker_held()); +} + +void test_CurrentVsNextWindowBlocker() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const c0 = rt.machine(sid).confirmed_window_close(); + TEST_ASSERT_TRUE(rt.machine(sid).CanSuspend()); + auto const prefix1 = + ComputePrefix1Time(rt.machine(sid).confirmed_window_open(), Dur(100)); + rt.AdvanceTo(prefix1); + TEST_ASSERT_TRUE(rt.machine(sid).current_window_blocker_held()); + TEST_ASSERT_EQUAL(ToMs(c0), ToMs(rt.machine(sid).current_promised_close())); + rt.AdvanceTo(prefix1 + Dur(20)); + auto const c1 = rt.machine(sid).confirmed_window_close(); + TEST_ASSERT_TRUE(ToMs(c1) > ToMs(c0)); + TEST_ASSERT_EQUAL(ToMs(c0), ToMs(rt.machine(sid).current_promised_close())); +} + +void test_Prefix1LatePongAfterPrefix2() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const open = rt.machine(sid).confirmed_window_open(); + auto const target_before = rt.machine(sid).last_following_target(); + static_cast(target_before); + rt.SetDelayFn(sid, [](PingAttemptKind kind, int) { + if (kind == PingAttemptKind::kPrefix1) { + return Dur(250); + } + return Dur(20); + }); + auto const prefix1 = ComputePrefix1Time(open, Dur(100)); + rt.AdvanceTo(prefix1 + Dur(260)); + TEST_ASSERT_TRUE(rt.counters(sid).prefix2 >= 1); + TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_samples); + TEST_ASSERT_TRUE(rt.machine(sid).outstanding_attempt_count() == 0); + TEST_ASSERT_TRUE(rt.counters(sid).late_pongs >= 1); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_RttTailEntersStatistics() { + StatisticsCounter stats; + for (int i = 0; i < 100; ++i) { + stats.Add(Dur(100)); + } + TEST_ASSERT_EQUAL(100, ToMs(stats.PercentileValue(99))); + stats.Add(Dur(300)); + TEST_ASSERT_TRUE(ToMs(stats.PercentileValue(99)) >= 300); + + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + rt.SetDelayFn(sid, [](PingAttemptKind kind, int) { + if (kind == PingAttemptKind::kPrefix1) { + return Dur(300); + } + return Dur(20); + }); + auto const prefix1 = + ComputePrefix1Time(rt.machine(sid).confirmed_window_open(), Dur(100)); + rt.AdvanceTo(prefix1 + Dur(310)); + TEST_ASSERT_TRUE(ToMs(rt.SelectedRtt(sid)) >= 300); +} + +void test_PxxMissDoesNotRestream() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_EQUAL(0, rt.counters(sid).restreams); + rt.SetDelayFn(sid, [](PingAttemptKind kind, int) { + if (kind == PingAttemptKind::kPrefix1) { + return Dur(250); + } + return Dur(20); + }); + auto const prefix1 = + ComputePrefix1Time(rt.machine(sid).confirmed_window_open(), Dur(100)); + rt.AdvanceTo(prefix1 + Dur(110)); + TEST_ASSERT_TRUE(rt.counters(sid).prefix2 >= 1); + TEST_ASSERT_EQUAL(0, rt.counters(sid).restreams); + TEST_ASSERT_TRUE(rt.machine(sid).outstanding_attempt_count() >= 1); +} + +void test_EnsureLinkedErrorReleasesBlocker() { + LocalPresenceMachine machine; + machine.ArmInitial(Tp(0)); + auto tick = machine.TickNow(Tp(0), Dur(100)); + TEST_ASSERT_TRUE(tick.want_send); + machine.OnSendStarting(); + TEST_ASSERT_TRUE(machine.request_blocker_held()); + machine.OnStartFailed(Tp(0), Dur(100), + PresenceRestreamReason::kConnectionUnavailable); + TEST_ASSERT_FALSE(machine.request_blocker_held()); + TEST_ASSERT_TRUE(machine.CanSuspend()); + TEST_ASSERT_EQUAL(1, machine.counters().restreams); +} + +void test_QuarantineKeepsConfirmedUntilClose() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const open = rt.machine(sid).confirmed_window_open(); + auto const deadline = + LocalOfflineDeadline(open, rt.policy().offline_detection_timeout()); + TEST_ASSERT_TRUE(ToMs(deadline) >= 2000); + rt.machine(sid).OnQuarantine(Tp(1000)); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(sid, Tp(1500))); + TEST_ASSERT_TRUE(rt.policy().IsLocallyOnline(Tp(1500))); + TEST_ASSERT_TRUE(rt.policy().IsLocallyOnline(deadline)); + TEST_ASSERT_FALSE(rt.policy().IsLocallyOnline(deadline + Dur(1))); +} + +void test_HardRemovalDropsAggregateImmediately() { + ClientConnectivityPolicy policy; + ServerId const a{1}; + ServerId const b{2}; + policy.ConfirmServerPong(a, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + policy.ConfirmServerPong(b, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); + policy.RemoveServerFromCloud(a); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(a, Tp(50))); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(b, Tp(50))); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); + policy.RemoveServerFromCloud(b); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(50))); +} + +void test_RuntimeConfigChangeKeepsOldUntilPong() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const close_old = rt.machine(sid).confirmed_window_close(); + rt.policy().ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); + rt.machine(sid).SetDesired( + rt.now(), RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200)), Percentile::FromPercent(99.0)); + TEST_ASSERT_EQUAL(1000, ToMs(rt.machine(sid).confirmed_interval())); + TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_close() == close_old); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_RuntimePercentileOnlyChangeKeepsSchedule() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + auto const conf = RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)); + rt.AddServer(sid, conf, Dur(100), Percentile::FromPercent(95.0)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); + auto const close_before = rt.machine(sid).confirmed_window_close(); + auto const open_before = rt.machine(sid).confirmed_window_open(); + rt.policy().ConfigureServerRxTiming(sid, conf, + Percentile::FromPercent(99.99)); + rt.machine(sid).SetDesired(rt.now(), conf, Percentile::FromPercent(99.99)); + TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); + TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_close() == close_before); + TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_open() == open_before); + TEST_ASSERT_TRUE(rt.machine(sid).percentile() == + Percentile::FromPercent(99.99)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_Prefix1SuccessNoPrefix2() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const prefix1 = + ComputePrefix1Time(rt.machine(sid).confirmed_window_open(), Dur(100)); + rt.AdvanceTo(prefix1 + Dur(20)); + TEST_ASSERT_EQUAL(1, rt.counters(sid).prefix1); + TEST_ASSERT_EQUAL(0, rt.counters(sid).prefix2); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_MultiServerIndependentSchedules() { + PresenceHarness rt{Tp(0)}; + ServerId const a{1}; + ServerId const b{2}; + rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), Percentile::FromPercent(99.0), 0); + rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), Percentile::FromPercent(95.0), 1); + rt.SetFixedDelay(a, Dur(20)); + rt.SetFixedDelay(b, Dur(20)); + rt.AdvanceTo(Tp(40)); + TEST_ASSERT_EQUAL(1000, ToMs(rt.machine(a).confirmed_interval())); + TEST_ASSERT_EQUAL(3000, ToMs(rt.machine(b).confirmed_interval())); + rt.SetConnectivity(a, false); + auto const offline_deadline = LocalOfflineDeadline( + rt.machine(a).confirmed_window_open(), + rt.policy().offline_detection_timeout()); + rt.AdvanceTo(offline_deadline + Dur(1)); + TEST_ASSERT_FALSE(rt.policy().IsServerLocallyOnline(a, rt.now())); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +struct RuntimeReport { + int confirmed_cycles{}; + Duration duration{}; +}; + +RuntimeReport g_stat_report{}; + +void test_StatisticalRuntimePollingIsLocallyOnline() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + auto const interval = Dur(1000); + auto const window = Dur(1000); + auto const rtt = Dur(100); + // §25: start p95, then runtime switch p99, then p99.99 — no restart. + auto phase_pct = Percentile::FromPercent(95.0); + rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(window), rtt, phase_pct); + rt.SetDelayFn(sid, [&rt, sid](PingAttemptKind kind, int) { + auto const prefix1 = rt.counters(sid).prefix1; + if (kind == PingAttemptKind::kPrefix1) { + if ((prefix1 % 19) == 0) { + return Dur(100000); + } + if ((prefix1 % 11) == 0) { + return Dur(150); + } + if ((prefix1 % 17) == 0) { + return Dur(250); + } + } + if ((kind == PingAttemptKind::kPrefix2) && ((prefix1 % 19) == 0)) { + return Dur(100000); + } + return Dur(20); + }); + + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + auto const measure_start = rt.now(); + int phase = 0; // 0=p95, 1=p99, 2=p99.99 + struct PhaseSnap { + int prefix1{}; + int prefix2{}; + int retry{}; + int recoveries{}; + long long selected_rtt_ms{}; + }; + PhaseSnap phases[3]{}; + auto snap_phase = [&](int idx) { + phases[idx].prefix1 = rt.counters(sid).prefix1; + phases[idx].prefix2 = rt.counters(sid).prefix2; + phases[idx].retry = rt.counters(sid).retry; + phases[idx].recoveries = rt.counters(sid).recoveries_to_online; + phases[idx].selected_rtt_ms = ToMs(rt.SelectedRtt(sid)); + }; + while (true) { + rt.AdvancePolling(Dur(10), Dur(10), true); + auto const elapsed_ms = + std::chrono::duration_cast(rt.now() - measure_start).count(); + if (phase == 0 && elapsed_ms >= 100000) { + snap_phase(0); + phase_pct = Percentile::FromPercent(99.0); + rt.policy().ConfigureServerRxTiming( + sid, RxTimingConf::Every(interval).WithWindow(window), phase_pct); + rt.machine(sid).SetDesired( + rt.now(), RxTimingConf::Every(interval).WithWindow(window), phase_pct); + phase = 1; + } else if (phase == 1 && elapsed_ms >= 200000) { + snap_phase(1); + phase_pct = Percentile::FromPercent(99.99); + rt.policy().ConfigureServerRxTiming( + sid, RxTimingConf::Every(interval).WithWindow(window), phase_pct); + rt.machine(sid).SetDesired( + rt.now(), RxTimingConf::Every(interval).WithWindow(window), phase_pct); + phase = 2; + } + if (elapsed_ms >= 300000) { + break; + } + TEST_ASSERT_TRUE(elapsed_ms < 400000); + } + snap_phase(2); + TEST_ASSERT_EQUAL(2, phase); + TEST_ASSERT_TRUE(rt.machine(sid).percentile() == + Percentile::FromPercent(99.99)); + + g_stat_report.confirmed_cycles = rt.counters(sid).confirmed_pongs; + g_stat_report.duration = + std::chrono::duration_cast(rt.now() - measure_start); + auto const cycles = rt.counters(sid).confirmed_pongs; + + std::printf( + "STATISTICAL runtime (p95→p99→p99.99)\n" + " duration_ms=%lld confirmed_pongs=%d status_polls=%d online_samples=%d\n" + " false_offline_samples=%d false_offline_transitions=%d\n" + " totals: prefix1=%d prefix2=%d post_prefix_retry=%d late_pongs=%d " + "timeouts=%d recoveries=%d restreams=%d\n" + " phase p95: selected_rtt_ms=%lld prefix1=%d prefix2=%d retry=%d recoveries=%d\n" + " phase p99: selected_rtt_ms=%lld prefix1=%d prefix2=%d retry=%d recoveries=%d\n" + " phase p99.99: selected_rtt_ms=%lld prefix1=%d prefix2=%d retry=%d recoveries=%d\n", + static_cast(ToMs(g_stat_report.duration)), cycles, + rt.poll_stats().status_poll_count, rt.poll_stats().online_samples, + rt.poll_stats().false_offline_samples, + rt.poll_stats().false_offline_transitions, rt.counters(sid).prefix1, + rt.counters(sid).prefix2, rt.counters(sid).retry, + rt.counters(sid).late_pongs, rt.counters(sid).scheduler_timeouts, + rt.counters(sid).recoveries_to_online, rt.counters(sid).restreams, + phases[0].selected_rtt_ms, phases[0].prefix1, phases[0].prefix2, + phases[0].retry, phases[0].recoveries, phases[1].selected_rtt_ms, + phases[1].prefix1, phases[1].prefix2, phases[1].retry, + phases[1].recoveries, phases[2].selected_rtt_ms, phases[2].prefix1, + phases[2].prefix2, phases[2].retry, phases[2].recoveries); + + TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_samples); + TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_transitions); + TEST_ASSERT_TRUE(cycles >= 280); + TEST_ASSERT_TRUE(cycles <= 330); + TEST_ASSERT_TRUE(rt.counters(sid).prefix2 > 0); + TEST_ASSERT_TRUE(rt.counters(sid).retry > 0); + TEST_ASSERT_EQUAL(0, rt.counters(sid).restreams); +} + +void test_FaultOfflineNotBeforeWindowCloseThenRecovery() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + rt.AdvancePolling(Dur(2000), Dur(10), true); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + auto const deadline = LocalOfflineDeadline( + rt.machine(sid).confirmed_window_open(), + rt.policy().offline_detection_timeout()); + rt.SetConnectivity(sid, false); + + auto detected = TimePoint{}; + while (rt.now() < deadline + Dur(2000)) { + rt.AdvancePolling(Dur(10), Dur(10), false); + if (!rt.IsLocallyOnline()) { + detected = rt.now(); + break; + } + } + TEST_ASSERT_TRUE(detected > deadline); + rt.SetConnectivity(sid, true); + auto const recover_from = rt.now(); + while (rt.now() < recover_from + Dur(2000)) { + rt.AdvancePolling(Dur(10), Dur(10), false); + if (rt.IsLocallyOnline()) { + break; + } + } + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_RxWindowDoesNotAffectLocalPresenceDeadline() { + ClientConnectivityPolicy policy; + ServerId const sid{21}; + policy.SetOfflineDetectionTimeout(Dur(1000)); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(100), Dur(40)); + auto const open = policy.FindServerPresence(sid)->confirmed_window_open_local; + auto const deadline_before = + LocalOfflineDeadline(open, policy.offline_detection_timeout()); + policy.ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(10000))); + // Confirmed open unchanged; Presence deadline unchanged by rx_window. + TEST_ASSERT_EQUAL(ToMs(open), ToMs(policy.FindServerPresence(sid) + ->confirmed_window_open_local)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(deadline_before)); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(deadline_before + Dur(1))); +} + +void test_OfflineDetectionTimeoutRuntimeChangeAppliesImmediately() { + ClientConnectivityPolicy policy; + ServerId const sid{22}; + policy.SetOfflineDetectionTimeout(Dur(1000)); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(1000), Dur(40)); + auto const open = policy.FindServerPresence(sid)->confirmed_window_open_local; + TEST_ASSERT_FALSE(policy.IsLocallyOnline(open + Dur(1001))); + policy.SetOfflineDetectionTimeout(Dur(2000)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(open + Dur(1001))); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(open + Dur(2000))); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(open + Dur(2001))); +} + +void test_RetriesContinueAfterLocalOffline() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + rt.SetConnectivity(sid, false); + auto const deadline = LocalOfflineDeadline( + rt.machine(sid).confirmed_window_open(), + rt.policy().offline_detection_timeout()); + rt.AdvanceTo(deadline + Dur(1)); + TEST_ASSERT_FALSE(rt.IsLocallyOnline()); + auto const retries_before = rt.counters(sid).retry + rt.counters(sid).prefix2 + + rt.counters(sid).recovery; + rt.AdvanceTo(deadline + Dur(500)); + auto const retries_after = rt.counters(sid).retry + rt.counters(sid).prefix2 + + rt.counters(sid).recovery; + TEST_ASSERT_TRUE(retries_after > retries_before); + rt.SetConnectivity(sid, true); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(rt.now() + Dur(300)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_IntervalZeroWithoutPongKeepsConfirmed() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); + rt.machine(sid).SetDesired(rt.now(), + RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), Percentile::FromPercent(99.0)); + TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_IntervalZeroWithPongClearsFuturePresence() { + LocalPresenceMachine machine; + machine.SetDesired(Tp(0), RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), Percentile::FromPercent(99.0)); + machine.ArmInitial(Tp(0)); + auto tick = machine.TickNow(Tp(0), Dur(100)); + TEST_ASSERT_TRUE(tick.want_send); + machine.OnSendStarting(); + machine.OnAttemptSent(tick.send, Tp(0)); + auto outcome = machine.OnPong(tick.send.attempt_id, tick.send.cycle_id, Tp(0), + Tp(20), Dur(0), Dur(0), Dur(1000), + tick.send.following_open_target, Dur(100)); + TEST_ASSERT_EQUAL( + static_cast(LocalPresenceMachine::PongDisposition::kConfirmedSchedule), + static_cast(outcome.disposition)); + TEST_ASSERT_FALSE(machine.has_confirmed_schedule()); + TEST_ASSERT_FALSE(machine.IsOnline(Tp(20))); +} + +void test_RemoteTimingProjectionAndAggregation() { + ClientTiming timing{}; + timing.next_ping_delta_ms = 500; + timing.last_connect_delta_ms = 0; + TimePoint expected{}; + TimePoint deadline{}; + auto status = ClassifyRemoteServerPresence( + Tp(2550), Tp(1000), Tp(1100), timing, Dur(1000), &expected, &deadline); + TEST_ASSERT_EQUAL(1550, ToMs(expected)); + TEST_ASSERT_EQUAL(2550, ToMs(deadline)); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOnline), + static_cast(status)); + status = ClassifyRemoteServerPresence(Tp(2551), Tp(1000), Tp(1100), timing, + Dur(1000), &expected, &deadline); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOffline), + static_cast(status)); + + timing.next_ping_delta_ms = -200; + status = ClassifyRemoteServerPresence(Tp(1850), Tp(1000), Tp(1100), timing, + Dur(1000), &expected, &deadline); + TEST_ASSERT_EQUAL(850, ToMs(expected)); + TEST_ASSERT_EQUAL(1850, ToMs(deadline)); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOnline), + static_cast(status)); + status = ClassifyRemoteServerPresence(Tp(1851), Tp(1000), Tp(1100), timing, + Dur(1000), &expected, &deadline); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOffline), + static_cast(status)); + + timing.next_ping_delta_ms = 0; + status = ClassifyRemoteServerPresence(Tp(0), Tp(1000), Tp(1100), timing, + Dur(1000)); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOffline), + static_cast(status)); + + std::vector samples(3); + samples[0] = {ServerId{1}, RemoteServerPresence::kOnline, {}, {}, 0, true}; + samples[1] = {ServerId{2}, RemoteServerPresence::kOnline, {}, {}, 0, true}; + samples[2] = {ServerId{3}, RemoteServerPresence::kOnline, {}, {}, 0, true}; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); + samples[1].status = RemoteServerPresence::kOffline; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOffline), + static_cast(AggregateRemotePresence(samples).state)); + samples[1].status = RemoteServerPresence::kUnknown; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(samples).state)); + samples[1].status = RemoteServerPresence::kExcluded; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); + samples[0].status = RemoteServerPresence::kExcluded; + samples[1].status = RemoteServerPresence::kExcluded; + samples[2].status = RemoteServerPresence::kExcluded; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(samples).state)); + TEST_ASSERT_TRUE(RemotePresenceCanEarlyCompleteOffline( + {{ServerId{1}, RemoteServerPresence::kOffline, {}, {}, 0, true}})); +} + +void test_NoObserverCloudFallbackAndAuthoritativeSet() { + TEST_ASSERT_FALSE(AllowObserverCloudFallbackForPeerPresence()); + + // Peer Personal Cloud with 4 servers; Local Presence contract is bounded by + // AE_CLOUD_MAX_SERVER_CONNECTIONS (default 3) via selected_servers(). + std::vector peer_cloud{10, 11, 12, 13}; + auto const contract = AuthoritativePresenceServerIds( + peer_cloud, AE_CLOUD_MAX_SERVER_CONNECTIONS); + TEST_ASSERT_EQUAL(4, peer_cloud.size()); + TEST_ASSERT_EQUAL(AE_CLOUD_MAX_SERVER_CONNECTIONS, contract.size()); + TEST_ASSERT_EQUAL(10, contract[0]); + TEST_ASSERT_EQUAL(11, contract[1]); + TEST_ASSERT_EQUAL(12, contract[2]); + // Server 13 is in peer cloud but outside the Local Presence contract when + // N > AE_CLOUD_MAX_SERVER_CONNECTIONS — Remote AND must not require it. + TEST_ASSERT_TRUE(std::find(contract.begin(), contract.end(), + static_cast(13)) == contract.end()); + + // When N <= max, authoritative set equals the full peer cloud. + std::vector peer_small{21, 22}; + auto const full = AuthoritativePresenceServerIds( + peer_small, AE_CLOUD_MAX_SERVER_CONNECTIONS); + TEST_ASSERT_EQUAL(2, full.size()); + TEST_ASSERT_EQUAL(21, full[0]); + TEST_ASSERT_EQUAL(22, full[1]); + + // Observer cloud IDs must never be treated as peer Presence substitutes. + std::vector observer{90, 91}; + auto const observer_contract = + AuthoritativePresenceServerIds(observer, AE_CLOUD_MAX_SERVER_CONNECTIONS); + for (auto const id : observer_contract) { + TEST_ASSERT_TRUE(std::find(contract.begin(), contract.end(), id) == + contract.end()); + TEST_ASSERT_TRUE(std::find(full.begin(), full.end(), id) == full.end()); + } + + // Peer cloud unavailable => aggregate UNKNOWN with zero usable samples. + // QueryPeerPresence::OnCloud(Error) completes with this result and never + // binds the observer cloud (used_observer_cloud remains false; queried + // server list stays empty — no observer servers contacted). + std::vector empty; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(empty).state)); + TEST_ASSERT_EQUAL(0, empty.size()); +} + +void test_PeerCloudNotObserverCloudAuthoritativeIds() { + // Deterministic peer vs observer cloud sets (integration contract). + std::vector peer{101, 102}; + std::vector observer{201, 202}; + auto const auth = + AuthoritativePresenceServerIds(peer, AE_CLOUD_MAX_SERVER_CONNECTIONS); + TEST_ASSERT_EQUAL(2, auth.size()); + TEST_ASSERT_EQUAL(101, auth[0]); + TEST_ASSERT_EQUAL(102, auth[1]); + for (auto const oid : observer) { + TEST_ASSERT_TRUE(std::find(auth.begin(), auth.end(), oid) == auth.end()); + } + // Unknown peer cloud: no authoritative servers => UNKNOWN, no fallback set. + auto const none = + AuthoritativePresenceServerIds({}, AE_CLOUD_MAX_SERVER_CONNECTIONS); + TEST_ASSERT_EQUAL(0, none.size()); + TEST_ASSERT_EQUAL( + static_cast(PeerPresenceState::kUnknown), + static_cast( + AggregateRemotePresence(std::vector{}) + .state)); +} + +void test_RecoveredServerRequiresFreshOnline() { + std::vector samples{ + {1, RemoteServerPresence::kOnline, {}, {}, 1, true}, + {2, RemoteServerPresence::kExcluded, {}, {}, 0, false}, + }; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); + // Recovered server re-enters as UNKNOWN — cannot keep stale ONLINE. + samples[1].status = RemoteServerPresence::kUnknown; + samples[1].has_timing = false; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(samples).state)); + samples[1].status = RemoteServerPresence::kOnline; + samples[1].has_timing = true; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); +} + +void test_QueryFailureIsUnknownNotOffline() { + // Usable server with failed timing (UNKNOWN after retries) must not force + // peer Offline; zero Offline contributions + incomplete ONLINE => UNKNOWN. + std::vector samples{ + {1, RemoteServerPresence::kOnline, {}, {}, 1, true}, + {2, RemoteServerPresence::kUnknown, {}, {}, 0, true}, + }; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(samples).state)); + // After quarantine/unselect, remaining ONLINE servers may aggregate ONLINE. + samples[1].status = RemoteServerPresence::kExcluded; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); +} + +} // namespace ae::test_local_presence + +void setUp() {} +void tearDown() {} + +int main() { + UNITY_BEGIN(); + RUN_TEST(ae::test_local_presence::test_PrefixFormula); + RUN_TEST(ae::test_local_presence::test_ConfirmOnlyAfterPong); + RUN_TEST(ae::test_local_presence::test_SelectedRttProjectionIgnoresMeasuredPong); + RUN_TEST(ae::test_local_presence::test_PerServerIndependence); + RUN_TEST(ae::test_local_presence::test_OfflineOnlyAfterOfflineDetectionTimeout); + RUN_TEST(ae::test_local_presence::test_RuntimeIntervalChangeKeepsOldConfirmed); + RUN_TEST(ae::test_local_presence::test_RuntimePercentileOnlyChangeKeepsSchedule); + RUN_TEST(ae::test_local_presence::test_RuntimePercentile); + RUN_TEST(ae::test_local_presence::test_ReliabilityP95VsP99PrefixTimes); + RUN_TEST(ae::test_local_presence::test_AggregateIgnoresDeselected); + RUN_TEST(ae::test_local_presence::test_OneWayProjection); + RUN_TEST(ae::test_local_presence::test_MakeConfirmedScheduleDeterministic); + RUN_TEST(ae::test_local_presence::test_ConfigScopeOverrideAndPriority); + RUN_TEST(ae::test_local_presence::test_SendWithoutPongDoesNotConfirm); + RUN_TEST(ae::test_local_presence::test_LongSleepSuspendBetweenPongAndPrefix1); + RUN_TEST(ae::test_local_presence::test_CurrentVsNextWindowBlocker); + RUN_TEST(ae::test_local_presence::test_Prefix1LatePongAfterPrefix2); + RUN_TEST(ae::test_local_presence::test_RttTailEntersStatistics); + RUN_TEST(ae::test_local_presence::test_PxxMissDoesNotRestream); + RUN_TEST(ae::test_local_presence::test_EnsureLinkedErrorReleasesBlocker); + RUN_TEST(ae::test_local_presence::test_QuarantineKeepsConfirmedUntilClose); + RUN_TEST(ae::test_local_presence::test_HardRemovalDropsAggregateImmediately); + RUN_TEST(ae::test_local_presence::test_RuntimeConfigChangeKeepsOldUntilPong); + RUN_TEST(ae::test_local_presence::test_Prefix1SuccessNoPrefix2); + RUN_TEST(ae::test_local_presence::test_MultiServerIndependentSchedules); + RUN_TEST(ae::test_local_presence::test_RxWindowDoesNotAffectLocalPresenceDeadline); + RUN_TEST(ae::test_local_presence::test_OfflineDetectionTimeoutRuntimeChangeAppliesImmediately); + RUN_TEST(ae::test_local_presence::test_RetriesContinueAfterLocalOffline); + RUN_TEST(ae::test_local_presence::test_IntervalZeroWithoutPongKeepsConfirmed); + RUN_TEST(ae::test_local_presence::test_IntervalZeroWithPongClearsFuturePresence); + RUN_TEST(ae::test_local_presence::test_RemoteTimingProjectionAndAggregation); + RUN_TEST(ae::test_local_presence::test_NoObserverCloudFallbackAndAuthoritativeSet); + RUN_TEST(ae::test_local_presence::test_PeerCloudNotObserverCloudAuthoritativeIds); + RUN_TEST(ae::test_local_presence::test_RecoveredServerRequiresFreshOnline); + RUN_TEST(ae::test_local_presence::test_QueryFailureIsUnknownNotOffline); + RUN_TEST(ae::test_local_presence::test_StatisticalRuntimePollingIsLocallyOnline); + RUN_TEST(ae::test_local_presence::test_FaultOfflineNotBeforeWindowCloseThenRecovery); + return UNITY_END(); +}