From cc17ca2c16a4a27cf9e74333c52ed16aff226420 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Thu, 11 Jun 2026 21:49:02 +0300 Subject: [PATCH 001/112] common: monitoring: provide runtime id in average monitoring Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/common/monitoring/average.cpp | 8 ++++++++ src/core/common/monitoring/average.hpp | 7 ++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/core/common/monitoring/average.cpp b/src/core/common/monitoring/average.cpp index ce9b90712..f5b6d6dc3 100644 --- a/src/core/common/monitoring/average.cpp +++ b/src/core/common/monitoring/average.cpp @@ -80,6 +80,10 @@ Error Average::Update(const NodeMonitoringData& data) !err.IsNone()) { return err; } + + if (auto err = averageInstance->mSecond.mRuntimeID.Assign(instance.mRuntimeID); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } } return ErrorEnum::eNone; @@ -98,6 +102,10 @@ Error Average::GetData(NodeMonitoringData& data) const return AOS_ERROR_WRAP(err); } + if (auto err = data.mInstances.Back().mRuntimeID.Assign(averageMonitoringData.mRuntimeID); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + if (auto err = GetMonitoringData(data.mInstances.Back().mMonitoringData, averageMonitoringData.mMonitoringData); !err.IsNone()) { return err; diff --git a/src/core/common/monitoring/average.hpp b/src/core/common/monitoring/average.hpp index 7b08defdb..45419e8ef 100644 --- a/src/core/common/monitoring/average.hpp +++ b/src/core/common/monitoring/average.hpp @@ -63,9 +63,10 @@ class Average { private: struct AverageData { - bool mIsInitialized {}; - MonitoringData mMonitoringData; - PartitionInfoArray mMonitoredPartitions; + bool mIsInitialized {}; + StaticString mRuntimeID; + MonitoringData mMonitoringData; + PartitionInfoArray mMonitoredPartitions; }; static constexpr auto cAllocatorSize = sizeof(AverageData); From f1a221394d29ecbf9bbc42c8934069e1f2368e5a Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Thu, 11 Jun 2026 21:50:50 +0300 Subject: [PATCH 002/112] cm: launcher: consider balancing indicator in reserve runtime resources Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/balancer.cpp | 4 ++-- src/core/cm/launcher/instance.cpp | 38 ++++++++++++++++--------------- src/core/cm/launcher/instance.hpp | 34 ++++++++++++--------------- src/core/cm/launcher/node.hpp | 7 ------ src/core/cm/launcher/nodeitf.hpp | 8 +++++++ 5 files changed, 44 insertions(+), 47 deletions(-) diff --git a/src/core/cm/launcher/balancer.cpp b/src/core/cm/launcher/balancer.cpp index 6db0bb865..c3033a9c0 100644 --- a/src/core/cm/launcher/balancer.cpp +++ b/src/core/cm/launcher/balancer.cpp @@ -339,7 +339,7 @@ void Balancer::FilterByCPU(Instance& instance, NodeRuntimes& nodes) auto filter = [&instance](Node* node, const RuntimeInfo* runtime) { auto availCPU = node->GetAvailableCPU(runtime->mRuntimeID); - return instance.IsAvailableCpuOk(availCPU, node->GetConfig(), node->NeedBalancing()); + return instance.IsAvailableCpuOk(availCPU, *node); }; FilterRuntimes(nodes, filter); @@ -350,7 +350,7 @@ void Balancer::FilterByRAM(Instance& instance, NodeRuntimes& nodes) auto filter = [&instance](Node* node, const RuntimeInfo* runtime) { auto availRAM = node->GetAvailableRAM(runtime->mRuntimeID); - return instance.IsAvailableRamOk(availRAM, node->GetConfig(), node->NeedBalancing()); + return instance.IsAvailableRamOk(availRAM, *node); }; FilterRuntimes(nodes, filter); diff --git a/src/core/cm/launcher/instance.cpp b/src/core/cm/launcher/instance.cpp index 964405927..6d02a681e 100644 --- a/src/core/cm/launcher/instance.cpp +++ b/src/core/cm/launcher/instance.cpp @@ -316,20 +316,18 @@ Error ComponentInstance::Cache(bool disable) return ErrorEnum::eNone; } -bool ComponentInstance::IsAvailableCpuOk(size_t availableCPU, const NodeConfig& nodeConfig, bool useMonitoringData) +bool ComponentInstance::IsAvailableCpuOk(size_t availableCPU, const NodeItf& node) { (void)availableCPU; - (void)nodeConfig; - (void)useMonitoringData; + (void)node; return true; } -bool ComponentInstance::IsAvailableRamOk(size_t availableRAM, const NodeConfig& nodeConfig, bool useMonitoringData) +bool ComponentInstance::IsAvailableRamOk(size_t availableRAM, const NodeItf& node) { (void)availableRAM; - (void)nodeConfig; - (void)useMonitoringData; + (void)node; return true; } @@ -455,29 +453,29 @@ Error ServiceInstance::Cache(bool disable) return ErrorEnum::eNone; } -bool ServiceInstance::IsAvailableCpuOk(size_t availableCPU, const NodeConfig& nodeConfig, bool useMonitoringData) +bool ServiceInstance::IsAvailableCpuOk(size_t availableCPU, const NodeItf& node) { assert(mItemConfig); - auto requestedCPU = GetRequestedCPU(nodeConfig, useMonitoringData); + auto requestedCPU = GetRequestedCPU(node); bool ok = availableCPU >= requestedCPU; - LOG_DBG() << "Available CPU " << (ok ? "enough" : "not enough") << Log::Field("nodeID", nodeConfig.mNodeID) + LOG_DBG() << "Available CPU " << (ok ? "enough" : "not enough") << Log::Field("nodeID", node.GetConfig().mNodeID) << Log::Field("availableCPU", availableCPU) << Log::Field("requestedCPU", requestedCPU); return ok; } -bool ServiceInstance::IsAvailableRamOk(size_t availableRAM, const NodeConfig& nodeConfig, bool useMonitoringData) +bool ServiceInstance::IsAvailableRamOk(size_t availableRAM, const NodeItf& node) { assert(mItemConfig); - auto requestedRAM = GetRequestedRAM(nodeConfig, useMonitoringData); + auto requestedRAM = GetRequestedRAM(node); bool ok = availableRAM >= requestedRAM; - LOG_DBG() << "Available RAM " << (ok ? "enough" : "not enough") << Log::Field("nodeID", nodeConfig.mNodeID) + LOG_DBG() << "Available RAM " << (ok ? "enough" : "not enough") << Log::Field("nodeID", node.GetConfig().mNodeID) << Log::Field("availableRAM", availableRAM) << Log::Field("requestedRAM", requestedRAM); return ok; @@ -544,7 +542,7 @@ Error ServiceInstance::Schedule(NodeItf& node, const String& runtimeID) return ErrorEnum::eNone; } -size_t ServiceInstance::GetRequestedCPU(const NodeConfig& nodeConfig, bool useMonitoringData) +size_t ServiceInstance::GetRequestedCPU(const NodeItf& node) { assert(mItemConfig); @@ -552,6 +550,8 @@ size_t ServiceInstance::GetRequestedCPU(const NodeConfig& nodeConfig, bool useMo return 0; } + const auto& nodeConfig = node.GetConfig(); + size_t requestedCPU = 0; auto quota = mItemConfig->mQuotas.mCPUDMIPSLimit; @@ -561,7 +561,7 @@ size_t ServiceInstance::GetRequestedCPU(const NodeConfig& nodeConfig, bool useMo requestedCPU = GetReqCPUFromNodeConfig(quota, nodeConfig.mResourceRatios); } - if (useMonitoringData) { + if (node.NeedBalancing()) { if (mMonitoringData.mCPU > requestedCPU) { return mMonitoringData.mCPU; } @@ -570,7 +570,7 @@ size_t ServiceInstance::GetRequestedCPU(const NodeConfig& nodeConfig, bool useMo return requestedCPU; } -size_t ServiceInstance::GetRequestedRAM(const NodeConfig& nodeConfig, bool useMonitoringData) +size_t ServiceInstance::GetRequestedRAM(const NodeItf& node) { assert(mItemConfig); @@ -578,6 +578,8 @@ size_t ServiceInstance::GetRequestedRAM(const NodeConfig& nodeConfig, bool useMo return 0; } + const auto& nodeConfig = node.GetConfig(); + size_t requestedRAM = 0; auto quota = mItemConfig->mQuotas.mRAMLimit; @@ -587,7 +589,7 @@ size_t ServiceInstance::GetRequestedRAM(const NodeConfig& nodeConfig, bool useMo requestedRAM = GetReqRAMFromNodeConfig(quota, nodeConfig.mResourceRatios); } - if (useMonitoringData) { + if (node.NeedBalancing()) { if (mMonitoringData.mRAM > requestedRAM) { return mMonitoringData.mRAM; } @@ -747,8 +749,8 @@ Error ServiceInstance::SetupStateStorage(const NodeConfig& nodeConfig, String& s Error ServiceInstance::ReserveRuntimeResources(NodeItf& node, const String& runtimeID) { - auto requestedCPU = mItemConfig->mSkipResourceLimits ? 0 : GetRequestedCPU(node.GetConfig(), false); - auto requestedRAM = mItemConfig->mSkipResourceLimits ? 0 : GetRequestedRAM(node.GetConfig(), false); + auto requestedCPU = mItemConfig->mSkipResourceLimits ? 0 : GetRequestedCPU(node); + auto requestedRAM = mItemConfig->mSkipResourceLimits ? 0 : GetRequestedRAM(node); Array requestedResources = mItemConfig->mSkipResourceLimits ? Array() : mItemConfig->mResources; diff --git a/src/core/cm/launcher/instance.hpp b/src/core/cm/launcher/instance.hpp index 73cef93f0..64354e589 100644 --- a/src/core/cm/launcher/instance.hpp +++ b/src/core/cm/launcher/instance.hpp @@ -158,21 +158,19 @@ class Instance { * Checks whether available CPU fits instance requirements. * * @param availableCPU available CPU. - * @param nodeConfig node configuration. - * @param useMonitoringData whether to use monitoring data. + * @param node node. * @return bool. */ - virtual bool IsAvailableCpuOk(size_t availableCPU, const NodeConfig& nodeConfig, bool useMonitoringData) = 0; + virtual bool IsAvailableCpuOk(size_t availableCPU, const NodeItf& node) = 0; /** * Checks whether available RAM fits instance requirements. * * @param availableRAM available RAM. - * @param nodeConfig node configuration. - * @param useMonitoringData whether to use monitoring data. + * @param node node. * @return bool. */ - virtual bool IsAvailableRamOk(size_t availableRAM, const NodeConfig& nodeConfig, bool useMonitoringData) = 0; + virtual bool IsAvailableRamOk(size_t availableRAM, const NodeItf& node) = 0; /** * Checks whether runtime type fits instance requirements. @@ -300,21 +298,19 @@ class ComponentInstance : public Instance { * Checks whether available CPU fits instance requirements. * * @param availableCPU available CPU. - * @param nodeConfig node configuration. - * @param useMonitoringData whether to use monitoring data. + * @param node node. * @return bool. */ - bool IsAvailableCpuOk(size_t availableCPU, const NodeConfig& nodeConfig, bool useMonitoringData) override; + bool IsAvailableCpuOk(size_t availableCPU, const NodeItf& node) override; /** * Checks whether available RAM fits instance requirements. * * @param availableRAM available RAM. - * @param nodeConfig node configuration. - * @param useMonitoringData whether to use monitoring data. + * @param node node. * @return bool. */ - bool IsAvailableRamOk(size_t availableRAM, const NodeConfig& nodeConfig, bool useMonitoringData) override; + bool IsAvailableRamOk(size_t availableRAM, const NodeItf& node) override; /** * Checks whether node resources fit instance requirements. @@ -385,21 +381,19 @@ class ServiceInstance : public Instance { * Checks whether available CPU fits instance requirements. * * @param availableCPU available CPU. - * @param nodeConfig node configuration. - * @param useMonitoringData whether to use monitoring data. + * @param node node. * @return bool. */ - bool IsAvailableCpuOk(size_t availableCPU, const NodeConfig& nodeConfig, bool useMonitoringData) override; + bool IsAvailableCpuOk(size_t availableCPU, const NodeItf& node) override; /** * Checks whether available RAM fits instance requirements. * * @param availableRAM available RAM. - * @param nodeConfig node configuration. - * @param useMonitoringData whether to use monitoring data. + * @param node node. * @return bool. */ - bool IsAvailableRamOk(size_t availableRAM, const NodeConfig& nodeConfig, bool useMonitoringData) override; + bool IsAvailableRamOk(size_t availableRAM, const NodeItf& node) override; /** * Checks whether node resources fit instance requirements. @@ -429,8 +423,8 @@ class ServiceInstance : public Instance { private: static constexpr auto cDefaultResourceRation = 50.0; - size_t GetRequestedCPU(const NodeConfig& nodeConfig, bool useMonitoringData); - size_t GetRequestedRAM(const NodeConfig& nodeConfig, bool useMonitoringData); + size_t GetRequestedCPU(const NodeItf& node); + size_t GetRequestedRAM(const NodeItf& node); size_t GetReqStateSize(const NodeConfig& nodeConfig); size_t GetReqStorageSize(const NodeConfig& nodeConfig); diff --git a/src/core/cm/launcher/node.hpp b/src/core/cm/launcher/node.hpp index a0b25a403..9b69e7f2d 100644 --- a/src/core/cm/launcher/node.hpp +++ b/src/core/cm/launcher/node.hpp @@ -60,11 +60,6 @@ class Node : public NodeItf { */ const UnitNodeInfo& GetInfo() const { return mInfo; } - /** - * Indicates whether node requires rebalancing. - */ - bool NeedBalancing() const { return mNeedBalancing; } - /** * Updates node information. * @@ -188,8 +183,6 @@ class Node : public NodeItf { UnitNodeInfo mInfo {}; bool mIsNodeStatusReceived {}; - bool mNeedBalancing {}; - size_t mTotalCPUUsage {}; size_t mTotalRAMUsage {}; size_t mSystemCPUUsage {}; diff --git a/src/core/cm/launcher/nodeitf.hpp b/src/core/cm/launcher/nodeitf.hpp index a9db3cb48..a7ee0e1f3 100644 --- a/src/core/cm/launcher/nodeitf.hpp +++ b/src/core/cm/launcher/nodeitf.hpp @@ -55,8 +55,16 @@ class NodeItf { */ const NodeConfig& GetConfig() const { return mConfig; } + /** + * Indicates whether node requires rebalancing. + * + * @return bool. + */ + bool NeedBalancing() const { return mNeedBalancing; } + protected: NodeConfig mConfig {}; + bool mNeedBalancing {}; }; /** @}*/ From 5320f6572c9a64d2e888a7891a4fe119ce1f86f3 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Thu, 11 Jun 2026 21:51:56 +0300 Subject: [PATCH 003/112] cm: launcher: don't slice max threshold percent Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/node.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/core/cm/launcher/node.cpp b/src/core/cm/launcher/node.cpp index f92be7725..28eaf714b 100644 --- a/src/core/cm/launcher/node.cpp +++ b/src/core/cm/launcher/node.cpp @@ -120,9 +120,8 @@ void Node::PrepareForBalancing(bool rebalancing) const auto& alertRules = mConfig.mAlertRules.GetValue(); if (alertRules.mCPU.HasValue() || alertRules.mRAM.HasValue()) { if (alertRules.mCPU.HasValue()) { - const auto usedCPU = mTotalCPUUsage; - const auto maxTreshold - = mInfo.mMaxDMIPS * static_cast(alertRules.mCPU.GetValue().mMaxThreshold / 100.0); + const auto usedCPU = mTotalCPUUsage; + const auto maxTreshold = mInfo.mMaxDMIPS * alertRules.mCPU.GetValue().mMaxThreshold / 100.0; if (usedCPU > maxTreshold) { mNeedBalancing = true; @@ -130,9 +129,8 @@ void Node::PrepareForBalancing(bool rebalancing) } if (alertRules.mRAM.HasValue()) { - const auto usedRAM = mTotalRAMUsage; - const auto maxTreshold - = mInfo.mMaxDMIPS * static_cast(alertRules.mRAM.GetValue().mMaxThreshold / 100.0); + const auto usedRAM = mTotalRAMUsage; + const auto maxTreshold = mInfo.mTotalRAM * alertRules.mRAM.GetValue().mMaxThreshold / 100.0; if (usedRAM > maxTreshold) { mNeedBalancing = true; From c1336f6bcbf2a53f63a22d726413bc8359feab7c Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 12 Jun 2026 18:45:36 +0300 Subject: [PATCH 004/112] cm: launcher: launcher should keep status info for paused nodes as well Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/balancer.cpp | 6 ++ src/core/cm/launcher/launcher.cpp | 4 +- src/core/cm/launcher/node.cpp | 70 +----------------- src/core/cm/launcher/nodemanager.cpp | 30 ++++---- src/core/cm/launcher/utils.hpp | 102 +++++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 81 deletions(-) create mode 100644 src/core/cm/launcher/utils.hpp diff --git a/src/core/cm/launcher/balancer.cpp b/src/core/cm/launcher/balancer.cpp index c3033a9c0..fd8a8b87b 100644 --- a/src/core/cm/launcher/balancer.cpp +++ b/src/core/cm/launcher/balancer.cpp @@ -444,6 +444,12 @@ Error Balancer::PerformPolicyBalancing(Array>& instances) continue; } + if (!node->IsConnected() || node->GetInfo().mState != NodeStateEnum::eProvisioned) { + LOG_WRN() << "Node is skipped from balancing" << Log::Field("nodeID", info.mNodeID); + + continue; + } + if (auto err = mInstanceManager->ScheduleInstance(instance, *node, info.mRuntimeID); !err.IsNone()) { LOG_WRN() << "Can't schedule instance" << Log::Field("instance", id) << Log::Field(AOS_ERROR_WRAP(err)); diff --git a/src/core/cm/launcher/launcher.cpp b/src/core/cm/launcher/launcher.cpp index c1ed01a8e..125104247 100644 --- a/src/core/cm/launcher/launcher.cpp +++ b/src/core/cm/launcher/launcher.cpp @@ -517,7 +517,9 @@ void Launcher::ProcessUpdate() void Launcher::WaitAllNodesConnected(UniqueLock& lock) { auto allNodesConnected = [this]() { - auto notConnected = [](const Node& node) { return !node.IsConnected(); }; + auto notConnected = [](const Node& node) { + return !node.IsConnected() && node.GetInfo().mState == NodeStateEnum::eProvisioned; + }; return !mNodeManager.GetNodes().ContainsIf(notConnected) || !mIsRunning; }; diff --git a/src/core/cm/launcher/node.cpp b/src/core/cm/launcher/node.cpp index 28eaf714b..4eb880a0d 100644 --- a/src/core/cm/launcher/node.cpp +++ b/src/core/cm/launcher/node.cpp @@ -7,88 +7,24 @@ #include #include "node.hpp" +#include "utils.hpp" namespace aos::cm::launcher { -template -class Filter { -public: - class Iterator { - public: - Iterator(typename Array::ConstIterator it, typename Array::ConstIterator end, Cmp cmp) - : mIt(it) - , mEnd(end) - , mCmp(cmp) - { - while (mIt != mEnd && !mCmp(*mIt)) { - ++mIt; - } - } - - Iterator& operator++() - { - assert(mIt != mEnd); - - ++mIt; - - while (mIt != mEnd && !mCmp(*mIt)) { - ++mIt; - } - - return *this; - } - - Iterator operator++(int) - { - assert(mIt != mEnd); - - Iterator tmp = *this; - - ++(*this); - - return tmp; - } - - bool operator==(const Iterator& other) const { return mIt == other.mIt; } - bool operator!=(const Iterator& other) const { return mIt != other.mIt; } - - const T& operator*() const { return *mIt; } - const T* operator->() const { return mIt; } - - private: - typename Array::ConstIterator mIt; - typename Array::ConstIterator mEnd; - Cmp mCmp; - }; - - Filter(const Array& array, Cmp cmp) - : mArray(&array) - , mCmp(cmp) - { - } - - Iterator begin() const { return Iterator(mArray->begin(), mArray->end(), mCmp); } - Iterator end() const { return Iterator(mArray->end(), mArray->end(), mCmp); } - -private: - const Array* mArray; - Cmp mCmp; -}; - auto FilterActiveNodeInstances(const Array& array, const String& nodeID) { auto cmp = [nodeID](const InstanceStatus& status) { return status.mNodeID == nodeID && status.mState != aos::InstanceStateEnum::eInactive; }; - return Filter(array, cmp); + return Filter(array, cmp); } auto FilterByNode(const Array>& array, const String& nodeID) { auto cmp = [nodeID](const SharedPtr& instance) { return instance->GetInfo().mNodeID == nodeID; }; - return Filter, decltype(cmp)>(array, cmp); + return Filter(array, cmp); } /*********************************************************************************************************************** diff --git a/src/core/cm/launcher/nodemanager.cpp b/src/core/cm/launcher/nodemanager.cpp index 74591d184..c8cc05718 100644 --- a/src/core/cm/launcher/nodemanager.cpp +++ b/src/core/cm/launcher/nodemanager.cpp @@ -4,13 +4,21 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "nodemanager.hpp" - #include -#include + +#include "nodemanager.hpp" +#include "utils.hpp" namespace aos::cm::launcher { +auto FilterActiveNodes(Array& array) +{ + auto cmp + = [](const Node& node) { return node.IsConnected() && node.GetInfo().mState == NodeStateEnum::eProvisioned; }; + + return Filter(array, cmp); +} + /*********************************************************************************************************************** * Public **********************************************************************************************************************/ @@ -69,6 +77,8 @@ Error NodeManager::Stop() Error NodeManager::PrepareForBalancing(bool rebalancing) { + // Launcher utilizes scheduling implementation to load SM data for active instances on startup + // so we need to prepare for balancing all nodes. for (auto& node : mNodes) { node.PrepareForBalancing(rebalancing); } @@ -158,7 +168,7 @@ Error NodeManager::GetConnectedNodes(Array& nodes) { nodes.Clear(); - for (auto& node : mNodes) { + for (auto& node : FilterActiveNodes(mNodes)) { if (auto err = nodes.PushBack(&node); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -192,7 +202,7 @@ Error NodeManager::SendScheduledInstances(UniqueLock& lock, const Array& lock, const Array& lock, const ArrayUpdateInfo(info); } else { if (info.mState != NodeStateEnum::eProvisioned) { diff --git a/src/core/cm/launcher/utils.hpp b/src/core/cm/launcher/utils.hpp new file mode 100644 index 000000000..65acd5b15 --- /dev/null +++ b/src/core/cm/launcher/utils.hpp @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_CM_LAUNCHER_UTILS_HPP_ +#define AOS_CORE_CM_LAUNCHER_UTILS_HPP_ + +#include + +namespace aos::cm::launcher { + +template +class FilterIterator { +public: + FilterIterator(It it, It end, Cmp cmp) + : mIt(it) + , mEnd(end) + , mCmp(cmp) + { + while (mIt != mEnd && !mCmp(*mIt)) { + ++mIt; + } + } + + FilterIterator& operator++() + { + assert(mIt != mEnd); + + ++mIt; + + while (mIt != mEnd && !mCmp(*mIt)) { + ++mIt; + } + + return *this; + } + + FilterIterator operator++(int) + { + assert(mIt != mEnd); + + FilterIterator tmp = *this; + + ++(*this); + + return tmp; + } + + bool operator==(const FilterIterator& other) const { return mIt == other.mIt; } + bool operator!=(const FilterIterator& other) const { return mIt != other.mIt; } + + auto& operator*() const { return *mIt; } + auto operator->() const { return mIt; } + +private: + It mIt; + It mEnd; + Cmp mCmp; +}; + +template +class Filter { +public: + Filter(It begin, It end, Cmp cmp) + : mBegin(begin) + , mEnd(end) + , mCmp(cmp) + { + } + + template + Filter(Array& array, Cmp cmp) + : Filter(array.begin(), array.end(), cmp) + { + } + + template + Filter(const Array& array, Cmp cmp) + : Filter(array.begin(), array.end(), cmp) + { + } + + FilterIterator begin() const { return FilterIterator(mBegin, mEnd, mCmp); } + FilterIterator end() const { return FilterIterator(mEnd, mEnd, mCmp); } + +private: + It mBegin; + It mEnd; + Cmp mCmp; +}; + +template +Filter(Array&, Cmp) -> Filter::Iterator, Cmp>; + +template +Filter(const Array&, Cmp) -> Filter::ConstIterator, Cmp>; + +} // namespace aos::cm::launcher + +#endif From 4491aee709cfd697bf49fd692638fd3981c48807 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 12 Jun 2026 11:58:12 +0300 Subject: [PATCH 005/112] cm: launcher: add UID pool for the same instance ident Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/CMakeLists.txt | 3 +- src/core/cm/launcher/gidpool.cpp | 82 ----------- src/core/cm/launcher/gidpool.hpp | 83 ----------- src/core/cm/launcher/idpool.hpp | 176 ++++++++++++++++++++++++ src/core/cm/launcher/instance.cpp | 24 +--- src/core/cm/launcher/instance.hpp | 23 +--- src/core/cm/launcher/tests/launcher.cpp | 8 +- 7 files changed, 189 insertions(+), 210 deletions(-) delete mode 100644 src/core/cm/launcher/gidpool.cpp delete mode 100644 src/core/cm/launcher/gidpool.hpp create mode 100644 src/core/cm/launcher/idpool.hpp diff --git a/src/core/cm/launcher/CMakeLists.txt b/src/core/cm/launcher/CMakeLists.txt index a463c90a8..701910421 100644 --- a/src/core/cm/launcher/CMakeLists.txt +++ b/src/core/cm/launcher/CMakeLists.txt @@ -16,7 +16,6 @@ set(TARGET_NAME launcher) set(SOURCES balancer.cpp - gidpool.cpp imageinfoprovider.cpp instance.cpp instancemanager.cpp @@ -32,7 +31,7 @@ set(SOURCES # ###################################################################################################################### set(HEADERS - gidpool.hpp + idpool.hpp imageinfoprovider.hpp runrequestsloader.hpp itf/instancerunner.hpp diff --git a/src/core/cm/launcher/gidpool.cpp b/src/core/cm/launcher/gidpool.cpp deleted file mode 100644 index 8b2b463a6..000000000 --- a/src/core/cm/launcher/gidpool.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2025 EPAM Systems, Inc. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "gidpool.hpp" - -namespace aos::cm::launcher { - -RetWithError GIDPool::GetGID(const String& itemID, gid_t gid) -{ - if (auto existing = mItemGIDs.Find(itemID); existing != mItemGIDs.end()) { - if (gid != 0 && existing->mSecond.mGID != gid) { - return {0, AOS_ERROR_WRAP(ErrorEnum::eInvalidArgument)}; - } - - existing->mSecond.mRefCount++; - - return {existing->mSecond.mGID, ErrorEnum::eNone}; - } - - gid_t assigned = gid; - - if (gid != 0) { - if (auto err = mPool.TryAcquire(gid); !err.IsNone()) { - return {0, AOS_ERROR_WRAP(err)}; - } - } else { - auto [autoGID, acquireErr] = mPool.Acquire(); - if (!acquireErr.IsNone()) { - return {0, AOS_ERROR_WRAP(acquireErr)}; - } - - assigned = static_cast(autoGID); - } - - ItemEntry entry {assigned, 1}; - - if (auto err = mItemGIDs.Emplace(itemID, entry); !err.IsNone()) { - mPool.Release(assigned); - - return {0, AOS_ERROR_WRAP(err)}; - } - - return {assigned, ErrorEnum::eNone}; -} - -Error GIDPool::Release(const String& itemID) -{ - auto existing = mItemGIDs.Find(itemID); - if (existing == mItemGIDs.end()) { - return ErrorEnum::eNotFound; - } - - auto& entry = existing->mSecond; - - if (entry.mRefCount > 1) { - entry.mRefCount--; - - return ErrorEnum::eNone; - } - - if (auto err = mPool.Release(entry.mGID); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - return mItemGIDs.Remove(itemID); -} - -Error GIDPool::Clear() -{ - if (auto err = mPool.Clear(); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - mItemGIDs.Clear(); - - return ErrorEnum::eNone; -} - -} // namespace aos::cm::launcher diff --git a/src/core/cm/launcher/gidpool.hpp b/src/core/cm/launcher/gidpool.hpp deleted file mode 100644 index 53549d3df..000000000 --- a/src/core/cm/launcher/gidpool.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (C) 2025 EPAM Systems, Inc. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef AOS_CORE_CM_LAUNCHER_GIDPOOL_HPP_ -#define AOS_CORE_CM_LAUNCHER_GIDPOOL_HPP_ - -#include - -#include -#include -#include -#include - -namespace aos::cm::launcher { - -/** - * GID range start. - */ -static constexpr auto cGIDRangeBegin = 5000; - -/** - * GID range end. - */ -static constexpr auto cGIDRangeEnd = 10000; - -/** - * Max number of locked IDs simultaneously. - */ -static constexpr auto cMaxNumLockedGIDs = cMaxNumUpdateItems; - -/** - * Pool that manages group identifiers for update items. - */ -class GIDPool { -public: - /** - * Initializes the underlying identifier pool. - * - * @param validator validator callback. - * @return Error. - */ - Error Init(IdentifierPoolValidator validator) { return mPool.Init(validator); } - - /** - * Returns a GID for an update item. - * - * @param itemID item ID. - * @param gid requested GID. If 0, a new GID will be generated. - * @return RetWithError. - */ - RetWithError GetGID(const String& itemID, gid_t gid = 0); - - /** - * Releases a reference for the update item GID. - * - * @param itemID item ID. - * @return Error. - */ - Error Release(const String& itemID); - - /** - * Clears allocated GIDs. - * - * @return Error. - */ - Error Clear(); - -private: - struct ItemEntry { - gid_t mGID {}; - size_t mRefCount {}; - }; - - IdentifierRangePool mPool; - StaticMap, ItemEntry, cMaxNumUpdateItems> mItemGIDs; -}; - -} // namespace aos::cm::launcher - -#endif diff --git a/src/core/cm/launcher/idpool.hpp b/src/core/cm/launcher/idpool.hpp new file mode 100644 index 000000000..7d8fdbcd7 --- /dev/null +++ b/src/core/cm/launcher/idpool.hpp @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2025 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_CM_LAUNCHER_IDPOOL_HPP_ +#define AOS_CORE_CM_LAUNCHER_IDPOOL_HPP_ + +#include + +#include +#include +#include +#include + +namespace aos::cm::launcher { + +/** + * GID range start. + */ +static constexpr auto cGIDRangeBegin = 5000; + +/** + * GID range end. + */ +static constexpr auto cGIDRangeEnd = 10000; + +/** + * Max number of locked GIDs simultaneously. + */ +static constexpr auto cMaxNumLockedGIDs = cMaxNumUpdateItems; + +/** + * UID range start. + */ +static constexpr auto cUIDRangeBegin = 5000; + +/** + * UID range end. + */ +static constexpr auto cUIDRangeEnd = 10000; + +/** + * Max number of locked UIDs simultaneously. + */ +static constexpr auto cMaxNumLockedUIDs = cMaxNumInstances; + +/** + * Pool that manages identifiers with reference counting per key. + * + * @tparam K key type. + * @tparam I identifier type. + * @tparam cRangeBegin identifier range start. + * @tparam cRangeEnd identifier range end. + * @tparam cMaxNumLocked max number of locked IDs simultaneously. + * @tparam cMaxNumItems max number of tracked keys. + */ +template +class IDPool { +public: + /** + * Initializes the underlying identifier pool. + * + * @param validator validator callback. + * @return Error. + */ + Error Init(IdentifierPoolValidator validator) { return mPool.Init(validator); } + + /** + * Returns an identifier for a key. + * + * @param key key. + * @param defaultID requested identifier. If 0, a new identifier will be generated. + * @return RetWithError. + */ + RetWithError Acquire(const K& key, I defaultID = 0) + { + if (auto existing = mItems.Find(key); existing != mItems.end()) { + if (defaultID != 0 && existing->mSecond.mID != defaultID) { + return {0, AOS_ERROR_WRAP(ErrorEnum::eInvalidArgument)}; + } + + existing->mSecond.mRefCount++; + + return {existing->mSecond.mID, ErrorEnum::eNone}; + } + + I assigned = defaultID; + + if (defaultID != 0) { + if (auto err = mPool.TryAcquire(defaultID); !err.IsNone()) { + return {0, AOS_ERROR_WRAP(err)}; + } + } else { + auto [autoID, acquireErr] = mPool.Acquire(); + if (!acquireErr.IsNone()) { + return {0, AOS_ERROR_WRAP(acquireErr)}; + } + + assigned = static_cast(autoID); + } + + ItemEntry entry {assigned, 1}; + + if (auto err = mItems.Emplace(key, entry); !err.IsNone()) { + mPool.Release(assigned); + + return {0, AOS_ERROR_WRAP(err)}; + } + + return {assigned, ErrorEnum::eNone}; + } + + /** + * Releases a reference for the key identifier. + * + * @param key key. + * @return Error. + */ + Error Release(const K& key) + { + auto existing = mItems.Find(key); + if (existing == mItems.end()) { + return ErrorEnum::eNotFound; + } + + auto& entry = existing->mSecond; + + if (entry.mRefCount > 1) { + entry.mRefCount--; + + return ErrorEnum::eNone; + } + + if (auto err = mPool.Release(entry.mID); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return mItems.Remove(key); + } + + /** + * Clears allocated identifiers. + * + * @return Error. + */ + Error Clear() + { + if (auto err = mPool.Clear(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + mItems.Clear(); + + return ErrorEnum::eNone; + } + +private: + struct ItemEntry { + I mID {}; + size_t mRefCount {}; + }; + + IdentifierRangePool mPool; + StaticMap mItems; +}; + +using GIDPool + = IDPool, gid_t, cGIDRangeBegin, cGIDRangeEnd, cMaxNumLockedGIDs, cMaxNumUpdateItems>; + +using UIDPool = IDPool; + +} // namespace aos::cm::launcher + +#endif diff --git a/src/core/cm/launcher/instance.cpp b/src/core/cm/launcher/instance.cpp index 6d02a681e..2bcbd742f 100644 --- a/src/core/cm/launcher/instance.cpp +++ b/src/core/cm/launcher/instance.cpp @@ -385,29 +385,19 @@ ServiceInstance::ServiceInstance(const InstanceInfo& info, UIDPool& uidPool, GID Error ServiceInstance::Init() { - if (mInfo.mUID != 0) { - if (auto err = mUIDPool.TryAcquire(mInfo.mUID); !err.IsNone()) { - LOG_WRN() << "Can't add UID to pool" << Log::Field(err); - } - } else { - Error err; + Error uidErr; + Error gidErr; - Tie(mInfo.mUID, err) = mUIDPool.Acquire(); - if (!err.IsNone()) { - LOG_WRN() << "Can't add UID to pool" << Log::Field(err); - } + Tie(mInfo.mUID, uidErr) = mUIDPool.Acquire(mInfo.mInstanceIdent, mInfo.mUID); + if (!uidErr.IsNone()) { + return AOS_ERROR_WRAP(uidErr); } - gid_t gid; - Error gidErr; - - Tie(gid, gidErr) = mGIDPool.GetGID(mInfo.mInstanceIdent.mItemID, mInfo.mGID); + Tie(mInfo.mGID, gidErr) = mGIDPool.Acquire(mInfo.mInstanceIdent.mItemID, mInfo.mGID); if (!gidErr.IsNone()) { return AOS_ERROR_WRAP(gidErr); } - mInfo.mGID = gid; - return ErrorEnum::eNone; } @@ -424,7 +414,7 @@ Error ServiceInstance::Remove() return AOS_ERROR_WRAP(err); } - if (auto err = mUIDPool.Release(mInfo.mUID); !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { + if (auto err = mUIDPool.Release(mInfo.mInstanceIdent); !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { return AOS_ERROR_WRAP(err); } diff --git a/src/core/cm/launcher/instance.hpp b/src/core/cm/launcher/instance.hpp index 64354e589..ed2215a0d 100644 --- a/src/core/cm/launcher/instance.hpp +++ b/src/core/cm/launcher/instance.hpp @@ -11,13 +11,12 @@ #include #include -#include #include #include #include "itf/storage.hpp" -#include "gidpool.hpp" +#include "idpool.hpp" #include "imageinfoprovider.hpp" #include "nodeitf.hpp" #include "storagestate.hpp" @@ -28,26 +27,6 @@ namespace aos::cm::launcher { * @{ */ -/** - * UID range start. - */ -static constexpr auto cUIDRangeBegin = 5000; - -/** - * UID range end. - */ -static constexpr auto cUIDRangeEnd = 10000; - -/** - * Max number of locked IDs simultaneously. - */ -static constexpr auto cMaxNumLockedUIDs = cMaxNumInstances; - -/** - * User ID pool - */ -using UIDPool = IdentifierRangePool; - /** * Base instance class. */ diff --git a/src/core/cm/launcher/tests/launcher.cpp b/src/core/cm/launcher/tests/launcher.cpp index 219ca4cd9..47756faf9 100644 --- a/src/core/cm/launcher/tests/launcher.cpp +++ b/src/core/cm/launcher/tests/launcher.cpp @@ -1988,7 +1988,7 @@ TEST_F(CMLauncherTest, SetStatusOnStart) auto manifestDigest = BuildManifestDigest(cService1, cImageID1); auto instance1 = CreateInstanceInfo(CreateInstanceIdent(cService1, cSubject1, 0), manifestDigest, cRunnerRunc, cNodeIDLocalSM, - InstanceStateEnum::eActive, 5001, 0, Time::Now(), "1.0.0", false, "", SubjectTypeEnum::eGroup, 100); + InstanceStateEnum::eActive, 5000, 0, Time::Now(), "1.0.0", false, "", SubjectTypeEnum::eGroup, 100); auto instance2 = CreateInstanceInfo(CreateInstanceIdent(cService1, cSubject1, 1), manifestDigest, cRunnerRunc, cNodeIDLocalSM, @@ -2552,7 +2552,7 @@ TEST_F(CMLauncherTest, ServiceUpdate) EXPECT_TRUE(instanceStatusListener.WaitForNotifyCount(2, 2s)); // New version replaces the old one on the node: SM is told to stop 1.0.0 instances, then start 1.0.1. - // Cached v1.0.0 instances still hold the first UID pool allocation; v1.0.1 gets the next UIDs and IPs. + // Cached v1.0.0 instances still hold UID pool references; v1.0.1 reuses the same UIDs per InstanceIdent. auto stopV100Inst0 = CreateAosStopInstanceInfo(CreateInstanceIdent(cService1, cSubject1, 0), cRunnerRunc); stopV100Inst0.mVersion = "1.0.0"; auto stopV100Inst1 = CreateAosStopInstanceInfo(CreateInstanceIdent(cService1, cSubject1, 1), cRunnerRunc); @@ -2561,9 +2561,9 @@ TEST_F(CMLauncherTest, ServiceUpdate) std::map expectedAfterV101 = {{cNodeIDLocalSM, {{stopV100Inst0, stopV100Inst1}, {CreateServiceRunInfo( - CreateInstanceIdent(cService1, cSubject1, 0), cImageID1, cRunnerRunc, 5002, 5000, 50, "1.0.1"), + CreateInstanceIdent(cService1, cSubject1, 0), cImageID1, cRunnerRunc, 5000, 5000, 50, "1.0.1"), CreateServiceRunInfo( - CreateInstanceIdent(cService1, cSubject1, 1), cImageID1, cRunnerRunc, 5003, 5000, 50, "1.0.1")}}}}; + CreateInstanceIdent(cService1, cSubject1, 1), cImageID1, cRunnerRunc, 5001, 5000, 50, "1.0.1")}}}}; EXPECT_EQ(mInstanceRunner.GetRunRequests(), expectedAfterV101); From 2bb633358c5bd109621a2e5eed07538169cecc8d Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 17 Jun 2026 15:41:32 +0300 Subject: [PATCH 006/112] cm: config: increase alerts cache size to 64 Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/cm/config.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/cm/config.hpp b/src/core/cm/config.hpp index bfe1e7faa..ffb2677de 100644 --- a/src/core/cm/config.hpp +++ b/src/core/cm/config.hpp @@ -25,7 +25,7 @@ * Alerts cache size. */ #ifndef AOS_CONFIG_CM_ALERTS_CACHE_SIZE -#define AOS_CONFIG_CM_ALERTS_CACHE_SIZE 32 +#define AOS_CONFIG_CM_ALERTS_CACHE_SIZE 64 #endif /** From 049943f99305bca5701c3abcd220c9c18f3b11bc Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 17 Jun 2026 15:42:28 +0300 Subject: [PATCH 007/112] cm: launcher: fix static allocator size in instance manager Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/cm/launcher/instancemanager.hpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/core/cm/launcher/instancemanager.hpp b/src/core/cm/launcher/instancemanager.hpp index b1f97c7b6..70568706f 100644 --- a/src/core/cm/launcher/instancemanager.hpp +++ b/src/core/cm/launcher/instancemanager.hpp @@ -258,9 +258,14 @@ class InstanceManager { bool OverrideEnvVars(const OverrideEnvVarsRequest& envVars); private: - static constexpr auto cRemovePeriod = Time::cDay; - static constexpr auto cAllocatorSize = Max(sizeof(ComponentInstance), sizeof(ServiceInstance)) * cMaxNumInstances - + sizeof(InstanceInfo) * cMaxNumInstances + sizeof(InstanceInfo) + sizeof(oci::ImageIndex); + static constexpr auto cRemovePeriod = Time::cDay; + // LoadInstancesFromStorage: 1 StaticArray alive throughout loop + // + up to cMaxNumInstances instances. CreateInstance(RunInstanceRequest): up to (cMaxNumInstances-1) + // existing instances + 1 InstanceInfo (CreateInfo) + 1 new instance. Both paths peak at + // cMaxNumInstances+1 simultaneous allocations. + static constexpr auto cAllocatorSize = sizeof(StaticArray) + + Max(sizeof(ComponentInstance), sizeof(ServiceInstance)) * cMaxNumInstances; + static constexpr auto cMaxNumAllocations = cMaxNumInstances + 1; static constexpr auto cInstanceAllocatorSize = sizeof(oci::ImageConfig) + sizeof(oci::ItemConfig) + sizeof(InstanceStatus) + sizeof(oci::ImageIndex) + sizeof(EnvVarArray); @@ -304,8 +309,8 @@ class InstanceManager { Timer mCleanInstancesTimer; Timer mInitTimer; - StaticAllocator mAllocator; - StaticAllocator mInstanceAllocator; + StaticAllocator mAllocator; + StaticAllocator mInstanceAllocator; StaticArray, cMaxNumInstances> mActiveInstances; StaticArray, cMaxNumInstances> mScheduledInstances; From 25eada0d91d469ac51c70157938d7120bd7b20e5 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 17 Jun 2026 15:44:09 +0300 Subject: [PATCH 008/112] sm: imagemanager: fix static allocator size Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/sm/imagemanager/imagemanager.hpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/core/sm/imagemanager/imagemanager.hpp b/src/core/sm/imagemanager/imagemanager.hpp index 60c65cc90..137e15642 100644 --- a/src/core/sm/imagemanager/imagemanager.hpp +++ b/src/core/sm/imagemanager/imagemanager.hpp @@ -115,10 +115,17 @@ class ImageManager : public ImageManagerItf, public ItemInfoProviderItf, public // oci::cMaxNumLayers + 3 (layers + manifest + image config + aos service) static constexpr auto cMaxNumInstalledBlobs = cMaxNumUpdateItems * (oci::cMaxNumLayers + 3); static constexpr auto cMaxNumInstalledLayers = cMaxNumUpdateItems * oci::cMaxNumLayers; - static constexpr auto cAllocatorSize - = cMaxNumConcurrentItems * (sizeof(oci::ImageManifest) + sizeof(oci::ImageConfig)) - + sizeof(UpdateItemDataStaticArray) + sizeof(StaticArray, cMaxNumInstalledBlobs>) - + sizeof(StaticArray, cMaxNumInstalledLayers>); + // Worst case: RemoveItem (mutex held) runs RemoveOrphans->CalcItemBlobsAndLayers (1 manifest + 1 config) + // while cMaxNumConcurrentItems service installs are in their layer-download step inside the if(eService) + // block (manifest + config both alive, no mutex held). Config in InstallUpdateItem is scoped to the + // if(eService) block and freed before StoreUpdateItem, so only the +1 from CalcItemBlobsAndLayers adds + // to the manifest/config count beyond the N concurrent installs. + // Allocation count: 3 fixed (items array + usedBlobs + usedLayers) + 2 per slot (manifest+config). + static constexpr auto cAllocatorSize = sizeof(UpdateItemDataStaticArray) + + sizeof(StaticArray, cMaxNumInstalledBlobs>) + + sizeof(StaticArray, cMaxNumInstalledLayers>) + + (cMaxNumConcurrentItems + 1) * (sizeof(oci::ImageManifest) + sizeof(oci::ImageConfig)); + static constexpr auto cMaxNumAllocations = 3 + 2 * (cMaxNumConcurrentItems + 1); RetWithError RemoveItem(const String& id, const String& version) override; @@ -161,7 +168,7 @@ class ImageManager : public ImageManagerItf, public ItemInfoProviderItf, public ImageHandlerItf* mImageHandler {}; StorageItf* mStorage {}; - mutable StaticAllocator mAllocator; + mutable StaticAllocator mAllocator; Timer mTimer; mutable Mutex mMutex; From 468fee5a35022756174a8f5eb36c5451a369cc6b Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 17 Jun 2026 15:44:55 +0300 Subject: [PATCH 009/112] sm: imagemanager: postpone removing orphans till all item processed When multiple update items are installed at same time, removing orphans function that called at the end of update item install can remove already partially installed blobs from other items that leads to undefined behaviour. The fix is to perform removing orphans in dedicated thread after all items are processed. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/sm/imagemanager/imagemanager.cpp | 33 ++++++++++++++++------- src/core/sm/imagemanager/imagemanager.hpp | 1 + 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/core/sm/imagemanager/imagemanager.cpp b/src/core/sm/imagemanager/imagemanager.cpp index 5b9bc7d5d..924370290 100644 --- a/src/core/sm/imagemanager/imagemanager.cpp +++ b/src/core/sm/imagemanager/imagemanager.cpp @@ -173,6 +173,20 @@ Error ImageManager::InstallUpdateItem(const UpdateItemInfo& itemInfo) LOG_INF() << "Install item" << Log::Field("itemID", itemInfo.mID) << Log::Field("version", itemInfo.mVersion) << Log::Field("type", itemInfo.mType) << Log::Field("manifestDigest", itemInfo.mManifestDigest); + { + LockGuard lock {mMutex}; + + ++mNumActiveInstalls; + } + + auto decrementInstalls = DeferRelease(this, [](ImageManager* self) { + LockGuard lock {self->mMutex}; + + if (--self->mNumActiveInstalls == 0) { + self->mCV.NotifyAll(); + } + }); + oci::ContentDescriptor manifestDescriptor {"", itemInfo.mManifestDigest, 0}; LOG_DBG() << "Install manifest blob" << Log::Field("digest", itemInfo.mManifestDigest); @@ -857,14 +871,7 @@ Error ImageManager::StoreUpdateItem(const UpdateItemInfo& itemInfo) } if (removedItems) { - size_t removedSize = 0; - - Tie(removedSize, err) = RemoveOrphans(); - if (!err.IsNone()) { - LOG_ERR() << "Failed to remove orphans" << Log::Field(err); - } - - mSpaceAllocator->FreeSpace(removedSize); + mProcessOutdatedItems = true; } return ErrorEnum::eNone; @@ -1318,8 +1325,10 @@ void ImageManager::ProcessOutdatedItems() while (true) { UniqueLock lock {mMutex}; - if (auto err - = mCV.Wait(lock, [&]() { return (mClose || mProcessOutdatedItems) && mInProgressBlobs.IsEmpty(); }); + if (auto err = mCV.Wait(lock, + [&]() { + return (mClose || mProcessOutdatedItems) && mInProgressBlobs.IsEmpty() && mNumActiveInstalls == 0; + }); !err.IsNone()) { LOG_ERR() << "Wait failed" << Log::Field(err); continue; @@ -1333,6 +1342,8 @@ void ImageManager::ProcessOutdatedItems() continue; } + LOG_DBG() << "Start processing outdated items"; + mProcessOutdatedItems = false; if (auto err = HandleOutdatedItems(); !err.IsNone()) { @@ -1348,6 +1359,8 @@ void ImageManager::ProcessOutdatedItems() LOG_ERR() << "Remove orphans failed" << Log::Field(err); } + LOG_DBG() << "Removed orphans" << Log::Field("size", size); + mSpaceAllocator->FreeSpace(size); } } diff --git a/src/core/sm/imagemanager/imagemanager.hpp b/src/core/sm/imagemanager/imagemanager.hpp index 137e15642..9f186e826 100644 --- a/src/core/sm/imagemanager/imagemanager.hpp +++ b/src/core/sm/imagemanager/imagemanager.hpp @@ -177,6 +177,7 @@ class ImageManager : public ImageManagerItf, public ItemInfoProviderItf, public Thread<> mThread; bool mClose {}; bool mProcessOutdatedItems {}; + size_t mNumActiveInstalls {}; }; /** @}*/ From 964accf49fb8b818253dbccdae620c2b05d11edb Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 17 Jun 2026 16:16:00 +0300 Subject: [PATCH 010/112] sm: launcher: fix static allocator size Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/sm/launcher/launcher.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/sm/launcher/launcher.hpp b/src/core/sm/launcher/launcher.hpp index ffc415c8c..c75202eee 100644 --- a/src/core/sm/launcher/launcher.hpp +++ b/src/core/sm/launcher/launcher.hpp @@ -172,8 +172,8 @@ class Launcher : public LauncherItf, static constexpr auto cMaxNumSubscribers = 4; static constexpr auto cAllocatorSize = 2 * sizeof(StaticArray) + 2 * sizeof(InstanceInfoArray) + sizeof(InstanceStatusArray) + sizeof(oci::ImageManifest) - + sizeof(oci::ItemConfig) + sizeof(StaticString) + sizeof(networkmanager::InstanceNetworkConfig) - + sizeof(resourcemanager::ResourceInfo) + + sizeof(oci::ImageConfig) + sizeof(oci::ItemConfig) + sizeof(StaticString) + + sizeof(networkmanager::InstanceNetworkConfig) + sizeof(resourcemanager::ResourceInfo) + Max(sizeof(StaticArray), sizeof(StaticArray) + sizeof(StaticArray)); From 92a128154f81068d914b6da7d8384b032ba22182 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Tue, 16 Jun 2026 10:05:53 +0300 Subject: [PATCH 011/112] cm: launcher: fix instance status Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/cm/launcher/instancemanager.cpp | 60 +++++++++++++++--------- src/core/cm/launcher/instancemanager.hpp | 2 + 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index 885d98a1d..0e3cb732a 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -187,18 +187,30 @@ Error InstanceManager::UpdateStatus(const InstanceStatus& status) return ErrorEnum::eNone; } + Error firstErr = ErrorEnum::eNone; + auto instance = FindActiveInstance(static_cast(status), status.mVersion); - if (!instance) { - // Ignore inactive instance, SM sometimes sends inactive status for stopped instances. - if (status.mState == aos::InstanceStateEnum::eInactive) { - return ErrorEnum::eNone; + if (instance) { + if (auto err = instance->UpdateStatus(status); !err.IsNone()) { + firstErr = err; } + } else { + LOG_WRN() << "Received status for unknown instance" + << Log::Field("instance", static_cast(status)); + } - // Not expected instance received from SM. - return AOS_ERROR_WRAP(ErrorEnum::eNotFound); + if (status.mState != aos::InstanceStateEnum::eInactive) { + if (auto err = UpdateRunningInstance(status); !err.IsNone() && firstErr.IsNone()) { + firstErr = err; + } + } else { + mRunningInstances.RemoveIf([&status](const InstanceStatus& running) { + return static_cast(running) == static_cast(status) + && running.mVersion == status.mVersion; + }); } - return instance->UpdateStatus(status); + return firstErr; } RetWithError> InstanceManager::CreateInstance(const RunInstanceRequest& request, uint64_t index) @@ -357,6 +369,26 @@ void InstanceManager::UpdateMonitoringData(const Array(running) == static_cast(status) + && running.mVersion == status.mVersion; + }); + + if (existing != mRunningInstances.end()) { + *existing = status; + + return ErrorEnum::eNone; + } + + if (auto err = mRunningInstances.EmplaceBack(status); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + Error InstanceManager::LoadInstancesFromStorage() { mActiveInstances.Clear(); @@ -551,20 +583,6 @@ Error InstanceManager::UpdateRunningInstances(const String& nodeID, const Array< mRunningInstances.RemoveIf([&nodeID](const InstanceStatus& status) { return status.mNodeID == nodeID; }); mPreinstalledComponents.RemoveIf([&nodeID](const InstanceStatus& status) { return status.mNodeID == nodeID; }); - for (const auto& status : statuses) { - if (status.mNodeID == nodeID) { - if (status.mPreinstalled) { - if (auto err = mPreinstalledComponents.EmplaceBack(status); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - } else { - if (auto err = mRunningInstances.EmplaceBack(status); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - } - } - } - Error firstErr = ErrorEnum::eNone; for (const auto& status : statuses) { diff --git a/src/core/cm/launcher/instancemanager.hpp b/src/core/cm/launcher/instancemanager.hpp index 70568706f..7da8e5be8 100644 --- a/src/core/cm/launcher/instancemanager.hpp +++ b/src/core/cm/launcher/instancemanager.hpp @@ -269,6 +269,8 @@ class InstanceManager { static constexpr auto cInstanceAllocatorSize = sizeof(oci::ImageConfig) + sizeof(oci::ItemConfig) + sizeof(InstanceStatus) + sizeof(oci::ImageIndex) + sizeof(EnvVarArray); + Error UpdateRunningInstance(const InstanceStatus& status); + Error LoadInstancesFromStorage(); Error LoadInstanceFromStorage(const InstanceInfo& info); Error LoadInstanceStatuses(); From ecf53231752092bc76798dc3d4596b0d0f075bd3 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 18 Jun 2026 16:51:43 +0300 Subject: [PATCH 012/112] spaceallocator: fix mAllocationCount leak in ResizeSpace ResizeSpace called Partition::Free + Partition::Allocate which toggled mAllocationCount on every resize. Since mAllocationCount > 0 suppresses disk re-reads, mAvailableSize became stale and lazy eviction stopped triggering, leading to ENOSPC on subsequent allocations. Add Partition::AdjustSize that adjusts mAvailableSize and triggers eviction when needed without touching mAllocationCount. ResizeSpace now calls AdjustSize instead of the Free/Allocate pair on the partition, keeping mAllocationCount as a pure count of live Space objects. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../common/spaceallocator/spaceallocator.hpp | 44 +++++++++++- .../spaceallocator/tests/spaceallocator.cpp | 71 +++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/src/core/common/spaceallocator/spaceallocator.hpp b/src/core/common/spaceallocator/spaceallocator.hpp index 0aa69f56a..73c3532a4 100644 --- a/src/core/common/spaceallocator/spaceallocator.hpp +++ b/src/core/common/spaceallocator/spaceallocator.hpp @@ -157,6 +157,47 @@ class Partition { return ErrorEnum::eNone; } + /** + * Resizes allocated space without changing allocation count. + * Used by resize operations so that mAllocationCount tracks live Space objects only. + * + * @param deltaSize size delta: positive means more space is needed, negative means space is released. + * @return Error. + */ + Error Resize(int64_t deltaSize) + { + LockGuard lock {mMutex}; + + if (deltaSize <= 0) { + mAvailableSize += static_cast(-deltaSize); + + return ErrorEnum::eNone; + } + + const auto extra = static_cast(deltaSize); + + if (extra > mAvailableSize) { + if (mOutdatedItems.Size() == 0) { + return Error(ErrorEnum::eNoMemory, "not enough space"); + } + + auto [freedSize, err] = RemoveOutdatedItems(extra - mAvailableSize); + if (!err.IsNone()) { + return err; + } + + mAvailableSize += freedSize; + + if (extra > mAvailableSize) { + return Error(ErrorEnum::eNoMemory, "not enough space"); + } + } + + mAvailableSize -= extra; + + return ErrorEnum::eNone; + } + /** * Add outdated item. * @@ -501,13 +542,12 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { } Free(oldSize); - mPartition->Free(oldSize); if (auto err = Allocate(newSize); !err.IsNone()) { return err; } - if (auto err = mPartition->Allocate(newSize); !err.IsNone()) { + if (auto err = mPartition->Resize(static_cast(newSize) - static_cast(oldSize)); !err.IsNone()) { Free(newSize); return err; diff --git a/src/core/common/spaceallocator/tests/spaceallocator.cpp b/src/core/common/spaceallocator/tests/spaceallocator.cpp index 9ccb3cce5..ec420d0ed 100644 --- a/src/core/common/spaceallocator/tests/spaceallocator.cpp +++ b/src/core/common/spaceallocator/tests/spaceallocator.cpp @@ -334,4 +334,75 @@ TEST_F(SpaceallocatorTest, ResizeSpace) ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); } +TEST_F(SpaceallocatorTest, ResizeSpaceEviction) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + EXPECT_CALL(mPlatformFS, GetTotalSize(mMountPoint)) + .WillOnce(Return(RetWithError(mTotalSize, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mPath, mPlatformFS, mLimit, &mRemover).IsNone()); + + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillOnce(Return(RetWithError(mTotalSize, ErrorEnum::eNone))); + + // Leave only 100K free after initial allocation + const size_t initialSize = mTotalSize - 100 * cKilobyte; + + auto [space, err] = mSpaceAllocator.AllocateSpace(initialSize); + ASSERT_TRUE(err.IsNone()); + ASSERT_NE(space.Get(), nullptr); + + const size_t outdatedItemSize = 256 * cKilobyte; + + ASSERT_TRUE(mSpaceAllocator.AddOutdatedItem("file1", "", Time::Now()).IsNone()); + + EXPECT_CALL(mRemover, RemoveItem(String("file1"), String(""))) + .WillOnce(Return(RetWithError(outdatedItemSize, ErrorEnum::eNone))); + + // Resize needs 200K delta but only 100K available — requires evicting "file1" + const size_t newSize = initialSize + 200 * cKilobyte; + + ASSERT_TRUE(space->Resize(newSize).IsNone()); + ASSERT_EQ(space->Size(), newSize); + + ASSERT_TRUE(space->Accept().IsNone()); + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + +TEST_F(SpaceallocatorTest, ResizeSpaceInsufficientSpace) +{ + SpaceAllocator<5> mSpaceAllocator; + + EXPECT_CALL(mPlatformFS, GetMountPoint(mPath)) + .WillOnce(Return(RetWithError>(mMountPoint, ErrorEnum::eNone))); + + EXPECT_CALL(mPlatformFS, GetTotalSize(mMountPoint)) + .WillOnce(Return(RetWithError(mTotalSize, ErrorEnum::eNone))); + + ASSERT_TRUE(mSpaceAllocator.Init(mPath, mPlatformFS, mLimit).IsNone()); + + EXPECT_CALL(mPlatformFS, GetAvailableSize(mMountPoint)) + .WillOnce(Return(RetWithError(mTotalSize, ErrorEnum::eNone))); + + // Leave only 100K free after initial allocation + const size_t initialSize = mTotalSize - 100 * cKilobyte; + + auto [space, err] = mSpaceAllocator.AllocateSpace(initialSize); + ASSERT_TRUE(err.IsNone()); + ASSERT_NE(space.Get(), nullptr); + + // Resize needs 200K but only 100K available with no outdated items to evict + const size_t newSize = initialSize + 200 * cKilobyte; + + ASSERT_EQ(space->Resize(newSize), ErrorEnum::eNoMemory); + ASSERT_EQ(space->Size(), initialSize); + + ASSERT_TRUE(space->Release().IsNone()); + ASSERT_TRUE(mSpaceAllocator.Close().IsNone()); +} + } // namespace aos::spaceallocator From 901de8cac22fb53c01aa0ef27e1f2546b7e03e27 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Mon, 22 Jun 2026 18:47:44 +0300 Subject: [PATCH 013/112] imagemanager: fix stale cancel flag blocking new download after cancellation When a download is cancelled to process a new desired status, mCancel is set to true. Once the cancelled download finishes and mInProgress becomes false, a subsequent DownloadUpdateItems call would enter StartAction and immediately return false because mCancel was still set, causing the new download to fail with eCanceled without attempting any network activity. Only reject starting a new action when mCancel is true and mInProgress is also true, meaning there is an active download being cancelled. A stale mCancel with no in-progress action should not block the next download. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/cm/imagemanager/imagemanager.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/cm/imagemanager/imagemanager.cpp b/src/core/cm/imagemanager/imagemanager.cpp index b66eb8146..3c6ff7bff 100644 --- a/src/core/cm/imagemanager/imagemanager.cpp +++ b/src/core/cm/imagemanager/imagemanager.cpp @@ -1417,9 +1417,11 @@ bool ImageManager::StartAction() mCondVar.Wait(lock, [this]() { return !mInProgress || mCancel; }); - if (mCancel) { - mCancel = false; + const bool cancelledWhileRunning = mCancel && mInProgress; + + mCancel = false; + if (cancelledWhileRunning) { return false; } From 5a69313a7ada2dea2aa69046ad601cd0fd62c01c Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Thu, 18 Jun 2026 19:03:47 +0300 Subject: [PATCH 014/112] cm: launcher: instance status update should not add/rm new instances Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/cm/launcher/instancemanager.cpp | 92 +++++++++++++++--------- src/core/cm/launcher/instancemanager.hpp | 3 +- 2 files changed, 59 insertions(+), 36 deletions(-) diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index 0e3cb732a..f9f667e3d 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -171,43 +171,40 @@ Array& InstanceManager::GetRunningInstances() Error InstanceManager::UpdateStatus(const InstanceStatus& status) { - if (status.mPreinstalled) { - auto preinstalledComponent - = FindPreinstalledComponent(static_cast(status), status.mVersion); - if (preinstalledComponent == nullptr) { - if (auto err = mPreinstalledComponents.EmplaceBack(status); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } + Error firstErr = ErrorEnum::eNone; - return ErrorEnum::eNone; - } + auto& statuses = status.mPreinstalled ? mPreinstalledComponents : mRunningInstances; - *preinstalledComponent = status; + auto existing = statuses.FindIf([&status](const InstanceStatus& item) { + return static_cast(item) == static_cast(status) + && item.mVersion == status.mVersion; + }); - return ErrorEnum::eNone; + bool missingStatus = existing == statuses.end(); + + if (!missingStatus) { + *existing = status; } - Error firstErr = ErrorEnum::eNone; + bool missingActiveInstance = false; - auto instance = FindActiveInstance(static_cast(status), status.mVersion); - if (instance) { - if (auto err = instance->UpdateStatus(status); !err.IsNone()) { - firstErr = err; + if (!status.mPreinstalled) { + auto instance = FindActiveInstance(static_cast(status), status.mVersion); + if (instance) { + if (auto err = instance->UpdateStatus(status); !err.IsNone()) { + firstErr = err; + } + } else { + missingActiveInstance = true; } - } else { - LOG_WRN() << "Received status for unknown instance" - << Log::Field("instance", static_cast(status)); } - if (status.mState != aos::InstanceStateEnum::eInactive) { - if (auto err = UpdateRunningInstance(status); !err.IsNone() && firstErr.IsNone()) { - firstErr = err; - } - } else { - mRunningInstances.RemoveIf([&status](const InstanceStatus& running) { - return static_cast(running) == static_cast(status) - && running.mVersion == status.mVersion; - }); + if (missingActiveInstance || missingStatus) { + LOG_WRN() << "Received status for instance missing in" + << (missingActiveInstance ? " \'active instance list\'" : "") + << (missingStatus ? " \'status list\'" : "") + << Log::Field("instance", static_cast(status)) + << Log::Field("version", status.mVersion); } return firstErr; @@ -369,20 +366,20 @@ void InstanceManager::UpdateMonitoringData(const Array& statuses, const InstanceStatus& status) { - auto existing = mRunningInstances.FindIf([&status](const InstanceStatus& running) { - return static_cast(running) == static_cast(status) - && running.mVersion == status.mVersion; + auto existing = statuses.FindIf([&status](const InstanceStatus& item) { + return static_cast(item) == static_cast(status) + && item.mVersion == status.mVersion; }); - if (existing != mRunningInstances.end()) { + if (existing != statuses.end()) { *existing = status; return ErrorEnum::eNone; } - if (auto err = mRunningInstances.EmplaceBack(status); !err.IsNone()) { + if (auto err = statuses.EmplaceBack(status); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -586,9 +583,34 @@ Error InstanceManager::UpdateRunningInstances(const String& nodeID, const Array< Error firstErr = ErrorEnum::eNone; for (const auto& status : statuses) { - if (auto err = UpdateStatus(status); !err.IsNone() && firstErr.IsNone()) { + if (auto err = SetStatus(status); !err.IsNone() && firstErr.IsNone()) { + firstErr = err; + } + } + + return firstErr; +} + +Error InstanceManager::SetStatus(const InstanceStatus& status) +{ + if (status.mPreinstalled) { + return SetStatus(mPreinstalledComponents, status); + } + + Error firstErr = ErrorEnum::eNone; + if (auto err = SetStatus(mRunningInstances, status); !err.IsNone()) { + firstErr = err; + } + + auto instance = FindActiveInstance(static_cast(status), status.mVersion); + if (instance) { + if (auto err = instance->UpdateStatus(status); !err.IsNone() && firstErr.IsNone()) { firstErr = err; } + } else { + LOG_WRN() << "Received node instance status for not active instance" + << Log::Field("instance", static_cast(status)) + << Log::Field("version", status.mVersion); } return firstErr; diff --git a/src/core/cm/launcher/instancemanager.hpp b/src/core/cm/launcher/instancemanager.hpp index 7da8e5be8..d691fe559 100644 --- a/src/core/cm/launcher/instancemanager.hpp +++ b/src/core/cm/launcher/instancemanager.hpp @@ -269,7 +269,8 @@ class InstanceManager { static constexpr auto cInstanceAllocatorSize = sizeof(oci::ImageConfig) + sizeof(oci::ItemConfig) + sizeof(InstanceStatus) + sizeof(oci::ImageIndex) + sizeof(EnvVarArray); - Error UpdateRunningInstance(const InstanceStatus& status); + Error SetStatus(const InstanceStatus& status); + Error SetStatus(Array& statuses, const InstanceStatus& status); Error LoadInstancesFromStorage(); Error LoadInstanceFromStorage(const InstanceInfo& info); From d98d8654dba4d9fa2f23e30068e7765ad964239a Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Thu, 25 Jun 2026 11:52:06 +0300 Subject: [PATCH 015/112] common: clang format fix Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/common/spaceallocator/spaceallocator.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/common/spaceallocator/spaceallocator.hpp b/src/core/common/spaceallocator/spaceallocator.hpp index 73c3532a4..f64f00e6a 100644 --- a/src/core/common/spaceallocator/spaceallocator.hpp +++ b/src/core/common/spaceallocator/spaceallocator.hpp @@ -547,7 +547,8 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { return err; } - if (auto err = mPartition->Resize(static_cast(newSize) - static_cast(oldSize)); !err.IsNone()) { + if (auto err = mPartition->Resize(static_cast(newSize) - static_cast(oldSize)); + !err.IsNone()) { Free(newSize); return err; From 0d24946956371cfdc262d8c0baede497710845c9 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Wed, 17 Jun 2026 20:52:50 +0300 Subject: [PATCH 016/112] cm: launcher: separate sm info loading from scheduling Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/cm/launcher/instance.cpp | 27 ++++++++++++++++++--------- src/core/cm/launcher/instance.hpp | 27 +++++++++++++++++++++++++++ src/core/cm/launcher/nodemanager.cpp | 2 +- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/core/cm/launcher/instance.cpp b/src/core/cm/launcher/instance.cpp index 2bcbd742f..a92bb2f85 100644 --- a/src/core/cm/launcher/instance.cpp +++ b/src/core/cm/launcher/instance.cpp @@ -346,8 +346,11 @@ oci::BalancingPolicyEnum ComponentInstance::GetBalancingPolicy() Error ComponentInstance::Schedule(NodeItf& node, const String& runtimeID) { - auto releaseConfig = DeferRelease(reinterpret_cast(1), [&](int*) { mImageConfig = nullptr; }); + return LoadSMInfo(node, runtimeID); +} +Error ComponentInstance::LoadSMInfo(NodeItf& node, const String& runtimeID) +{ static_cast(mSMInfo) = mInfo.mInstanceIdent; mSMInfo.mVersion = mInfo.mVersion; mSMInfo.mManifestDigest = mInfo.mManifestDigest; @@ -495,10 +498,20 @@ Error ServiceInstance::Schedule(NodeItf& node, const String& runtimeID) { assert(mItemConfig); - auto releaseConfigs = DeferRelease(reinterpret_cast(1), [&](int*) { - mItemConfig.Reset(); - mImageConfig.Reset(); - }); + if (auto err = ReserveRuntimeResources(node, runtimeID); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (auto err = LoadSMInfo(node, runtimeID); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +Error ServiceInstance::LoadSMInfo(NodeItf& node, const String& runtimeID) +{ + assert(mItemConfig); static_cast(mSMInfo) = mInfo.mInstanceIdent; mSMInfo.mVersion = mInfo.mVersion; @@ -521,10 +534,6 @@ Error ServiceInstance::Schedule(NodeItf& node, const String& runtimeID) mSMInfo.mMonitoringParams.GetValue().mAlertRules = mItemConfig->mAlertRules.GetValue(); } - if (auto err = ReserveRuntimeResources(node, runtimeID); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - if (auto err = SetActive(node.GetConfig().mNodeID, runtimeID); !err.IsNone()) { return AOS_ERROR_WRAP(err); } diff --git a/src/core/cm/launcher/instance.hpp b/src/core/cm/launcher/instance.hpp index ed2215a0d..156b69724 100644 --- a/src/core/cm/launcher/instance.hpp +++ b/src/core/cm/launcher/instance.hpp @@ -209,6 +209,15 @@ class Instance { */ virtual Error Schedule(NodeItf& node, const String& runtimeID) = 0; + /** + * Loads SM instance info. + * + * @param node node interface. + * @param runtimeID runtime identifier. + * @return Error. + */ + virtual Error LoadSMInfo(NodeItf& node, const String& runtimeID) = 0; + /** * Overrides environment variables. * @@ -315,6 +324,15 @@ class ComponentInstance : public Instance { * @return Error. */ Error Schedule(NodeItf& node, const String& runtimeID) override; + + /** + * Loads SM instance info + * + * @param node node interface. + * @param runtimeID runtime identifier. + * @return Error. + */ + Error LoadSMInfo(NodeItf& node, const String& runtimeID) override; }; /** @@ -399,6 +417,15 @@ class ServiceInstance : public Instance { */ Error Schedule(NodeItf& node, const String& runtimeID) override; + /** + * Loads SM instance info. + * + * @param node node interface. + * @param runtimeID runtime identifier. + * @return Error. + */ + Error LoadSMInfo(NodeItf& node, const String& runtimeID) override; + private: static constexpr auto cDefaultResourceRation = 50.0; diff --git a/src/core/cm/launcher/nodemanager.cpp b/src/core/cm/launcher/nodemanager.cpp index c8cc05718..63745cd70 100644 --- a/src/core/cm/launcher/nodemanager.cpp +++ b/src/core/cm/launcher/nodemanager.cpp @@ -124,7 +124,7 @@ Error NodeManager::LoadSMDataForActiveInstances( continue; } - if (auto err = instance->Schedule(*node, runtimeID); !err.IsNone()) { + if (auto err = instance->LoadSMInfo(*node, runtimeID); !err.IsNone()) { LOG_ERR() << "Can't load instance" << Log::Field("nodeID", nodeID) << Log::Field("instanceID", instanceID) << Log::Field(AOS_ERROR_WRAP(err)); From 8fbbfe9c2e657c5b67eb180dd85d3e01a2a95378 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Thu, 18 Jun 2026 00:39:36 +0300 Subject: [PATCH 017/112] cm: launcher: don't return error if restore instance from storage failed Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/cm/launcher/instancemanager.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index f9f667e3d..bacca2682 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -400,7 +400,10 @@ Error InstanceManager::LoadInstancesFromStorage() for (const auto& instance : *instances) { if (auto err = LoadInstanceFromStorage(instance); !err.IsNone()) { - return AOS_ERROR_WRAP(err); + LOG_ERR() << "Can't load instance from storage" << Log::Field("instance", instance.mInstanceIdent) + << Log::Field(err); + + continue; } } @@ -426,8 +429,7 @@ Error InstanceManager::LoadInstanceFromStorage(const InstanceInfo& info) return AOS_ERROR_WRAP(err); } } else { - LOG_DBG() << "Load cached instance" << Log::Field("instanceID", instance->GetInfo().mInstanceIdent) - << Log::Field("nodeID", instance->GetStatus().mNodeID); + LOG_DBG() << "Load cached instance" << Log::Field("instanceID", instance->GetInfo().mInstanceIdent); if (auto err = mCachedInstances.EmplaceBack(instance); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -534,6 +536,11 @@ RetWithError> InstanceManager::CreateInstance(const Instance } if (auto err = newInstance->Init(); !err.IsNone()) { + // Do not leave invalid instance in storage. + if (auto err = newInstance->Remove(); !err.IsNone()) { + LOG_ERR() << "Can't remove instance" << Log::Field(err); + } + return {{}, AOS_ERROR_WRAP(err)}; } From 5c4bfb9702eed291191ffe17cd8e1763dcd34858 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Tue, 23 Jun 2026 21:40:15 +0300 Subject: [PATCH 018/112] sm: imagemanager: remove checking mInProgressBlobs for outdated items Checking mInProgressBlobs.IsEmpty is not necessary as when mNumActiveInstalls is zero then mInProgressBlobs is empty. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/imagemanager/imagemanager.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/core/sm/imagemanager/imagemanager.cpp b/src/core/sm/imagemanager/imagemanager.cpp index 924370290..34245d395 100644 --- a/src/core/sm/imagemanager/imagemanager.cpp +++ b/src/core/sm/imagemanager/imagemanager.cpp @@ -1325,10 +1325,7 @@ void ImageManager::ProcessOutdatedItems() while (true) { UniqueLock lock {mMutex}; - if (auto err = mCV.Wait(lock, - [&]() { - return (mClose || mProcessOutdatedItems) && mInProgressBlobs.IsEmpty() && mNumActiveInstalls == 0; - }); + if (auto err = mCV.Wait(lock, [&]() { return (mClose || mProcessOutdatedItems) && mNumActiveInstalls == 0; }); !err.IsNone()) { LOG_ERR() << "Wait failed" << Log::Field(err); continue; From acbf31cf8dc94a0097209bed75538ed2ba2ba4c6 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Tue, 23 Jun 2026 21:54:11 +0300 Subject: [PATCH 019/112] sm: imagemanager: move install layers to separate functions Move install service layers to InstallServiceLayers and install component layers to InstallComponentLayers to reduce size of InstallUpdateItem function. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/imagemanager/imagemanager.cpp | 94 ++++++++++++++--------- src/core/sm/imagemanager/imagemanager.hpp | 2 + 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/src/core/sm/imagemanager/imagemanager.cpp b/src/core/sm/imagemanager/imagemanager.cpp index 34245d395..48499faa7 100644 --- a/src/core/sm/imagemanager/imagemanager.cpp +++ b/src/core/sm/imagemanager/imagemanager.cpp @@ -219,46 +219,12 @@ Error ImageManager::InstallUpdateItem(const UpdateItemInfo& itemInfo) } if (itemInfo.mType == UpdateItemTypeEnum::eService) { - LOG_DBG() << "Install image config blob" << Log::Field("digest", manifest->mConfig.mDigest); - - if (auto err = InstallBlob(manifest->mConfig); !err.IsNone()) { + if (auto err = InstallServiceLayers(*manifest); !err.IsNone()) { return err; } - - auto config = MakeUnique(&mAllocator); - if (!config) { - return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); - } - - if (auto err = CreateBlobPath(manifest->mConfig.mDigest, path); !err.IsNone()) { - return err; - } - - if (auto err = mOCISpec->LoadImageConfig(path, *config); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - for (size_t i = 0; i < manifest->mLayers.Size(); ++i) { - const auto& layer = manifest->mLayers[i]; - - if (i >= config->mRootfs.mDiffIDs.Size()) { - return AOS_ERROR_WRAP(Error(ErrorEnum::eOutOfRange, "diff IDs size is less than layers size")); - } - - LOG_DBG() << "Install layer blob" << Log::Field("digest", layer.mDigest) - << Log::Field("diffDigest", config->mRootfs.mDiffIDs[i]); - - if (auto err = InstallLayer(layer, config->mRootfs.mDiffIDs[i]); !err.IsNone()) { - return err; - } - } } else { - for (const auto& layer : manifest->mLayers) { - LOG_DBG() << "Install layer blob" << Log::Field("digest", layer.mDigest); - - if (auto err = InstallBlob(layer); !err.IsNone()) { - return err; - } + if (auto err = InstallComponentLayers(*manifest); !err.IsNone()) { + return err; } } @@ -566,6 +532,57 @@ Error ImageManager::InstallBlob(const oci::ContentDescriptor& descriptor, bool w return ErrorEnum::eNone; } +Error ImageManager::InstallServiceLayers(const oci::ImageManifest& manifest) +{ + StaticString path; + + LOG_DBG() << "Install image config blob" << Log::Field("digest", manifest.mConfig.mDigest); + + if (auto err = InstallBlob(manifest.mConfig); !err.IsNone()) { + return err; + } + + auto config = MakeUnique(&mAllocator); + if (!config) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = CreateBlobPath(manifest.mConfig.mDigest, path); !err.IsNone()) { + return err; + } + + if (auto err = mOCISpec->LoadImageConfig(path, *config); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + for (size_t i = 0; i < manifest.mLayers.Size(); ++i) { + const auto& layer = manifest.mLayers[i]; + + if (i >= config->mRootfs.mDiffIDs.Size()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eOutOfRange, "diff IDs size is less than layers size")); + } + + if (auto err = InstallLayer(layer, config->mRootfs.mDiffIDs[i]); !err.IsNone()) { + return err; + } + } + + return ErrorEnum::eNone; +} + +Error ImageManager::InstallComponentLayers(const oci::ImageManifest& manifest) +{ + for (const auto& layer : manifest.mLayers) { + LOG_DBG() << "Install layer blob" << Log::Field("digest", layer.mDigest); + + if (auto err = InstallBlob(layer); !err.IsNone()) { + return err; + } + } + + return ErrorEnum::eNone; +} + Error ImageManager::CreateLayerMetadata(const String& path, size_t size, spaceallocator::SpaceItf* space) { auto [digest, err] = mImageHandler->GetUnpackedLayerDigest(fs::JoinPath(path, cUnpackedLayerFolder)); @@ -700,6 +717,9 @@ Error ImageManager::InstallLayer(const oci::ContentDescriptor& descriptor, const } } + LOG_DBG() << "Install layer blob" << Log::Field("digest", descriptor.mDigest) + << Log::Field("diffDigest", diffDigest); + if (err = InstallBlob(descriptor, false); !err.IsNone()) { return err; } diff --git a/src/core/sm/imagemanager/imagemanager.hpp b/src/core/sm/imagemanager/imagemanager.hpp index 9f186e826..8b32e9ae6 100644 --- a/src/core/sm/imagemanager/imagemanager.hpp +++ b/src/core/sm/imagemanager/imagemanager.hpp @@ -134,6 +134,8 @@ class ImageManager : public ImageManagerItf, public ItemInfoProviderItf, public Error ValidateBlob(const String& path, const String& digest) const; Error DownloadBlob(const String& path, const String& digest, size_t size); Error InstallBlob(const oci::ContentDescriptor& descriptor, bool waitInProgress = true); + Error InstallServiceLayers(const oci::ImageManifest& manifest); + Error InstallComponentLayers(const oci::ImageManifest& manifest); Error ValidateLayer(const String& path, const String& diffDigest) const; Error CreateLayerMetadata(const String& path, size_t size, spaceallocator::SpaceItf* space); Error UnpackLayer(const String& path, const oci::ContentDescriptor& descriptor, const String& diffDigest); From b675bf01884f809007d07f4bbe090b60cabd27b8 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 24 Jun 2026 19:15:05 +0300 Subject: [PATCH 020/112] sm: imagemanager: track installing blobs/layers for orphan removal Replace the mNumActiveInstalls counter and flat mInProgressBlobs list with per-item InstallItem structs that record which blobs and layers each concurrent install owns. Feed those into RemoveOrphans via AddInstallingItems so that blobs actively being downloaded are treated as used and not deleted as orphans. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/imagemanager/imagemanager.cpp | 265 ++++++++++++++-------- src/core/sm/imagemanager/imagemanager.hpp | 40 ++-- 2 files changed, 202 insertions(+), 103 deletions(-) diff --git a/src/core/sm/imagemanager/imagemanager.cpp b/src/core/sm/imagemanager/imagemanager.cpp index 48499faa7..02e6d192b 100644 --- a/src/core/sm/imagemanager/imagemanager.cpp +++ b/src/core/sm/imagemanager/imagemanager.cpp @@ -170,28 +170,24 @@ Error ImageManager::GetAllInstalledItems(Array& statuses) cons Error ImageManager::InstallUpdateItem(const UpdateItemInfo& itemInfo) { - LOG_INF() << "Install item" << Log::Field("itemID", itemInfo.mID) << Log::Field("version", itemInfo.mVersion) + LOG_INF() << "Install update item" << Log::Field("itemID", itemInfo.mID) << Log::Field("version", itemInfo.mVersion) << Log::Field("type", itemInfo.mType) << Log::Field("manifestDigest", itemInfo.mManifestDigest); - { - LockGuard lock {mMutex}; - - ++mNumActiveInstalls; + auto [installItemIt, err] = CreateInstallingItem(itemInfo); + if (!err.IsNone()) { + return err; } - auto decrementInstalls = DeferRelease(this, [](ImageManager* self) { - LockGuard lock {self->mMutex}; + auto& installItem = *installItemIt; - if (--self->mNumActiveInstalls == 0) { - self->mCV.NotifyAll(); - } - }); + auto releaseInstallingItem + = DeferRelease(this, [&](ImageManager* self) { self->ReleaseInstallingItem(installItemIt); }); oci::ContentDescriptor manifestDescriptor {"", itemInfo.mManifestDigest, 0}; LOG_DBG() << "Install manifest blob" << Log::Field("digest", itemInfo.mManifestDigest); - if (auto err = InstallBlob(manifestDescriptor); !err.IsNone()) { + if (err = InstallBlob(manifestDescriptor, &installItem); !err.IsNone()) { return err; } @@ -202,33 +198,33 @@ Error ImageManager::InstallUpdateItem(const UpdateItemInfo& itemInfo) StaticString path; - if (auto err = CreateBlobPath(itemInfo.mManifestDigest, path); !err.IsNone()) { + if (err = CreateBlobPath(itemInfo.mManifestDigest, path); !err.IsNone()) { return err; } - if (auto err = mOCISpec->LoadImageManifest(path, *manifest); !err.IsNone()) { + if (err = mOCISpec->LoadImageManifest(path, *manifest); !err.IsNone()) { return AOS_ERROR_WRAP(err); } if (manifest->mItemConfig.HasValue()) { LOG_DBG() << "Install item config blob" << Log::Field("digest", manifest->mItemConfig->mDigest); - if (auto err = InstallBlob(*manifest->mItemConfig); !err.IsNone()) { + if (err = InstallBlob(*manifest->mItemConfig, &installItem); !err.IsNone()) { return err; } } if (itemInfo.mType == UpdateItemTypeEnum::eService) { - if (auto err = InstallServiceLayers(*manifest); !err.IsNone()) { + if (err = InstallServiceLayers(*manifest, installItem); !err.IsNone()) { return err; } } else { - if (auto err = InstallComponentLayers(*manifest); !err.IsNone()) { + if (err = InstallComponentLayers(*manifest, installItem); !err.IsNone()) { return err; } } - if (auto err = StoreUpdateItem(itemInfo); !err.IsNone()) { + if (err = StoreUpdateItem(itemInfo); !err.IsNone()) { return err; } @@ -475,15 +471,23 @@ Error ImageManager::DownloadBlob(const String& path, const String& digest, size_ return ErrorEnum::eNone; } -Error ImageManager::InstallBlob(const oci::ContentDescriptor& descriptor, bool waitInProgress) +Error ImageManager::InstallBlob(const oci::ContentDescriptor& descriptor, InstallItem* installItem, bool waitInstalling) { - if (waitInProgress) { - if (auto err = WaitForInProgressBlob(descriptor.mDigest); !err.IsNone()) { + if (waitInstalling) { + if (auto err = WaitForInstallingBlob(descriptor.mDigest); !err.IsNone()) { return err; } - auto releaseInProgress - = DeferRelease(&descriptor.mDigest, [&](const String* digest) { ReleaseInProgressBlob(*digest); }); + auto releaseInstalling + = DeferRelease(&descriptor.mDigest, [&](const String* digest) { ReleaseInstallingBlob(*digest); }); + } + + if (installItem) { + LockGuard lock {mMutex}; + + if (auto err = installItem->mBlobs.EmplaceBack(descriptor.mDigest); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } } LOG_DBG() << "Install blob" << Log::Field("digest", descriptor.mDigest) << Log::Field("size", descriptor.mSize); @@ -532,57 +536,6 @@ Error ImageManager::InstallBlob(const oci::ContentDescriptor& descriptor, bool w return ErrorEnum::eNone; } -Error ImageManager::InstallServiceLayers(const oci::ImageManifest& manifest) -{ - StaticString path; - - LOG_DBG() << "Install image config blob" << Log::Field("digest", manifest.mConfig.mDigest); - - if (auto err = InstallBlob(manifest.mConfig); !err.IsNone()) { - return err; - } - - auto config = MakeUnique(&mAllocator); - if (!config) { - return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); - } - - if (auto err = CreateBlobPath(manifest.mConfig.mDigest, path); !err.IsNone()) { - return err; - } - - if (auto err = mOCISpec->LoadImageConfig(path, *config); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - for (size_t i = 0; i < manifest.mLayers.Size(); ++i) { - const auto& layer = manifest.mLayers[i]; - - if (i >= config->mRootfs.mDiffIDs.Size()) { - return AOS_ERROR_WRAP(Error(ErrorEnum::eOutOfRange, "diff IDs size is less than layers size")); - } - - if (auto err = InstallLayer(layer, config->mRootfs.mDiffIDs[i]); !err.IsNone()) { - return err; - } - } - - return ErrorEnum::eNone; -} - -Error ImageManager::InstallComponentLayers(const oci::ImageManifest& manifest) -{ - for (const auto& layer : manifest.mLayers) { - LOG_DBG() << "Install layer blob" << Log::Field("digest", layer.mDigest); - - if (auto err = InstallBlob(layer); !err.IsNone()) { - return err; - } - } - - return ErrorEnum::eNone; -} - Error ImageManager::CreateLayerMetadata(const String& path, size_t size, spaceallocator::SpaceItf* space) { auto [digest, err] = mImageHandler->GetUnpackedLayerDigest(fs::JoinPath(path, cUnpackedLayerFolder)); @@ -686,14 +639,27 @@ Error ImageManager::UnpackLayer(const String& path, const oci::ContentDescriptor return ErrorEnum::eNone; } -Error ImageManager::InstallLayer(const oci::ContentDescriptor& descriptor, const String& diffDigest) +Error ImageManager::InstallLayer( + const oci::ContentDescriptor& descriptor, const String& diffDigest, InstallItem& installItem) { - if (auto err = WaitForInProgressBlob(descriptor.mDigest); !err.IsNone()) { + if (auto err = WaitForInstallingBlob(descriptor.mDigest); !err.IsNone()) { return err; } - auto releaseInProgress - = DeferRelease(&descriptor.mDigest, [&](const String* digest) { ReleaseInProgressBlob(*digest); }); + auto releaseInstalling + = DeferRelease(&descriptor.mDigest, [&](const String* digest) { ReleaseInstallingBlob(*digest); }); + + { + LockGuard lock {mMutex}; + + if (auto err = installItem.mBlobs.PushBack(descriptor.mDigest); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (auto err = installItem.mLayers.PushBack(diffDigest); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } LOG_DBG() << "Install layer" << Log::Field("digest", descriptor.mDigest); @@ -720,7 +686,7 @@ Error ImageManager::InstallLayer(const oci::ContentDescriptor& descriptor, const LOG_DBG() << "Install layer blob" << Log::Field("digest", descriptor.mDigest) << Log::Field("diffDigest", diffDigest); - if (err = InstallBlob(descriptor, false); !err.IsNone()) { + if (err = InstallBlob(descriptor, nullptr, false); !err.IsNone()) { return err; } @@ -771,36 +737,36 @@ void ImageManager::ReleaseSpace(const String& path, spaceallocator::SpaceItf* sp } } -Error ImageManager::WaitForInProgressBlob(const String& digest) +Error ImageManager::WaitForInstallingBlob(const String& digest) { UniqueLock lock {mMutex}; if (auto err = mCV.Wait(lock, [&]() { - auto it = mInProgressBlobs.FindIf([&digest](const StaticString& inProgressDigest) { - return inProgressDigest == digest; + auto it = mInstallingBlobs.FindIf([&digest](const StaticString& installingDigest) { + return installingDigest == digest; }); - return it == mInProgressBlobs.end(); + return it == mInstallingBlobs.end(); }); !err.IsNone()) { return AOS_ERROR_WRAP(err); } - if (auto err = mInProgressBlobs.PushBack(digest); !err.IsNone()) { + if (auto err = mInstallingBlobs.PushBack(digest); !err.IsNone()) { return AOS_ERROR_WRAP(err); } return ErrorEnum::eNone; } -Error ImageManager::ReleaseInProgressBlob(const String& digest) +Error ImageManager::ReleaseInstallingBlob(const String& digest) { UniqueLock lock {mMutex}; - auto it = mInProgressBlobs.FindIf( - [&digest](const StaticString& inProgressDigest) { return inProgressDigest == digest; }); - if (it != mInProgressBlobs.end()) { - mInProgressBlobs.Erase(it); + auto it = mInstallingBlobs.FindIf( + [&digest](const StaticString& installingDigest) { return installingDigest == digest; }); + if (it != mInstallingBlobs.end()) { + mInstallingBlobs.Erase(it); } else { return AOS_ERROR_WRAP(ErrorEnum::eNotFound); } @@ -823,6 +789,90 @@ RetWithError ImageManager::RemoveOldItemVersions(Array& return RemoveOldUpdateItems(itemData); } +Error ImageManager::InstallServiceLayers(const oci::ImageManifest& manifest, InstallItem& installItem) +{ + StaticString path; + + LOG_DBG() << "Install image config blob" << Log::Field("digest", manifest.mConfig.mDigest); + + if (auto err = InstallBlob(manifest.mConfig, &installItem); !err.IsNone()) { + return err; + } + + auto config = MakeUnique(&mAllocator); + if (!config) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + if (auto err = CreateBlobPath(manifest.mConfig.mDigest, path); !err.IsNone()) { + return err; + } + + if (auto err = mOCISpec->LoadImageConfig(path, *config); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + for (size_t i = 0; i < manifest.mLayers.Size(); ++i) { + const auto& layer = manifest.mLayers[i]; + + if (i >= config->mRootfs.mDiffIDs.Size()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eOutOfRange, "diff IDs size is less than layers size")); + } + + if (auto err = InstallLayer(layer, config->mRootfs.mDiffIDs[i], installItem); !err.IsNone()) { + return err; + } + } + + return ErrorEnum::eNone; +} + +Error ImageManager::InstallComponentLayers(const oci::ImageManifest& manifest, InstallItem& installItem) +{ + for (const auto& layer : manifest.mLayers) { + LOG_DBG() << "Install layer blob" << Log::Field("digest", layer.mDigest); + + if (auto err = InstallBlob(layer, &installItem); !err.IsNone()) { + return err; + } + } + + return ErrorEnum::eNone; +} + +RetWithError::Iterator> ImageManager::CreateInstallingItem( + const UpdateItemInfo& itemInfo) +{ + LockGuard lock {mMutex}; + + if (auto err = mInstallingItems.EmplaceBack(); !err.IsNone()) { + return {mInstallingItems.end(), AOS_ERROR_WRAP(err)}; + } + + mInstallingItems.Back().mID = itemInfo.mID; + mInstallingItems.Back().mVersion = itemInfo.mVersion; + + auto it = mInstallingItems.FindIf( + [&](const InstallItem& item) { return item.mID == itemInfo.mID && item.mVersion == itemInfo.mVersion; }); + + if (it == mInstallingItems.end()) { + return {it, AOS_ERROR_WRAP(ErrorEnum::eNotFound)}; + } + + return it; +} + +void ImageManager::ReleaseInstallingItem(List::Iterator it) +{ + LockGuard lock {mMutex}; + + mInstallingItems.Erase(it); + + if (mInstallingItems.IsEmpty()) { + mCV.NotifyAll(); + } +} + RetWithError ImageManager::CropUpdateItems() { auto [itemsCount, err] = mStorage->GetUpdateItemsCount(); @@ -1119,6 +1169,36 @@ Error ImageManager::HandleItemsIntegrity() return ErrorEnum::eNone; } +Error ImageManager::AddInstallingItems( + Array>& usedBlobs, Array>& usedLayers) +{ + for (const auto& installingItem : mInstallingItems) { + StaticString path; + + for (const auto& blob : installingItem.mBlobs) { + if (auto err = CreateBlobPath(blob, path); !err.IsNone()) { + return err; + } + + if (auto err = AddPathIfNotExist(usedBlobs, path); !err.IsNone()) { + return err; + } + } + + for (const auto& layer : installingItem.mLayers) { + if (auto err = CreateLayerPath(layer, path); !err.IsNone()) { + return err; + } + + if (auto err = AddPathIfNotExist(usedLayers, path); !err.IsNone()) { + return err; + } + } + } + + return ErrorEnum::eNone; +} + Error ImageManager::CalcItemBlobsAndLayers(const UpdateItemData& itemData, Array>& itemBlobs, Array>& itemLayers) { @@ -1312,6 +1392,10 @@ RetWithError ImageManager::RemoveOrphans() return {0, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; } + if (auto err = AddInstallingItems(*usedBlobs, *usedLayers); !err.IsNone()) { + LOG_ERR() << "Failed to add installing items" << Log::Field(err); + } + for (const auto& itemData : *itemsData) { if (auto err = CalcItemBlobsAndLayers(itemData, *usedBlobs, *usedLayers); !err.IsNone()) { LOG_ERR() << "Failed to calculate item blobs and layers" << Log::Field("itemID", itemData.mID) @@ -1345,7 +1429,8 @@ void ImageManager::ProcessOutdatedItems() while (true) { UniqueLock lock {mMutex}; - if (auto err = mCV.Wait(lock, [&]() { return (mClose || mProcessOutdatedItems) && mNumActiveInstalls == 0; }); + if (auto err + = mCV.Wait(lock, [&]() { return (mClose || mProcessOutdatedItems) && mInstallingItems.IsEmpty(); }); !err.IsNone()) { LOG_ERR() << "Wait failed" << Log::Field(err); continue; diff --git a/src/core/sm/imagemanager/imagemanager.hpp b/src/core/sm/imagemanager/imagemanager.hpp index 8b32e9ae6..92ff0f405 100644 --- a/src/core/sm/imagemanager/imagemanager.hpp +++ b/src/core/sm/imagemanager/imagemanager.hpp @@ -12,13 +12,14 @@ #include #include -#include "config.hpp" #include "itf/blobinfoprovider.hpp" #include "itf/imagehandler.hpp" #include "itf/imagemanager.hpp" #include "itf/iteminfoprovider.hpp" #include "itf/storage.hpp" +#include "config.hpp" + namespace aos::sm::imagemanager { /** @addtogroup sm Service Manager @@ -113,7 +114,8 @@ class ImageManager : public ImageManagerItf, public ItemInfoProviderItf, public static constexpr auto cSizeFile = "size"; static constexpr auto cMaxNumItemVersions = 2; // oci::cMaxNumLayers + 3 (layers + manifest + image config + aos service) - static constexpr auto cMaxNumInstalledBlobs = cMaxNumUpdateItems * (oci::cMaxNumLayers + 3); + static constexpr auto cMaxNumItemBlobs = oci::cMaxNumLayers + 3; + static constexpr auto cMaxNumInstalledBlobs = cMaxNumUpdateItems * (cMaxNumItemBlobs); static constexpr auto cMaxNumInstalledLayers = cMaxNumUpdateItems * oci::cMaxNumLayers; // Worst case: RemoveItem (mutex held) runs RemoveOrphans->CalcItemBlobsAndLayers (1 manifest + 1 config) // while cMaxNumConcurrentItems service installs are in their layer-download step inside the if(eService) @@ -127,32 +129,44 @@ class ImageManager : public ImageManagerItf, public ItemInfoProviderItf, public + (cMaxNumConcurrentItems + 1) * (sizeof(oci::ImageManifest) + sizeof(oci::ImageConfig)); static constexpr auto cMaxNumAllocations = 3 + 2 * (cMaxNumConcurrentItems + 1); + struct InstallItem { + StaticString mID; + StaticString mVersion; + StaticArray, cMaxNumItemBlobs> mBlobs; + StaticArray, oci::cMaxNumLayers> mLayers; + }; + RetWithError RemoveItem(const String& id, const String& version) override; Error CreateBlobPath(const String& digest, String& path) const; Error CreateLayerPath(const String& digest, String& path) const; Error ValidateBlob(const String& path, const String& digest) const; Error DownloadBlob(const String& path, const String& digest, size_t size); - Error InstallBlob(const oci::ContentDescriptor& descriptor, bool waitInProgress = true); - Error InstallServiceLayers(const oci::ImageManifest& manifest); - Error InstallComponentLayers(const oci::ImageManifest& manifest); + Error InstallBlob( + const oci::ContentDescriptor& descriptor, InstallItem* installItem = nullptr, bool waitInstalling = true); Error ValidateLayer(const String& path, const String& diffDigest) const; Error CreateLayerMetadata(const String& path, size_t size, spaceallocator::SpaceItf* space); Error UnpackLayer(const String& path, const oci::ContentDescriptor& descriptor, const String& diffDigest); - Error InstallLayer(const oci::ContentDescriptor& descriptor, const String& diffDigest); + Error InstallLayer(const oci::ContentDescriptor& descriptor, const String& diffDigest, InstallItem& installItem); Error GetBlobURL(const String& digest, String& url) const; void ReleaseSpace(const String& path, spaceallocator::SpaceItf* space, Error err); - Error WaitForInProgressBlob(const String& digest); - Error ReleaseInProgressBlob(const String& digest); - Error AddNewUpdateItem(const UpdateItemInfo& itemInfo); - Error StoreUpdateItem(const UpdateItemInfo& itemInfo); - Error RemoveUpdateItem(const UpdateItemData& itemData); + Error WaitForInstallingBlob(const String& digest); + Error ReleaseInstallingBlob(const String& digest); + RetWithError::Iterator> CreateInstallingItem(const UpdateItemInfo& itemInfo); + void ReleaseInstallingItem(List::Iterator it); + Error InstallServiceLayers(const oci::ImageManifest& manifest, InstallItem& installItem); + Error InstallComponentLayers(const oci::ImageManifest& manifest, InstallItem& installItem); + Error AddNewUpdateItem(const UpdateItemInfo& itemInfo); + Error StoreUpdateItem(const UpdateItemInfo& itemInfo); + Error RemoveUpdateItem(const UpdateItemData& itemData); RetWithError RemoveOldUpdateItems(Array& itemsData); RetWithError RemoveOldItemVersions(Array& itemData); RetWithError CropUpdateItems(); Error UpdateOutdatedItems(); Error HandleOutdatedItems(); Error HandleItemsIntegrity(); + Error AddInstallingItems( + Array>& usedBlobs, Array>& usedLayers); Error CalcItemBlobsAndLayers(const UpdateItemData& itemData, Array>& itemBlobs, Array>& itemLayers); RetWithError RemoveOrphanBlobs(const Array>& usedBlobs); @@ -175,11 +189,11 @@ class ImageManager : public ImageManagerItf, public ItemInfoProviderItf, public Timer mTimer; mutable Mutex mMutex; ConditionalVariable mCV; - StaticList, cMaxNumConcurrentItems> mInProgressBlobs; + StaticList mInstallingItems; + StaticList, cMaxNumConcurrentItems> mInstallingBlobs; Thread<> mThread; bool mClose {}; bool mProcessOutdatedItems {}; - size_t mNumActiveInstalls {}; }; /** @}*/ From a7dd8f18b2f93d69a2690e24aeef67b9bff4bcc5 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 24 Jun 2026 19:40:51 +0300 Subject: [PATCH 021/112] Revert "sm: imagemanager: postpone removing orphans till all item processed" This reverts commit 468fee5a35022756174a8f5eb36c5451a369cc6b. Introducing installing items handling resolve this issue as well. So this fix is not required anymore. Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/imagemanager/imagemanager.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/core/sm/imagemanager/imagemanager.cpp b/src/core/sm/imagemanager/imagemanager.cpp index 02e6d192b..3b3666622 100644 --- a/src/core/sm/imagemanager/imagemanager.cpp +++ b/src/core/sm/imagemanager/imagemanager.cpp @@ -941,7 +941,14 @@ Error ImageManager::StoreUpdateItem(const UpdateItemInfo& itemInfo) } if (removedItems) { - mProcessOutdatedItems = true; + size_t removedSize = 0; + + Tie(removedSize, err) = RemoveOrphans(); + if (!err.IsNone()) { + LOG_ERR() << "Failed to remove orphans" << Log::Field(err); + } + + mSpaceAllocator->FreeSpace(removedSize); } return ErrorEnum::eNone; @@ -1444,8 +1451,6 @@ void ImageManager::ProcessOutdatedItems() continue; } - LOG_DBG() << "Start processing outdated items"; - mProcessOutdatedItems = false; if (auto err = HandleOutdatedItems(); !err.IsNone()) { @@ -1461,8 +1466,6 @@ void ImageManager::ProcessOutdatedItems() LOG_ERR() << "Remove orphans failed" << Log::Field(err); } - LOG_DBG() << "Removed orphans" << Log::Field("size", size); - mSpaceAllocator->FreeSpace(size); } } From 2d7d7740a29f2388c2c4773f088e234cf70b5cc8 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 24 Jun 2026 19:49:19 +0300 Subject: [PATCH 022/112] sm: imagemanager: tests: fix empty digest in RemoveOutdatedItems test RemoveOutdatedItems test items had empty manifest digests, causing CalcItemBlobsAndLayers to fail with eInvalidArgument when RemoveOrphans called CreateBlobPath on them. Provide valid sha256 digests and add a LoadImageManifest mock returning a valid manifest to fix the error. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- .../sm/imagemanager/tests/imagemanager.cpp | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/core/sm/imagemanager/tests/imagemanager.cpp b/src/core/sm/imagemanager/tests/imagemanager.cpp index 5fd90747f..31650913b 100644 --- a/src/core/sm/imagemanager/tests/imagemanager.cpp +++ b/src/core/sm/imagemanager/tests/imagemanager.cpp @@ -459,16 +459,31 @@ TEST_F(ImageManagerTest, GetAllInstalledItems) TEST_F(ImageManagerTest, RemoveOutdatedItems) { std::vector initialItems = { - {"item1", UpdateItemTypeEnum::eService, "1.0.0", "", ItemStateEnum::eRemoved, + {"item1", UpdateItemTypeEnum::eService, "1.0.0", + "sha256:1111111111111111111111111111111111111111111111111111111111111111", ItemStateEnum::eRemoved, Time::Now().Add(-(cUpdateItemTTL + 1 * Time::cSeconds))}, - {"item2", UpdateItemTypeEnum::eService, "1.0.0", "", ItemStateEnum::eRemoved, + {"item2", UpdateItemTypeEnum::eService, "1.0.0", + "sha256:2222222222222222222222222222222222222222222222222222222222222222", ItemStateEnum::eRemoved, Time::Now().Add(-(cUpdateItemTTL + 1 * Time::cSeconds))}, - {"item3", UpdateItemTypeEnum::eService, "1.0.0", "", ItemStateEnum::eRemoved, Time::Now()}, - {"item4", UpdateItemTypeEnum::eService, "1.0.0", "", ItemStateEnum::eRemoved, Time::Now()}, + {"item3", UpdateItemTypeEnum::eService, "1.0.0", + "sha256:3333333333333333333333333333333333333333333333333333333333333333", ItemStateEnum::eRemoved, + Time::Now()}, + {"item4", UpdateItemTypeEnum::eService, "1.0.0", + "sha256:4444444444444444444444444444444444444444444444444444444444444444", ItemStateEnum::eRemoved, + Time::Now()}, }; mStorageStub.Init(initialItems); + auto imageManifest = std::make_unique(); + + imageManifest->mConfig.mMediaType = "application/vnd.oci.image.config.v1+json"; + imageManifest->mConfig.mDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"; + imageManifest->mConfig.mSize = 512; + + EXPECT_CALL(mOCISpecMock, LoadImageManifest(_, _)) + .WillRepeatedly(DoAll(SetArgReferee<1>(*imageManifest), Return(ErrorEnum::eNone))); + // Expect adding outdated items to space allocator for all deleted items EXPECT_CALL(mSpaceAllocatorMock, AddOutdatedItem(String("item1"), String("1.0.0"), _)).Times(1); From 75d1cff505a655c59ba21aab99e14ee71b2e865f Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 24 Jun 2026 20:04:58 +0300 Subject: [PATCH 023/112] sm: imagemanager: tests: test orphan removal during active install Verify that blobs being downloaded by an active install are not deleted as orphans when RemoveOrphans runs concurrently via RemoveItem. The test blocks the downloader after the manifest file is created, triggers RemoveItem through ItemRemoverItf, and asserts the file survives before allowing the install to complete. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- .../sm/imagemanager/tests/imagemanager.cpp | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/core/sm/imagemanager/tests/imagemanager.cpp b/src/core/sm/imagemanager/tests/imagemanager.cpp index 31650913b..b6e5f3793 100644 --- a/src/core/sm/imagemanager/tests/imagemanager.cpp +++ b/src/core/sm/imagemanager/tests/imagemanager.cpp @@ -986,4 +986,70 @@ TEST_F(ImageManagerTest, RemoveOrphanLayers) } } +TEST_F(ImageManagerTest, RemoveOrphansPreservesInstallingBlobs) +{ + // Verify that blobs being downloaded during an active install are not deleted as orphans + // when RemoveOrphans runs concurrently via RemoveItem (called by the space allocator). + + constexpr auto cManifestDigest = "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"; + + UpdateItemInfo itemInfo {"component1", UpdateItemTypeEnum::eComponent, "1.0.0", cManifestDigest}; + + auto manifestPath = GetBlobPath(cManifestDigest); + + auto imageManifest = std::make_unique(); + + imageManifest->mConfig.mMediaType = "application/vnd.oci.empty.v1+json"; + imageManifest->mConfig.mDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"; + imageManifest->mConfig.mSize = 2; + + // Block the manifest download after the file is created so RemoveOrphans can run + // while the install is still in progress. + + std::promise downloadStartedPromise; + std::promise downloadResumePromise; + + auto downloadStartedFuture = downloadStartedPromise.get_future(); + auto downloadResumeFuture = downloadResumePromise.get_future(); + + EXPECT_CALL(mDownloaderMock, Download(String(cManifestDigest), _, manifestPath)) + .WillOnce(Invoke([&](const String&, const String&, const String& path) -> Error { + CreateFile(path.CStr()); + downloadStartedPromise.set_value(); + downloadResumeFuture.wait(); + return ErrorEnum::eNone; + })); + + EXPECT_CALL(mFileInfoProviderMock, GetFileInfo(_, _, _)) + .WillOnce(DoAll(SetArgReferee<1>(GetFileInfoByDigest(cManifestDigest)), Return(ErrorEnum::eNone))); + + EXPECT_CALL(mOCISpecMock, LoadImageManifest(manifestPath, _)) + .WillOnce(DoAll(SetArgReferee<1>(*imageManifest), Return(ErrorEnum::eNone))); + + // Install in background: will block inside the manifest downloader. + + auto installFuture = std::async(std::launch::async, [&]() { return mImageManager.InstallUpdateItem(itemInfo); }); + + // Wait until the manifest file exists on disk and the digest is already recorded + // in the installing item's blob list (added by InstallBlob before calling Download). + + ASSERT_EQ(downloadStartedFuture.wait_for(std::chrono::seconds(5)), std::future_status::ready); + + // Trigger RemoveOrphans via ItemRemoverItf (as the space allocator would do). + // The manifest blob exists on disk but is not yet in storage — without AddInstallingItems + // it would be treated as an orphan and deleted. + + static_cast(&mImageManager)->RemoveItem("nonexistent", "1.0.0"); + + EXPECT_TRUE(std::filesystem::exists(manifestPath.CStr())) + << "Manifest blob was incorrectly deleted as orphan during active install"; + + // Let the download finish and verify the install completes successfully. + + downloadResumePromise.set_value(); + + auto err = installFuture.get(); + EXPECT_TRUE(err.IsNone()) << "Install failed: " << tests::utils::ErrorToStr(err); +} + } // namespace aos::sm::imagemanager From 2f1b261578e947e5f3c6c7574733599a8bfbb248 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 25 Jun 2026 15:20:57 +0300 Subject: [PATCH 024/112] sm: imagemanager: fix premature blob install lock release The `releaseInstalling` RAII guard was scoped inside the `if (waitInstalling)` block, causing `ReleaseInstallingBlob` to fire immediately on block exit rather than at function return. Move the guard to function scope and add the `waitInstalling` check inside the lambda to preserve conditional behavior. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/imagemanager/imagemanager.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/core/sm/imagemanager/imagemanager.cpp b/src/core/sm/imagemanager/imagemanager.cpp index 3b3666622..e6b1841f3 100644 --- a/src/core/sm/imagemanager/imagemanager.cpp +++ b/src/core/sm/imagemanager/imagemanager.cpp @@ -477,11 +477,14 @@ Error ImageManager::InstallBlob(const oci::ContentDescriptor& descriptor, Instal if (auto err = WaitForInstallingBlob(descriptor.mDigest); !err.IsNone()) { return err; } - - auto releaseInstalling - = DeferRelease(&descriptor.mDigest, [&](const String* digest) { ReleaseInstallingBlob(*digest); }); } + auto releaseInstalling = DeferRelease(&descriptor.mDigest, [&](const String* digest) { + if (waitInstalling) { + ReleaseInstallingBlob(*digest); + } + }); + if (installItem) { LockGuard lock {mMutex}; From d5b05b620517699112ee5efa62518630d834a900 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 25 Jun 2026 15:44:15 +0300 Subject: [PATCH 025/112] cm: launcher: fix lint warning Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- src/core/cm/launcher/instancemanager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index bacca2682..302403cc1 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -537,7 +537,7 @@ RetWithError> InstanceManager::CreateInstance(const Instance if (auto err = newInstance->Init(); !err.IsNone()) { // Do not leave invalid instance in storage. - if (auto err = newInstance->Remove(); !err.IsNone()) { + if (err = newInstance->Remove(); !err.IsNone()) { LOG_ERR() << "Can't remove instance" << Log::Field(err); } From b7c4febfecab5db7a94cd1f7763d897ad024176a Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 25 Jun 2026 17:05:41 +0300 Subject: [PATCH 026/112] sm: networkmanager: pass instance subnet to firewall Add mSubnet to InstanceFirewallParams and populate it from the instance network allocation. This lets the service manager firewall recognise the instance's own network (subnet) and allow unrestricted communication between instances that share it, while still filtering cross-network traffic. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/itf/firewall.hpp | 6 +++++- src/core/sm/networkmanager/networkmanager.cpp | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/sm/networkmanager/itf/firewall.hpp b/src/core/sm/networkmanager/itf/firewall.hpp index 796f34845..1520b25a5 100644 --- a/src/core/sm/networkmanager/itf/firewall.hpp +++ b/src/core/sm/networkmanager/itf/firewall.hpp @@ -43,7 +43,11 @@ struct OutputAccessConfig { * Per-instance firewall parameters. */ struct InstanceFirewallParams { - StaticString mIP; + StaticString mIP; + // Subnet (CIDR) of the instance network. Instances sharing it (same + // network) communicate without restrictions; only cross-network traffic + // is filtered by the access rules below. + StaticString mSubnet; bool mAllowPublic {}; StaticArray mInput; StaticArray mOutput; diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index eadb7ad5e..ff732fe6e 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -1395,6 +1395,7 @@ Error NetworkManager::PrepareInstanceFirewallParams(const InstanceNetworkConfig& const aos::InstanceNetworkAllocation& networkParams, InstanceFirewallParams& params) const { params.mIP = networkParams.mIP; + params.mSubnet = networkParams.mSubnet; params.mAllowPublic = true; StaticArray, cMaxExposedPort> portConfig; From 3a23e268f6ff452bfcfb73ddb3d4757115f0e2c4 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Fri, 26 Jun 2026 12:26:37 +0300 Subject: [PATCH 027/112] cm: updatemanager: store instance statuses in flat pointer-based array Replace per-item StaticArray with a flat StaticArray in UnitInstancesStatuses, backed by mUnitInstancesStatuses owned by UnitStatusHandler. This removes the per-item instance limit and avoids large inline storage inside each UnitInstancesStatuses entry. Remove cMaxNumUpdateItemInstances constant as it is no longer needed. Update UnitInstancesStatuses::operator== to dereference pointers for deep value comparison. Update tests accordingly. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Kobets --- .../cm/updatemanager/tests/updatemanager.cpp | 49 +++++++++++++------ .../cm/updatemanager/unitstatushandler.cpp | 34 ++++++++----- .../cm/updatemanager/unitstatushandler.hpp | 11 +++-- src/core/common/config.hpp | 7 --- src/core/common/types/common.hpp | 5 -- src/core/common/types/unitstatus.hpp | 14 +++++- 6 files changed, 73 insertions(+), 47 deletions(-) diff --git a/src/core/cm/updatemanager/tests/updatemanager.cpp b/src/core/cm/updatemanager/tests/updatemanager.cpp index 6e551220a..b0d38895a 100644 --- a/src/core/cm/updatemanager/tests/updatemanager.cpp +++ b/src/core/cm/updatemanager/tests/updatemanager.cpp @@ -41,6 +41,13 @@ const auto cCVTimeout = std::chrono::seconds(5); * Static **********************************************************************************************************************/ +static StaticArray sInstanceStatusStorage; + +void ResetInstanceStatusStorage() +{ + sInstanceStatusStorage.Clear(); +} + void SetNodeInfo(UnitNodeInfo& nodeInfo, const String& nodeID, const String& nodeType, const NodeState& state = NodeStateEnum::eProvisioned, bool isConnected = true, Error error = ErrorEnum::eNone) { @@ -151,15 +158,19 @@ void CreateInstancesStatuses(UnitStatus& unitStatus, const String& itemID, const instancesStatuses.mPreinstalled = preinstalled; for (size_t i = 0; i < numInstances; i++) { - UnitInstanceStatus instanceStatus; + auto err = sInstanceStatusStorage.EmplaceBack(); + EXPECT_TRUE(err.IsNone()); + auto& instanceStatus = sInstanceStatusStorage.Back(); + + instanceStatus = {}; instanceStatus.mInstance = i; instanceStatus.mManifestDigest = "digest1"; instanceStatus.mNodeID = "node1"; instanceStatus.mRuntimeID = "runtime1"; instanceStatus.mState = state; - auto err = instancesStatuses.mInstances.PushBack(instanceStatus); + err = instancesStatuses.mInstances.PushBack(&instanceStatus); EXPECT_TRUE(err.IsNone()); } } @@ -179,15 +190,19 @@ void ChangeInstancesStatuses(UnitStatus& unitStatus, const String& itemID, const instancesStatuses.mInstances.Clear(); for (size_t i = 0; i < numInstances; i++) { - UnitInstanceStatus instanceStatus; + auto err = sInstanceStatusStorage.EmplaceBack(); + EXPECT_TRUE(err.IsNone()); + auto& instanceStatus = sInstanceStatusStorage.Back(); + + instanceStatus = {}; instanceStatus.mInstance = i; instanceStatus.mManifestDigest = "digest1"; instanceStatus.mNodeID = "node1"; instanceStatus.mRuntimeID = "runtime1"; instanceStatus.mState = state; - auto err = instancesStatuses.mInstances.PushBack(instanceStatus); + err = instancesStatuses.mInstances.PushBack(&instanceStatus); EXPECT_TRUE(err.IsNone()); } } @@ -295,11 +310,11 @@ void ConvertInstancesStatuses( status.mType = unitInstanceStatus.mType; status.mSubjectID = unitInstanceStatus.mSubjectID; status.mVersion = unitInstanceStatus.mVersion; - status.mInstance = instanceStatus.mInstance; - status.mNodeID = instanceStatus.mNodeID; - status.mRuntimeID = instanceStatus.mRuntimeID; - status.mManifestDigest = instanceStatus.mManifestDigest; - status.mState = instanceStatus.mState; + status.mInstance = instanceStatus->mInstance; + status.mNodeID = instanceStatus->mNodeID; + status.mRuntimeID = instanceStatus->mRuntimeID; + status.mManifestDigest = instanceStatus->mManifestDigest; + status.mState = instanceStatus->mState; status.mPreinstalled = unitInstanceStatus.mPreinstalled; instancesStatuses.PushBack(status); @@ -324,6 +339,8 @@ class UpdateManagerTest : public Test { void SetUp() override { + ResetInstanceStatusStorage(); + Config config {cUnitStatusSendTimeout}; auto err = mUpdateManager.Init(config, mIdentProviderMock, mNodeHandlerMock, mUnitConfigMock, @@ -608,7 +625,7 @@ TEST_F(UpdateManagerTest, SendDeltaUnitStatus) EXPECT_TRUE(err.IsNone()); CreateInstanceStatus(statuses->Back(), instancesStatuses.mItemID, instancesStatuses.mSubjectID, - instanceStatus.mInstance, instancesStatuses.mVersion, instanceStatus); + instanceStatus->mInstance, instancesStatuses.mVersion, *instanceStatus); } } @@ -627,7 +644,7 @@ TEST_F(UpdateManagerTest, SendDeltaUnitStatus) EXPECT_TRUE(err.IsNone()); CreateInstanceStatus(statuses->Back(), instancesStatuses.mItemID, instancesStatuses.mSubjectID, - instanceStatus.mInstance, instancesStatuses.mVersion, instanceStatus); + instanceStatus->mInstance, instancesStatuses.mVersion, *instanceStatus); } } @@ -811,11 +828,11 @@ TEST_F(UpdateManagerTest, ProcessFullDesiredStatus) status.mItemID = instancesStatuses.mItemID; status.mSubjectID = instancesStatuses.mSubjectID; status.mVersion = instancesStatuses.mVersion; - status.mInstance = instanceStatus.mInstance; - status.mNodeID = instanceStatus.mNodeID; - status.mRuntimeID = instanceStatus.mRuntimeID; - status.mManifestDigest = instanceStatus.mManifestDigest; - status.mState = instanceStatus.mState; + status.mInstance = instanceStatus->mInstance; + status.mNodeID = instanceStatus->mNodeID; + status.mRuntimeID = instanceStatus->mRuntimeID; + status.mManifestDigest = instanceStatus->mManifestDigest; + status.mState = instanceStatus->mState; instances.PushBack(status); } diff --git a/src/core/cm/updatemanager/unitstatushandler.cpp b/src/core/cm/updatemanager/unitstatushandler.cpp index 8d04069c8..da0f8fbf5 100644 --- a/src/core/cm/updatemanager/unitstatushandler.cpp +++ b/src/core/cm/updatemanager/unitstatushandler.cpp @@ -311,20 +311,25 @@ void UnitStatusHandler::OnInstancesStatusesChanged(const Array& itemIt = &mUnitStatus.mInstances->Back(); } - auto instanceIt = itemIt->mInstances.FindIf([&status](const UnitInstanceStatus& instanceStatus) { - return instanceStatus.mInstance == status.mInstance; + auto instanceIt = itemIt->mInstances.FindIf([&status](const UnitInstanceStatus* instanceStatus) { + return instanceStatus->mInstance == status.mInstance; }); if (instanceIt == itemIt->mInstances.end()) { - if (auto err = itemIt->mInstances.EmplaceBack(); !err.IsNone()) { + if (auto err = mUnitInstancesStatuses.EmplaceBack(); !err.IsNone()) { LOG_ERR() << "Failed to emplace instance status" << Log::Field(err); return; } + if (auto err = itemIt->mInstances.PushBack(&mUnitInstancesStatuses.Back()); !err.IsNone()) { + LOG_ERR() << "Failed to push instance status pointer" << Log::Field(err); + return; + } + instanceIt = &itemIt->mInstances.Back(); } - static_cast(*instanceIt) = static_cast(status); - instanceIt->mInstance = status.mInstance; + static_cast(**instanceIt) = static_cast(status); + (*instanceIt)->mInstance = status.mInstance; } StartTimer(); @@ -460,6 +465,7 @@ Error UnitStatusHandler::SetUpdateItemsStatus() Error UnitStatusHandler::SetInstancesStatus() { mUnitStatus.mInstances.EmplaceValue(); + mUnitInstancesStatuses.Clear(); auto instancesStatuses = MakeUnique>(&mAllocator); @@ -482,12 +488,16 @@ Error UnitStatusHandler::SetInstancesStatus() it = &mUnitStatus.mInstances->Back(); } - UnitInstanceStatus instanceStatus {}; + if (auto err = mUnitInstancesStatuses.EmplaceBack(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto& instanceStatus = mUnitInstancesStatuses.Back(); static_cast(instanceStatus) = static_cast(status); instanceStatus.mInstance = status.mInstance; - it->mInstances.PushBack(instanceStatus); + it->mInstances.PushBack(&instanceStatus); } return ErrorEnum::eNone; @@ -534,11 +544,11 @@ void UnitStatusHandler::LogUnitStatus() << Log::Field("version", instanceStatuses.mVersion); for (const auto& instanceStatus : instanceStatuses.mInstances) { - LOG_INF() << "Unit status instance" << Log::Field("instance", instanceStatus.mInstance) - << Log::Field("manifestDigest", instanceStatus.mManifestDigest) - << Log::Field("nodeID", instanceStatus.mNodeID) - << Log::Field("runtimeID", instanceStatus.mRuntimeID) - << Log::Field("state", instanceStatus.mState) << Log::Field(instanceStatus.mError); + LOG_INF() << "Unit status instance" << Log::Field("instance", instanceStatus->mInstance) + << Log::Field("manifestDigest", instanceStatus->mManifestDigest) + << Log::Field("nodeID", instanceStatus->mNodeID) + << Log::Field("runtimeID", instanceStatus->mRuntimeID) + << Log::Field("state", instanceStatus->mState) << Log::Field(instanceStatus->mError); } } } diff --git a/src/core/cm/updatemanager/unitstatushandler.hpp b/src/core/cm/updatemanager/unitstatushandler.hpp index 413d974a2..7f07d8831 100644 --- a/src/core/cm/updatemanager/unitstatushandler.hpp +++ b/src/core/cm/updatemanager/unitstatushandler.hpp @@ -131,11 +131,12 @@ class UnitStatusHandler : private nodeinfoprovider::NodeInfoListenerItf, cloudconnection::CloudConnectionItf* mCloudConnection {}; SenderItf* mSender {}; - Mutex mMutex; - UnitStatus mUnitStatus; - StaticAllocator mAllocator; - bool mCloudConnected {}; - bool mIsStatusProcessing {}; + Mutex mMutex; + UnitStatus mUnitStatus; + StaticArray mUnitInstancesStatuses; + StaticAllocator mAllocator; + bool mCloudConnected {}; + bool mIsStatusProcessing {}; Timer mTimer; bool mTimerStarted {}; diff --git a/src/core/common/config.hpp b/src/core/common/config.hpp index b594527f1..5d05376ef 100644 --- a/src/core/common/config.hpp +++ b/src/core/common/config.hpp @@ -148,13 +148,6 @@ #define AOS_CONFIG_TYPES_MAX_NUM_BLOBS 16 #endif -/** - * Max number of instances per update item. - */ -#ifndef AOS_CONFIG_TYPES_MAX_NUM_UPDATE_ITEM_INSTANCES -#define AOS_CONFIG_TYPES_MAX_NUM_UPDATE_ITEM_INSTANCES 16 -#endif - /** * Error message len. */ diff --git a/src/core/common/types/common.hpp b/src/core/common/types/common.hpp index 5ca21a321..6c10d7cd5 100644 --- a/src/core/common/types/common.hpp +++ b/src/core/common/types/common.hpp @@ -44,11 +44,6 @@ constexpr auto cMaxNumUpdateItems = AOS_CONFIG_TYPES_MAX_NUM_UPDATE_ITEMS; */ constexpr auto cMaxNumBlobs = AOS_CONFIG_TYPES_MAX_NUM_BLOBS; -/** - * Max number of instances per update item. - */ -constexpr auto cMaxNumUpdateItemInstances = AOS_CONFIG_TYPES_MAX_NUM_UPDATE_ITEM_INSTANCES; - /** * Max number of instances. */ diff --git a/src/core/common/types/unitstatus.hpp b/src/core/common/types/unitstatus.hpp index c917b1acf..6b750d1f4 100644 --- a/src/core/common/types/unitstatus.hpp +++ b/src/core/common/types/unitstatus.hpp @@ -109,7 +109,7 @@ struct UnitInstanceStatus : public InstanceStatusData { bool operator!=(const UnitInstanceStatus& rhs) const { return !operator==(rhs); } }; -using UnitInstanceStatusArray = StaticArray; +using UnitInstanceStatusArray = StaticArray; /** * Instances statuses. @@ -153,8 +153,18 @@ struct UnitInstancesStatuses { */ bool operator==(const UnitInstancesStatuses& rhs) const { + if (mInstances.Size() != rhs.mInstances.Size()) { + return false; + } + + for (size_t i = 0; i < mInstances.Size(); i++) { + if (*mInstances[i] != *rhs.mInstances[i]) { + return false; + } + } + return mItemID == rhs.mItemID && mType == rhs.mType && mSubjectID == rhs.mSubjectID && mVersion == rhs.mVersion - && mPreinstalled == rhs.mPreinstalled && mInstances == rhs.mInstances; + && mPreinstalled == rhs.mPreinstalled; } /** From cf64d3edd20de5de3f7feff1b2e3fd55ff896f07 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 26 Jun 2026 02:25:49 +0300 Subject: [PATCH 028/112] cm: launcher: remove cache when allowed number of instances exceeded Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/instancemanager.cpp | 30 ++++++++++++++++++++++-- src/core/cm/launcher/instancemanager.hpp | 9 +++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index 302403cc1..da83bfe41 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -300,10 +300,12 @@ Error InstanceManager::SubmitScheduledInstances() mScheduledInstances.Clear(); + ClearCacheIfLimitReached(); + return ErrorEnum::eNone; } -Error InstanceManager::DisableInstance(SharedPtr& instance) +void InstanceManager::DisableInstance(SharedPtr& instance) { if (auto err = instance->Cache(true); !err.IsNone()) { const auto& id = instance->GetInfo().mInstanceIdent; @@ -316,7 +318,29 @@ Error InstanceManager::DisableInstance(SharedPtr& instance) mScheduledInstances.Remove(instance); mActiveInstances.Remove(instance); - return ErrorEnum::eNone; + ClearCacheIfLimitReached(); +} + +void InstanceManager::ClearCacheIfLimitReached() +{ + // Cache shares the allocator budget with active/scheduled instances. Drop the whole cache once the + // allowed number of instances is reached. + if (mScheduledInstances.Size() + mActiveInstances.Size() + mCachedInstances.Size() < 2 * cMaxNumInstances - 1) { + // Storage can hold at most cMaxNumInstances instances (active + cached are persisted), so keep their + // total within that limit and drop the cache once it is reached. + if (mActiveInstances.Size() + mCachedInstances.Size() <= cMaxNumInstances) { + return; + } + } + + for (auto& instance : mCachedInstances) { + if (auto err = instance->Remove(); !err.IsNone()) { + LOG_ERR() << "Remove cached instance failed" << Log::Field("instanceID", instance->GetInfo().mInstanceIdent) + << AOS_ERROR_WRAP(err); + } + } + + mCachedInstances.Clear(); } SharedPtr InstanceManager::FindActiveInstance(const InstanceIdent& id, const String& version) @@ -518,6 +542,8 @@ Error InstanceManager::ClearInstancesWithDeletedImages() RetWithError> InstanceManager::CreateInstance(const InstanceInfo& info) { + ClearCacheIfLimitReached(); + SharedPtr newInstance; switch (info.mInstanceIdent.mType.GetValue()) { diff --git a/src/core/cm/launcher/instancemanager.hpp b/src/core/cm/launcher/instancemanager.hpp index d691fe559..8600fdd38 100644 --- a/src/core/cm/launcher/instancemanager.hpp +++ b/src/core/cm/launcher/instancemanager.hpp @@ -186,9 +186,8 @@ class InstanceManager { * Disables instance. * * @param instance instance. - * @return Error. */ - Error DisableInstance(SharedPtr& instance); + void DisableInstance(SharedPtr& instance); /** * Updates monitoring data for active instances. @@ -264,8 +263,8 @@ class InstanceManager { // existing instances + 1 InstanceInfo (CreateInfo) + 1 new instance. Both paths peak at // cMaxNumInstances+1 simultaneous allocations. static constexpr auto cAllocatorSize = sizeof(StaticArray) - + Max(sizeof(ComponentInstance), sizeof(ServiceInstance)) * cMaxNumInstances; - static constexpr auto cMaxNumAllocations = cMaxNumInstances + 1; + + Max(sizeof(ComponentInstance), sizeof(ServiceInstance)) * cMaxNumInstances * 2; + static constexpr auto cMaxNumAllocations = 2 * cMaxNumInstances + 1; static constexpr auto cInstanceAllocatorSize = sizeof(oci::ImageConfig) + sizeof(oci::ItemConfig) + sizeof(InstanceStatus) + sizeof(oci::ImageIndex) + sizeof(EnvVarArray); @@ -282,6 +281,8 @@ class InstanceManager { template Error RemoveInstances(Array>& instances, Predicate predicate) const; + void ClearCacheIfLimitReached(); + RetWithError> CreateInstance(const InstanceInfo& info); SharedPtr FindReadyInstance(const InstanceIdent& id, const String& version); From 6c94e9b11f9239a8fa2a55eced66d28260ed32f4 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Tue, 30 Jun 2026 20:25:10 +0300 Subject: [PATCH 029/112] sm: networkmanager: clear only physical networks on teardown On destruction iterate mPhysicalNetworks (the networks whose bridge was actually created) instead of every persisted provider, and treat eNotFound from DeleteLink as success in ClearNetwork. This stops the spurious "Can't clear network: err=link not found" error on systemctl restart aos.target, where the destructor tried to delete bridges that were never created this session or already removed when the last instance left. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index ff732fe6e..b5a0d229e 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -66,8 +66,13 @@ NetworkManager::~NetworkManager() { mRuntimeCache.Clear(); - for (const auto& provider : mNetworkProviders) { - if (auto err = ClearNetwork(provider.mSecond); !err.IsNone()) { + for (const auto& networkID : mPhysicalNetworks) { + auto it = mNetworkProviders.Find(networkID); + if (it == mNetworkProviders.end()) { + continue; + } + + if (auto err = ClearNetwork(it->mSecond); !err.IsNone()) { LOG_ERR() << "Can't clear network" << Log::Field(err); } } @@ -1065,13 +1070,15 @@ Error NetworkManager::ClearNetwork(const NetworkInfo& networkInfo) } if (!networkInfo.mBridgeIfName.IsEmpty()) { - if (auto errDel = mNetIf->DeleteLink(networkInfo.mBridgeIfName); !errDel.IsNone() && err.IsNone()) { + if (auto errDel = mNetIf->DeleteLink(networkInfo.mBridgeIfName); + !errDel.IsNone() && errDel.Value() != ErrorEnum::eNotFound && err.IsNone()) { err = AOS_ERROR_WRAP(errDel); } } if (!networkInfo.mVlanIfName.IsEmpty()) { - if (auto errDel = mNetIf->DeleteLink(networkInfo.mVlanIfName); !errDel.IsNone() && err.IsNone()) { + if (auto errDel = mNetIf->DeleteLink(networkInfo.mVlanIfName); + !errDel.IsNone() && errDel.Value() != ErrorEnum::eNotFound && err.IsNone()) { err = AOS_ERROR_WRAP(errDel); } } From 31a5b90346acfeffc878a5eeaddfce13e1f0a25e Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Tue, 30 Jun 2026 20:25:44 +0300 Subject: [PATCH 030/112] sm: networkmanager: create vlan enslaved to bridge in one netlink op Add a master parameter to InterfaceFactoryItf::CreateVlan so the vlan is enslaved to the bridge via IFLA_MASTER in the same RTM_NEWLINK message. CreateNetwork no longer issues a separate SetMasterLink, saving a netlink round-trip on network creation. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../sm/networkmanager/itf/interfacefactory.hpp | 6 ++++-- src/core/sm/networkmanager/networkmanager.cpp | 8 +++----- .../tests/mocks/interfacefactorymock.hpp | 2 +- .../sm/networkmanager/tests/networkmanager.cpp | 14 +++++--------- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/core/sm/networkmanager/itf/interfacefactory.hpp b/src/core/sm/networkmanager/itf/interfacefactory.hpp index 29083a871..6a2a0f728 100644 --- a/src/core/sm/networkmanager/itf/interfacefactory.hpp +++ b/src/core/sm/networkmanager/itf/interfacefactory.hpp @@ -31,13 +31,15 @@ class InterfaceFactoryItf { virtual Error CreateBridge(const String& name, const String& ip, const String& subnet) = 0; /** - * Creates vlan interface. + * Creates vlan interface, optionally enslaved to a master bridge in the + * same operation. * * @param name vlan name. * @param vlanID vlan ID. + * @param master master bridge name to enslave to; empty for none. * @return Error. */ - virtual Error CreateVlan(const String& name, uint64_t vlanID) = 0; + virtual Error CreateVlan(const String& name, uint64_t vlanID, const String& master) = 0; /** * Creates a parameter-less link of the given kind (e.g. "ifb", "dummy"). diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index b5a0d229e..6e18eadd2 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -1485,7 +1485,9 @@ Error NetworkManager::CreateNetwork(const NetworkInfo& network) } }); - if (err = mNetIfFactory->CreateVlan(network.mVlanIfName, network.mVlanID); !err.IsNone()) { + // Create the vlan already enslaved to the bridge (master) in one operation, + // avoiding a separate SetMasterLink round-trip. + if (err = mNetIfFactory->CreateVlan(network.mVlanIfName, network.mVlanID, network.mBridgeIfName); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -1495,10 +1497,6 @@ Error NetworkManager::CreateNetwork(const NetworkInfo& network) } }); - if (err = mNetIf->SetMasterLink(network.mVlanIfName, network.mBridgeIfName); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - // Masquerade is a per-network property (one rule per subnet/bridge), so it // is installed here on network creation rather than per instance. if (err = mFirewall->AddMasquerade(network.mSubnet, network.mBridgeIfName); !err.IsNone()) { diff --git a/src/core/sm/networkmanager/tests/mocks/interfacefactorymock.hpp b/src/core/sm/networkmanager/tests/mocks/interfacefactorymock.hpp index a710a3ca4..8f42d4c03 100644 --- a/src/core/sm/networkmanager/tests/mocks/interfacefactorymock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/interfacefactorymock.hpp @@ -16,7 +16,7 @@ namespace aos::sm::networkmanager { class InterfaceFactoryMock : public InterfaceFactoryItf { public: MOCK_METHOD(Error, CreateBridge, (const String&, const String&, const String&), (override)); - MOCK_METHOD(Error, CreateVlan, (const String&, uint64_t), (override)); + MOCK_METHOD(Error, CreateVlan, (const String&, uint64_t, const String&), (override)); MOCK_METHOD(Error, CreateLink, (const String&, const String&), (override)); }; diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index 3f2a5c9b2..7bf27e742 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -165,8 +165,7 @@ class NetworkManagerTest : public Test { void SetupEnsureNodeNetworkPhysicalMocks(const aos::String& ip, const aos::String& subnet, uint64_t vlanID) { EXPECT_CALL(mNetIfFactory, CreateBridge(_, ip, subnet)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetIfFactory, CreateVlan(_, vlanID)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetIf, SetMasterLink(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetIfFactory, CreateVlan(_, vlanID, _)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mDNSName, CreateServer(_, _)) .WillOnce(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); } @@ -308,8 +307,7 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_VerifyHostsFile) EXPECT_CALL(mStorage, AddInstanceNetworkInfo(_)).Times(numInstances).WillRepeatedly(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mNetIfFactory, CreateBridge(_, _, _)).Times(numInstances).WillRepeatedly(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetIfFactory, CreateVlan(_, _)).Times(numInstances).WillRepeatedly(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetIf, SetMasterLink(_, _)).Times(numInstances).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetIfFactory, CreateVlan(_, _, _)).Times(numInstances).WillRepeatedly(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mDNSName, CreateServer(_, _)) .Times(numInstances) .WillRepeatedly(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); @@ -806,8 +804,7 @@ TEST_F(NetworkManagerTest, StopReleaseAndRecreateInstance) EXPECT_CALL(mStorage, AddInstanceNetworkInfo(_)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mNetIfFactory, CreateBridge(_, _, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetIfFactory, CreateVlan(_, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetIf, SetMasterLink(_, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetIfFactory, CreateVlan(_, _, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mDNSName, CreateServer(_, _)) .Times(2) .WillRepeatedly(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); @@ -1077,9 +1074,8 @@ TEST_F(NetworkManagerTest, InitWithExistingNetworks) EXPECT_CALL( mNetIfFactory, CreateBridge(existingNetwork.mBridgeIfName, existingNetwork.mIP, existingNetwork.mSubnet)) .WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetIfFactory, CreateVlan(existingNetwork.mVlanIfName, existingNetwork.mVlanID)) - .WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetIf, SetMasterLink(existingNetwork.mVlanIfName, existingNetwork.mBridgeIfName)) + EXPECT_CALL( + mNetIfFactory, CreateVlan(existingNetwork.mVlanIfName, existingNetwork.mVlanID, existingNetwork.mBridgeIfName)) .WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mDNSName, CreateServer(_, _)) .WillOnce(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); From 6fb90735a932fad62d9ba9cea958009077f1af99 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Tue, 30 Jun 2026 20:26:08 +0300 Subject: [PATCH 031/112] sm: networkmanager: add combined veth/instance-config interface methods Add CreateVethToNamespace and ConfigureInstanceInterface to InterfaceManagerItf. CreateVethToNamespace creates the veth pair with the peer placed directly into the instance netns (named, up) and the host side brought up and enslaved to the bridge in a single operation; ConfigureInstanceInterface brings the interface up, assigns the address and installs the default route in a single namespace entry. These let the SM bridge attach path replace the create+move+rename+setmaster+setup sequence and the three separate namespace switches, cutting netlink round-trips. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../networkmanager/itf/interfacemanager.hpp | 31 +++++++++++++++++++ .../tests/mocks/interfacemanagermock.hpp | 3 ++ 2 files changed, 34 insertions(+) diff --git a/src/core/sm/networkmanager/itf/interfacemanager.hpp b/src/core/sm/networkmanager/itf/interfacemanager.hpp index dda57f124..e552eb3f8 100644 --- a/src/core/sm/networkmanager/itf/interfacemanager.hpp +++ b/src/core/sm/networkmanager/itf/interfacemanager.hpp @@ -64,6 +64,37 @@ class InterfaceManagerItf { */ virtual Error CreateVeth(const String& hostIfName, const String& peerIfName) = 0; + /** + * Creates a veth pair with the peer placed directly into the given network + * namespace, already named as peerIfName. The host side stays in the + * current namespace. Combines veth creation, namespace move and rename into + * a single netlink operation. + * + * @param hostIfName host-side veth name (current namespace). + * @param peerIfName peer-side veth name inside the target namespace. + * @param netNSPath path to the target netns (e.g. /run/netns/). + * @param master master bridge name to enslave the host side to in the same + * operation; empty for none. + * @return Error. + */ + virtual Error CreateVethToNamespace( + const String& hostIfName, const String& peerIfName, const String& netNSPath, const String& master) + = 0; + + /** + * Configures an interface inside a network namespace in a single namespace + * entry: brings it up, assigns the address and installs the default route. + * + * @param ifname interface name inside the namespace. + * @param ipWithMask IP in CIDR form, e.g. "10.0.0.5/24". + * @param gateway default-route gateway IP. + * @param netNSPath path to the instance netns. + * @return Error. + */ + virtual Error ConfigureInstanceInterface( + const String& ifname, const String& ipWithMask, const String& gateway, const String& netNSPath) + = 0; + /** * Moves a link into a network namespace identified by its /run/netns path. * diff --git a/src/core/sm/networkmanager/tests/mocks/interfacemanagermock.hpp b/src/core/sm/networkmanager/tests/mocks/interfacemanagermock.hpp index 73472b3d1..9084ea755 100644 --- a/src/core/sm/networkmanager/tests/mocks/interfacemanagermock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/interfacemanagermock.hpp @@ -19,6 +19,9 @@ class InterfaceManagerMock : public InterfaceManagerItf { MOCK_METHOD(Error, SetupLink, (const String&, const String&), (override)); MOCK_METHOD(Error, SetMasterLink, (const String&, const String&), (override)); MOCK_METHOD(Error, CreateVeth, (const String&, const String&), (override)); + MOCK_METHOD(Error, CreateVethToNamespace, (const String&, const String&, const String&, const String&), (override)); + MOCK_METHOD( + Error, ConfigureInstanceInterface, (const String&, const String&, const String&, const String&), (override)); MOCK_METHOD(Error, MoveLinkToNamespace, (const String&, const String&), (override)); MOCK_METHOD(Error, RenameLink, (const String&, const String&, const String&), (override)); MOCK_METHOD(Error, AddAddress, (const String&, const String&, const String&), (override)); From 63c2459b03dba1d6268352d8dfb2dacd86dd7f26 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 1 Jul 2026 20:48:48 +0300 Subject: [PATCH 032/112] sm: launcher: stop all instances on close including inactive Inactive instances still have runtime created and also should be stopped to properly cleanup their runtime. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/sm/launcher/launcher.cpp | 3 +-- src/core/sm/launcher/tests/launcher.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index 22dc86355..a932a4783 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -694,8 +694,7 @@ void Launcher::StopAllInstances() } for (auto& instance : mInstances) { - if (instance.mStatus.mState != InstanceStateEnum::eActive - || instance.mInfo.mType == UpdateItemTypeEnum::eComponent) { + if (instance.mInfo.mType == UpdateItemTypeEnum::eComponent) { continue; } diff --git a/src/core/sm/launcher/tests/launcher.cpp b/src/core/sm/launcher/tests/launcher.cpp index 0581a79f9..069c63dea 100644 --- a/src/core/sm/launcher/tests/launcher.cpp +++ b/src/core/sm/launcher/tests/launcher.cpp @@ -473,6 +473,11 @@ TEST_F(LauncherTest, StopInstancesWithExpiredOfflineTTL) ASSERT_TRUE(stop1promise.get_future().wait_for(std::chrono::seconds(5)) == std::future_status::ready) << "Runtime1 StopInstance was not called"; + EXPECT_CALL(mRuntime0, StopInstance(static_cast(cStoredInfos[0]), _)) + .WillOnce(Return(ErrorEnum::eNone)); + EXPECT_CALL(mRuntime1, StopInstance(static_cast(cStoredInfos[1]), _)) + .WillOnce(Return(ErrorEnum::eNone)); + err = mLauncher.Stop(); ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); } @@ -794,6 +799,8 @@ TEST_F(LauncherTest, GetInstancesStatuses) EXPECT_CALL(mRuntime0, StopInstance(static_cast(cStartInstanceInfos[0]), _)) .WillOnce(Return(ErrorEnum::eNone)); + EXPECT_CALL(mRuntime1, StopInstance(static_cast(cStartInstanceInfos[1]), _)) + .WillOnce(Return(ErrorEnum::eNone)); err = mLauncher.Stop(); ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); From 8a56008f7971a92e37f8cdde4e43846c7971620c Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 2 Jul 2026 11:53:45 +0300 Subject: [PATCH 033/112] cm: updatemanager: clear instance statuses array on clearing unit status Instance statuses array should be cleared when unit status is reset. Otherwise it leads to no mem error when processing instance statues change. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/cm/updatemanager/unitstatushandler.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/cm/updatemanager/unitstatushandler.cpp b/src/core/cm/updatemanager/unitstatushandler.cpp index da0f8fbf5..14c0a61d0 100644 --- a/src/core/cm/updatemanager/unitstatushandler.cpp +++ b/src/core/cm/updatemanager/unitstatushandler.cpp @@ -568,6 +568,7 @@ void UnitStatusHandler::ClearUnitStatus() mUnitStatus.mUpdateItems.Reset(); mUnitStatus.mInstances.Reset(); mUnitStatus.mUnitSubjects.Reset(); + mUnitInstancesStatuses.Clear(); }; void UnitStatusHandler::ClearUpdateStatuses() From ff89a643c45585dd92ea67c6fdb193f6f1238485 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 2 Jul 2026 11:55:20 +0300 Subject: [PATCH 034/112] cm: launcher: remove extra space in instance manager log Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/cm/launcher/instancemanager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index da83bfe41..64b1d4ae3 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -66,7 +66,7 @@ Error InstanceManager::Start() } if (auto err = LoadInstancesFromStorage(); !err.IsNone()) { - LOG_ERR() << "Can't load instances from storage " << Log::Field(err); + LOG_ERR() << "Can't load instances from storage" << Log::Field(err); return err; } From 68daa8e07d0425bd1eae331a58987d6e74a385f7 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 2 Jul 2026 11:55:59 +0300 Subject: [PATCH 035/112] sm: launcher: add info log on start and stop instances Ass info log to measure start/stop time. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/sm/launcher/launcher.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index a932a4783..6bcb98cd0 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -556,7 +556,7 @@ Error Launcher::HandleComponentStatus(const aos::InstanceStatus& status) void Launcher::UpdateInstancesImpl(Array& stopInstances, const Array& startInstances) { - LOG_INF() << "Update instances" << Log::Field("stopCount", stopInstances.Size()) + LOG_INF() << "Update instances start" << Log::Field("stopCount", stopInstances.Size()) << Log::Field("startCount", startInstances.Size()); auto sendStatus = DeferRelease(&mInstances, [this](Array*) { @@ -614,6 +614,9 @@ void Launcher::UpdateInstancesImpl(Array& stopInstances, const Ar if (auto err = mLaunchPool.Shutdown(); !err.IsNone()) { LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } + + LOG_INF() << "Update instances finished" << Log::Field("stopCount", stopInstances.Size()) + << Log::Field("startCount", startInstances.Size()); } void Launcher::StopInstances(const Array& stopInstances) @@ -687,6 +690,8 @@ void Launcher::StopInstanceTask(aos::sm::launcher::RuntimeItf* runtime, Instance void Launcher::StopAllInstances() { + LOG_INF() << "Stop all instances start" << Log::Field("count", mInstances.Size()); + if (auto err = mLaunchPool.Run(); !err.IsNone()) { LOG_ERR() << "Can't start thread pool" << Log::Field(AOS_ERROR_WRAP(err)); @@ -712,6 +717,8 @@ void Launcher::StopAllInstances() if (auto err = mLaunchPool.Shutdown(); !err.IsNone()) { LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } + + LOG_INF() << "Stop all instances finished" << Log::Field("count", mInstances.Size()); } void Launcher::PrepareInstances(const Array& startInstances) From 10d9c5a6b12e42846df32a671748396bf6c0d7ca Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 2 Jul 2026 23:07:56 +0300 Subject: [PATCH 036/112] sm: launcher: parallelize instance start/stop via thread pool - Run stop, network-prepare and start tasks for independent instances concurrently on mLaunchPool instead of sequentially - Split instance removal out into RemoveInstance/RemoveInstances, including network release, and dispatch it through the thread pool - Add LoadInstanceData/LoadInstancesData to load offline TTL for stored instances on start via the thread pool - Resize mAllocator size and allocation count to cover the new concurrent per-task allocations - Update launcher unit tests to match the new start-failure/state propagation and add a missing stop expectation Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/launcher/launcher.cpp | 368 ++++++++++++++---------- src/core/sm/launcher/launcher.hpp | 52 ++-- src/core/sm/launcher/tests/launcher.cpp | 2 +- 3 files changed, 244 insertions(+), 178 deletions(-) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index 6bcb98cd0..175df3d07 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -81,6 +81,8 @@ Error Launcher::Start() lock.Unlock(); + LoadInstancesData(*storedInstances); + if (auto err = UpdateInstances({}, *storedInstances); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -554,6 +556,60 @@ Error Launcher::HandleComponentStatus(const aos::InstanceStatus& status) return ErrorEnum::eNone; } +Error Launcher::LoadInstanceData(InstanceData& instanceData) +{ + auto itemConfig = MakeUnique(&mAllocator); + auto imageConfig = MakeUnique(&mAllocator); + + if (auto err = GetInstanceConfigs(instanceData.mInfo, *itemConfig, *imageConfig); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + instanceData.mOfflineTTL = itemConfig->mOfflineTTL; + + return ErrorEnum::eNone; +} + +void Launcher::LoadInstancesData(const Array& storedInstances) +{ + LOG_DBG() << "Load instances data" << Log::Field("count", storedInstances.Size()); + + if (auto err = mLaunchPool.Run(); !err.IsNone()) { + LOG_ERR() << "Can't start thread pool" << Log::Field(AOS_ERROR_WRAP(err)); + + return; + } + + for (const auto& instanceInfo : storedInstances) { + auto [instanceData, err] = AddInstanceData(instanceInfo); + if (!err.IsNone()) { + LOG_ERR() << "Failed to add instance data" << Log::Field("instance", instanceInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + continue; + } + + if (err = mLaunchPool.AddTask([this, instanceData](void*) { + if (auto err = LoadInstanceData(*instanceData); !err.IsNone()) { + LOG_ERR() << "Failed to load instance data" << Log::Field("instance", instanceData->mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + } + }); + !err.IsNone()) { + LOG_ERR() << "Failed to load instance data" << Log::Field("instance", instanceInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + } + } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } + + if (auto err = mLaunchPool.Shutdown(); !err.IsNone()) { + LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); + } +} + void Launcher::UpdateInstancesImpl(Array& stopInstances, const Array& startInstances) { LOG_INF() << "Update instances start" << Log::Field("stopCount", stopInstances.Size()) @@ -588,29 +644,16 @@ void Launcher::UpdateInstancesImpl(Array& stopInstances, const Ar } StopInstances(stopInstances); - - if (auto err = mLaunchPool.Wait(); !err.IsNone()) { - LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); - } - - RemoveInstancesData(stopInstances); + RemoveInstances(stopInstances); if (!mFirstStart) { RemoveUpdateItems(*removeItems); InstallUpdateItems(startInstances); - - if (auto err = mLaunchPool.Wait(); !err.IsNone()) { - LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); - } + PrepareInstances(startInstances); } - PrepareInstances(startInstances); StartInstances(startInstances); - if (auto err = mLaunchPool.Wait(); !err.IsNone()) { - LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); - } - if (auto err = mLaunchPool.Shutdown(); !err.IsNone()) { LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } @@ -625,30 +668,42 @@ void Launcher::StopInstances(const Array& stopInstances) auto instanceData = FindInstanceData(instance); if (!instanceData) { LOG_ERR() << "Failed to stop instance" << Log::Field("instance", instance) - << Log::Field(AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance not found"))); + << Log::Field(AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance data not found"))); continue; } - if (auto err = StopInstance(*instanceData, true); !err.IsNone()) { + if (auto err = AddStopInstanceTask(*instanceData); !err.IsNone()) { LOG_ERR() << "Failed to stop instance" << Log::Field("instance", instance) << Log::Field(err); SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); - } else { - SetInstanceState(*instanceData, InstanceStateEnum::eInactive); } } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } } -Error Launcher::StopInstance(InstanceData& instanceData, bool isRemoval) +Error Launcher::AddStopInstanceTask(InstanceData& instanceData) { auto runtime = FindInstanceRuntime(instanceData.mStatus.mRuntimeID); if (runtime == nullptr) { return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "runtime not found")); } - if (auto err = mLaunchPool.AddTask( - [this, runtime, &instanceData, isRemoval](void*) { StopInstanceTask(runtime, instanceData, isRemoval); }); + if (auto err = mLaunchPool.AddTask([this, runtime, &instanceData](void*) { + if (auto err = StopInstance(runtime, instanceData); !err.IsNone()) { + LOG_ERR() << "Failed to stop instance" << Log::Field("instance", instanceData.mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + + return; + } + + SetInstanceState(instanceData, InstanceStateEnum::eInactive); + }); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -656,36 +711,16 @@ Error Launcher::StopInstance(InstanceData& instanceData, bool isRemoval) return ErrorEnum::eNone; } -void Launcher::StopInstanceTask(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData, bool isRemoval) +Error Launcher::StopInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData) { LOG_INF() << "Stop instance" << Log::Field("instance", instanceData.mInfo) - << Log::Field("version", instanceData.mInfo.mVersion) - << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID) << Log::Field("isRemoval", isRemoval); + << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID); if (auto err = runtime->StopInstance(instanceData.mInfo, instanceData.mStatus); !err.IsNone()) { - LOG_ERR() << "Failed to stop instance" << Log::Field("instance", instanceData.mInfo) - << Log::Field(AOS_ERROR_WRAP(err)); - - return; - } - - if (instanceData.mInfo.mType != UpdateItemTypeEnum::eService || !isRemoval) { - return; - } - - StaticString instanceID; - - if (auto err = mInstanceIDProvider->GetInstanceID(instanceData.mInfo, instanceID); !err.IsNone()) { - LOG_ERR() << "Failed to generate instance ID" << Log::Field("instance", instanceData.mInfo) - << Log::Field(AOS_ERROR_WRAP(err)); - - return; + return AOS_ERROR_WRAP(err); } - if (auto err = mNetworkManager->ReleaseInstanceNetwork(instanceID, instanceData.mInfo.mOwnerID); !err.IsNone()) { - LOG_ERR() << "Failed to release instance network" << Log::Field("instance", instanceData.mInfo) - << Log::Field(AOS_ERROR_WRAP(err)); - } + return ErrorEnum::eNone; } void Launcher::StopAllInstances() @@ -703,7 +738,7 @@ void Launcher::StopAllInstances() continue; } - if (auto err = StopInstance(instance, false); !err.IsNone()) { + if (auto err = AddStopInstanceTask(instance); !err.IsNone()) { LOG_ERR() << "Failed to stop instance" << Log::Field("instance", instance.mInfo) << Log::Field(err); SetInstanceState(instance, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); @@ -721,46 +756,75 @@ void Launcher::StopAllInstances() LOG_INF() << "Stop all instances finished" << Log::Field("count", mInstances.Size()); } +Error Launcher::PrepareInstance(InstanceData& instanceData) +{ + LOG_DBG() << "Prepare instance" << Log::Field("instance", instanceData.mInfo); + + if (auto err = mStorage->UpdateInstanceInfo(instanceData.mInfo); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (instanceData.mInfo.mType != UpdateItemTypeEnum::eService) { + return ErrorEnum::eNone; + } + + auto itemConfig = MakeUnique(&mAllocator); + auto imageConfig = MakeUnique(&mAllocator); + + if (auto err = GetInstanceConfigs(instanceData.mInfo, *itemConfig, *imageConfig); !err.IsNone()) { + return err; + } + + instanceData.mOfflineTTL = itemConfig->mOfflineTTL; + + if (auto err = CreateNetwork(instanceData.mInfo, *itemConfig, *imageConfig); !err.IsNone()) { + return err; + } + + return ErrorEnum::eNone; +} + void Launcher::PrepareInstances(const Array& startInstances) { for (const auto& instance : startInstances) { auto instanceData = FindInstanceData(instance); - if (!instanceData) { - Error err; + if (instanceData) { + LOG_WRN() << "Instance data already exists" << Log::Field("instance", instance); - Tie(instanceData, err) = AddInstanceData(instance); - if (!err.IsNone()) { - LOG_ERR() << "Failed to add instance data" << Log::Field("instance", instance) - << Log::Field(AOS_ERROR_WRAP(err)); - - continue; - } - } else { SetInstanceState(*instanceData, InstanceStateEnum::eInactive); continue; } - if (instanceData->mInfo.mType != UpdateItemTypeEnum::eService) { - continue; - } - - auto itemConfig = MakeUnique(&mAllocator); - auto imageConfig = MakeUnique(&mAllocator); + Error err; - if (auto err = GetInstanceConfigs(instanceData->mInfo, *itemConfig, *imageConfig); !err.IsNone()) { - SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); - - LOG_ERR() << "Failed to get instance configs" << Log::Field("instance", instanceData->mInfo) + Tie(instanceData, err) = AddInstanceData(instance); + if (!err.IsNone()) { + LOG_ERR() << "Failed to add instance data" << Log::Field("instance", instance) << Log::Field(AOS_ERROR_WRAP(err)); continue; } - if (auto err = CreateNetwork(instanceData->mInfo, *itemConfig, *imageConfig); !err.IsNone()) { + if (err = mLaunchPool.AddTask([this, instanceData](void*) { + if (auto err = PrepareInstance(*instanceData); !err.IsNone()) { + LOG_ERR() << "Failed to start instance" << Log::Field("instance", instanceData->mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + } + }); + !err.IsNone()) { + LOG_ERR() << "Failed to prepare instance" << Log::Field("instance", instance) + << Log::Field(AOS_ERROR_WRAP(err)); + SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); } } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } } void Launcher::StartInstances(const Array& startInstances) @@ -768,36 +832,53 @@ void Launcher::StartInstances(const Array& startInstances) for (const auto& instance : startInstances) { auto instanceData = FindInstanceData(instance); if (!instanceData) { - LOG_ERR() << "Failed to find instance data" << Log::Field("instance", instance) + LOG_ERR() << "Failed to start instance" << Log::Field("instance", instance) << Log::Field(AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance data not found"))); continue; } if (instanceData->mStatus.mState != InstanceStateEnum::eInactive) { + LOG_ERR() << "Failed to start instance" << Log::Field("instance", instance) + << Log::Field(AOS_ERROR_WRAP(Error(ErrorEnum::eWrongState, "instance not inactive"))); + continue; } - if (auto err = StartInstance(*instanceData); !err.IsNone()) { + if (auto err = AddStartInstanceTask(*instanceData); !err.IsNone()) { LOG_ERR() << "Failed to start instance" << Log::Field("instance", instance) << Log::Field(AOS_ERROR_WRAP(err)); SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); } } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } } -Error Launcher::StartInstance(InstanceData& instanceData) +Error Launcher::AddStartInstanceTask(InstanceData& instanceData) { - SetInstanceState(instanceData, InstanceStateEnum::eActivating); - auto runtime = FindInstanceRuntime(instanceData.mInfo.mRuntimeID); if (runtime == nullptr) { return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "runtime not found")); } - if (auto err - = mLaunchPool.AddTask([this, runtime, &instanceData](void*) { StartInstanceTask(runtime, instanceData); }); + if (auto err = mLaunchPool.AddTask([this, runtime, &instanceData](void*) { + SetInstanceState(instanceData, InstanceStateEnum::eActivating); + + if (auto err = StartInstance(runtime, instanceData); !err.IsNone()) { + LOG_ERR() << "Failed to start instance" << Log::Field("instance", instanceData.mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + + return; + } + + SetInstanceState(instanceData, InstanceStateEnum::eActive); + }); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -805,17 +886,17 @@ Error Launcher::StartInstance(InstanceData& instanceData) return ErrorEnum::eNone; } -void Launcher::StartInstanceTask(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData) +Error Launcher::StartInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData) { LOG_INF() << "Start instance" << Log::Field("instance", instanceData.mInfo) - << Log::Field("version", instanceData.mInfo.mVersion) << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID) << Log::Field("manifestDigest", instanceData.mInfo.mManifestDigest); if (auto err = runtime->StartInstance(instanceData.mInfo, instanceData.mStatus); !err.IsNone()) { - LOG_ERR() << "Failed to start instance" << Log::Field("instance", instanceData.mInfo) - << Log::Field(AOS_ERROR_WRAP(err)); + return AOS_ERROR_WRAP(err); } + + return ErrorEnum::eNone; } Error Launcher::AppendInstancesWithModifiedParams( @@ -914,37 +995,6 @@ RuntimeItf* Launcher::FindInstanceRuntime(const InstanceIdent& instanceIdent) co return const_cast(this)->FindInstanceRuntime(instanceIdent); } -RetWithError Launcher::GetOfflineTTL(const InstanceInfo& instanceInfo) -{ - auto path = MakeUnique>(&mAllocator); - - if (auto err = mItemInfoProvider->GetBlobPath(instanceInfo.mManifestDigest, *path); !err.IsNone()) { - return {{}, AOS_ERROR_WRAP(err)}; - } - - auto manifest = MakeUnique(&mAllocator); - - if (auto err = mOCISpec->LoadImageManifest(*path, *manifest); !err.IsNone()) { - return {{}, AOS_ERROR_WRAP(err)}; - } - - if (!manifest->mItemConfig.HasValue()) { - return {0}; - } - - if (auto err = mItemInfoProvider->GetBlobPath(manifest->mItemConfig->mDigest, *path); !err.IsNone()) { - return {{}, AOS_ERROR_WRAP(err)}; - } - - auto itemConfig = MakeUnique(&mAllocator); - - if (auto err = mOCISpec->LoadItemConfig(*path, *itemConfig); !err.IsNone()) { - return {{}, AOS_ERROR_WRAP(err)}; - } - - return itemConfig->mOfflineTTL; -} - void Launcher::GetRemoveUpdateItems(const Array& stopInstances, const Array& startInstances, Array& removeItems) { @@ -989,6 +1039,10 @@ void Launcher::RemoveUpdateItems(const Array& removeItems) continue; } } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } } void Launcher::InstallUpdateItems(const Array& startInstances) @@ -1031,25 +1085,19 @@ void Launcher::InstallUpdateItems(const Array& startInstances) continue; } } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } } RetWithError Launcher::AddInstanceData(const InstanceInfo& instanceInfo) { LockGuard lock {mMutex}; - LOG_DBG() << "Add instance data" << Log::Field("instance", instanceInfo) - << Log::Field("runtimeID", instanceInfo.mRuntimeID); - - if (auto err = mStorage->UpdateInstanceInfo(instanceInfo); !err.IsNone()) { - LOG_ERR() << "Failed to update instance info in storage" << Log::Field("instance", instanceInfo) - << Log::Field(AOS_ERROR_WRAP(err)); - } - - Duration offlineTTL = 0; - Error err; + LOG_DBG() << "Add instance data" << Log::Field("instance", instanceInfo); - err = mInstances.EmplaceBack(); - if (!err.IsNone()) { + if (auto err = mInstances.EmplaceBack(); !err.IsNone()) { return {nullptr, AOS_ERROR_WRAP(err)}; } @@ -1061,36 +1109,34 @@ RetWithError Launcher::AddInstanceData(const InstanceIn itInstance->mStatus.mRuntimeID = instanceInfo.mRuntimeID; itInstance->mStatus.mState = InstanceStateEnum::eInactive; - if (!instanceInfo.mPreinstalled) { - Tie(offlineTTL, err) = GetOfflineTTL(instanceInfo); - if (!err.IsNone()) { - LOG_ERR() << "Failed to get offline TTL for instance" << Log::Field("instance", instanceInfo) - << Log::Field(AOS_ERROR_WRAP(err)); - - itInstance->mStatus.mState = InstanceStateEnum::eFailed; - itInstance->mStatus.mError = AOS_ERROR_WRAP(err); - } else { - LOG_DBG() << "Offline TTL for instance" << Log::Field("instance", instanceInfo) - << Log::Field("offlineTTL", offlineTTL); - } - } - - itInstance->mOfflineTTL = offlineTTL; - return itInstance; } -Error Launcher::RemoveInstanceData(const InstanceIdent& instanceIdent) +Error Launcher::RemoveInstance(InstanceData& instanceData) { - LOG_DBG() << "Remove instance data" << Log::Field("instance", instanceIdent); + LOG_DBG() << "Remove instance" << Log::Field("instance", instanceData.mInfo); + + if (auto err = mStorage->RemoveInstanceInfo(instanceData.mInfo); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + StaticString instanceID; + + if (auto err = mInstanceIDProvider->GetInstanceID(instanceData.mInfo, instanceID); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } - if (auto err = mStorage->RemoveInstanceInfo(instanceIdent); !err.IsNone()) { - LOG_ERR() << "Remove instance info from storage failed" << Log::Field("instance", instanceIdent) - << Log::Field(AOS_ERROR_WRAP(err)); + if (auto err = mNetworkManager->ReleaseInstanceNetwork(instanceID, instanceData.mInfo.mOwnerID); !err.IsNone()) { + return AOS_ERROR_WRAP(err); } - if (auto count = mInstances.RemoveIf([this, &instanceIdent](const auto& instanceData) { - return static_cast(instanceData.mInfo) == instanceIdent; + LockGuard lock {mMutex}; + + LOG_DBG() << "Remove instance data" << Log::Field("instance", instanceData.mInfo); + + if (auto count = mInstances.RemoveIf([this, &instanceData](const auto& instance) { + return static_cast(instance.mInfo) + == static_cast(instanceData.mInfo); }); count == 0) { return AOS_ERROR_WRAP(ErrorEnum::eNotFound); @@ -1099,28 +1145,42 @@ Error Launcher::RemoveInstanceData(const InstanceIdent& instanceIdent) return ErrorEnum::eNone; } -void Launcher::RemoveInstancesData(const Array& instances) +void Launcher::RemoveInstances(const Array& instances) { - LockGuard lock {mMutex}; - for (const auto& instanceIdent : instances) { auto instanceData = FindInstanceData(instanceIdent); if (!instanceData) { LOG_ERR() << "Instance data not found, skip removing" << Log::Field("instance", instanceIdent); continue; - } else if (instanceData->mStatus.mState != InstanceStateEnum::eInactive) { + } + + if (instanceData->mStatus.mState != InstanceStateEnum::eInactive) { LOG_ERR() << "Instance is not inactive, skip removing" << Log::Field("instance", instanceIdent) << Log::Field("state", instanceData->mStatus.mState); continue; } - if (auto err = RemoveInstanceData(instanceIdent); !err.IsNone()) { - LOG_ERR() << "Failed to remove instance data" << Log::Field("instance", instanceIdent) + if (auto err = mLaunchPool.AddTask([this, instanceData](void*) { + if (auto err = RemoveInstance(*instanceData); !err.IsNone()) { + LOG_ERR() << "Failed to remove instance" << Log::Field("instance", instanceData->mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + } + }); + !err.IsNone()) { + LOG_ERR() << "Failed to remove instance" << Log::Field("instance", instanceData->mInfo) << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); } } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } } void Launcher::SetInstanceState(InstanceData& instance, const InstanceState& state, const Error& error) @@ -1185,7 +1245,7 @@ Error Launcher::GetInstanceNetworkConfig(const InstanceInfo& instance, const oci for (const auto& resource : itemConfig.mResources) { if (auto err = mResourceInfoProvider->GetResourceInfo(resource.mName, *resourceInfo); !err.IsNone()) { - return err; + return AOS_ERROR_WRAP(err); } if (auto err = networkConfig.mHosts.Insert( @@ -1232,7 +1292,7 @@ Error Launcher::CreateNetwork( StaticString instanceID; if (auto err = mInstanceIDProvider->GetInstanceID(instance, instanceID); !err.IsNone()) { - return err; + return AOS_ERROR_WRAP(err); } auto networkConfig = MakeUnique(&mAllocator); @@ -1243,7 +1303,7 @@ Error Launcher::CreateNetwork( if (auto err = mNetworkManager->CreateInstanceNetwork(instanceID, instance.mOwnerID, *networkConfig); !err.IsNone() && !err.Is(ErrorEnum::eAlreadyExist)) { - return err; + return AOS_ERROR_WRAP(err); } return ErrorEnum::eNone; diff --git a/src/core/sm/launcher/launcher.hpp b/src/core/sm/launcher/launcher.hpp index c75202eee..6f29933c6 100644 --- a/src/core/sm/launcher/launcher.hpp +++ b/src/core/sm/launcher/launcher.hpp @@ -170,28 +170,35 @@ class Launcher : public LauncherItf, static constexpr auto cOIDNamespace = "6ba7b812-9dad-11d1-80b4-00c04fd430c8"; static constexpr auto cThreadTaskSize = 512; static constexpr auto cMaxNumSubscribers = 4; - static constexpr auto cAllocatorSize = 2 * sizeof(StaticArray) - + 2 * sizeof(InstanceInfoArray) + sizeof(InstanceStatusArray) + sizeof(oci::ImageManifest) - + sizeof(oci::ImageConfig) + sizeof(oci::ItemConfig) + sizeof(StaticString) - + sizeof(networkmanager::InstanceNetworkConfig) + sizeof(resourcemanager::ResourceInfo) + + static constexpr auto cAllocatorSize = 2 * sizeof(StaticArray) + + 2 * sizeof(InstanceInfoArray) + sizeof(InstanceStatusArray) + + cMaxNumConcurrentItems + * (sizeof(oci::ImageConfig) + sizeof(oci::ItemConfig) + + Max(sizeof(StaticString) + sizeof(oci::ImageManifest), + sizeof(networkmanager::InstanceNetworkConfig) + sizeof(resourcemanager::ResourceInfo))) + Max(sizeof(StaticArray), sizeof(StaticArray) + sizeof(StaticArray)); + static constexpr auto cMaxNumAllocations = 4 + cMaxNumConcurrentItems * 4; void OnConnect() override; void OnDisconnect() override; void RunRebootThread(); void HandleOfflineTTLs(); + Error LoadInstanceData(InstanceData& instanceData); + void LoadInstancesData(const Array& storedInstances); Error HandleComponentStatus(const aos::InstanceStatus& status); void UpdateInstancesImpl(Array& stopInstances, const Array& startInstances); void StopInstances(const Array& stopInstances); - Error StopInstance(InstanceData& instanceData, bool isRemoval); - void StopInstanceTask(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData, bool isRemoval); + Error AddStopInstanceTask(InstanceData& instanceData); + Error StopInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData); void StopAllInstances(); + Error PrepareInstance(InstanceData& instanceData); void PrepareInstances(const Array& startInstances); void StartInstances(const Array& startInstances); - Error StartInstance(InstanceData& instanceData); - void StartInstanceTask(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData); + Error AddStartInstanceTask(InstanceData& instanceData); + Error StartInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData); Error AppendInstancesWithModifiedParams( const Array& startInstances, Array& stopInstances); Error StartLaunch(); @@ -201,8 +208,8 @@ class Launcher : public LauncherItf, void RemoveUpdateItems(const Array& removeItems); void InstallUpdateItems(const Array& startInstances); RetWithError AddInstanceData(const InstanceInfo& instanceInfo); - Error RemoveInstanceData(const InstanceIdent& instanceIdent); - void RemoveInstancesData(const Array& instances); + Error RemoveInstance(InstanceData& instanceData); + void RemoveInstances(const Array& instances); void SetInstanceState(InstanceData& instance, const InstanceState& state, const Error& error = ErrorEnum::eNone); Error GetInstanceConfigs(const InstanceInfo& instance, oci::ItemConfig& itemConfig, oci::ImageConfig& imageConfig); Error GetInstanceNetworkConfig(const InstanceInfo& instance, const oci::ItemConfig& itemConfig, @@ -210,19 +217,18 @@ class Launcher : public LauncherItf, Error CreateNetwork( const InstanceInfo& instance, const oci::ItemConfig& itemConfig, const oci::ImageConfig& imageConfig); - InstanceData* FindInstanceData(const InstanceIdent& instanceIdent); - InstanceData* FindInstanceData(const InstanceIdent& instanceIdent) const; - RuntimeItf* FindInstanceRuntime(const String& runtimeID); - RuntimeItf* FindInstanceRuntime(const String& runtimeID) const; - RuntimeItf* FindInstanceRuntime(const InstanceIdent& instanceIdent); - RuntimeItf* FindInstanceRuntime(const InstanceIdent& instanceIdent) const; - RetWithError GetOfflineTTL(const InstanceInfo& instanceInfo); - Optional GetMinOfflineTTL() const; - void StartTTLTimer(); - void StopExpiredInstances(UniqueLock& lock); - void SendNodeInstancesStatuses(); - - mutable StaticAllocator mAllocator; + InstanceData* FindInstanceData(const InstanceIdent& instanceIdent); + InstanceData* FindInstanceData(const InstanceIdent& instanceIdent) const; + RuntimeItf* FindInstanceRuntime(const String& runtimeID); + RuntimeItf* FindInstanceRuntime(const String& runtimeID) const; + RuntimeItf* FindInstanceRuntime(const InstanceIdent& instanceIdent); + RuntimeItf* FindInstanceRuntime(const InstanceIdent& instanceIdent) const; + Optional GetMinOfflineTTL() const; + void StartTTLTimer(); + void StopExpiredInstances(UniqueLock& lock); + void SendNodeInstancesStatuses(); + + StaticAllocator mAllocator; StaticArray mSubscribers; Thread mThread; Thread mRebootThread; diff --git a/src/core/sm/launcher/tests/launcher.cpp b/src/core/sm/launcher/tests/launcher.cpp index 069c63dea..36e8b471b 100644 --- a/src/core/sm/launcher/tests/launcher.cpp +++ b/src/core/sm/launcher/tests/launcher.cpp @@ -766,7 +766,7 @@ TEST_F(LauncherTest, GetInstancesStatuses) .WillOnce(Invoke([](const InstanceInfo& instance, InstanceStatus& status) { SetInstanceStatus(instance, InstanceStateEnum::eFailed, status); - return ErrorEnum::eNone; + return ErrorEnum::eFailed; })); err = mLauncher.Start(); From 62705ff9faba365f8016415ea0e18c911671f1e5 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Sun, 5 Jul 2026 17:04:14 +0300 Subject: [PATCH 037/112] sm: networkmanager: reduce stack usage in AddInstanceToNetwork Reduce stack usage in AddInstanceToNetwork by using allocator for big structs and update max allocator size. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 30 +++++++++--------- src/core/sm/networkmanager/networkmanager.hpp | 31 +++++++++++++------ 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 6e18eadd2..86e7a5c9e 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -560,9 +560,9 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin } }); - StaticArray, cMaxNumHosts> host; + auto hosts = MakeUnique, cMaxNumHosts>>(&mAllocator); - if (err = PrepareHosts(instanceID, networkID, networkConfig, host); !err.IsNone()) { + if (err = PrepareHosts(instanceID, networkID, networkConfig, *hosts); !err.IsNone()) { return err; } @@ -573,23 +573,23 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin return err; } - BridgeParams bridgeParams; + auto bridgeParams = MakeUnique(&mAllocator); - if (err = PrepareBridgeParams(networkID, networkParams, bridgeParams); !err.IsNone()) { + if (err = PrepareBridgeParams(networkID, networkParams, *bridgeParams); !err.IsNone()) { return err; } - bridgeParams.mNetNSPath = netNSPath; + bridgeParams->mNetNSPath = netNSPath; BridgeAttachResult attachResult; - if (err = mBridgeNetwork->Attach(instanceID, bridgeParams, attachResult); !err.IsNone()) { + if (err = mBridgeNetwork->Attach(instanceID, *bridgeParams, attachResult); !err.IsNone()) { return AOS_ERROR_WRAP(err); } auto cleanupBridge = DeferRelease(&instanceID, [this, &bridgeParams, &err](const String* id) { if (!err.IsNone()) { - if (auto errDetach = mBridgeNetwork->Detach(*id, bridgeParams.mBridgeIfName); !errDetach.IsNone()) { + if (auto errDetach = mBridgeNetwork->Detach(*id, bridgeParams->mBridgeIfName); !errDetach.IsNone()) { LOG_ERR() << "Failed to detach bridge" << Log::Field("instanceID", *id) << Log::Field(errDetach); } } @@ -614,13 +614,13 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin } }); - BandwidthParams bandwidthParams; + auto bandwidthParams = MakeUnique(&mAllocator); - if (err = PrepareBandwidthParams(networkConfig, bandwidthParams); !err.IsNone()) { + if (err = PrepareBandwidthParams(networkConfig, *bandwidthParams); !err.IsNone()) { return err; } - if (err = mBandwidth->Apply(attachResult.mHostIfName, bandwidthParams); !err.IsNone()) { + if (err = mBandwidth->Apply(attachResult.mHostIfName, *bandwidthParams); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -647,13 +647,13 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin dnsServer = it->mSecond; } - DNSAliasesParams dnsParams; + auto dnsParams = MakeUnique(&mAllocator); - if (err = PrepareDNSAliasesParams(networkParams, host, dnsParams); !err.IsNone()) { + if (err = PrepareDNSAliasesParams(networkParams, *hosts, *dnsParams); !err.IsNone()) { return err; } - if (err = dnsServer->AddHost(instanceID, dnsParams); !err.IsNone()) { + if (err = dnsServer->AddHost(instanceID, *dnsParams); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -686,13 +686,13 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin return err; } - if (err = CreateResolvConfFile(networkID, runtimeParams.mResolvConfFilePath, bridgeParams.mGateway, networkParams, + if (err = CreateResolvConfFile(networkID, runtimeParams.mResolvConfFilePath, bridgeParams->mGateway, networkParams, networkParams.mDNSServers); !err.IsNone()) { return err; } - if (err = UpdateInstanceNetworkCache(instanceID, networkID, host); !err.IsNone()) { + if (err = UpdateInstanceNetworkCache(instanceID, networkID, *hosts); !err.IsNone()) { return err; } diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index b40539924..ca373080f 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -177,13 +177,31 @@ class NetworkManager : public NetworkManagerItf { using InstanceCache = StaticMap, InstanceHosts, cMaxNumInstances>; using NetworkCache = StaticMap, InstanceCache, cMaxNumOwners>; + // StartInstanceNetwork keeps its cached InstanceNetworkInfo alive across the nested call to + // AddInstanceToNetwork, which in turn allocates hosts, bridge/firewall/bandwidth/DNS params and + // its own InstanceNetworkInfo before returning. That is the largest concurrent footprint of any + // mAllocator call chain, so it sizes the per-concurrent-item budget below (it dominates the + // smaller CreateInstanceNetwork and OnPendingFirewallUpdate chains). + static constexpr auto cMaxOperationAllocatorSize = 2 * sizeof(InstanceNetworkInfo) + sizeof(InstanceHosts) + + sizeof(BridgeParams) + sizeof(InstanceFirewallParams) + sizeof(BandwidthParams) + sizeof(DNSAliasesParams); + + // Start()/OnConnect() run once, outside the concurrent instance-operation hot path, so their + // allocations are added rather than multiplied by cMaxNumConcurrentItems: RemoveDNSOrphans' known + // networks list, CleanupLeftoverInstances' leftover instance/network ID pairs plus the + // InstanceNetworkInfo DeleteInstanceNetworkConfig allocates while clearing a host interface, and + // OnConnect's state sync snapshot. + static constexpr auto cAllocatorSize = cMaxOperationAllocatorSize * cMaxNumConcurrentItems + + sizeof(StaticArray, cMaxNumOwners>) + + sizeof(StaticArray, cMaxNumInstances>) * 2 + sizeof(InstanceNetworkInfo) + + sizeof(StaticArray); + static constexpr auto cNumAllocations = 8 * cMaxNumConcurrentItems; + static constexpr uint64_t cBurstLen = 12800; static constexpr auto cMaxExposedPort = 2; static constexpr auto cCountRetriesIfNameGen = 10; static constexpr auto cMaxNetworkIDLen = 8; static constexpr auto cBridgePrefix = "br-"; static constexpr auto cVlanIfPrefix = "vlan-"; - static constexpr auto cNumAllocations = 8 * cMaxNumConcurrentItems; static constexpr auto cResolvConfLineLen = AOS_CONFIG_NETWORKMANAGER_RESOLV_CONF_LINE_LEN; Error IsInstanceInNetwork(const String& instanceID, const String& networkID) const; @@ -263,15 +281,8 @@ class NetworkManager : public NetworkManagerItf { StaticAllocator)> mNetworkInfosAllocator; StaticAllocator)> mInstanceNetworkInfosAllocator; - mutable Mutex mMutex; - StaticAllocator<(sizeof(InstanceFirewallParams) + sizeof(UpdateItemNetworkParams) - + sizeof(aos::InstanceNetworkAllocation) + sizeof(InstanceNetworkInfo) - + sizeof(InstanceNetworkStateInfo)) - * cMaxNumConcurrentItems - + sizeof(StaticArray, cMaxNumInstances>) - + sizeof(StaticArray), - cNumAllocations> - mAllocator; + mutable Mutex mMutex; + StaticAllocator mAllocator; mutable StaticAllocator<(sizeof(Host) * 3) * cMaxNumConcurrentItems, cNumAllocations> mHostAllocator; }; From 5043b97b5773f01e34a3c25b1331503f52a00334 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 8 Jul 2026 19:16:31 +0300 Subject: [PATCH 038/112] cm: launcher: fix env var status lost to assert side effect ConvertEnvVarsToStatuses called PushBack inside assert(), so in release builds (NDEBUG) the call was compiled out entirely and env var statuses were never populated. This made OverrideEnvVars overrides invisible to instance status change detection, causing CMLauncherTest.OverrideEnvVars to fail its notification wait and skip Stop(), hanging on teardown. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/cm/launcher/tests/stubs/instancerunnerstub.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/cm/launcher/tests/stubs/instancerunnerstub.hpp b/src/core/cm/launcher/tests/stubs/instancerunnerstub.hpp index a9defbb52..80d3d1fa2 100644 --- a/src/core/cm/launcher/tests/stubs/instancerunnerstub.hpp +++ b/src/core/cm/launcher/tests/stubs/instancerunnerstub.hpp @@ -152,7 +152,10 @@ class InstanceRunnerStub : public InstanceRunnerItf { envVarStatus.mName = envVar.mName; envVarStatus.mError = ErrorEnum::eNone; - assert(envVarsStatuses.PushBack(envVarStatus).IsNone()); + + auto err = envVarsStatuses.PushBack(envVarStatus); + assert(err.IsNone()); + (void)err; } } From 689715aba76130e1f6a164dc8a384312a9374d9a Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 8 Jul 2026 19:49:08 +0300 Subject: [PATCH 039/112] sm: networkmanager: drop duplicate cMaxNumHosts constant sm::networkmanager kept its own cMaxNumHosts (10), separate from the common aos::cMaxNumHosts (8) used by the same Host-typed fields. Drop the duplicate and bump the common constant to 10 so DNSAliasesParams, InstanceNetworkConfig, and the derived host-name buffers in NetworkManager all size off the single shared constant. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/common/config.hpp | 2 +- src/core/sm/config.hpp | 4 ---- src/core/sm/networkmanager/itf/dnsname.hpp | 2 ++ src/core/sm/networkmanager/itf/types.hpp | 5 ----- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/core/common/config.hpp b/src/core/common/config.hpp index 5d05376ef..51c326a9a 100644 --- a/src/core/common/config.hpp +++ b/src/core/common/config.hpp @@ -299,7 +299,7 @@ * Max number of hosts. */ #ifndef AOS_CONFIG_TYPES_MAX_NUM_HOSTS -#define AOS_CONFIG_TYPES_MAX_NUM_HOSTS 8 +#define AOS_CONFIG_TYPES_MAX_NUM_HOSTS 10 #endif /** diff --git a/src/core/sm/config.hpp b/src/core/sm/config.hpp index 030154d11..9d96142a1 100644 --- a/src/core/sm/config.hpp +++ b/src/core/sm/config.hpp @@ -22,10 +22,6 @@ #define AOS_CONFIG_NETWORKMANAGER_RESOLV_CONF_LINE_LEN 256 #endif -#ifndef AOS_CONFIG_NETWORKMANAGER_MAX_NUM_HOSTS -#define AOS_CONFIG_NETWORKMANAGER_MAX_NUM_HOSTS 10 -#endif - /** * Max CNI name length. */ diff --git a/src/core/sm/networkmanager/itf/dnsname.hpp b/src/core/sm/networkmanager/itf/dnsname.hpp index d51b9c68e..3fc7964ff 100644 --- a/src/core/sm/networkmanager/itf/dnsname.hpp +++ b/src/core/sm/networkmanager/itf/dnsname.hpp @@ -12,6 +12,8 @@ #include #include +#include "types.hpp" + namespace aos::sm::networkmanager { /** @addtogroup sm Service Manager diff --git a/src/core/sm/networkmanager/itf/types.hpp b/src/core/sm/networkmanager/itf/types.hpp index cc8e30886..ebe785c71 100644 --- a/src/core/sm/networkmanager/itf/types.hpp +++ b/src/core/sm/networkmanager/itf/types.hpp @@ -22,11 +22,6 @@ namespace aos::sm::networkmanager { */ static constexpr auto cMaxNumAliases = AOS_CONFIG_NETWORKMANAGER_MAX_NUM_ALIASES; -/** - * Max number of hosts. - */ -static constexpr auto cMaxNumHosts = AOS_CONFIG_NETWORKMANAGER_MAX_NUM_HOSTS; - /** * Instance network config for Create (stored in DB). */ From f16113fe5f16993e3b1d9773c350843b8454f4e5 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 26 Jun 2026 14:45:42 +0300 Subject: [PATCH 040/112] common: crypto: fix trailing % and simplify DecodeToPKCS11ID logic Because we don't need to fully support PKCS11 encoding for ids, simplify logic and make the implementation of DecodeToPKCS11ID alligned with encoding function. Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/common/crypto/certloader.cpp | 65 +++++++++-------------- src/core/common/crypto/itf/certloader.hpp | 1 + 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/src/core/common/crypto/certloader.cpp b/src/core/common/crypto/certloader.cpp index d2cbf2627..c64e32d37 100644 --- a/src/core/common/crypto/certloader.cpp +++ b/src/core/common/crypto/certloader.cpp @@ -355,45 +355,32 @@ Error DecodeToPKCS11ID(const String& idStr, Array& id) { id.Clear(); - auto percentDetected = false; - aos::StaticString<2> hexByte; - - for (const auto& ch : idStr) { - if (ch == '%') { - if (percentDetected || hexByte.Size()) { - return aos::ErrorEnum::eInvalidArgument; - } - - percentDetected = true; - } else if (percentDetected) { - auto err = hexByte.PushBack(ch); - if (!err.IsNone()) { - return err; - } - - if (hexByte.Size() == hexByte.MaxSize()) { - percentDetected = false; - - uint8_t byte; - - aos::Tie(byte, err) = hexByte.HexToByte(); - if (!err.IsNone()) { - return err; - } - - err = id.PushBack(byte); - if (!err.IsNone()) { - return err; - } - - hexByte.Clear(); - } - - } else { - auto err = id.PushBack(static_cast(ch)); - if (!err.IsNone()) { - return err; - } + if (idStr.Size() % 3 != 0) { + return aos::ErrorEnum::eInvalidArgument; + } + + for (size_t i = 0; i < idStr.Size(); i += 3) { + if (idStr[i] != '%') { + return ErrorEnum::eInvalidArgument; + } + + aos::StaticString<2> hexByte; + + auto err = hexByte.Insert(hexByte.end(), idStr.begin() + i + 1, idStr.begin() + i + 3); + if (!err.IsNone()) { + return err; + } + + uint8_t byte; + + aos::Tie(byte, err) = hexByte.HexToByte(); + if (!err.IsNone()) { + return err; + } + + err = id.PushBack(byte); + if (!err.IsNone()) { + return err; } } diff --git a/src/core/common/crypto/itf/certloader.hpp b/src/core/common/crypto/itf/certloader.hpp index 3c2246491..b3a859988 100644 --- a/src/core/common/crypto/itf/certloader.hpp +++ b/src/core/common/crypto/itf/certloader.hpp @@ -67,6 +67,7 @@ Error EncodePKCS11ID(const Array& id, String& idStr); /** * Decodes PKCS11 ID from percent-encoded string. + * Only fully percent-encoded input is supported, which is aligned with EncodePKCS11ID implementation. * * @param idStr percent-encoded string. * @param id PKCS11 ID. From 89181dfeab02b61780a9130a78a6b6f41d94ef2b Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 26 Jun 2026 16:01:34 +0300 Subject: [PATCH 041/112] common: pkcs11: FindCertificateByKeyID check for error fix Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/common/pkcs11/pkcs11.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/common/pkcs11/pkcs11.cpp b/src/core/common/pkcs11/pkcs11.cpp index b963200e5..28656c67b 100644 --- a/src/core/common/pkcs11/pkcs11.cpp +++ b/src/core/common/pkcs11/pkcs11.cpp @@ -1453,7 +1453,7 @@ RetWithError> Utils::FindCertificateByKeyID StaticArray handles; auto err = mSession->FindObjects(certTempl, handles); - if (err.IsNone()) { + if (!err.IsNone()) { return {nullptr, err}; } From d1f583c9919f49e2715081b6ee794fa81ce4c4bc Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 26 Jun 2026 16:02:23 +0300 Subject: [PATCH 042/112] common: pkcs11: don't return certificate object if DERToX509Cert fails Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/common/pkcs11/pkcs11.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/common/pkcs11/pkcs11.cpp b/src/core/common/pkcs11/pkcs11.cpp index 28656c67b..a0720b792 100644 --- a/src/core/common/pkcs11/pkcs11.cpp +++ b/src/core/common/pkcs11/pkcs11.cpp @@ -1492,8 +1492,11 @@ RetWithError> Utils::GetCertificate(ObjectH certificate->mRaw.Resize(attrValues[0].Size()); err = mCryptoProvider.DERToX509Cert(certificate->mRaw, *certificate); + if (!err.IsNone()) { + return {nullptr, err}; + } - return {certificate, err}; + return {certificate, ErrorEnum::eNone}; } } // namespace aos::pkcs11 From 1b3b8c3cc46a5075e23d639313484b9649a77bd5 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Tue, 30 Jun 2026 15:32:34 +0300 Subject: [PATCH 043/112] common: iam: add traces to print crypto algorithms Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/common/crypto/mbedtls/cryptoprovider.cpp | 5 ++++- src/core/common/crypto/openssl/cryptoprovider.cpp | 5 ++++- src/core/common/pkcs11/pkcs11.cpp | 15 +++++++++++---- .../iam/certhandler/certmodules/pkcs11/pkcs11.cpp | 5 +++-- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/core/common/crypto/mbedtls/cryptoprovider.cpp b/src/core/common/crypto/mbedtls/cryptoprovider.cpp index 52e41220a..ec3980795 100644 --- a/src/core/common/crypto/mbedtls/cryptoprovider.cpp +++ b/src/core/common/crypto/mbedtls/cryptoprovider.cpp @@ -1915,7 +1915,10 @@ Error MbedTLSCryptoProvider::ParseX509CertPublicKey(const mbedtls_pk_context* pk return ParseECKey(mbedtls_pk_ec(*pk), cert); default: - return ErrorEnum::eNotFound; + LOG_ERR() << "Unsupported certificate public key algorithm: type=" << static_cast(mbedtls_pk_get_type(pk)) + << ", only RSA and ECDSA are supported"; + + return AOS_ERROR_WRAP(ErrorEnum::eNotSupported); } } diff --git a/src/core/common/crypto/openssl/cryptoprovider.cpp b/src/core/common/crypto/openssl/cryptoprovider.cpp index 5ace0c91a..5fa63dd78 100644 --- a/src/core/common/crypto/openssl/cryptoprovider.cpp +++ b/src/core/common/crypto/openssl/cryptoprovider.cpp @@ -350,7 +350,10 @@ Error ConvertEvpPKey(const EVP_PKEY* src, Variant& return SetECDSAPubKey(src, dst); } - return ErrorEnum::eNotSupported; + LOG_ERR() << "Unsupported certificate public key algorithm: type=" << EVP_PKEY_base_id(src) + << ", only RSA and ECDSA are supported"; + + return AOS_ERROR_WRAP(ErrorEnum::eNotSupported); } Error ConvertX509ToDER(const X509* cert, Array& derBlob) diff --git a/src/core/common/pkcs11/pkcs11.cpp b/src/core/common/pkcs11/pkcs11.cpp index a0720b792..9b4c7ee51 100644 --- a/src/core/common/pkcs11/pkcs11.cpp +++ b/src/core/common/pkcs11/pkcs11.cpp @@ -1041,10 +1041,15 @@ RetWithError Utils::GenerateRSAKeyPairWithLabel( } RetWithError Utils::GenerateECDSAKeyPairWithLabel( - const Array& id, const String& label, [[maybe_unused]] EllipticCurve curve) + const Array& id, const String& label, EllipticCurve curve) { - // only P384 curve is supported for now - assert(curve == EllipticCurve::eP384); + // only P384 (secp384r1) curve is supported for now + if (curve != EllipticCurve::eP384) { + LOG_ERR() << "Unsupported elliptic curve: curve=" << static_cast(curve) + << ", only P384 (secp384r1) is supported"; + + return {{}, AOS_ERROR_WRAP(ErrorEnum::eNotSupported)}; + } auto funcList = mSession->GetFunctionList(); @@ -1385,7 +1390,9 @@ RetWithError Utils::ExportPrivateKey( } } - return {{}, ErrorEnum::eInvalidArgument}; + LOG_ERR() << "Unsupported key type: keyType=" << keyType << ", only RSA and ECDSA (secp384r1) are supported"; + + return {{}, AOS_ERROR_WRAP(ErrorEnum::eNotSupported)}; } Error Utils::FindCertificates(const Array& id, const String& label, Array& handles) diff --git a/src/core/iam/certhandler/certmodules/pkcs11/pkcs11.cpp b/src/core/iam/certhandler/certmodules/pkcs11/pkcs11.cpp index bbdbc7a29..e5508e9ee 100644 --- a/src/core/iam/certhandler/certmodules/pkcs11/pkcs11.cpp +++ b/src/core/iam/certhandler/certmodules/pkcs11/pkcs11.cpp @@ -227,9 +227,10 @@ RetWithError> PKCS11Module::CreateKey(const Str break; default: - LOG_ERR() << "Unsupported algorithm"; + LOG_ERR() << "Unsupported algorithm: certType=" << mCertType << ", keyType=" << keyType + << ", only RSA and ECDSA (secp384r1) are supported"; - return {nullptr, AOS_ERROR_WRAP(ErrorEnum::eInvalidArgument)}; + return {nullptr, AOS_ERROR_WRAP(ErrorEnum::eNotSupported)}; } err = TokenMemInfo(); From 779a012e4b4b264d41ba0f736fe9c0ffae844113 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 9 Jul 2026 14:36:10 +0300 Subject: [PATCH 044/112] sm: launcher: fix instance data invalidation on remove RemoveInstances scheduled each instance removal as a concurrent task on the launch pool, and every task captured a raw InstanceData* pointing into mInstances. RemoveInstance erased its own entry from mInstances as soon as it finished, which shifts trailing elements down to fill the gap. Other tasks still running concurrently held pointers into those now-shifted (or destructed) slots, so they ended up reading and writing the wrong instance, or freed memory. Split the work instead: tasks now only release external resources (storage entry, network) via the renamed ReleaseInstance, without touching mInstances. The actual removal from mInstances happens in a single serial pass after mLaunchPool.Wait(), under mMutex, once no task can be holding a pointer into the array anymore. Each entry is re-checked for the eInactive state at removal time, so instances that failed to stop or failed release (now marked eFailed) are left in place instead of being purged. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Kobets --- src/core/sm/launcher/launcher.cpp | 37 +++++++++++++++++++------------ src/core/sm/launcher/launcher.hpp | 2 +- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index 175df3d07..b9b9bcc45 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -1112,7 +1112,7 @@ RetWithError Launcher::AddInstanceData(const InstanceIn return itInstance; } -Error Launcher::RemoveInstance(InstanceData& instanceData) +Error Launcher::ReleaseInstance(const InstanceData& instanceData) { LOG_DBG() << "Remove instance" << Log::Field("instance", instanceData.mInfo); @@ -1130,18 +1130,6 @@ Error Launcher::RemoveInstance(InstanceData& instanceData) return AOS_ERROR_WRAP(err); } - LockGuard lock {mMutex}; - - LOG_DBG() << "Remove instance data" << Log::Field("instance", instanceData.mInfo); - - if (auto count = mInstances.RemoveIf([this, &instanceData](const auto& instance) { - return static_cast(instance.mInfo) - == static_cast(instanceData.mInfo); - }); - count == 0) { - return AOS_ERROR_WRAP(ErrorEnum::eNotFound); - } - return ErrorEnum::eNone; } @@ -1163,7 +1151,7 @@ void Launcher::RemoveInstances(const Array& instances) } if (auto err = mLaunchPool.AddTask([this, instanceData](void*) { - if (auto err = RemoveInstance(*instanceData); !err.IsNone()) { + if (auto err = ReleaseInstance(*instanceData); !err.IsNone()) { LOG_ERR() << "Failed to remove instance" << Log::Field("instance", instanceData->mInfo) << Log::Field(AOS_ERROR_WRAP(err)); @@ -1181,6 +1169,27 @@ void Launcher::RemoveInstances(const Array& instances) if (auto err = mLaunchPool.Wait(); !err.IsNone()) { LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); } + + LockGuard lock {mMutex}; + + for (const auto& instanceIdent : instances) { + LOG_DBG() << "Remove instance data" << Log::Field("instance", instanceIdent); + + mInstances.RemoveIf([this, &instanceIdent](const auto& instance) { + if (static_cast(instance.mInfo) != instanceIdent) { + return false; + } + + if (instance.mStatus.mState != InstanceStateEnum::eInactive) { + LOG_ERR() << "Instance is not inactive, skip removing" << Log::Field("instance", instanceIdent) + << Log::Field("state", instance.mStatus.mState); + + return false; + } + + return true; + }); + } } void Launcher::SetInstanceState(InstanceData& instance, const InstanceState& state, const Error& error) diff --git a/src/core/sm/launcher/launcher.hpp b/src/core/sm/launcher/launcher.hpp index 6f29933c6..d64690a72 100644 --- a/src/core/sm/launcher/launcher.hpp +++ b/src/core/sm/launcher/launcher.hpp @@ -208,7 +208,7 @@ class Launcher : public LauncherItf, void RemoveUpdateItems(const Array& removeItems); void InstallUpdateItems(const Array& startInstances); RetWithError AddInstanceData(const InstanceInfo& instanceInfo); - Error RemoveInstance(InstanceData& instanceData); + Error ReleaseInstance(const InstanceData& instanceData); void RemoveInstances(const Array& instances); void SetInstanceState(InstanceData& instance, const InstanceState& state, const Error& error = ErrorEnum::eNone); Error GetInstanceConfigs(const InstanceInfo& instance, oci::ItemConfig& itemConfig, oci::ImageConfig& imageConfig); From 74c24257892f4cfa2bf6d23a3d82be1041bd2e7c Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Fri, 3 Jul 2026 14:24:16 +0300 Subject: [PATCH 045/112] sm: networkmanager: return resolv/hosts data instead of writing files Drop the file-path InstanceNetworkRuntimeParams from StartInstanceNetwork and stop writing resolv.conf / hosts inside AddInstanceToNetwork. Instead expose GetResolvServers (resolver IPs) and GetHosts (IP + hostname entries) so the caller - which owns the container's filesystem layout - writes the files at its own paths and in its own format ("nameserver " / "\t"). Removes the imaginary dependency of the network API on caller file paths. Caller (container runtime) and mocks to be updated separately. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../sm/networkmanager/itf/networkmanager.hpp | 28 +- src/core/sm/networkmanager/networkmanager.cpp | 301 ++++++------------ src/core/sm/networkmanager/networkmanager.hpp | 46 +-- 3 files changed, 148 insertions(+), 227 deletions(-) diff --git a/src/core/sm/networkmanager/itf/networkmanager.hpp b/src/core/sm/networkmanager/itf/networkmanager.hpp index 982ac16ad..6c6333194 100644 --- a/src/core/sm/networkmanager/itf/networkmanager.hpp +++ b/src/core/sm/networkmanager/itf/networkmanager.hpp @@ -79,12 +79,32 @@ class NetworkManagerItf : public SystemTrafficProviderItf, * * @param instanceID instance ID. * @param networkID network ID. - * @param runtimeParams runtime parameters (file paths). * @return Error. */ - virtual Error StartInstanceNetwork( - const String& instanceID, const String& networkID, const InstanceNetworkRuntimeParams& runtimeParams) - = 0; + virtual Error StartInstanceNetwork(const String& instanceID, const String& networkID) = 0; + + /** + * Returns the resolver IPs for the instance. Only the addresses are + * returned; the caller writes resolv.conf itself and must prefix each entry + * with "nameserver " (e.g. "nameserver 10.0.0.1"). Call after + * StartInstanceNetwork. + * + * @param instanceID instance ID. + * @param[out] servers resolver IP addresses (no "nameserver" prefix). + * @return Error. + */ + virtual Error GetResolvServers(const String& instanceID, Array>& servers) const = 0; + + /** + * Returns the host entries (IP + hostname) for the instance. The caller + * writes the hosts file itself, formatting each entry as "\t". + * Call after StartInstanceNetwork. + * + * @param instanceID instance ID. + * @param[out] hosts host entries. + * @return Error. + */ + virtual Error GetHosts(const String& instanceID, Array& hosts) const = 0; /** * Stops instance network: tears down DNS/bandwidth/firewall/bridge for the diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 86e7a5c9e..5278bb98e 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -4,9 +4,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include -#include - #include #include @@ -242,8 +239,7 @@ Error NetworkManager::CreateInstanceNetwork( return ErrorEnum::eNone; } -Error NetworkManager::StartInstanceNetwork( - const String& instanceID, const String& networkID, const InstanceNetworkRuntimeParams& runtimeParams) +Error NetworkManager::StartInstanceNetwork(const String& instanceID, const String& networkID) { LOG_DBG() << "Start instance network" << Log::Field("instanceID", instanceID) << Log::Field("networkID", networkID); @@ -283,12 +279,105 @@ Error NetworkManager::StartInstanceNetwork( return err; } - err = AddInstanceToNetwork( - instanceID, networkID, cachedInfo->mNetworkConfig, cachedInfo->mAllocatedParams, runtimeParams); + err = AddInstanceToNetwork(instanceID, networkID, cachedInfo->mNetworkConfig, cachedInfo->mAllocatedParams); return err; } +Error NetworkManager::GetResolvServers(const String& instanceID, Array>& servers) const +{ + StaticString networkID; + StaticString bridgeIP; + auto dns = MakeUnique, cMaxNumDNSServers>>(&mResolvHostsAllocator); + + { + LockGuard lock {mMutex}; + + auto it = mInstanceNetworkInfos.Find(instanceID); + if (it == mInstanceNetworkInfos.end()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance network info not found")); + } + + networkID = it->mSecond.mNetworkID; + *dns = it->mSecond.mAllocatedParams.mDNSServers; + + if (auto np = mNetworkProviders.Find(networkID); np != mNetworkProviders.end()) { + bridgeIP = np->mSecond.mIP; + } + } + + // Per-bridge dnsmasq listens on the bridge IP - make it the primary resolver. + if (!bridgeIP.IsEmpty()) { + if (auto err = servers.PushBack(bridgeIP); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + + for (const auto& server : *dns) { + if (servers.Find(server) == servers.end()) { + if (auto err = servers.PushBack(server); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + } + + if (servers.IsEmpty()) { + if (auto err = servers.EmplaceBack("8.8.8.8"); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + + return ErrorEnum::eNone; +} + +Error NetworkManager::GetHosts(const String& instanceID, Array& hosts) const +{ + StaticString networkID; + StaticString instanceIP; + StaticString hostname; + auto customHosts = MakeUnique>(&mResolvHostsAllocator); + + { + LockGuard lock {mMutex}; + + auto it = mInstanceNetworkInfos.Find(instanceID); + if (it == mInstanceNetworkInfos.end()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance network info not found")); + } + + networkID = it->mSecond.mNetworkID; + instanceIP = it->mSecond.mAllocatedParams.mIP; + hostname = it->mSecond.mNetworkConfig.mHostname; + *customHosts = it->mSecond.mNetworkConfig.mHosts; + } + + if (auto err = hosts.EmplaceBack("127.0.0.1", "localhost"); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + if (auto err = hosts.EmplaceBack("::1", "localhost ip6-localhost ip6-loopback"); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + StaticString ownHosts {networkID}; + + if (!hostname.IsEmpty()) { + ownHosts.Append(" ").Append(hostname); + } + + if (auto err = hosts.EmplaceBack(instanceIP, ownHosts); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + for (const auto& host : *customHosts) { + if (auto err = hosts.PushBack(host); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + + return ErrorEnum::eNone; +} + Error NetworkManager::StopInstanceNetwork(const String& instanceID, const String& networkID) { LOG_DBG() << "Stop instance network" << Log::Field("instanceID", instanceID) << Log::Field("networkID", networkID); @@ -539,8 +628,7 @@ Error NetworkManager::EnsureNodeNetwork(const String& networkID) } Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const String& networkID, - const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams, - const InstanceNetworkRuntimeParams& runtimeParams) + const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams) { LOG_DBG() << "Add instance to network" << Log::Field("instanceID", instanceID) << Log::Field("networkID", networkID); @@ -681,16 +769,8 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin } }); - if (err = CreateHostsFile(networkID, networkParams.mIP, networkConfig, runtimeParams.mHostsFilePath); - !err.IsNone()) { - return err; - } - - if (err = CreateResolvConfFile(networkID, runtimeParams.mResolvConfFilePath, bridgeParams->mGateway, networkParams, - networkParams.mDNSServers); - !err.IsNone()) { - return err; - } + // resolv.conf / hosts are no longer written here; the caller fetches the + // data via GetResolvServers/GetHosts and writes the files at its own paths. if (err = UpdateInstanceNetworkCache(instanceID, networkID, *hosts); !err.IsNone()) { return err; @@ -1179,189 +1259,6 @@ Error NetworkManager::IsHostnameExist( return ErrorEnum::eNone; } -Error NetworkManager::CreateResolvConfFile(const String& networkID, const String& resolvConfFilePath, - const String& bridgeIP, const aos::InstanceNetworkAllocation& networkParams, - const Array>& dns) const -{ - LOG_DBG() << "Create resolv.conf file" << Log::Field("networkID", networkID); - - if (resolvConfFilePath.IsEmpty()) { - return ErrorEnum::eNone; - } - - StaticArray, cMaxNumDNSServers> mainServers; - - // Per-bridge dnsmasq listens on the bridge IP — make it the primary - // resolver so each container's queries land on its own network's DNS. - if (!bridgeIP.IsEmpty()) { - if (auto err = mainServers.PushBack(bridgeIP); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - } - - for (const auto& server : dns) { - if (mainServers.Find(server) == mainServers.end()) { - if (auto err = mainServers.PushBack(server); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - } - } - - if (mainServers.IsEmpty()) { - if (auto err = mainServers.PushBack("8.8.8.8"); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - } - - return WriteResolvConfFile(resolvConfFilePath, mainServers, networkParams); -} - -Error NetworkManager::WriteResolvConfFile(const String& filePath, const Array>& mainServers, - const aos::InstanceNetworkAllocation& networkParams) const -{ - LOG_DBG() << "Write resolv.conf file" << Log::Field("filePath", filePath); - - auto fd = open(filePath.CStr(), O_CREAT | O_WRONLY, 0644); - if (fd < 0) { - return Error(errno); - } - - auto closeFile = DeferRelease(&fd, [](const int* fd) { close(*fd); }); - - auto writeNameServers = [&fd](const Array>& servers) -> Error { - for (const auto& server : servers) { - StaticString line; - - if (auto err = line.Format("nameserver\t%s\n", server.CStr()); !err.IsNone()) { - return err; - } - - const auto buff = Array(reinterpret_cast(line.Get()), line.Size()); - - size_t pos = 0; - - while (pos < buff.Size()) { - auto chunkSize = write(fd, buff.Get() + pos, buff.Size() - pos); - if (chunkSize < 0) { - return Error(errno); - } - - pos += chunkSize; - } - } - - return ErrorEnum::eNone; - }; - - if (auto err = writeNameServers(mainServers); !err.IsNone()) { - return err; - } - - return writeNameServers(networkParams.mDNSServers); -} - -Error NetworkManager::CreateHostsFile(const String& networkID, const String& instanceIP, - const InstanceNetworkConfig& network, const String& hostsFilePath) const -{ - LOG_DBG() << "Create hosts file" << Log::Field("networkID", networkID); - - if (hostsFilePath.IsEmpty()) { - return ErrorEnum::eNone; - } - - StaticArray, cMaxNumHosts * 3> hosts; - - auto localhost = MakeShared(&mHostAllocator, String("127.0.0.1"), String("localhost")); - - if (auto err = hosts.PushBack(localhost); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - auto localhost6 = MakeShared(&mHostAllocator, String("::1"), String("localhost ip6-localhost ip6-loopback")); - - if (auto err = hosts.PushBack(localhost6); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - StaticString ownHosts {networkID}; - - if (!network.mHostname.IsEmpty()) { - ownHosts.Append(" ").Append(network.mHostname); - } - - auto instanceHost = MakeShared(&mHostAllocator, instanceIP, ownHosts); - - if (auto err = hosts.PushBack(instanceHost); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - return WriteHostsFile(hostsFilePath, hosts, network.mHosts); -} - -Error NetworkManager::WriteHostsFile( - const String& filePath, const Array>& hosts, const Array& additionalHosts) const -{ - LOG_DBG() << "Write hosts file" << Log::Field("filePath", filePath); - - auto fd = open(filePath.CStr(), O_CREAT | O_WRONLY, 0644); - if (fd < 0) { - return Error(errno); - } - - auto closeFile = DeferRelease(&fd, [](const int* fd) { close(*fd); }); - - if (auto err = WriteHosts(hosts, fd); !err.IsNone()) { - return err; - } - - return WriteHosts(additionalHosts, fd); -} - -Error NetworkManager::WriteHost(const Host& host, int fd) const -{ - StaticString line; - - if (auto err = line.Format("%s\t%s\n", host.mIP.CStr(), host.mHostname.CStr()); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - const auto buff = Array(reinterpret_cast(line.Get()), line.Size()); - - size_t pos = 0; - while (pos < buff.Size()) { - auto chunkSize = write(fd, buff.Get() + pos, buff.Size() - pos); - if (chunkSize < 0) { - return Error(errno); - } - - pos += chunkSize; - } - - return ErrorEnum::eNone; -} - -Error NetworkManager::WriteHosts(const Array>& hosts, int fd) const -{ - for (const auto& host : hosts) { - if (auto err = WriteHost(*host, fd); !err.IsNone()) { - return err; - } - } - - return ErrorEnum::eNone; -}; - -Error NetworkManager::WriteHosts(const Array& hosts, int fd) const -{ - for (const auto& host : hosts) { - if (auto err = WriteHost(host, fd); !err.IsNone()) { - return err; - } - } - - return ErrorEnum::eNone; -}; - Error NetworkManager::PrepareBridgeParams( const String& networkID, const aos::InstanceNetworkAllocation& networkParams, BridgeParams& params) const { diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index ca373080f..b8ca51c39 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -125,11 +125,27 @@ class NetworkManager : public NetworkManagerItf { * * @param instanceID instance ID. * @param networkID network ID. - * @param runtimeParams runtime parameters. * @return Error. */ - Error StartInstanceNetwork( - const String& instanceID, const String& networkID, const InstanceNetworkRuntimeParams& runtimeParams) override; + Error StartInstanceNetwork(const String& instanceID, const String& networkID) override; + + /** + * Returns resolver IPs for the instance (caller prefixes each with "nameserver"). + * + * @param instanceID instance ID. + * @param[out] servers resolver IP addresses. + * @return Error. + */ + Error GetResolvServers(const String& instanceID, Array>& servers) const override; + + /** + * Returns host entries (IP + hostname) for the instance. + * + * @param instanceID instance ID. + * @param[out] hosts host entries. + * @return Error. + */ + Error GetHosts(const String& instanceID, Array& hosts) const override; /** * Stops instance network. @@ -170,8 +186,7 @@ class NetworkManager : public NetworkManagerItf { const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); Error AddInstanceToNetwork(const String& instanceID, const String& networkID, - const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams, - const InstanceNetworkRuntimeParams& runtimeParams); + const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); using InstanceHosts = StaticArray, cMaxNumHosts>; using InstanceCache = StaticMap, InstanceHosts, cMaxNumInstances>; @@ -228,19 +243,6 @@ class NetworkManager : public NetworkManagerItf { Error IsHostnameExist(const InstanceCache& instanceCache, const Array>& hosts) const; Error PushHostWithDomain( const String& host, const String& networkID, Array>& hosts) const; - Error CreateHostsFile(const String& networkID, const String& instanceIP, const InstanceNetworkConfig& network, - const String& hostsFilePath) const; - Error WriteHost(const Host& host, int fd) const; - Error WriteHosts(const Array>& hosts, int fd) const; - Error WriteHosts(const Array& hosts, int fd) const; - Error WriteHostsFile( - const String& filePath, const Array>& hosts, const Array& additionalHosts) const; - - Error CreateResolvConfFile(const String& networkID, const String& resolvConfFilePath, const String& bridgeIP, - const aos::InstanceNetworkAllocation& networkParams, const Array>& dns) const; - Error WriteResolvConfFile(const String& filePath, const Array>& mainServers, - const aos::InstanceNetworkAllocation& networkParams) const; - Error CreateNetwork(const NetworkInfo& network); Error DeleteInstanceNetworkConfig(const String& instanceID, const String& networkID); Error GenerateIfName(String& ifName, const String& ifPrefix); @@ -281,9 +283,11 @@ class NetworkManager : public NetworkManagerItf { StaticAllocator)> mNetworkInfosAllocator; StaticAllocator)> mInstanceNetworkInfosAllocator; - mutable Mutex mMutex; - StaticAllocator mAllocator; - mutable StaticAllocator<(sizeof(Host) * 3) * cMaxNumConcurrentItems, cNumAllocations> mHostAllocator; + mutable Mutex mMutex; + StaticAllocator mAllocator; + mutable StaticAllocator) + + sizeof(StaticArray, cMaxNumDNSServers>)> + mResolvHostsAllocator; }; /** @}*/ From b5948b39ca1eb017e1f71fe56af71a713deab7b1 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Mon, 6 Jul 2026 12:43:04 +0300 Subject: [PATCH 046/112] sm: networkmanager: cover resolv/hosts getters in tests Migrate the resolv.conf / hosts assertions from reading files written by StartInstanceNetwork to the new GetResolvServers / GetHosts getters (the caller now writes the files), drop the obsolete file-writing tests, and switch the network manager mock to the file-path-free StartInstanceNetwork signature plus the two getters. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../networkmanager/tests/networkmanager.cpp | 164 +++++------------- .../sm/tests/mocks/networkmanagermock.hpp | 7 +- 2 files changed, 43 insertions(+), 128 deletions(-) diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index 7bf27e742..55859a460 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -224,12 +224,11 @@ class NetworkManagerTest : public Test { TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_VerifyHostsFile) { - const int numInstances = 4; - std::vector threads; - std::vector instanceIDs; - std::vector networkIDs; - std::vector paramsVec; - std::vector runtimeParamsVec; + const int numInstances = 4; + std::vector threads; + std::vector instanceIDs; + std::vector networkIDs; + std::vector paramsVec; std::vector networkParamsVec; std::vector allocatedParamsVec; @@ -244,11 +243,6 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_VerifyHostsFile) params.mHostname = aos::String(hostname.c_str()); paramsVec.push_back(params); - InstanceNetworkRuntimeParams runtimeParams; - std::string hostsFilePath = "hosts_" + std::to_string(i); - runtimeParams.mHostsFilePath = aos::fs::JoinPath(mWorkingDir, hostsFilePath.c_str()); - runtimeParamsVec.push_back(runtimeParams); - auto allocated = CreateTestAllocatedParams(); std::string ip = "192.168.1." + std::to_string(i + 2); allocated.mIP = aos::String(ip.c_str()); @@ -326,11 +320,10 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_VerifyHostsFile) .WillRepeatedly(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); for (int i = 0; i < numInstances; i++) { - threads.emplace_back([this, i, &instanceIDs, &networkIDs, ¶msVec, &runtimeParamsVec]() { + threads.emplace_back([this, i, &instanceIDs, &networkIDs, ¶msVec]() { ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceIDs[i].c_str(), networkIDs[i].c_str(), paramsVec[i]), aos::ErrorEnum::eNone); - ASSERT_EQ( - mNetManager->StartInstanceNetwork(instanceIDs[i].c_str(), networkIDs[i].c_str(), runtimeParamsVec[i]), + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceIDs[i].c_str(), networkIDs[i].c_str()), aos::ErrorEnum::eNone); }); } @@ -340,7 +333,13 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_VerifyHostsFile) } for (int i = 0; i < numInstances; i++) { - std::string hostsContent = ReadFile(runtimeParamsVec[i].mHostsFilePath.CStr()); + aos::StaticArray hosts; + ASSERT_EQ(mNetManager->GetHosts(instanceIDs[i].c_str(), hosts), aos::ErrorEnum::eNone); + + std::string hostsContent; + for (const auto& host : hosts) { + hostsContent += std::string(host.mIP.CStr()) + "\t" + host.mHostname.CStr() + "\n"; + } EXPECT_THAT(hostsContent, HasSubstr("127.0.0.1\tlocalhost")); EXPECT_THAT(hostsContent, HasSubstr("::1\tlocalhost ip6-localhost ip6-loopback")); @@ -420,8 +419,7 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_ValidateAllPluginConfig .WillOnce(Return(aos::RetWithError> { {"/var/run/netns/test-instance"}, aos::ErrorEnum::eNone})); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); EXPECT_EQ(capturedAttachInstance, instanceID); EXPECT_EQ(std::string(capturedBridgeParams.mBridgeIfName.CStr()).substr(0, 3), "br-"); @@ -482,12 +480,17 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_VerifyResolvConfFile) EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); - InstanceNetworkRuntimeParams runtimeParams; - runtimeParams.mResolvConfFilePath = aos::fs::JoinPath(mWorkingDir, "resolv.conf"); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); + aos::StaticArray, aos::cMaxNumDNSServers + 1> servers; + ASSERT_EQ(mNetManager->GetResolvServers(instanceID, servers), aos::ErrorEnum::eNone); - std::string resolvContent = ReadFile(runtimeParams.mResolvConfFilePath.CStr()); + // The caller builds resolv.conf from the returned IPs; reproduce it here to + // reuse the content assertions. + std::string resolvContent; + for (const auto& server : servers) { + resolvContent += "nameserver\t" + std::string(server.CStr()) + "\n"; + } // Per-bridge dnsmasq listens on the bridge IP — must be the primary nameserver. EXPECT_THAT(resolvContent, HasSubstr("nameserver\t192.168.1.1")); @@ -499,78 +502,6 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_VerifyResolvConfFile) } } -TEST_F(NetworkManagerTest, StartInstanceNetwork_NoConfigFiles) -{ - const aos::String instanceID = "test-instance"; - const aos::String networkID = "test-network"; - auto params = CreateTestInstanceNetworkConfig(); - auto allocatedParams = CreateTestAllocatedParams(); - - SetupEnsureNodeNetworkCreateMocks(networkID, allocatedParams.mSubnet, "192.168.1.1", 100ULL); - - EXPECT_CALL(mNetworkProvider, AllocateInstanceNetwork(_, networkID, aos::String("test-node"), _, _)) - .WillOnce(DoAll(SetArgReferee<4>(allocatedParams), Return(aos::ErrorEnum::eNone))); - - EXPECT_CALL(mStorage, AddInstanceNetworkInfo(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - - ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID, networkID, params), aos::ErrorEnum::eNone); - - SetupEnsureNodeNetworkPhysicalMocks("192.168.1.1", allocatedParams.mSubnet, 100ULL); - - ExpectAddInstanceCalls(); - ExpectPersistInstanceCalls(); - - EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(_, _, _, _)).WillOnce(Return(aos::ErrorEnum::eNone)); - - EXPECT_CALL(mNetns, CreateNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) - .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); - - InstanceNetworkRuntimeParams runtimeParams; - runtimeParams.mHostsFilePath = ""; - runtimeParams.mResolvConfFilePath = ""; - - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); - - EXPECT_FALSE(std::filesystem::exists(aos::fs::JoinPath(mWorkingDir, "hosts").CStr())); - EXPECT_FALSE(std::filesystem::exists(aos::fs::JoinPath(mWorkingDir, "resolv.conf").CStr())); -} - -TEST_F(NetworkManagerTest, StartInstanceNetwork_FileCreationError) -{ - const aos::String instanceID = "test-instance"; - const aos::String networkID = "test-network"; - auto params = CreateTestInstanceNetworkConfig(); - auto allocatedParams = CreateTestAllocatedParams(); - - SetupEnsureNodeNetworkCreateMocks(networkID, allocatedParams.mSubnet, "192.168.1.1", 100ULL); - - EXPECT_CALL(mNetworkProvider, AllocateInstanceNetwork(_, networkID, aos::String("test-node"), _, _)) - .WillOnce(DoAll(SetArgReferee<4>(allocatedParams), Return(aos::ErrorEnum::eNone))); - - EXPECT_CALL(mStorage, AddInstanceNetworkInfo(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - - ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID, networkID, params), aos::ErrorEnum::eNone); - - SetupEnsureNodeNetworkPhysicalMocks("192.168.1.1", allocatedParams.mSubnet, 100ULL); - - ExpectAddInstanceCalls(); - - EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(_, _, _, _)).WillOnce(Return(aos::ErrorEnum::eNone)); - - EXPECT_CALL(mNetns, CreateNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) - .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); - EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - ExpectDeleteInstanceCalls(); - - InstanceNetworkRuntimeParams runtimeParams; - runtimeParams.mHostsFilePath = "/nonexistent/directory/hosts"; - runtimeParams.mResolvConfFilePath = "/nonexistent/directory/resolv.conf"; - - EXPECT_NE(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); -} - TEST_F(NetworkManagerTest, StartInstanceNetwork_FailOnAttachError) { const aos::String instanceID = "test-instance"; @@ -596,9 +527,7 @@ TEST_F(NetworkManagerTest, StartInstanceNetwork_FailOnAttachError) .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - InstanceNetworkRuntimeParams runtimeParams; - EXPECT_EQ( - mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eInvalidArgument); + EXPECT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eInvalidArgument); } TEST_F(NetworkManagerTest, StartInstanceNetwork_FailOnTrafficMonitorError) @@ -631,8 +560,7 @@ TEST_F(NetworkManagerTest, StartInstanceNetwork_FailOnTrafficMonitorError) EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); ExpectDeleteInstanceCalls(); - InstanceNetworkRuntimeParams runtimeParams; - EXPECT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eRuntime); + EXPECT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eRuntime); } TEST_F(NetworkManagerTest, CreateInstanceNetwork_Idempotent) @@ -681,8 +609,7 @@ TEST_F(NetworkManagerTest, StopAndReleaseInstanceNetwork) .Times(1) .WillRepeatedly(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); EXPECT_CALL(mTrafficMonitor, StopInstanceMonitoring(instanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); ExpectDeleteInstanceCalls(); @@ -746,9 +673,8 @@ TEST_F(NetworkManagerTest, StopAndReleaseInstanceNetwork_MultipleInstances) .Times(2) .WillRepeatedly(Return(aos::ErrorEnum::eNone)); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID1, networkID, runtimeParams), aos::ErrorEnum::eNone); - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID2, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID1, networkID), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID2, networkID), aos::ErrorEnum::eNone); // Stop instance1 EXPECT_CALL(mTrafficMonitor, StopInstanceMonitoring(instanceID1)).WillOnce(Return(aos::ErrorEnum::eNone)); @@ -822,8 +748,7 @@ TEST_F(NetworkManagerTest, StopReleaseAndRecreateInstance) ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID, networkID, params), aos::ErrorEnum::eNone); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); EXPECT_CALL(mTrafficMonitor, StopInstanceMonitoring(instanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); ExpectDeleteInstanceCalls(); @@ -844,7 +769,7 @@ TEST_F(NetworkManagerTest, StopReleaseAndRecreateInstance) EXPECT_EQ(mNetManager->ReleaseInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID, networkID, params), aos::ErrorEnum::eNone); - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); } TEST_F(NetworkManagerTest, StopInstanceNetwork_FailOnDetachError) @@ -875,8 +800,7 @@ TEST_F(NetworkManagerTest, StopInstanceNetwork_FailOnDetachError) .Times(1) .WillRepeatedly(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); EXPECT_CALL(mTrafficMonitor, StopInstanceMonitoring(instanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mDNSServer, RemoveHost(_)).WillOnce(Return(aos::ErrorEnum::eNone)); @@ -905,9 +829,7 @@ TEST_F(NetworkManagerTest, StartInstanceNetwork_NetworkIDMismatch) ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID, networkID, params), aos::ErrorEnum::eNone); - InstanceNetworkRuntimeParams runtimeParams; - EXPECT_EQ(mNetManager->StartInstanceNetwork(instanceID, "wrong-network", runtimeParams), - aos::ErrorEnum::eInvalidArgument); + EXPECT_EQ(mNetManager->StartInstanceNetwork(instanceID, "wrong-network"), aos::ErrorEnum::eInvalidArgument); } TEST_F(NetworkManagerTest, ReleaseInstanceNetwork_NetworkIDMismatch) @@ -954,8 +876,7 @@ TEST_F(NetworkManagerTest, ReleaseInstanceNetwork_WithoutStop) EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); EXPECT_EQ(mNetManager->ReleaseInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eInvalidArgument); } @@ -1052,9 +973,8 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_EnsureNodeNetworkCreate .Times(2) .WillRepeatedly(Return(aos::ErrorEnum::eNone)); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID1, networkID, runtimeParams), aos::ErrorEnum::eNone); - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID2, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID1, networkID), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID2, networkID), aos::ErrorEnum::eNone); } TEST_F(NetworkManagerTest, InitWithExistingNetworks) @@ -1106,8 +1026,7 @@ TEST_F(NetworkManagerTest, InitWithExistingNetworks) .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(_, _, _, _)).WillOnce(Return(aos::ErrorEnum::eNone)); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, "network1", runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, "network1"), aos::ErrorEnum::eNone); } TEST_F(NetworkManagerTest, CreateInstanceNetwork_VerifyUpdateItemNetworkParams) @@ -1210,11 +1129,7 @@ TEST_F(NetworkManagerTest, OnPendingFirewallUpdate_RunningInstance_CallsFirewall ExpectPersistInstanceCalls(); EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(_, _, _, _)).WillOnce(Return(aos::ErrorEnum::eNone)); - InstanceNetworkRuntimeParams runtimeParams; - runtimeParams.mHostsFilePath = "/tmp/networkmanager_test/hosts"; - runtimeParams.mResolvConfFilePath = "/tmp/networkmanager_test/resolv.conf"; - - err = mNetManager->StartInstanceNetwork("test-instance", "test-network", runtimeParams); + err = mNetManager->StartInstanceNetwork("test-instance", "test-network"); ASSERT_EQ(err, aos::ErrorEnum::eNone); aos::networkmanager::PendingFirewallUpdate update; @@ -1262,8 +1177,7 @@ TEST_F(NetworkManagerTest, OnConnect_SyncsNetworkStateWithCM) .WillOnce(Return(aos::RetWithError> { {"/var/run/netns/test-instance"}, aos::ErrorEnum::eNone})); - InstanceNetworkRuntimeParams runtimeParams; - ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID, runtimeParams), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); // OnConnect should sync only running instances EXPECT_CALL(mNetworkProvider, SyncNetworkState(aos::String("test-node"), _)) diff --git a/src/core/sm/tests/mocks/networkmanagermock.hpp b/src/core/sm/tests/mocks/networkmanagermock.hpp index c95fb69bc..352deab33 100644 --- a/src/core/sm/tests/mocks/networkmanagermock.hpp +++ b/src/core/sm/tests/mocks/networkmanagermock.hpp @@ -22,9 +22,10 @@ class NetworkManagerMock : public NetworkManagerItf { MOCK_METHOD(Error, SetTrafficPeriod, (TrafficPeriod period), (override)); MOCK_METHOD(Error, CreateInstanceNetwork, (const String& instanceID, const String& networkID, const InstanceNetworkConfig& networkConfig), (override)); - MOCK_METHOD(Error, StartInstanceNetwork, - (const String& instanceID, const String& networkID, const InstanceNetworkRuntimeParams& runtimeParams), - (override)); + MOCK_METHOD(Error, StartInstanceNetwork, (const String& instanceID, const String& networkID), (override)); + MOCK_METHOD( + Error, GetResolvServers, (const String& instanceID, Array>& servers), (const, override)); + MOCK_METHOD(Error, GetHosts, (const String& instanceID, Array& hosts), (const, override)); MOCK_METHOD(Error, StopInstanceNetwork, (const String& instanceID, const String& networkID), (override)); MOCK_METHOD(Error, ReleaseInstanceNetwork, (const String& instanceID, const String& networkID), (override)); MOCK_METHOD(void, OnPendingFirewallUpdate, From 312e7bffc1b82b283d2cf3e4d118d88c43b177e1 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 8 Jul 2026 21:37:40 +0300 Subject: [PATCH 047/112] sm: launcher: start/stop instance network on instance start/stop Call NetworkManager StartInstanceNetwork/StopInstanceNetwork when starting and stopping service instances, and cache the resolved instance ID in InstanceData instead of re-resolving it via InstanceIDProvider on each use. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/sm/launcher/launcher.cpp | 55 ++++++++++++++++--------- src/core/sm/launcher/launcher.hpp | 9 ++-- src/core/sm/launcher/tests/launcher.cpp | 2 + 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index b9b9bcc45..47fecb27f 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -716,11 +716,22 @@ Error Launcher::StopInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceDat LOG_INF() << "Stop instance" << Log::Field("instance", instanceData.mInfo) << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID); - if (auto err = runtime->StopInstance(instanceData.mInfo, instanceData.mStatus); !err.IsNone()) { - return AOS_ERROR_WRAP(err); + Error err; + + if (auto stopErr = runtime->StopInstance(instanceData.mInfo, instanceData.mStatus); + !stopErr.IsNone() && err.IsNone()) { + err = AOS_ERROR_WRAP(stopErr); } - return ErrorEnum::eNone; + if (instanceData.mInfo.mType == UpdateItemTypeEnum::eService) { + if (auto networkErr + = mNetworkManager->StopInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); + !networkErr.IsNone() && err.IsNone()) { + err = AOS_ERROR_WRAP(networkErr); + } + } + + return err; } void Launcher::StopAllInstances() @@ -777,7 +788,7 @@ Error Launcher::PrepareInstance(InstanceData& instanceData) instanceData.mOfflineTTL = itemConfig->mOfflineTTL; - if (auto err = CreateNetwork(instanceData.mInfo, *itemConfig, *imageConfig); !err.IsNone()) { + if (auto err = CreateNetwork(instanceData, *itemConfig, *imageConfig); !err.IsNone()) { return err; } @@ -892,6 +903,13 @@ Error Launcher::StartInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceDa << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID) << Log::Field("manifestDigest", instanceData.mInfo.mManifestDigest); + if (instanceData.mInfo.mType == UpdateItemTypeEnum::eService) { + if (auto err = mNetworkManager->StartInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + if (auto err = runtime->StartInstance(instanceData.mInfo, instanceData.mStatus); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -1109,6 +1127,12 @@ RetWithError Launcher::AddInstanceData(const InstanceIn itInstance->mStatus.mRuntimeID = instanceInfo.mRuntimeID; itInstance->mStatus.mState = InstanceStateEnum::eInactive; + if (auto err = mInstanceIDProvider->GetInstanceID(instanceInfo, itInstance->mInstanceID); !err.IsNone()) { + mInstances.Erase(itInstance); + + return {nullptr, AOS_ERROR_WRAP(err)}; + } + return itInstance; } @@ -1120,13 +1144,8 @@ Error Launcher::ReleaseInstance(const InstanceData& instanceData) return AOS_ERROR_WRAP(err); } - StaticString instanceID; - - if (auto err = mInstanceIDProvider->GetInstanceID(instanceData.mInfo, instanceID); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - if (auto err = mNetworkManager->ReleaseInstanceNetwork(instanceID, instanceData.mInfo.mOwnerID); !err.IsNone()) { + if (auto err = mNetworkManager->ReleaseInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); + !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -1296,21 +1315,17 @@ Error Launcher::GetInstanceNetworkConfig(const InstanceInfo& instance, const oci } Error Launcher::CreateNetwork( - const InstanceInfo& instance, const oci::ItemConfig& itemConfig, const oci::ImageConfig& imageConfig) + const InstanceData& instanceData, const oci::ItemConfig& itemConfig, const oci::ImageConfig& imageConfig) { - StaticString instanceID; - - if (auto err = mInstanceIDProvider->GetInstanceID(instance, instanceID); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - auto networkConfig = MakeUnique(&mAllocator); - if (auto err = GetInstanceNetworkConfig(instance, itemConfig, imageConfig, *networkConfig); !err.IsNone()) { + if (auto err = GetInstanceNetworkConfig(instanceData.mInfo, itemConfig, imageConfig, *networkConfig); + !err.IsNone()) { return err; } - if (auto err = mNetworkManager->CreateInstanceNetwork(instanceID, instance.mOwnerID, *networkConfig); + if (auto err + = mNetworkManager->CreateInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID, *networkConfig); !err.IsNone() && !err.Is(ErrorEnum::eAlreadyExist)) { return AOS_ERROR_WRAP(err); } diff --git a/src/core/sm/launcher/launcher.hpp b/src/core/sm/launcher/launcher.hpp index d64690a72..51c2be03a 100644 --- a/src/core/sm/launcher/launcher.hpp +++ b/src/core/sm/launcher/launcher.hpp @@ -157,9 +157,10 @@ class Launcher : public LauncherItf, private: struct InstanceData { - InstanceInfo mInfo; - InstanceStatus mStatus; - Duration mOfflineTTL; + InstanceInfo mInfo; + InstanceStatus mStatus; + StaticString mInstanceID; + Duration mOfflineTTL; }; struct UpdateItemInfo { @@ -215,7 +216,7 @@ class Launcher : public LauncherItf, Error GetInstanceNetworkConfig(const InstanceInfo& instance, const oci::ItemConfig& itemConfig, const oci::ImageConfig& imageConfig, networkmanager::InstanceNetworkConfig& networkConfig); Error CreateNetwork( - const InstanceInfo& instance, const oci::ItemConfig& itemConfig, const oci::ImageConfig& imageConfig); + const InstanceData& instanceData, const oci::ItemConfig& itemConfig, const oci::ImageConfig& imageConfig); InstanceData* FindInstanceData(const InstanceIdent& instanceIdent); InstanceData* FindInstanceData(const InstanceIdent& instanceIdent) const; diff --git a/src/core/sm/launcher/tests/launcher.cpp b/src/core/sm/launcher/tests/launcher.cpp index 36e8b471b..429ca0b6b 100644 --- a/src/core/sm/launcher/tests/launcher.cpp +++ b/src/core/sm/launcher/tests/launcher.cpp @@ -189,6 +189,8 @@ class LauncherTest : public Test { EXPECT_CALL(mInstanceIDProvider, GetInstanceID).WillRepeatedly(Return(ErrorEnum::eNone)); EXPECT_CALL(mNetworkManager, CreateInstanceNetwork).WillRepeatedly(Return(ErrorEnum::eNone)); + EXPECT_CALL(mNetworkManager, StartInstanceNetwork).WillRepeatedly(Return(ErrorEnum::eNone)); + EXPECT_CALL(mNetworkManager, StopInstanceNetwork).WillRepeatedly(Return(ErrorEnum::eNone)); EXPECT_CALL(mNetworkManager, ReleaseInstanceNetwork).WillRepeatedly(Return(ErrorEnum::eNone)); } From 5f0b5a5a57e4cfa90c6ab19388e2f09bd296c594 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Sat, 11 Jul 2026 11:03:24 +0300 Subject: [PATCH 048/112] sm: launcher: run instance network start/stop as pool tasks Move network setup/teardown out of StartInstance/StopInstance and into new StartNetworks/StopNetworks helpers that queue StartInstanceNetwork/ StopInstanceNetwork calls on the launch thread pool, so networks are started/stopped in parallel rather than serialized with the instance start/stop itself. Add StopAllNetworks, invoked from Stop() after StopAllInstances, to tear down networks for all service instances on launcher shutdown, and run it via the launch pool like the other batch operations. Tag the UpdateInstancesImpl and instance/network start/stop log messages with a "[profiling]" marker to make timing of these stages easier to trace. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko --- src/core/sm/launcher/launcher.cpp | 192 ++++++++++++++++++++++++------ src/core/sm/launcher/launcher.hpp | 5 + 2 files changed, 160 insertions(+), 37 deletions(-) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index 47fecb27f..efdc3e63c 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -115,10 +115,21 @@ Error Launcher::Stop() lock.Unlock(); - StopAllInstances(); + auto err = mLaunchPool.Run(); + + if (err.IsNone()) { + StopAllInstances(); + StopAllNetworks(); + + err = mLaunchPool.Shutdown(); + } + + if (!err.IsNone() && stopErr.IsNone()) { + stopErr = AOS_ERROR_WRAP(err); + } for (auto& it : mRuntimes) { - if (auto err = it.mFirst->Stop(); !err.IsNone() && stopErr.IsNone()) { + if (err = it.mFirst->Stop(); !err.IsNone() && stopErr.IsNone()) { stopErr = AOS_ERROR_WRAP(err); } } @@ -612,7 +623,7 @@ void Launcher::LoadInstancesData(const Array& storedInstances) void Launcher::UpdateInstancesImpl(Array& stopInstances, const Array& startInstances) { - LOG_INF() << "Update instances start" << Log::Field("stopCount", stopInstances.Size()) + LOG_INF() << "[profiling] Update instances begin" << Log::Field("stopCount", stopInstances.Size()) << Log::Field("startCount", startInstances.Size()); auto sendStatus = DeferRelease(&mInstances, [this](Array*) { @@ -644,6 +655,7 @@ void Launcher::UpdateInstancesImpl(Array& stopInstances, const Ar } StopInstances(stopInstances); + StopNetworks(stopInstances); RemoveInstances(stopInstances); if (!mFirstStart) { @@ -652,18 +664,20 @@ void Launcher::UpdateInstancesImpl(Array& stopInstances, const Ar PrepareInstances(startInstances); } + StartNetworks(startInstances); StartInstances(startInstances); if (auto err = mLaunchPool.Shutdown(); !err.IsNone()) { LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } - LOG_INF() << "Update instances finished" << Log::Field("stopCount", stopInstances.Size()) - << Log::Field("startCount", startInstances.Size()); + LOG_INF() << "[profiling] Update instances end"; } void Launcher::StopInstances(const Array& stopInstances) { + LOG_INF() << "[profiling] Stop instances begin" << Log::Field("count", stopInstances.Size()); + for (const auto& instance : stopInstances) { auto instanceData = FindInstanceData(instance); if (!instanceData) { @@ -683,6 +697,8 @@ void Launcher::StopInstances(const Array& stopInstances) if (auto err = mLaunchPool.Wait(); !err.IsNone()) { LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); } + + LOG_INF() << "[profiling] Stop instances end"; } Error Launcher::AddStopInstanceTask(InstanceData& instanceData) @@ -716,36 +732,19 @@ Error Launcher::StopInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceDat LOG_INF() << "Stop instance" << Log::Field("instance", instanceData.mInfo) << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID); - Error err; - - if (auto stopErr = runtime->StopInstance(instanceData.mInfo, instanceData.mStatus); - !stopErr.IsNone() && err.IsNone()) { - err = AOS_ERROR_WRAP(stopErr); - } - - if (instanceData.mInfo.mType == UpdateItemTypeEnum::eService) { - if (auto networkErr - = mNetworkManager->StopInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); - !networkErr.IsNone() && err.IsNone()) { - err = AOS_ERROR_WRAP(networkErr); - } + if (auto err = runtime->StopInstance(instanceData.mInfo, instanceData.mStatus); !err.IsNone()) { + return AOS_ERROR_WRAP(err); } - return err; + return ErrorEnum::eNone; } void Launcher::StopAllInstances() { - LOG_INF() << "Stop all instances start" << Log::Field("count", mInstances.Size()); - - if (auto err = mLaunchPool.Run(); !err.IsNone()) { - LOG_ERR() << "Can't start thread pool" << Log::Field(AOS_ERROR_WRAP(err)); - - return; - } + LOG_INF() << "[profiling] Stop all instances begin" << Log::Field("count", mInstances.Size()); for (auto& instance : mInstances) { - if (instance.mInfo.mType == UpdateItemTypeEnum::eComponent) { + if (instance.mInfo.mType != UpdateItemTypeEnum::eService) { continue; } @@ -760,11 +759,26 @@ void Launcher::StopAllInstances() LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); } - if (auto err = mLaunchPool.Shutdown(); !err.IsNone()) { - LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); + LOG_INF() << "[profiling] Stop all instances end"; +} + +void Launcher::StopAllNetworks() +{ + LOG_INF() << "[profiling] Stop all networks begin" << Log::Field("count", mInstances.Size()); + + for (auto& instance : mInstances) { + if (instance.mInfo.mType != UpdateItemTypeEnum::eService) { + continue; + } + + if (auto err = AddStopNetworkTask(instance); !err.IsNone()) { + LOG_ERR() << "Failed to stop network" << Log::Field("instance", instance.mInfo) << Log::Field(err); + + SetInstanceState(instance, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + } } - LOG_INF() << "Stop all instances finished" << Log::Field("count", mInstances.Size()); + LOG_INF() << "[profiling] Stop all networks end"; } Error Launcher::PrepareInstance(InstanceData& instanceData) @@ -838,8 +852,117 @@ void Launcher::PrepareInstances(const Array& startInstances) } } +Error Launcher::AddStartNetworkTask(InstanceData& instanceData) +{ + if (auto err = mLaunchPool.AddTask([this, &instanceData](void*) { + if (auto err = mNetworkManager->StartInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); + !err.IsNone()) { + LOG_ERR() << "Failed to start network" << Log::Field("instance", instanceData.mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + } + }); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +void Launcher::StartNetworks(const Array& startInstances) +{ + LOG_INF() << "[profiling] Start networks begin" << Log::Field("count", startInstances.Size()); + + for (const auto& instance : startInstances) { + auto instanceData = FindInstanceData(instance); + if (!instanceData) { + LOG_ERR() << "Failed to start network" << Log::Field("instance", instance) + << Log::Field(AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance data not found"))); + + continue; + } + + if (instanceData->mInfo.mType != UpdateItemTypeEnum::eService) { + continue; + } + + if (instanceData->mStatus.mState != InstanceStateEnum::eInactive) { + LOG_ERR() << "Failed to start network" << Log::Field("instance", instance) + << Log::Field(AOS_ERROR_WRAP(Error(ErrorEnum::eWrongState, "instance not inactive"))); + + continue; + } + + if (auto err = AddStartNetworkTask(*instanceData); !err.IsNone()) { + LOG_ERR() << "Failed to start network" << Log::Field("instance", instanceData->mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + } + } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } + + LOG_INF() << "[profiling] Start networks end"; +} + +Error Launcher::AddStopNetworkTask(InstanceData& instanceData) +{ + if (auto err = mLaunchPool.AddTask([this, &instanceData](void*) { + if (auto err = mNetworkManager->StopInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); + !err.IsNone()) { + LOG_ERR() << "Failed to stop network" << Log::Field("instance", instanceData.mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + } + }); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +void Launcher::StopNetworks(const Array& stopInstances) +{ + LOG_INF() << "[profiling] Stop networks begin" << Log::Field("count", stopInstances.Size()); + + for (const auto& instance : stopInstances) { + auto instanceData = FindInstanceData(instance); + if (!instanceData) { + LOG_ERR() << "Failed to stop network" << Log::Field("instance", instance) + << Log::Field(AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance data not found"))); + + continue; + } + + if (instanceData->mInfo.mType != UpdateItemTypeEnum::eService) { + continue; + } + + if (auto err = AddStopNetworkTask(*instanceData); !err.IsNone()) { + LOG_ERR() << "Failed to stop network" << Log::Field("instance", instanceData->mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + SetInstanceState(*instanceData, InstanceStateEnum::eFailed, AOS_ERROR_WRAP(err)); + } + } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } + + LOG_INF() << "[profiling] Stop networks end"; +} + void Launcher::StartInstances(const Array& startInstances) { + LOG_INF() << "[profiling] Start instances begin" << Log::Field("count", startInstances.Size()); + for (const auto& instance : startInstances) { auto instanceData = FindInstanceData(instance); if (!instanceData) { @@ -867,6 +990,8 @@ void Launcher::StartInstances(const Array& startInstances) if (auto err = mLaunchPool.Wait(); !err.IsNone()) { LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); } + + LOG_INF() << "[profiling] Start instances end"; } Error Launcher::AddStartInstanceTask(InstanceData& instanceData) @@ -903,13 +1028,6 @@ Error Launcher::StartInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceDa << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID) << Log::Field("manifestDigest", instanceData.mInfo.mManifestDigest); - if (instanceData.mInfo.mType == UpdateItemTypeEnum::eService) { - if (auto err = mNetworkManager->StartInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); - !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - } - if (auto err = runtime->StartInstance(instanceData.mInfo, instanceData.mStatus); !err.IsNone()) { return AOS_ERROR_WRAP(err); } diff --git a/src/core/sm/launcher/launcher.hpp b/src/core/sm/launcher/launcher.hpp index 51c2be03a..954213834 100644 --- a/src/core/sm/launcher/launcher.hpp +++ b/src/core/sm/launcher/launcher.hpp @@ -193,10 +193,15 @@ class Launcher : public LauncherItf, void UpdateInstancesImpl(Array& stopInstances, const Array& startInstances); void StopInstances(const Array& stopInstances); Error AddStopInstanceTask(InstanceData& instanceData); + void StopNetworks(const Array& stopInstances); + Error AddStopNetworkTask(InstanceData& instanceData); Error StopInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData); void StopAllInstances(); + void StopAllNetworks(); Error PrepareInstance(InstanceData& instanceData); void PrepareInstances(const Array& startInstances); + void StartNetworks(const Array& startInstances); + Error AddStartNetworkTask(InstanceData& instanceData); void StartInstances(const Array& startInstances); Error AddStartInstanceTask(InstanceData& instanceData); Error StartInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceData& instanceData); From 466039a21c359d9d8d79d17bce5cf5ac9fc60f7a Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Wed, 8 Jul 2026 20:05:52 +0300 Subject: [PATCH 049/112] common: wait for callbacks in Timer::Stop Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/common/tools/tests/thread.cpp | 4 +- src/core/common/tools/tests/timer.cpp | 6 +-- src/core/common/tools/thread.hpp | 32 +++++++++--- src/core/common/tools/timer.cpp | 69 +++++++++++++++++++++++--- src/core/common/tools/timer.hpp | 59 ++++++++++++++++------ src/core/sm/launcher/launcher.cpp | 20 +++++--- 6 files changed, 147 insertions(+), 43 deletions(-) diff --git a/src/core/common/tools/tests/thread.cpp b/src/core/common/tools/tests/thread.cpp index a05edae8b..43059539f 100644 --- a/src/core/common/tools/tests/thread.cpp +++ b/src/core/common/tools/tests/thread.cpp @@ -240,13 +240,11 @@ TEST(ThreadTest, ThreadPool) } EXPECT_TRUE(threadPool.Run().IsNone()); - EXPECT_TRUE(threadPool.Wait().IsNone()); + EXPECT_TRUE(threadPool.Shutdown().IsNone()); EXPECT_EQ(value1, i); EXPECT_EQ(value2, i); EXPECT_EQ(value3, i); - - EXPECT_TRUE(threadPool.Shutdown().IsNone()); } TEST(ThreadTest, SemaphoreTest) diff --git a/src/core/common/tools/tests/timer.cpp b/src/core/common/tools/tests/timer.cpp index de39545a2..1a784612b 100644 --- a/src/core/common/tools/tests/timer.cpp +++ b/src/core/common/tools/tests/timer.cpp @@ -57,7 +57,7 @@ TEST(TimerTest, RunOneShot) EXPECT_THAT(invokeTime, ApproxEqualTime(Time(now).Add(cTimeout))); - EXPECT_TRUE(timer.Stop().IsNone()); + EXPECT_TRUE(timer.Stop(Timer::StopMode::WaitForCallbacks).IsNone()); } TEST(TimerTest, RunMultiShot) @@ -78,7 +78,7 @@ TEST(TimerTest, RunMultiShot) EXPECT_TRUE(timer.Start(cTimeout, WrapCallback(cb), false).IsNone()); sleep(1); - EXPECT_TRUE(timer.Stop().IsNone()); + EXPECT_TRUE(timer.Stop(Timer::StopMode::WaitForCallbacks).IsNone()); EXPECT_THAT(invokeTimes, ElementsAre(ApproxEqualTime(expInvTimes[0]), ApproxEqualTime(expInvTimes[1]), ApproxEqualTime(expInvTimes[2]))); @@ -97,7 +97,7 @@ TEST(TimerTest, CreateResetStop) sleep(1); - EXPECT_TRUE(timer.Stop().IsNone()); + EXPECT_TRUE(timer.Stop(Timer::StopMode::WaitForCallbacks).IsNone()); sleep(2); diff --git a/src/core/common/tools/thread.hpp b/src/core/common/tools/thread.hpp index 912ff64da..81ffbc378 100644 --- a/src/core/common/tools/thread.hpp +++ b/src/core/common/tools/thread.hpp @@ -539,6 +539,10 @@ class ThreadPool : private NonCopyable { { LockGuard lock {mMutex}; + if (mShutdown) { + return AOS_ERROR_WRAP(ErrorEnum::eCanceled); + } + auto err = mQueue.Push(Function()); if (!err.IsNone()) { return err; @@ -569,6 +573,10 @@ class ThreadPool : private NonCopyable { { LockGuard lock {mMutex}; + if (mShutdown) { + return AOS_ERROR_WRAP(ErrorEnum::eCanceled); + } + auto err = mQueue.Push(functor); if (!err.IsNone()) { return err; @@ -605,7 +613,7 @@ class ThreadPool : private NonCopyable { auto err = mTaskCondVar.Wait(lock, [this]() { return mShutdown || !mQueue.IsEmpty(); }); assert(err.IsNone()); - if (mShutdown) { + if (mShutdown && mQueue.IsEmpty()) { return; } @@ -646,7 +654,7 @@ class ThreadPool : private NonCopyable { } /** - * Waits for all current tasks are finished. + * Waits for all pool threads to be finished. */ Error Wait() { @@ -662,13 +670,15 @@ class ThreadPool : private NonCopyable { /** * Shutdowns all pool threads. + * + * @param waitAllTasks if true, waits for all pending tasks to be executed. + * @return Error. */ - Error Shutdown() + Error Shutdown(bool waitAllTasks = true) { UniqueLock lock(mMutex); mShutdown = true; - mQueue.Clear(); lock.Unlock(); @@ -677,11 +687,17 @@ class ThreadPool : private NonCopyable { return err; } - for (auto& thread : mThreads) { - auto joinErr = thread.Join(); - if (!joinErr.IsNone() && err.IsNone()) { - err = joinErr; + if (waitAllTasks) { + Error err = ErrorEnum::eNone; + + for (auto& thread : mThreads) { + auto joinErr = thread.Join(); + if (!joinErr.IsNone() && err.IsNone()) { + err = joinErr; + } } + + return err; } return err; diff --git a/src/core/common/tools/timer.cpp b/src/core/common/tools/timer.cpp index c29150c33..9a416196d 100644 --- a/src/core/common/tools/timer.cpp +++ b/src/core/common/tools/timer.cpp @@ -20,10 +20,57 @@ ConditionalVariable Timer::mCommonCondVar; Thread<> Timer::mManagementThread; ThreadPool Timer::mInvocationThreads; +/*********************************************************************************************************************** + * Public + **********************************************************************************************************************/ + +Error Timer::Stop(StopMode mode) +{ + { + LockGuard lock {mMutex}; + + mStopped = true; + } + + if (auto err = UnregisterTimer(this, mode); !err.IsNone()) { + return err; + } + + if (mode == StopMode::NoWait) { + return ErrorEnum::eNone; + } + + UniqueLock lock {mMutex}; + + if (auto err = mCondVar.Wait(lock, [this]() { return mActiveCallbacks == 0; }); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + /*********************************************************************************************************************** * Private **********************************************************************************************************************/ +void Timer::AcquireActiveCallback() +{ + LockGuard lock {mMutex}; + + mActiveCallbacks++; +} + +void Timer::ReleaseActiveCallback() +{ + LockGuard lock {mMutex}; + + mActiveCallbacks--; + + if (mActiveCallbacks == 0) { + mCondVar.NotifyAll(); + } +} + Error Timer::RegisterTimer(Timer* timer) { LockGuard lock {mCommonMutex}; @@ -45,8 +92,10 @@ Error Timer::RegisterTimer(Timer* timer) return ErrorEnum::eNone; } -Error Timer::UnregisterTimer(Timer* timer) +Error Timer::UnregisterTimer(Timer* timer, StopMode mode) { + bool stopThreads = false; + { LockGuard lock {mCommonMutex}; @@ -60,12 +109,14 @@ Error Timer::UnregisterTimer(Timer* timer) return AOS_ERROR_WRAP(err); } - if (!mRegisteredTimers.IsEmpty()) { - return ErrorEnum::eNone; - } + stopThreads = mRegisteredTimers.IsEmpty(); } - return StopThreads(); + if (stopThreads) { + return StopThreads(mode); + } + + return ErrorEnum::eNone; } Error Timer::StartThreads() @@ -81,13 +132,13 @@ Error Timer::StartThreads() return ErrorEnum::eNone; } -Error Timer::StopThreads() +Error Timer::StopThreads(StopMode mode) { if (auto err = mManagementThread.Join(); !err.IsNone()) { return AOS_ERROR_WRAP(err); } - if (auto err = mInvocationThreads.Shutdown(); !err.IsNone()) { + if (auto err = mInvocationThreads.Shutdown(mode == StopMode::WaitForCallbacks); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -96,7 +147,11 @@ Error Timer::StopThreads() void Timer::InvokeTimerCallback(Timer* timer) { + timer->AcquireActiveCallback(); + if (auto err = mInvocationThreads.AddTask(timer->mFunction); !err.IsNone()) { + timer->ReleaseActiveCallback(); + LOG_ERR() << "Invoke timer callback failure: err=" << AOS_ERROR_WRAP(err); } } diff --git a/src/core/common/tools/timer.hpp b/src/core/common/tools/timer.hpp index 886ae3324..77e9d8215 100644 --- a/src/core/common/tools/timer.hpp +++ b/src/core/common/tools/timer.hpp @@ -21,6 +21,14 @@ namespace aos { */ class Timer { public: + /** + * Timer stop mode. + */ + enum class StopMode { + NoWait, ///< Stop timer without waiting for running callbacks. + WaitForCallbacks, ///< Wait for running callbacks and stop timer. + }; + /** * Constructs timer instance. */ @@ -29,7 +37,7 @@ class Timer { /** * Destructs timer instance. */ - ~Timer() { Stop(); } + ~Timer() { Stop(StopMode::WaitForCallbacks); } /** * Starts timer. @@ -47,28 +55,40 @@ class Timer { return AOS_ERROR_WRAP(ErrorEnum::eInvalidArgument); } - if (auto err = Stop(); !err.IsNone()) { + if (auto err = Stop(StopMode::NoWait); !err.IsNone()) { return AOS_ERROR_WRAP(err); } - LockGuard lock {mMutex}; - const auto wrappedCallback = [this, callback](void* arg) { bool oneshot = false; + bool stopped = false; { LockGuard lock {mMutex}; + stopped = mStopped; oneshot = mOneShot; } + if (stopped) { + ReleaseActiveCallback(); + + return; + } + if (oneshot) { - Stop(); + Stop(StopMode::NoWait); } callback(arg); + + ReleaseActiveCallback(); }; + LockGuard lock {mMutex}; + + mStopped = false; + if (auto err = mFunction.Capture(wrappedCallback, arg); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -82,9 +102,10 @@ class Timer { /** * Stops timer. * + * @param mode specifies whether to wait for currently running callbacks. * @return Error code. */ - Error Stop() { return UnregisterTimer(this); } + Error Stop(StopMode mode); /** * Restarts timer. @@ -93,25 +114,35 @@ class Timer { */ Error Restart() { - LockGuard lock {mMutex}; - - if (!mFunction) { - return ErrorEnum::eNone; + if (auto err = Stop(StopMode::NoWait); !err.IsNone()) { + return AOS_ERROR_WRAP(err); } - if (auto err = Stop(); !err.IsNone()) { - return AOS_ERROR_WRAP(err); + { + LockGuard lock {mMutex}; + + if (!mFunction) { + return ErrorEnum::eNone; + } + + mStopped = false; } return RegisterTimer(this); } private: + void AcquireActiveCallback(); + void ReleaseActiveCallback(); + Duration mInterval {}; bool mOneShot {}; + bool mStopped {}; + size_t mActiveCallbacks {}; StaticFunction mFunction; Time mWakeupTime; Mutex mMutex; + ConditionalVariable mCondVar; // Set two threads for callbacks: in case if any executes for a long time, another will hedge. static constexpr auto cInvocationThreadsCount = 2; @@ -119,10 +150,10 @@ class Timer { static constexpr Duration cTimerResolution = Time::cMicroseconds * 500; static Error RegisterTimer(Timer* timer); - static Error UnregisterTimer(Timer* timer); + static Error UnregisterTimer(Timer* timer, StopMode mode); static Error StartThreads(); - static Error StopThreads(); + static Error StopThreads(StopMode mode); static void ProcessTimers(void* arg); static void UpdateWakeupTime(const Time& now, Timer* timer); diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index efdc3e63c..08b42eb8c 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -499,14 +499,14 @@ void Launcher::StopExpiredInstances(UniqueLock& lock) lock.Unlock(); - if (auto err = mOfflineTTLPool.Wait(); !err.IsNone()) { - LOG_ERR() << "Offline TTL thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); - } - if (auto err = mOfflineTTLPool.Shutdown(); !err.IsNone()) { LOG_ERR() << "Offline TTL thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } + if (auto err = mOfflineTTLPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Offline TTL thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } + lock.Lock(); } @@ -612,13 +612,13 @@ void Launcher::LoadInstancesData(const Array& storedInstances) } } - if (auto err = mLaunchPool.Wait(); !err.IsNone()) { - LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); - } - if (auto err = mLaunchPool.Shutdown(); !err.IsNone()) { LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } + + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } } void Launcher::UpdateInstancesImpl(Array& stopInstances, const Array& startInstances) @@ -671,6 +671,10 @@ void Launcher::UpdateInstancesImpl(Array& stopInstances, const Ar LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } + LOG_INF() << "[profiling] Update instances end"; } From 98c372e2eb32a7b3986045f443970877737e61fe Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Thu, 9 Jul 2026 11:24:07 +0300 Subject: [PATCH 050/112] cm: common: specify stop mode for Timer::Stop Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/cm/alerts/alerts.cpp | 2 +- src/core/cm/imagemanager/imagemanager.cpp | 2 +- src/core/cm/launcher/instancemanager.cpp | 4 ++-- src/core/cm/launcher/launcher.cpp | 2 +- src/core/cm/monitoring/monitoring.cpp | 2 +- src/core/cm/storagestate/storagestate.cpp | 6 +++++- src/core/cm/updatemanager/unitstatushandler.cpp | 4 ++-- src/core/common/monitoring/monitoring.cpp | 2 +- src/core/sm/imagemanager/imagemanager.cpp | 2 +- src/core/sm/launcher/launcher.cpp | 14 +------------- 10 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/core/cm/alerts/alerts.cpp b/src/core/cm/alerts/alerts.cpp index 54fa21cc2..54d63e5c1 100644 --- a/src/core/cm/alerts/alerts.cpp +++ b/src/core/cm/alerts/alerts.cpp @@ -102,7 +102,7 @@ Error Alerts::Stop() err = AOS_ERROR_WRAP(unsubscribeErr); } - if (auto stopErr = mSendTimer.Stop(); !stopErr.IsNone()) { + if (auto stopErr = mSendTimer.Stop(Timer::StopMode::WaitForCallbacks); !stopErr.IsNone()) { LOG_ERR() << "Failed to stop alerts send timer" << Log::Field(stopErr); if (err.IsNone()) { diff --git a/src/core/cm/imagemanager/imagemanager.cpp b/src/core/cm/imagemanager/imagemanager.cpp index 3c6ff7bff..9f2c46bb7 100644 --- a/src/core/cm/imagemanager/imagemanager.cpp +++ b/src/core/cm/imagemanager/imagemanager.cpp @@ -97,7 +97,7 @@ Error ImageManager::Stop() { LOG_DBG() << "Stop image manager"; - return mTimer.Stop(); + return mTimer.Stop(Timer::StopMode::WaitForCallbacks); } Error ImageManager::DownloadUpdateItems(const Array& itemsInfo, diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index 64b1d4ae3..9773fefe9 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -104,11 +104,11 @@ Error InstanceManager::Start() Error InstanceManager::Stop() { - if (auto err = mCleanInstancesTimer.Stop(); !err.IsNone()) { + if (auto err = mCleanInstancesTimer.Stop(Timer::StopMode::WaitForCallbacks); !err.IsNone()) { return AOS_ERROR_WRAP(err); } - if (auto err = mInitTimer.Stop(); !err.IsNone()) { + if (auto err = mInitTimer.Stop(Timer::StopMode::WaitForCallbacks); !err.IsNone()) { return AOS_ERROR_WRAP(err); } diff --git a/src/core/cm/launcher/launcher.cpp b/src/core/cm/launcher/launcher.cpp index 125104247..f2b305c58 100644 --- a/src/core/cm/launcher/launcher.cpp +++ b/src/core/cm/launcher/launcher.cpp @@ -202,7 +202,7 @@ Error Launcher::Stop() return err; } - if (auto err = mEnvVarsTTLTimer.Stop(); !err.IsNone()) { + if (auto err = mEnvVarsTTLTimer.Stop(Timer::StopMode::WaitForCallbacks); !err.IsNone()) { return AOS_ERROR_WRAP(err); } diff --git a/src/core/cm/monitoring/monitoring.cpp b/src/core/cm/monitoring/monitoring.cpp index d3220b9b4..f6be59c77 100644 --- a/src/core/cm/monitoring/monitoring.cpp +++ b/src/core/cm/monitoring/monitoring.cpp @@ -97,7 +97,7 @@ Error Monitoring::Stop() mIsRunning = false; - return mSendTimer.Stop(); + return mSendTimer.Stop(Timer::StopMode::WaitForCallbacks); } Error Monitoring::OnMonitoringReceived(const aos::monitoring::NodeMonitoringData& monitoring) diff --git a/src/core/cm/storagestate/storagestate.cpp b/src/core/cm/storagestate/storagestate.cpp index c9f43117b..4f1bf36e8 100644 --- a/src/core/cm/storagestate/storagestate.cpp +++ b/src/core/cm/storagestate/storagestate.cpp @@ -109,7 +109,11 @@ Error StorageState::Stop() } } - return mThreadPool.Shutdown(); + if (auto err = mThreadPool.Shutdown(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; } Error StorageState::UpdateState(const aos::UpdateState& state) diff --git a/src/core/cm/updatemanager/unitstatushandler.cpp b/src/core/cm/updatemanager/unitstatushandler.cpp index 14c0a61d0..5cd6b40a3 100644 --- a/src/core/cm/updatemanager/unitstatushandler.cpp +++ b/src/core/cm/updatemanager/unitstatushandler.cpp @@ -142,7 +142,7 @@ Error UnitStatusHandler::SendFullUnitStatus() ClearUnitStatus(); ClearUpdateStatuses(); - mTimer.Stop(); + mTimer.Stop(Timer::StopMode::NoWait); LockGuard lock {mMutex}; @@ -389,7 +389,7 @@ void UnitStatusHandler::OnDisconnect() LockGuard lock {mMutex}; mCloudConnected = false; - mTimer.Stop(); + mTimer.Stop(Timer::StopMode::NoWait); } Error UnitStatusHandler::SetUnitConfigStatus() diff --git a/src/core/common/monitoring/monitoring.cpp b/src/core/common/monitoring/monitoring.cpp index 92c842c76..75aae24a4 100644 --- a/src/core/common/monitoring/monitoring.cpp +++ b/src/core/common/monitoring/monitoring.cpp @@ -208,7 +208,7 @@ Error Monitoring::Stop() Error err; - if (auto stopErr = mTimer.Stop(); err.IsNone() && !stopErr.IsNone()) { + if (auto stopErr = mTimer.Stop(Timer::StopMode::WaitForCallbacks); err.IsNone() && !stopErr.IsNone()) { err = AOS_ERROR_WRAP(stopErr); } diff --git a/src/core/sm/imagemanager/imagemanager.cpp b/src/core/sm/imagemanager/imagemanager.cpp index e6b1841f3..55fa9f846 100644 --- a/src/core/sm/imagemanager/imagemanager.cpp +++ b/src/core/sm/imagemanager/imagemanager.cpp @@ -123,7 +123,7 @@ Error ImageManager::Stop() LOG_DBG() << "Stop image manager"; - if (auto err = mTimer.Stop(); !err.IsNone() && stopErr.IsNone()) { + if (auto err = mTimer.Stop(Timer::StopMode::WaitForCallbacks); !err.IsNone() && stopErr.IsNone()) { stopErr = AOS_ERROR_WRAP(err); } diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index 08b42eb8c..be3f256a1 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -143,7 +143,7 @@ Error Launcher::Stop() mThread.Join(); mRebootThread.Join(); - mOfflineTTLHandler.Stop(); + mOfflineTTLHandler.Stop(Timer::StopMode::WaitForCallbacks); return stopErr; } @@ -503,10 +503,6 @@ void Launcher::StopExpiredInstances(UniqueLock& lock) LOG_ERR() << "Offline TTL thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } - if (auto err = mOfflineTTLPool.Wait(); !err.IsNone()) { - LOG_ERR() << "Offline TTL thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); - } - lock.Lock(); } @@ -615,10 +611,6 @@ void Launcher::LoadInstancesData(const Array& storedInstances) if (auto err = mLaunchPool.Shutdown(); !err.IsNone()) { LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } - - if (auto err = mLaunchPool.Wait(); !err.IsNone()) { - LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); - } } void Launcher::UpdateInstancesImpl(Array& stopInstances, const Array& startInstances) @@ -671,10 +663,6 @@ void Launcher::UpdateInstancesImpl(Array& stopInstances, const Ar LOG_ERR() << "Thread pool shutdown failed" << Log::Field(AOS_ERROR_WRAP(err)); } - if (auto err = mLaunchPool.Wait(); !err.IsNone()) { - LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); - } - LOG_INF() << "[profiling] Update instances end"; } From 80c65fb03a6135617acfff8f68b3e2139cd6b251 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Mon, 13 Jul 2026 13:27:31 +0300 Subject: [PATCH 051/112] cm: common: sm: fix lint warnings Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykola Solianko --- src/core/cm/alerts/alerts.cpp | 4 ++-- src/core/cm/launcher/balancer.cpp | 6 +++--- src/core/cm/launcher/balancer.hpp | 4 ++-- src/core/common/tools/thread.hpp | 4 ---- src/core/sm/networkmanager/networkmanager.cpp | 18 +++++++++--------- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/core/cm/alerts/alerts.cpp b/src/core/cm/alerts/alerts.cpp index 54d63e5c1..994461f0e 100644 --- a/src/core/cm/alerts/alerts.cpp +++ b/src/core/cm/alerts/alerts.cpp @@ -186,8 +186,8 @@ Error Alerts::UnsubscribeListener(AlertsListenerItf& listener) size_t removed = 0; - for (auto& [tag, listeners] : mListeners) { - removed += listeners.Remove(&listener); + for (auto& item : mListeners) { + removed += item.mSecond.Remove(&listener); } return removed > 0 ? ErrorEnum::eNone : ErrorEnum::eNotFound; diff --git a/src/core/cm/launcher/balancer.cpp b/src/core/cm/launcher/balancer.cpp index fd8a8b87b..6c1d15138 100644 --- a/src/core/cm/launcher/balancer.cpp +++ b/src/core/cm/launcher/balancer.cpp @@ -149,7 +149,7 @@ Error Balancer::ScheduleInstance(SharedPtr& instance, const oci::Index } // Schedule instance - auto& node = nodeRuntime.mFirst; + const auto& node = nodeRuntime.mFirst; const auto& runtime = nodeRuntime.mSecond; if (auto err = mInstanceManager->ScheduleInstance(instance, *node, runtime->mRuntimeID); !err.IsNone()) { @@ -194,7 +194,7 @@ void Balancer::FilterNodesByResources(Instance& instance, Array& nodes) nodes.RemoveIf([&instance](const Node* node) { return !instance.AreNodeResourcesOk(*node); }); } -RetWithError> Balancer::SelectRuntime(Instance& instance, Array& nodes) +RetWithError> Balancer::SelectRuntime(Instance& instance, const Array& nodes) { auto nodeRuntimes = MakeUnique(&mAllocator); @@ -265,7 +265,7 @@ RetWithError> Balancer::SelectRuntime(Instance& return {result, ErrorEnum::eNone}; } -Error Balancer::CreateRuntimes(Array& nodes, NodeRuntimes& runtimes) +Error Balancer::CreateRuntimes(const Array& nodes, NodeRuntimes& runtimes) { for (auto node : nodes) { if (auto err = runtimes.Emplace(node); !err.IsNone()) { diff --git a/src/core/cm/launcher/balancer.hpp b/src/core/cm/launcher/balancer.hpp index 6a4b21d28..9d0846b49 100644 --- a/src/core/cm/launcher/balancer.hpp +++ b/src/core/cm/launcher/balancer.hpp @@ -75,9 +75,9 @@ class Balancer { void FilterNodesByResources(Instance& instance, Array& nodes); // Selects runtime - RetWithError> SelectRuntime(Instance& instance, Array& nodes); + RetWithError> SelectRuntime(Instance& instance, const Array& nodes); - Error CreateRuntimes(Array& nodes, NodeRuntimes& runtimes); + Error CreateRuntimes(const Array& nodes, NodeRuntimes& runtimes); template void FilterRuntimes(NodeRuntimes& runtimes, Filter& filter); diff --git a/src/core/common/tools/thread.hpp b/src/core/common/tools/thread.hpp index 81ffbc378..1140fc2e4 100644 --- a/src/core/common/tools/thread.hpp +++ b/src/core/common/tools/thread.hpp @@ -688,16 +688,12 @@ class ThreadPool : private NonCopyable { } if (waitAllTasks) { - Error err = ErrorEnum::eNone; - for (auto& thread : mThreads) { auto joinErr = thread.Join(); if (!joinErr.IsNone() && err.IsNone()) { err = joinErr; } } - - return err; } return err; diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 5278bb98e..874e7f1d1 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -987,8 +987,8 @@ Error NetworkManager::CleanupLeftoverInstances() { LockGuard lock {mMutex}; - for (const auto& [id, info] : mInstanceNetworkInfos) { - if (auto err = entries->PushBack({id, info.mNetworkID}); !err.IsNone()) { + for (const auto& item : mInstanceNetworkInfos) { + if (auto err = entries->PushBack({item.mFirst, item.mSecond.mNetworkID}); !err.IsNone()) { return AOS_ERROR_WRAP(err); } } @@ -1483,13 +1483,13 @@ void NetworkManager::OnPendingFirewallUpdate( { LockGuard lock {mMutex}; - for (const auto& [id, info] : mInstanceNetworkInfos) { - if (info.mNetworkConfig.mInstanceIdent == update.mInstanceIdent) { - instanceID = id; - networkID = info.mNetworkID; - *networkConfig = info.mNetworkConfig; - *allocatedParams = info.mAllocatedParams; - hostIfName = info.mHostIfName; + for (const auto& item : mInstanceNetworkInfos) { + if (item.mSecond.mNetworkConfig.mInstanceIdent == update.mInstanceIdent) { + instanceID = item.mFirst; + networkID = item.mSecond.mNetworkID; + *networkConfig = item.mSecond.mNetworkConfig; + *allocatedParams = item.mSecond.mAllocatedParams; + hostIfName = item.mSecond.mHostIfName; auto network = mRuntimeCache.Find(networkID); if (network != mRuntimeCache.end()) { From 91ba5419a14928a3a9a6a31fdf91e3d130b58707 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Sun, 12 Jul 2026 11:55:22 +0300 Subject: [PATCH 052/112] sm: launcher: skip loading data for non-service instances LoadInstancesData iterated over all stored instances and scheduled LoadInstanceData for each of them, including components. Component instances do not have image/item configs to load, so this caused load failures to be logged for every stored component on startup. Skip instances whose type is not eService before scheduling the load task. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/sm/launcher/launcher.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index be3f256a1..b83a360b0 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -596,6 +596,10 @@ void Launcher::LoadInstancesData(const Array& storedInstances) continue; } + if (instanceData->mInfo.mType != UpdateItemTypeEnum::eService) { + continue; + } + if (err = mLaunchPool.AddTask([this, instanceData](void*) { if (auto err = LoadInstanceData(*instanceData); !err.IsNone()) { LOG_ERR() << "Failed to load instance data" << Log::Field("instance", instanceData->mInfo) From 1404ddc24765d117ccef60e2e69108e77fe44011 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Mon, 13 Jul 2026 12:33:18 +0300 Subject: [PATCH 053/112] sm: launcher: remove instance state checks in RemoveInstances RemoveInstances skipped releasing/removing instance data whose state was not eInactive. By the time RemoveInstances runs, matching instances have already been stopped via the launch pool, so the check only caused otherwise removable instances to be silently left in mInstances. Drop the redundant state checks. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/sm/launcher/launcher.cpp | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index b83a360b0..2db99cf27 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -1276,13 +1276,6 @@ void Launcher::RemoveInstances(const Array& instances) continue; } - if (instanceData->mStatus.mState != InstanceStateEnum::eInactive) { - LOG_ERR() << "Instance is not inactive, skip removing" << Log::Field("instance", instanceIdent) - << Log::Field("state", instanceData->mStatus.mState); - - continue; - } - if (auto err = mLaunchPool.AddTask([this, instanceData](void*) { if (auto err = ReleaseInstance(*instanceData); !err.IsNone()) { LOG_ERR() << "Failed to remove instance" << Log::Field("instance", instanceData->mInfo) @@ -1309,18 +1302,7 @@ void Launcher::RemoveInstances(const Array& instances) LOG_DBG() << "Remove instance data" << Log::Field("instance", instanceIdent); mInstances.RemoveIf([this, &instanceIdent](const auto& instance) { - if (static_cast(instance.mInfo) != instanceIdent) { - return false; - } - - if (instance.mStatus.mState != InstanceStateEnum::eInactive) { - LOG_ERR() << "Instance is not inactive, skip removing" << Log::Field("instance", instanceIdent) - << Log::Field("state", instance.mStatus.mState); - - return false; - } - - return true; + return static_cast(instance.mInfo) == instanceIdent; }); } } From 768d023a73de67950478109e17fa5e06282457a2 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Mon, 13 Jul 2026 13:07:03 +0300 Subject: [PATCH 054/112] sm: networkmanager: size resolv/hosts allocator for concurrency mResolvHostsAllocator backs GetHosts/GetResolvServers, both const methods that the launcher now calls concurrently from its instance start/stop pool tasks (one call per in-flight instance). Each call keeps its allocation alive for the call's duration, but the allocator was sized for a single caller, so more than one instance starting at once overran mMaxSize and hit the assertion in Allocator::Allocate, aborting the process. Scale the allocator's byte budget and allocation-slot count by cMaxNumConcurrentItems, matching the pattern already used for mAllocator in this class. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/sm/networkmanager/networkmanager.hpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index b8ca51c39..49b6229cc 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -211,6 +211,13 @@ class NetworkManager : public NetworkManagerItf { + sizeof(StaticArray); static constexpr auto cNumAllocations = 8 * cMaxNumConcurrentItems; + // GetHosts/GetResolvServers are const methods that instance start/stop pool tasks call + // concurrently (one call per in-flight instance), each making a single allocation off + // mResolvHostsAllocator that stays alive for the call's duration. Size it for + // cMaxNumConcurrentItems concurrent callers instead of just one. + static constexpr auto cResolvHostsAllocatorSize = cMaxNumConcurrentItems + * (sizeof(StaticArray) + sizeof(StaticArray, cMaxNumDNSServers>)); + static constexpr uint64_t cBurstLen = 12800; static constexpr auto cMaxExposedPort = 2; static constexpr auto cCountRetriesIfNameGen = 10; @@ -283,11 +290,9 @@ class NetworkManager : public NetworkManagerItf { StaticAllocator)> mNetworkInfosAllocator; StaticAllocator)> mInstanceNetworkInfosAllocator; - mutable Mutex mMutex; - StaticAllocator mAllocator; - mutable StaticAllocator) - + sizeof(StaticArray, cMaxNumDNSServers>)> - mResolvHostsAllocator; + mutable Mutex mMutex; + StaticAllocator mAllocator; + mutable StaticAllocator mResolvHostsAllocator; }; /** @}*/ From bebd07321480cf16e2dd688c3f24499a4a6b4286 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Mon, 13 Jul 2026 13:03:21 +0300 Subject: [PATCH 055/112] sm: networkmanager: guard concurrent last-instance network teardown RemoveInstances releases instances as parallel launch-pool tasks, so several ReleaseInstanceNetwork calls for the last instances on a shared network can run concurrently. Each removes its own entry from mInstanceNetworkInfos first, so both then observe no instances left and each tears the network down (storage RemoveNetworkInfo + CM ReleaseNodeNetwork), the second failing with "not found". Claim the teardown atomically via mNetworkProviders under the lock: only the release that still finds the network present removes it and proceeds to clean up storage and the CM. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 874e7f1d1..11e293632 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -493,6 +493,10 @@ Error NetworkManager::ReleaseInstanceNetwork(const String& instanceID, const Str return ErrorEnum::eNone; } + if (mNetworkProviders.Find(networkID) == mNetworkProviders.end()) { + return ErrorEnum::eNone; + } + mNetworkProviders.Remove(networkID); } From 477d873d0d4632f82a071e9e8d037582f9c7c3a1 Mon Sep 17 00:00:00 2001 From: Vasyl Samoilov Date: Tue, 14 Jul 2026 14:30:19 +0300 Subject: [PATCH 056/112] ci: update badges to point to develop branch results --- .github/workflows/build-test.yaml | 1 + README.md | 20 +++++++++++++++++--- codecov.yml | 21 +++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 codecov.yml diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index f2ac69c37..c30354fa2 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -54,6 +54,7 @@ jobs: ./build.sh coverage - name: Upload codecov report + if: github.event_name == 'push' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index 959b49306..091d94a13 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,21 @@ -[![ci](https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build_test.yaml/badge.svg)](https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build_test.yaml) -[![codecov](https://codecov.io/gh/aosedge/aos_core_lib_cpp/graph/badge.svg?token=kg8h7ATd9S)](https://codecov.io/gh/aosedge/aos_core_lib_cpp) -[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=aosedge_aos_core_lib_cpp&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=aosedge_aos_core_lib_cpp) +| Branch | CI | Coverage | Quality Gate | +|--------|----|----------|--------------| +| main | [![ci][ci-main]][ci-main-link] | [![coverage][cov-main]][cov-main-link] | [![Quality Gate][qg-main]][qg-main-link] | +| develop | [![ci][ci-dev]][ci-dev-link] | [![coverage][cov-dev]][cov-dev-link] | [![Quality Gate][qg-dev]][qg-dev-link] | + +[ci-main]: https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build-test.yaml/badge.svg?branch=main +[ci-main-link]: https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build-test.yaml?query=branch%3Amain +[cov-main]: https://sonarcloud.io/api/project_badges/measure?project=aosedge_aos_core_lib_cpp&metric=coverage&branch=main +[cov-main-link]: https://sonarcloud.io/summary/new_code?id=aosedge_aos_core_lib_cpp&branch=main +[qg-main]: https://sonarcloud.io/api/project_badges/measure?project=aosedge_aos_core_lib_cpp&metric=alert_status&branch=main +[qg-main-link]: https://sonarcloud.io/summary/new_code?id=aosedge_aos_core_lib_cpp&branch=main +[ci-dev]: https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build-test.yaml/badge.svg?branch=develop +[ci-dev-link]: https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build-test.yaml?query=branch%3Adevelop +[cov-dev]: https://sonarcloud.io/api/project_badges/measure?project=aosedge_aos_core_lib_cpp&metric=coverage&branch=develop +[cov-dev-link]: https://sonarcloud.io/summary/new_code?id=aosedge_aos_core_lib_cpp&branch=develop +[qg-dev]: https://sonarcloud.io/api/project_badges/measure?project=aosedge_aos_core_lib_cpp&metric=alert_status&branch=develop +[qg-dev-link]: https://sonarcloud.io/summary/new_code?id=aosedge_aos_core_lib_cpp&branch=develop # Aos core cpp libraries diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..c4d7e1e48 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,21 @@ +codecov: + branch: main + +coverage: + status: + project: + default: + target: auto + threshold: 1% + informational: true + patch: + default: + target: 80% + informational: true + +ignore: + - "**/tests/**" + +branches: + - main + - develop From 60e1a4dbf8054da1105204dd7670ba2c6d767dac Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 8 Jul 2026 14:29:19 +0300 Subject: [PATCH 057/112] cm: unitconfig: handle absent state on node info change Signed-off-by: Mykola Solianko --- src/core/cm/unitconfig/tests/unitconfig.cpp | 14 ++++++++++++++ src/core/cm/unitconfig/unitconfig.cpp | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/src/core/cm/unitconfig/tests/unitconfig.cpp b/src/core/cm/unitconfig/tests/unitconfig.cpp index 9e98b52cc..658a5bd34 100644 --- a/src/core/cm/unitconfig/tests/unitconfig.cpp +++ b/src/core/cm/unitconfig/tests/unitconfig.cpp @@ -395,6 +395,20 @@ TEST_F(UnitConfigTest, OnNodeInfoChangedSkipsIfVersionMatches) mUnitConfig.OnNodeInfoChanged(nodeInfo); } +TEST_F(UnitConfigTest, OnNodeInfoChangedSkipsWhenUnitConfigAbsent) +{ + ASSERT_TRUE(mUnitConfig.Init({cTestConfigFile}, mNodeInfoProvider, mNodeConfigHandler, mJSONProvider).IsNone()); + + UnitConfigStatus status; + ASSERT_TRUE(mUnitConfig.GetUnitConfigStatus(status).IsNone()); + ASSERT_EQ(status.mState, UnitConfigStateEnum::eAbsent); + + UnitNodeInfo nodeInfo = CreateTestNodeInfo(); + + // Strict mocks: neither GetNodeConfigStatus nor UpdateNodeConfig must be called when unit config is absent. + mUnitConfig.OnNodeInfoChanged(nodeInfo); +} + TEST_F(UnitConfigTest, OnNodeInfoChangedWithUnitConfigError) { CreateTestConfigFile(cInvalidTestUnitConfig); diff --git a/src/core/cm/unitconfig/unitconfig.cpp b/src/core/cm/unitconfig/unitconfig.cpp index f2ba9aa2d..98aee4419 100644 --- a/src/core/cm/unitconfig/unitconfig.cpp +++ b/src/core/cm/unitconfig/unitconfig.cpp @@ -196,6 +196,12 @@ void UnitConfig::OnNodeInfoChanged(const UnitNodeInfo& info) << Log::Field("state", info.mState) << Log::Field("isConnected", info.mIsConnected) << Log::Field(info.mError); + if (mUnitConfigState == UnitConfigStateEnum::eAbsent) { + LOG_DBG() << "Skip node config update due to unit config is absent" << Log::Field("nodeID", info.mNodeID); + + return; + } + if (mUnitConfigState != UnitConfigStateEnum::eInstalled) { LOG_WRN() << "Can't update node config due to state" << Log::Field("nodeID", info.mNodeID) << Log::Field("state", mUnitConfigState) << Log::Field(mUnitConfigError); From 5ca3edbcb95624e9ab37a0555d0c219b9a882331 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Tue, 14 Jul 2026 16:15:00 +0300 Subject: [PATCH 058/112] common: timer: use default NoWait param in Timer::Stop Signed-off-by: Mykola Kobets --- src/core/cm/updatemanager/unitstatushandler.cpp | 4 ++-- src/core/common/tools/timer.hpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/cm/updatemanager/unitstatushandler.cpp b/src/core/cm/updatemanager/unitstatushandler.cpp index 5cd6b40a3..14c0a61d0 100644 --- a/src/core/cm/updatemanager/unitstatushandler.cpp +++ b/src/core/cm/updatemanager/unitstatushandler.cpp @@ -142,7 +142,7 @@ Error UnitStatusHandler::SendFullUnitStatus() ClearUnitStatus(); ClearUpdateStatuses(); - mTimer.Stop(Timer::StopMode::NoWait); + mTimer.Stop(); LockGuard lock {mMutex}; @@ -389,7 +389,7 @@ void UnitStatusHandler::OnDisconnect() LockGuard lock {mMutex}; mCloudConnected = false; - mTimer.Stop(Timer::StopMode::NoWait); + mTimer.Stop(); } Error UnitStatusHandler::SetUnitConfigStatus() diff --git a/src/core/common/tools/timer.hpp b/src/core/common/tools/timer.hpp index 77e9d8215..c468c532c 100644 --- a/src/core/common/tools/timer.hpp +++ b/src/core/common/tools/timer.hpp @@ -55,7 +55,7 @@ class Timer { return AOS_ERROR_WRAP(ErrorEnum::eInvalidArgument); } - if (auto err = Stop(StopMode::NoWait); !err.IsNone()) { + if (auto err = Stop(); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -77,7 +77,7 @@ class Timer { } if (oneshot) { - Stop(StopMode::NoWait); + Stop(); } callback(arg); @@ -105,7 +105,7 @@ class Timer { * @param mode specifies whether to wait for currently running callbacks. * @return Error code. */ - Error Stop(StopMode mode); + Error Stop(StopMode mode = StopMode::NoWait); /** * Restarts timer. @@ -114,7 +114,7 @@ class Timer { */ Error Restart() { - if (auto err = Stop(StopMode::NoWait); !err.IsNone()) { + if (auto err = Stop(); !err.IsNone()) { return AOS_ERROR_WRAP(err); } From d8582c71003e1e05c040eccedd73051390ecea87 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Tue, 14 Jul 2026 16:15:51 +0300 Subject: [PATCH 059/112] common: timer: set 5 ms tolerance for timer unit tests Signed-off-by: Mykola Kobets --- src/core/common/tools/tests/timer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/common/tools/tests/timer.cpp b/src/core/common/tools/tests/timer.cpp index 1a784612b..4be146a99 100644 --- a/src/core/common/tools/tests/timer.cpp +++ b/src/core/common/tools/tests/timer.cpp @@ -24,7 +24,7 @@ std::function WrapCallback(MockFunction& cb) MATCHER_P(ApproxEqualTime, expected, "") { - Duration tolerance = 1 * Time::cMilliseconds; + Duration tolerance = 5 * Time::cMilliseconds; Duration diff; if (arg > expected) { diff = arg.UnixNano() - expected.UnixNano(); From 08ddca8439365ed34c102f75b69e712a1b514e19 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Fri, 17 Jul 2026 16:23:00 +0300 Subject: [PATCH 060/112] ci: simplify README badges to develop branch only Drop the main-branch row from the CI/coverage/quality-gate badge table and keep a single set of badges for develop. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- README.md | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 091d94a13..e907893d8 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,8 @@ -| Branch | CI | Coverage | Quality Gate | -|--------|----|----------|--------------| -| main | [![ci][ci-main]][ci-main-link] | [![coverage][cov-main]][cov-main-link] | [![Quality Gate][qg-main]][qg-main-link] | -| develop | [![ci][ci-dev]][ci-dev-link] | [![coverage][cov-dev]][cov-dev-link] | [![Quality Gate][qg-dev]][qg-dev-link] | - -[ci-main]: https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build-test.yaml/badge.svg?branch=main -[ci-main-link]: https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build-test.yaml?query=branch%3Amain -[cov-main]: https://sonarcloud.io/api/project_badges/measure?project=aosedge_aos_core_lib_cpp&metric=coverage&branch=main -[cov-main-link]: https://sonarcloud.io/summary/new_code?id=aosedge_aos_core_lib_cpp&branch=main -[qg-main]: https://sonarcloud.io/api/project_badges/measure?project=aosedge_aos_core_lib_cpp&metric=alert_status&branch=main -[qg-main-link]: https://sonarcloud.io/summary/new_code?id=aosedge_aos_core_lib_cpp&branch=main +[![ci][ci-dev]][ci-dev-link] +[![coverage][cov-dev]][cov-dev-link] +[![Quality Gate][qg-dev]][qg-dev-link] + [ci-dev]: https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build-test.yaml/badge.svg?branch=develop [ci-dev-link]: https://github.com/aosedge/aos_core_lib_cpp/actions/workflows/build-test.yaml?query=branch%3Adevelop [cov-dev]: https://sonarcloud.io/api/project_badges/measure?project=aosedge_aos_core_lib_cpp&metric=coverage&branch=develop From cbdbcc525aac5c2129894bc03fc5d291dd41e054 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Fri, 17 Jul 2026 16:06:41 +0300 Subject: [PATCH 061/112] sm: launcher: tolerate already-exists/not-found errors Start/stop instance and network operations can race with runtime or network manager state that was already updated by a previous operation. Treat eAlreadyExist on start and eNotFound on stop as non-fatal instead of failing the operation, and downgrade the corresponding duplicate/missing instance data log lines from warning/error to debug since they no longer indicate a problem. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Solianko Reviewed-by: Mykola Kobets --- src/core/sm/launcher/launcher.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index 2db99cf27..edcfdfa34 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -728,7 +728,8 @@ Error Launcher::StopInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceDat LOG_INF() << "Stop instance" << Log::Field("instance", instanceData.mInfo) << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID); - if (auto err = runtime->StopInstance(instanceData.mInfo, instanceData.mStatus); !err.IsNone()) { + if (auto err = runtime->StopInstance(instanceData.mInfo, instanceData.mStatus); + !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { return AOS_ERROR_WRAP(err); } @@ -810,7 +811,7 @@ void Launcher::PrepareInstances(const Array& startInstances) for (const auto& instance : startInstances) { auto instanceData = FindInstanceData(instance); if (instanceData) { - LOG_WRN() << "Instance data already exists" << Log::Field("instance", instance); + LOG_DBG() << "Instance data already exists" << Log::Field("instance", instance); SetInstanceState(*instanceData, InstanceStateEnum::eInactive); @@ -852,7 +853,7 @@ Error Launcher::AddStartNetworkTask(InstanceData& instanceData) { if (auto err = mLaunchPool.AddTask([this, &instanceData](void*) { if (auto err = mNetworkManager->StartInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); - !err.IsNone()) { + !err.IsNone() && !err.Is(ErrorEnum::eAlreadyExist)) { LOG_ERR() << "Failed to start network" << Log::Field("instance", instanceData.mInfo) << Log::Field(AOS_ERROR_WRAP(err)); @@ -909,7 +910,7 @@ Error Launcher::AddStopNetworkTask(InstanceData& instanceData) { if (auto err = mLaunchPool.AddTask([this, &instanceData](void*) { if (auto err = mNetworkManager->StopInstanceNetwork(instanceData.mInstanceID, instanceData.mInfo.mOwnerID); - !err.IsNone()) { + !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { LOG_ERR() << "Failed to stop network" << Log::Field("instance", instanceData.mInfo) << Log::Field(AOS_ERROR_WRAP(err)); @@ -930,8 +931,7 @@ void Launcher::StopNetworks(const Array& stopInstances) for (const auto& instance : stopInstances) { auto instanceData = FindInstanceData(instance); if (!instanceData) { - LOG_ERR() << "Failed to stop network" << Log::Field("instance", instance) - << Log::Field(AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance data not found"))); + LOG_DBG() << "Instance already removed" << Log::Field("instance", instance); continue; } @@ -1024,7 +1024,8 @@ Error Launcher::StartInstance(aos::sm::launcher::RuntimeItf* runtime, InstanceDa << Log::Field("runtimeID", instanceData.mInfo.mRuntimeID) << Log::Field("manifestDigest", instanceData.mInfo.mManifestDigest); - if (auto err = runtime->StartInstance(instanceData.mInfo, instanceData.mStatus); !err.IsNone()) { + if (auto err = runtime->StartInstance(instanceData.mInfo, instanceData.mStatus); + !err.IsNone() && !err.Is(ErrorEnum::eAlreadyExist)) { return AOS_ERROR_WRAP(err); } From 2e08ecd1dfc007e1aa3eac907d36664ff15a59a0 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 22 Jul 2026 11:05:30 +0300 Subject: [PATCH 062/112] ci: allow unsafe checkout of fork PR head in build-test pull_request_target checks out the PR head ref/repo directly, which actions/checkout now refuses by default since it can run untrusted fork code with elevated workflow permissions. Explicitly opt in since this checkout is required for the build/test/SonarQube steps. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykola Kobets Reviewed-by: Mykola Solianko Reviewed-by: Mykhailo Lohvynenko --- .github/workflows/build-release.yaml | 2 +- .github/workflows/build-test.yaml | 3 ++- .github/workflows/check-format.yaml | 2 +- .github/workflows/docker-build.yml | 2 +- .github/workflows/lint.yaml | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 445742aa0..353260574 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index c30354fa2..8c1211e78 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -36,11 +36,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} fetch-depth: 0 + allow-unsafe-pr-checkout: true - name: Install build wrapper uses: SonarSource/sonarqube-scan-action/install-build-wrapper@v7 diff --git a/.github/workflows/check-format.yaml b/.github/workflows/check-format.yaml index f6ad6298f..194d6ffbc 100644 --- a/.github/workflows/check-format.yaml +++ b/.github/workflows/check-format.yaml @@ -18,7 +18,7 @@ jobs: name: Formatting Check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index cd2a7cab0..377c23322 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Log in to github container registry uses: docker/login-action@v3 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2672e7ca4..860f44ecf 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 From b769dad294b84ecf127a43b2941b4f08a7722a68 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Mon, 27 Jul 2026 18:04:53 +0300 Subject: [PATCH 063/112] cm: launcher: double the amount of gid-s uid-s for instance removal Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/idpool.hpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/core/cm/launcher/idpool.hpp b/src/core/cm/launcher/idpool.hpp index 7d8fdbcd7..d2cb5868a 100644 --- a/src/core/cm/launcher/idpool.hpp +++ b/src/core/cm/launcher/idpool.hpp @@ -28,8 +28,9 @@ static constexpr auto cGIDRangeEnd = 10000; /** * Max number of locked GIDs simultaneously. + * Double the number of update items to avoid exhausting the range when removing instances. */ -static constexpr auto cMaxNumLockedGIDs = cMaxNumUpdateItems; +static constexpr auto cMaxNumLockedGIDs = 2 * cMaxNumUpdateItems; /** * UID range start. @@ -43,8 +44,9 @@ static constexpr auto cUIDRangeEnd = 10000; /** * Max number of locked UIDs simultaneously. + * Double the number of update items to avoid exhausting the range when removing instances. */ -static constexpr auto cMaxNumLockedUIDs = cMaxNumInstances; +static constexpr auto cMaxNumLockedUIDs = 2 * cMaxNumInstances; /** * Pool that manages identifiers with reference counting per key. @@ -166,10 +168,9 @@ class IDPool { StaticMap mItems; }; -using GIDPool - = IDPool, gid_t, cGIDRangeBegin, cGIDRangeEnd, cMaxNumLockedGIDs, cMaxNumUpdateItems>; +using GIDPool = IDPool, gid_t, cGIDRangeBegin, cGIDRangeEnd, cMaxNumLockedGIDs, cMaxNumLockedGIDs>; -using UIDPool = IDPool; +using UIDPool = IDPool; } // namespace aos::cm::launcher From ea68141a43f07b42ce0bea407c4cc06a5ca17603 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Mon, 27 Jul 2026 17:57:54 +0300 Subject: [PATCH 064/112] cm: launcher: fix crash in cm::launcher nullptr returned with eNoError, which caused a crash in IsSubjectEnabled Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/instancemanager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index 9773fefe9..e69b57c3e 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -563,11 +563,11 @@ RetWithError> InstanceManager::CreateInstance(const Instance if (auto err = newInstance->Init(); !err.IsNone()) { // Do not leave invalid instance in storage. - if (err = newInstance->Remove(); !err.IsNone()) { - LOG_ERR() << "Can't remove instance" << Log::Field(err); + if (auto rmErr = newInstance->Remove(); !rmErr.IsNone()) { + LOG_ERR() << "Can't remove instance" << Log::Field(AOS_ERROR_WRAP(rmErr)); } - return {{}, AOS_ERROR_WRAP(err)}; + return {nullptr, err}; } if (auto [_, err] = newInstance->OverrideEnvVars(mEnvVarsOverrides); !err.IsNone()) { From 1b8775cd66c5df79859a0e9114bc8fe7f239f1d7 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Mon, 27 Jul 2026 18:17:50 +0300 Subject: [PATCH 065/112] cm: launcher: release id pool even storage RemoveInstance failed Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/instance.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/core/cm/launcher/instance.cpp b/src/core/cm/launcher/instance.cpp index a92bb2f85..48fbf62a7 100644 --- a/src/core/cm/launcher/instance.cpp +++ b/src/core/cm/launcher/instance.cpp @@ -408,24 +408,32 @@ Error ServiceInstance::Remove() { LOG_DBG() << "Remove instance" << Log::Field("instanceID", mInfo.mInstanceIdent); + Error firstErr = ErrorEnum::eNone; + if (auto err = mStorageState.Remove(mInfo.mInstanceIdent); !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { - return AOS_ERROR_WRAP(err); + firstErr = AOS_ERROR_WRAP(err); } if (auto err = mStorage.RemoveInstance(mInfo.mInstanceIdent, mInfo.mVersion); !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { - return AOS_ERROR_WRAP(err); + if (firstErr.IsNone()) { + firstErr = AOS_ERROR_WRAP(err); + } } if (auto err = mUIDPool.Release(mInfo.mInstanceIdent); !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { - return AOS_ERROR_WRAP(err); + if (firstErr.IsNone()) { + firstErr = AOS_ERROR_WRAP(err); + } } if (auto err = mGIDPool.Release(mInfo.mInstanceIdent.mItemID); !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { - return AOS_ERROR_WRAP(err); + if (firstErr.IsNone()) { + firstErr = AOS_ERROR_WRAP(err); + } } - return ErrorEnum::eNone; + return firstErr; } Error ServiceInstance::Cache(bool disable) From 49215e4f9c0c0bc659cbc22019330a0fd1ac959b Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 17 Jul 2026 01:37:40 +0300 Subject: [PATCH 066/112] cm: launcher: move override env vars implementation to a dedicated class Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/CMakeLists.txt | 1 + src/core/cm/launcher/balancer.cpp | 5 +- src/core/cm/launcher/balancer.hpp | 4 +- src/core/cm/launcher/instancemanager.cpp | 19 --- src/core/cm/launcher/instancemanager.hpp | 11 +- src/core/cm/launcher/launcher.cpp | 125 ++++----------- src/core/cm/launcher/launcher.hpp | 31 ++-- src/core/cm/launcher/nodemanager.cpp | 32 +++- src/core/cm/launcher/nodemanager.hpp | 9 +- .../cm/launcher/overrideenvvarsprocessor.cpp | 149 ++++++++++++++++++ .../cm/launcher/overrideenvvarsprocessor.hpp | 114 ++++++++++++++ 11 files changed, 360 insertions(+), 140 deletions(-) create mode 100644 src/core/cm/launcher/overrideenvvarsprocessor.cpp create mode 100644 src/core/cm/launcher/overrideenvvarsprocessor.hpp diff --git a/src/core/cm/launcher/CMakeLists.txt b/src/core/cm/launcher/CMakeLists.txt index 701910421..26d1f67ce 100644 --- a/src/core/cm/launcher/CMakeLists.txt +++ b/src/core/cm/launcher/CMakeLists.txt @@ -22,6 +22,7 @@ set(SOURCES launcher.cpp node.cpp nodemanager.cpp + overrideenvvarsprocessor.cpp runrequestsloader.cpp storagestate.cpp ) diff --git a/src/core/cm/launcher/balancer.cpp b/src/core/cm/launcher/balancer.cpp index 6c1d15138..8f8b4e3f5 100644 --- a/src/core/cm/launcher/balancer.cpp +++ b/src/core/cm/launcher/balancer.cpp @@ -24,7 +24,8 @@ void Balancer::Init(InstanceManager& instanceManager, ImageInfoProvider& imageIn mRunner = &runner; } -Error Balancer::RunInstances(UniqueLock& lock, Array>& instances, bool rebalancing) +Error Balancer::RunInstances(UniqueLock& lock, Array>& instances, bool rebalancing, + const OverrideEnvVarsRequest& overrideEnvVars) { if (auto err = PrepareForBalancing(rebalancing); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -47,7 +48,7 @@ Error Balancer::RunInstances(UniqueLock& lock, Array> } if (auto err = mNodeManager->SendScheduledInstances( - lock, mInstanceManager->GetActiveInstances(), mInstanceManager->GetRunningInstances()); + lock, mInstanceManager->GetActiveInstances(), mInstanceManager->GetRunningInstances(), overrideEnvVars); !err.IsNone()) { return AOS_ERROR_WRAP(err); } diff --git a/src/core/cm/launcher/balancer.hpp b/src/core/cm/launcher/balancer.hpp index 9d0846b49..b3b9de83a 100644 --- a/src/core/cm/launcher/balancer.hpp +++ b/src/core/cm/launcher/balancer.hpp @@ -43,9 +43,11 @@ class Balancer { * * @param lock lock on the balancing mutex. * @param rebalancing flag indicating rebalancing. + * @param overrideEnvVars override environment variables applied to scheduled instances. * @return Error. */ - Error RunInstances(UniqueLock& lock, Array>& instances, bool rebalancing); + Error RunInstances(UniqueLock& lock, Array>& instances, bool rebalancing, + const OverrideEnvVarsRequest& overrideEnvVars); /** * Loads Service Manager (SM) data for active instances that were loaded from storage. diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index e69b57c3e..c4f024be2 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -570,10 +570,6 @@ RetWithError> InstanceManager::CreateInstance(const Instance return {nullptr, err}; } - if (auto [_, err] = newInstance->OverrideEnvVars(mEnvVarsOverrides); !err.IsNone()) { - return {{}, AOS_ERROR_WRAP(err)}; - } - return newInstance; } @@ -651,10 +647,6 @@ Error InstanceManager::SetStatus(const InstanceStatus& status) Error InstanceManager::ScheduleInstance(SharedPtr& instance, NodeItf& node, const String& runtimeID) { - if (auto [_, overrideErr] = instance->OverrideEnvVars(mEnvVarsOverrides); !overrideErr.IsNone()) { - return AOS_ERROR_WRAP(overrideErr); - } - if (auto err = instance->Schedule(node, runtimeID); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -677,17 +669,6 @@ Error InstanceManager::ScheduleInstance(SharedPtr& instance, const Err return ErrorEnum::eNone; } -bool InstanceManager::OverrideEnvVars(const OverrideEnvVarsRequest& envVars) -{ - if (mEnvVarsOverrides.mItems == envVars.mItems) { - return false; - } - - mEnvVarsOverrides = envVars; - - return true; -} - SharedPtr InstanceManager::FindReadyInstance(const InstanceIdent& id, const String& version) { auto instance = FindScheduledInstance(id, version); diff --git a/src/core/cm/launcher/instancemanager.hpp b/src/core/cm/launcher/instancemanager.hpp index 8600fdd38..29827465c 100644 --- a/src/core/cm/launcher/instancemanager.hpp +++ b/src/core/cm/launcher/instancemanager.hpp @@ -248,14 +248,6 @@ class InstanceManager { */ Error ScheduleInstance(SharedPtr& instance, const Error& error); - /** - * Overrides environment variables. - * - * @param envVars environment variables. - * @return bool true if env vars changed, false otherwise. - */ - bool OverrideEnvVars(const OverrideEnvVarsRequest& envVars); - private: static constexpr auto cRemovePeriod = Time::cDay; // LoadInstancesFromStorage: 1 StaticArray alive throughout loop @@ -323,8 +315,7 @@ class InstanceManager { StaticArray mPreinstalledComponents; StaticArray mRunningInstances; - SubjectArray mSubjects; - OverrideEnvVarsRequest mEnvVarsOverrides; + SubjectArray mSubjects; }; /** diff --git a/src/core/cm/launcher/launcher.cpp b/src/core/cm/launcher/launcher.cpp index f2b305c58..632899fbf 100644 --- a/src/core/cm/launcher/launcher.cpp +++ b/src/core/cm/launcher/launcher.cpp @@ -38,7 +38,7 @@ Error Launcher::Init(const Config& config, nodeinfoprovider::NodeInfoProviderItf unitconfig::NodeConfigProviderItf& nodeConfigProvider, storagestate::StorageStateItf& storageState, MonitoringProviderItf& monitorProvider, alerts::AlertsProviderItf& alertsProvider, iamclient::IdentProviderItf& identProvider, IdentifierPoolValidator gidValidator, - IdentifierPoolValidator uidValidator, StorageItf& storage) + IdentifierPoolValidator uidValidator, StorageItf& storage, SenderItf& sender) { LOG_DBG() << "Init Launcher"; @@ -51,6 +51,7 @@ Error Launcher::Init(const Config& config, nodeinfoprovider::NodeInfoProviderItf mMonitorProvider = &monitorProvider; mAlertsProvider = &alertsProvider; mIdentProvider = &identProvider; + mSender = &sender; auto err = mInstanceManager.Init(config, itemInfoProvider, storageState, ociSpec, gidValidator, uidValidator, storage); @@ -64,6 +65,10 @@ Error Launcher::Init(const Config& config, nodeinfoprovider::NodeInfoProviderItf mNodeManager.Init(*mNodeInfoProvider, *mNodeConfigProvider, *mRunner); mBalancer.Init(mInstanceManager, mImageInfoProvider, mNodeManager, *mMonitorProvider, *mRunner); + if (err = mOverrideEnvVarsProcessor.Init(config, storage, sender, *this); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + return ErrorEnum::eNone; } @@ -119,41 +124,24 @@ Error Launcher::Start() return AOS_ERROR_WRAP(err); } - // Load env vars overrides. - if (auto err = LoadEnvVarsOverrides(); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - auto onEnvVarsTTLTimerTick = [this](void*) { - UniqueLock updateLock {mUpdateMutex}; - - if (auto err = ProcessOverrideEnvVars(mOverrideEnvVars); !err.IsNone()) { - LOG_ERR() << "Update override env vars failed" << Log::Field(err); - } - }; - - if (auto err = mEnvVarsTTLTimer.Start(mConfig.mCheckOverrideEnvVarsPeriod, onEnvVarsTTLTimerTick, false); - !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - // Load SM data for active instances. if (auto err = mBalancer.LoadSMDataForActiveInstances(); !err.IsNone()) { LOG_ERR() << "Can't load SM data for active instances" << Log::Field(err); } + // Load env vars overrides and start TTL check timer; flag an update if some expired while offline. + if (auto [changed, err] = mOverrideEnvVarsProcessor.Start(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } else { + mIsOverrideEnvVarsChanged = changed; + } + // Start process updates thread. mDisableProcessUpdates = false; mUpdatedNodes.Clear(); mNewSubjects.SetValue(*subjects); // Check subjects after startup. UpdateInstanceStatuses(); - - // Check for override env var TTL and setup update if needed. - if (auto err = ProcessOverrideEnvVars(mOverrideEnvVars); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - ProcessNotScheduledInstances(); if (auto err = mWorkerThread.Run([this](void*) { ProcessUpdate(); }); !err.IsNone()) { @@ -202,7 +190,7 @@ Error Launcher::Stop() return err; } - if (auto err = mEnvVarsTTLTimer.Stop(Timer::StopMode::WaitForCallbacks); !err.IsNone()) { + if (auto err = mOverrideEnvVarsProcessor.Stop(); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -295,9 +283,10 @@ Error Launcher::OverrideEnvVars(const OverrideEnvVarsRequest& envVars) { LOG_DBG() << "Override env vars"; - LockGuard updateLock {mUpdateMutex}; + // Don't hold mUpdateMutex: the listener takes it, which would deadlock. + auto [_, err] = mOverrideEnvVarsProcessor.OverrideEnvVars(envVars); - return ProcessOverrideEnvVars(envVars); + return err; } /*********************************************************************************************************************** @@ -402,7 +391,7 @@ Error Launcher::BalanceInstances(UniqueLock& lock, bool rebalance) auto instances = MakeUnique, cMaxNumInstances>>(&mAllocator); mRunRequestsLoader.CreateInstances(mNodeManager.GetNodes(), *instances); - auto runErr = mBalancer.RunInstances(lock, *instances, rebalance); + auto runErr = mBalancer.RunInstances(lock, *instances, rebalance, mOverrideEnvVarsProcessor.GetOverrideEnvVars()); FailActivatingInstances(); UpdateInstanceStatuses(); @@ -461,29 +450,16 @@ void Launcher::ProcessUpdate() doRebalance = true; } - // Process override environment variables changed. + // On override env vars change resend all nodes; bool forceRestart = false; if (mIsOverrideEnvVarsChanged) { mIsOverrideEnvVarsChanged = false; - mUpdatedNodes.Clear(); - - for (auto& instance : mInstanceManager.GetActiveInstances()) { - if (auto [changed, overrideErr] = instance->OverrideEnvVars(mOverrideEnvVars); !overrideErr.IsNone()) { - LOG_ERR() << "Failed to override env vars" << Log::Field(AOS_ERROR_WRAP(overrideErr)); + forceRestart = true; - continue; - } else { - if (changed) { - err = PushUnique(mUpdatedNodes, instance->GetInfo().mNodeID); - if (!err.IsNone()) { - LOG_ERR() << "Failed to add node ID to updated nodes" << Log::Field(AOS_ERROR_WRAP(err)); - - continue; - } - - forceRestart = true; - } + for (const auto& node : mNodeManager.GetNodes()) { + if (auto pushErr = PushUnique(mUpdatedNodes, node.GetInfo().mNodeID); !pushErr.IsNone()) { + LOG_ERR() << "Failed to add node to updated nodes" << Log::Field(AOS_ERROR_WRAP(pushErr)); } } } @@ -492,7 +468,8 @@ void Launcher::ProcessUpdate() if (!mUpdatedNodes.IsEmpty()) { if (!doRebalance) { err = mNodeManager.ResendInstances(updateLock, mUpdatedNodes, mInstanceManager.GetActiveInstances(), - mInstanceManager.GetRunningInstances(), forceRestart); + mInstanceManager.GetRunningInstances(), mOverrideEnvVarsProcessor.GetOverrideEnvVars(), + forceRestart); if (!err.IsNone()) { LOG_ERR() << "Failed to resend instances" << Log::Field(AOS_ERROR_WRAP(err)); } @@ -527,49 +504,6 @@ void Launcher::WaitAllNodesConnected(UniqueLock& lock) mAllNodesConnectedCondVar.Wait(lock, allNodesConnected); } -Error Launcher::LoadEnvVarsOverrides() -{ - // Restore override environment variables without TTL check, so we can detect changes in ProcessOverrideEnvVars(). - if (auto err = mStorage->LoadOverrideEnvVars(mOverrideEnvVars); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - mInstanceManager.OverrideEnvVars(mOverrideEnvVars); - - return ErrorEnum::eNone; -} - -Error Launcher::ProcessOverrideEnvVars(const OverrideEnvVarsRequest& envVars) -{ - mOverrideEnvVars = envVars; - - // Remove variables with expired TTLs. - auto now = Time::Now(); - - for (auto& item : mOverrideEnvVars.mItems) { - item.mVariables.RemoveIf([&now](const EnvVarInfo& envVarInfo) { - return envVarInfo.mTTL.HasValue() && envVarInfo.mTTL.GetValue() < now; - }); - } - - mOverrideEnvVars.mItems.RemoveIf([](const EnvVarsInstanceInfo& item) { return item.mVariables.IsEmpty(); }); - - // Save override environment variables. - if (!mInstanceManager.OverrideEnvVars(mOverrideEnvVars)) { - return ErrorEnum::eNone; - } - - if (auto err = mStorage->SaveOverrideEnvVars(mOverrideEnvVars); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - mIsOverrideEnvVarsChanged = true; - - mProcessUpdatesCondVar.NotifyAll(); - - return ErrorEnum::eNone; -} - void Launcher::ProcessNotScheduledInstances() { bool hasNotScheduledInstance = mInstanceManager.GetActiveInstances().ContainsIf( @@ -687,4 +621,13 @@ void Launcher::SubjectsChanged(const Array>& subjects) mProcessUpdatesCondVar.NotifyAll(); } +void Launcher::OnOverrideEnvVarsChanged() +{ + LockGuard updateLock {mUpdateMutex}; + + mIsOverrideEnvVarsChanged = true; + + mProcessUpdatesCondVar.NotifyAll(); +} + } // namespace aos::cm::launcher diff --git a/src/core/cm/launcher/launcher.hpp b/src/core/cm/launcher/launcher.hpp index 6969dbeaf..f717d5d61 100644 --- a/src/core/cm/launcher/launcher.hpp +++ b/src/core/cm/launcher/launcher.hpp @@ -18,11 +18,13 @@ #include "itf/envvarhandler.hpp" #include "itf/instancestatusreceiver.hpp" #include "itf/launcher.hpp" +#include "itf/sender.hpp" #include "itf/storage.hpp" #include "balancer.hpp" #include "instancemanager.hpp" #include "nodemanager.hpp" +#include "overrideenvvarsprocessor.hpp" #include "runrequestsloader.hpp" namespace aos::cm::launcher { @@ -39,7 +41,8 @@ class Launcher : public LauncherItf, public EnvVarHandlerItf, private nodeinfoprovider::NodeInfoListenerItf, private alerts::AlertsListenerItf, - private iamclient::SubjectsListenerItf { + private iamclient::SubjectsListenerItf, + private OverrideEnvVarsListenerItf { public: /** * Initializes launcher object instance. @@ -57,6 +60,7 @@ class Launcher : public LauncherItf, * @param gidValidator GID validator. * @param uidValidator UID validator. * @param storage storage interface. + * @param sender sender interface. * @return Error. */ Error Init(const Config& config, nodeinfoprovider::NodeInfoProviderItf& nodeInfoProvider, InstanceRunnerItf& runner, @@ -64,7 +68,7 @@ class Launcher : public LauncherItf, unitconfig::NodeConfigProviderItf& nodeConfigProvider, storagestate::StorageStateItf& storageState, MonitoringProviderItf& monitorProvider, alerts::AlertsProviderItf& alertsProvider, iamclient::IdentProviderItf& identProvider, IdentifierPoolValidator gidValidator, - IdentifierPoolValidator uidValidator, StorageItf& storage); + IdentifierPoolValidator uidValidator, StorageItf& storage, SenderItf& sender); /** * Starts launcher instance. @@ -146,9 +150,7 @@ class Launcher : public LauncherItf, void ProcessUpdate(); void WaitAllNodesConnected(UniqueLock& lock); - Error LoadEnvVarsOverrides(); - Error ProcessOverrideEnvVars(const OverrideEnvVarsRequest& envVars); - void ProcessNotScheduledInstances(); + void ProcessNotScheduledInstances(); // InstanceStatusReceiverItf implementation Error OnInstanceStatusReceived(const InstanceStatus& status) override; @@ -163,6 +165,9 @@ class Launcher : public LauncherItf, // iamclient::SubjectsListenerItf implementation void SubjectsChanged(const Array>& subjects) override; + // OverrideEnvVarsListenerItf implementation + void OnOverrideEnvVarsChanged() override; + // External dependencies Config mConfig; StorageItf* mStorage {}; @@ -173,14 +178,16 @@ class Launcher : public LauncherItf, storagestate::StorageStateItf* mStorageState {}; MonitoringProviderItf* mMonitorProvider {}; alerts::AlertsProviderItf* mAlertsProvider {}; + SenderItf* mSender {}; StaticArray mInstanceStatusListeners; // Managers - RunRequestsLoader mRunRequestsLoader {}; - InstanceManager mInstanceManager {}; - NodeManager mNodeManager {}; - ImageInfoProvider mImageInfoProvider {}; - Balancer mBalancer {}; + RunRequestsLoader mRunRequestsLoader {}; + InstanceManager mInstanceManager {}; + NodeManager mNodeManager {}; + ImageInfoProvider mImageInfoProvider {}; + Balancer mBalancer {}; + OverrideEnvVarsProcessor mOverrideEnvVarsProcessor {}; // Process update thread Thread<> mWorkerThread; @@ -194,9 +201,7 @@ class Launcher : public LauncherItf, bool mForceRebalance {}; // Override environment variables - OverrideEnvVarsRequest mOverrideEnvVars; - Timer mEnvVarsTTLTimer; - bool mIsOverrideEnvVarsChanged {}; + bool mIsOverrideEnvVarsChanged {}; // Misc StaticArray mInstanceStatuses; diff --git a/src/core/cm/launcher/nodemanager.cpp b/src/core/cm/launcher/nodemanager.cpp index 63745cd70..499077918 100644 --- a/src/core/cm/launcher/nodemanager.cpp +++ b/src/core/cm/launcher/nodemanager.cpp @@ -197,11 +197,34 @@ Array& NodeManager::GetNodes() return mNodes; } +Error NodeManager::ApplyOverrideEnvVars( + const Array>& instances, const OverrideEnvVarsRequest& overrideEnvVars) +{ + Error firstErr = ErrorEnum::eNone; + + for (auto& instance : instances) { + if (auto [changed, err] = instance->OverrideEnvVars(overrideEnvVars); !err.IsNone()) { + LOG_ERR() << "Can't override env vars" << Log::Field("instance", instance->GetInfo().mInstanceIdent) + << Log::Field(err); + + if (firstErr.IsNone()) { + firstErr = err; + } + } + } + + return firstErr; +} + Error NodeManager::SendScheduledInstances(UniqueLock& lock, const Array>& scheduledInstances, - const Array& runningInstances) + const Array& runningInstances, const OverrideEnvVarsRequest& overrideEnvVars) { Error firstErr = ErrorEnum::eNone; + if (auto err = ApplyOverrideEnvVars(scheduledInstances, overrideEnvVars); !err.IsNone()) { + return err; + } + for (auto& node : FilterActiveNodes(mNodes)) { auto err = node.SendScheduledInstances(scheduledInstances, runningInstances); if (!err.IsNone()) { @@ -237,10 +260,15 @@ Error NodeManager::SendScheduledInstances(UniqueLock& lock, const Array& lock, const Array>& updatedNodes, - const Array>& activeInstances, const Array& runningInstances, bool forceRestart) + const Array>& activeInstances, const Array& runningInstances, + const OverrideEnvVarsRequest& overrideEnvVars, bool forceRestart) { Error firstErr = ErrorEnum::eNone; + if (auto err = ApplyOverrideEnvVars(activeInstances, overrideEnvVars); !err.IsNone()) { + return err; + } + mNodesExpectedToSendStatus.Clear(); for (auto& node : FilterActiveNodes(mNodes)) { diff --git a/src/core/cm/launcher/nodemanager.hpp b/src/core/cm/launcher/nodemanager.hpp index 83ba69dc5..1fc0162c4 100644 --- a/src/core/cm/launcher/nodemanager.hpp +++ b/src/core/cm/launcher/nodemanager.hpp @@ -120,10 +120,11 @@ class NodeManager { * @param lock mutex lock. * @param scheduledInstances scheduled instances. * @param runningInstances running instances. + * @param overrideEnvVars override environment variables applied to scheduled instances. * @return Error. */ Error SendScheduledInstances(UniqueLock& lock, const Array>& scheduledInstances, - const Array& runningInstances); + const Array& runningInstances, const OverrideEnvVarsRequest& overrideEnvVars); /** * Resends instances to nodes and waits for instance statuses from them. @@ -132,12 +133,13 @@ class NodeManager { * @param updatedNodes updated nodes. * @param activeInstances active instances. * @param runningInstances running instances. + * @param overrideEnvVars override environment variables applied to active instances. * @param forceRestart force restart instances. * @return Error. */ Error ResendInstances(UniqueLock& lock, const Array>& updatedNodes, const Array>& activeInstances, const Array& runningInstances, - bool forceRestart = false); + const OverrideEnvVarsRequest& overrideEnvVars, bool forceRestart = false); private: static constexpr auto cStatusUpdateTimeout = Time::cMinutes * 10; @@ -150,6 +152,9 @@ class NodeManager { Error FindImageDescriptor(const String& itemID, const String& version, const String& manifestDigest, ImageInfoProvider& imageInfoProvider, oci::IndexContentDescriptor& imageDescriptor); + Error ApplyOverrideEnvVars( + const Array>& instances, const OverrideEnvVarsRequest& overrideEnvVars); + nodeinfoprovider::NodeInfoProviderItf* mNodeInfoProvider {}; unitconfig::NodeConfigProviderItf* mNodeConfigProvider {}; InstanceRunnerItf* mRunner {}; diff --git a/src/core/cm/launcher/overrideenvvarsprocessor.cpp b/src/core/cm/launcher/overrideenvvarsprocessor.cpp new file mode 100644 index 000000000..92cd50c13 --- /dev/null +++ b/src/core/cm/launcher/overrideenvvarsprocessor.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "overrideenvvarsprocessor.hpp" + +namespace aos::cm::launcher { + +/*********************************************************************************************************************** + * Public + **********************************************************************************************************************/ + +Error OverrideEnvVarsProcessor::Init( + const Config& config, StorageItf& storage, SenderItf& envVarStatusSender, OverrideEnvVarsListenerItf& listener) +{ + LOG_DBG() << "Init override env vars processor"; + + mCheckPeriod = config.mCheckOverrideEnvVarsPeriod; + mStorage = &storage; + mEnvVarStatusSender = &envVarStatusSender; + mListener = &listener; + + return ErrorEnum::eNone; +} + +RetWithError OverrideEnvVarsProcessor::Start() +{ + LOG_DBG() << "Start override env vars processor"; + + bool changed = false; + Error err; + + { + LockGuard lock {mMutex}; + + if (auto loadErr = mStorage->LoadOverrideEnvVars(mOverrideEnvVars); !loadErr.IsNone()) { + return {false, AOS_ERROR_WRAP(loadErr)}; + } + + // Drop variables that expired while offline. + Tie(changed, err) = ProcessOverrideEnvVars(mOverrideEnvVars); + if (!err.IsNone()) { + return {changed, err}; + } + } + + if (auto timerErr = mTimer.Start( + mCheckPeriod, [this](void*) { OnTTLTimerTick(); }, false); + !timerErr.IsNone()) { + return {changed, AOS_ERROR_WRAP(timerErr)}; + } + + return {changed, ErrorEnum::eNone}; +} + +Error OverrideEnvVarsProcessor::Stop() +{ + LOG_DBG() << "Stop override env vars processor"; + + if (auto err = mTimer.Stop(Timer::StopMode::WaitForCallbacks); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + +RetWithError OverrideEnvVarsProcessor::OverrideEnvVars(const OverrideEnvVarsRequest& envVars) +{ + bool changed = false; + Error err; + + { + LockGuard lock {mMutex}; + + Tie(changed, err) = ProcessOverrideEnvVars(envVars); + } + + // Notify outside mMutex to avoid lock order inversion. + if (changed && err.IsNone()) { + mListener->OnOverrideEnvVarsChanged(); + } + + return {changed, err}; +} + +/*********************************************************************************************************************** + * Private + **********************************************************************************************************************/ + +void OverrideEnvVarsProcessor::OnTTLTimerTick() +{ + if (auto [_, err] = OverrideEnvVars(mOverrideEnvVars); !err.IsNone()) { + LOG_ERR() << "Update override env vars failed" << Log::Field(err); + } +} + +RetWithError OverrideEnvVarsProcessor::ProcessOverrideEnvVars(const OverrideEnvVarsRequest& envVars) +{ + auto now = Time::Now(); + bool changed = mOverrideEnvVars.mItems != envVars.mItems || HasExpiredVariables(envVars, now); + + LOG_DBG() << "Process override env vars" << Log::Field("changed", changed) + << Log::Field("count", envVars.mItems.Size()); + + if (changed) { + // envVars may alias mOverrideEnvVars on TTL recheck. + if (&mOverrideEnvVars != &envVars) { + mOverrideEnvVars = envVars; + } + + RemoveExpiredVariables(mOverrideEnvVars, now); + + if (auto err = mStorage->SaveOverrideEnvVars(mOverrideEnvVars); !err.IsNone()) { + return {changed, AOS_ERROR_WRAP(err)}; + } + } + + return {changed, ErrorEnum::eNone}; +} + +bool OverrideEnvVarsProcessor::HasExpiredVariables(const OverrideEnvVarsRequest& envVars, const Time& now) +{ + for (const auto& item : envVars.mItems) { + for (const auto& envVar : item.mVariables) { + if (envVar.mTTL.HasValue() && envVar.mTTL.GetValue() < now) { + return true; + } + } + } + + return false; +} + +void OverrideEnvVarsProcessor::RemoveExpiredVariables(OverrideEnvVarsRequest& envVars, const Time& now) +{ + for (auto& item : envVars.mItems) { + item.mVariables.RemoveIf([&now](const EnvVarInfo& envVarInfo) { + return envVarInfo.mTTL.HasValue() && envVarInfo.mTTL.GetValue() < now; + }); + } + + envVars.mItems.RemoveIf([](const EnvVarsInstanceInfo& item) { return item.mVariables.IsEmpty(); }); +} + +} // namespace aos::cm::launcher diff --git a/src/core/cm/launcher/overrideenvvarsprocessor.hpp b/src/core/cm/launcher/overrideenvvarsprocessor.hpp new file mode 100644 index 000000000..6eb5853de --- /dev/null +++ b/src/core/cm/launcher/overrideenvvarsprocessor.hpp @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_CM_LAUNCHER_OVERRIDEENVVARSPROCESSOR_HPP_ +#define AOS_CORE_CM_LAUNCHER_OVERRIDEENVVARSPROCESSOR_HPP_ + +#include +#include +#include + +#include "itf/sender.hpp" +#include "itf/storage.hpp" + +#include "config.hpp" + +namespace aos::cm::launcher { + +/** @addtogroup cm Communication Manager + * @{ + */ + +/** + * Override environment variables listener interface. + */ +class OverrideEnvVarsListenerItf { +public: + /** + * Destructor. + */ + virtual ~OverrideEnvVarsListenerItf() = default; + + /** + * Notifies that the override env vars has changed (new request or TTL expiry). + */ + virtual void OnOverrideEnvVarsChanged() = 0; +}; + +/** + * Processes override environment variables requests. + */ +class OverrideEnvVarsProcessor { +public: + /** + * Initializes override env vars processor. + * + * @param config launcher configuration. + * @param storage storage interface. + * @param envVarStatusSender override env vars status sender interface. + * @param listener override env vars change listener. + * @return Error. + */ + Error Init( + const Config& config, StorageItf& storage, SenderItf& envVarStatusSender, OverrideEnvVarsListenerItf& listener); + + /** + * Starts override env vars processor. + * + * The change is reported via the return value instead of the listener: on start the caller holds the launcher + * mutex (which the listener also takes, so notifying would deadlock) and the worker thread is not running yet. + * + * @return RetWithError true if the restored override set changed due to expired variables. + */ + RetWithError Start(); + + /** + * Stops override env vars processor. + * + * @return Error. + */ + Error Stop(); + + /** + * Overrides environment variables. + * + * @param envVars requested override environment variables. + * @return RetWithError true if the override env vars changed. + */ + RetWithError OverrideEnvVars(const OverrideEnvVarsRequest& envVars); + + /** + * Returns current effective override environment variables. + * + * @return const OverrideEnvVarsRequest&. + */ + const OverrideEnvVarsRequest& GetOverrideEnvVars() const { return mOverrideEnvVars; } + +private: + static void RemoveExpiredVariables(OverrideEnvVarsRequest& envVars, const Time& now); + static bool HasExpiredVariables(const OverrideEnvVarsRequest& envVars, const Time& now); + + void OnTTLTimerTick(); + RetWithError ProcessOverrideEnvVars(const OverrideEnvVarsRequest& envVars); + Error SendStatuses(const OverrideEnvVarsRequest& envVars); + + Duration mCheckPeriod {}; + StorageItf* mStorage {}; + SenderItf* mEnvVarStatusSender {}; + OverrideEnvVarsListenerItf* mListener {}; + + Mutex mMutex; + Timer mTimer; + + OverrideEnvVarsRequest mOverrideEnvVars; + OverrideEnvVarsStatuses mEnvVarStatuses; +}; + +/** @}*/ + +} // namespace aos::cm::launcher + +#endif From fe9a50bf4071ffd90174863ee95e24f686236c0a Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 17 Jul 2026 01:55:44 +0300 Subject: [PATCH 067/112] cm: launcher: fix unit tests Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/tests/launcher.cpp | 164 +++++++++--------- .../cm/launcher/tests/stubs/senderstub.hpp | 31 ++++ 2 files changed, 114 insertions(+), 81 deletions(-) create mode 100644 src/core/cm/launcher/tests/stubs/senderstub.hpp diff --git a/src/core/cm/launcher/tests/launcher.cpp b/src/core/cm/launcher/tests/launcher.cpp index 47756faf9..754952343 100644 --- a/src/core/cm/launcher/tests/launcher.cpp +++ b/src/core/cm/launcher/tests/launcher.cpp @@ -26,6 +26,7 @@ #include "stubs/monitoringproviderstub.hpp" #include "stubs/nodeinfoproviderstub.hpp" #include "stubs/resourcemanagerstub.hpp" +#include "stubs/senderstub.hpp" #include "stubs/storagestatestub.hpp" #include "stubs/storagestub.hpp" @@ -153,6 +154,7 @@ class CMLauncherTest : public testing::Test { resourcemanager::ResourceManagerStub mResourceManager; StorageStub mStorage; storagestate::StorageStateStub mStorageState; + SenderStub mSender; Launcher mLauncher; }; @@ -580,11 +582,11 @@ TEST_F(CMLauncherTest, InstancesWithInvalidImageAreRemovedOnStart) mInstanceRunner.Init(mLauncher); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -626,11 +628,11 @@ TEST_F(CMLauncherTest, InstancesWithOutdatedTTLRemovedOnStart) mInstanceRunner.Init(mLauncher); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -691,11 +693,11 @@ TEST_F(CMLauncherTest, CacheInstances) mInstanceRunner.Init(mLauncher); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -789,11 +791,11 @@ TEST_F(CMLauncherTest, Components) mInstanceRunner.Init(mLauncher); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -1481,7 +1483,7 @@ TEST_F(CMLauncherTest, Balancing) ASSERT_TRUE(mLauncher .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, - ValidateGID, ValidateUID, mStorage) + ValidateGID, ValidateUID, mStorage, mSender) .IsNone()); InstanceStatusListenerStub instanceStatusListener; @@ -1583,11 +1585,11 @@ TEST_F(CMLauncherTest, PlatformFiltering) AddItem(cService3, cImageID1, *itemConfig3, CreateImageConfig("x86_64", "generic", "linux", "5.4.0", "feature1")); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -1676,11 +1678,11 @@ TEST_F(CMLauncherTest, ResendInstancesOnMismatchedNodeStatus) }); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -1751,11 +1753,11 @@ TEST_F(CMLauncherTest, SubjectChanged) mInstanceRunner.Init(mLauncher); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); InstanceStatusListenerStub instanceStatusListener; mLauncher.SubscribeListener(instanceStatusListener); @@ -1824,11 +1826,11 @@ TEST_F(CMLauncherTest, TestSentInstanceInfo) mInstanceRunner.Init(mLauncher, true, aos::InstanceStateEnum::eActive); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -1910,11 +1912,11 @@ TEST_F(CMLauncherTest, PreinstalledComponents) mInstanceRunner.SetPreinstalledComponents({preinstalledStatus}); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -1998,11 +2000,11 @@ TEST_F(CMLauncherTest, SetStatusOnStart) ASSERT_TRUE(mStorage.AddInstance(instance2).IsNone()); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); InstanceStatusListenerStub instanceStatusListener; mLauncher.SubscribeListener(instanceStatusListener); @@ -2065,11 +2067,11 @@ TEST_F(CMLauncherTest, OverrideEnvVars) mInstanceRunner.Init(mLauncher, true, aos::InstanceStateEnum::eActive); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); InstanceStatusListenerStub instanceStatusListener; mLauncher.SubscribeListener(instanceStatusListener); @@ -2180,11 +2182,11 @@ TEST_F(CMLauncherTest, MultiNodeInstance) AddItem(cComponent1, cImageID1, *componentConfig, CreateImageConfig()); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); @@ -2310,11 +2312,11 @@ TEST_F(CMLauncherTest, RebalancingWithStoredNotScheduledInstances) mMonitoringProvider.SetAverageMonitoring(cNodeIDRemoteSM1, *remoteMonitoring); // Init launcher. - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); InstanceStatusListenerStub instanceStatusListener; mLauncher.SubscribeListener(instanceStatusListener); @@ -2407,11 +2409,11 @@ TEST_F(CMLauncherTest, CpuAlertRebalancingMovesLowerPriorityService) } // Init launcher. - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); InstanceStatusListenerStub instanceStatusListener; mLauncher.SubscribeListener(instanceStatusListener); @@ -2511,11 +2513,11 @@ TEST_F(CMLauncherTest, ServiceUpdate) mInstanceRunner.Init(mLauncher, true, aos::InstanceStateEnum::eActive); // Init launcher - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); InstanceStatusListenerStub instanceStatusListener; mLauncher.SubscribeListener(instanceStatusListener); @@ -2611,11 +2613,11 @@ TEST_F(CMLauncherTest, UnlimitedSharedResource) mInstanceRunner.Init(mLauncher, true, aos::InstanceStateEnum::eActive); - ASSERT_TRUE( - mLauncher - .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, mResourceManager, - mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, ValidateGID, ValidateUID, mStorage) - .IsNone()); + ASSERT_TRUE(mLauncher + .Init(CreateConfig(), mNodeInfoProvider, mInstanceRunner, mImageStore, mImageStore, + mResourceManager, mStorageState, mMonitoringProvider, mAlertsProvider, mIdentProvider, + ValidateGID, ValidateUID, mStorage, mSender) + .IsNone()); ASSERT_TRUE(mLauncher.Start().IsNone()); diff --git a/src/core/cm/launcher/tests/stubs/senderstub.hpp b/src/core/cm/launcher/tests/stubs/senderstub.hpp new file mode 100644 index 000000000..59cfaad4b --- /dev/null +++ b/src/core/cm/launcher/tests/stubs/senderstub.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2026 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CM_LAUNCHER_STUBS_SENDERSTUB_HPP_ +#define AOS_CM_LAUNCHER_STUBS_SENDERSTUB_HPP_ + +#include + +namespace aos::cm::launcher { + +class SenderStub : public SenderItf { +public: + Error SendOverrideEnvsStatuses(const OverrideEnvVarsStatuses& statuses) override + { + mStatuses = statuses; + + return ErrorEnum::eNone; + } + + const OverrideEnvVarsStatuses& GetOverrideEnvVarsStatuses() const { return mStatuses; } + +private: + OverrideEnvVarsStatuses mStatuses; +}; + +} // namespace aos::cm::launcher + +#endif From d58a08190583a6cbda31ebaf37dca3156e094be8 Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Fri, 17 Jul 2026 02:32:10 +0300 Subject: [PATCH 068/112] cm: launcher: add override env var accessor Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/balancer.cpp | 5 ++- src/core/cm/launcher/balancer.hpp | 4 +-- src/core/cm/launcher/launcher.cpp | 7 ++-- src/core/cm/launcher/nodemanager.cpp | 26 +++++++------- src/core/cm/launcher/nodemanager.hpp | 16 ++++----- .../cm/launcher/overrideenvvarsprocessor.hpp | 35 +++++++++++++++++-- 6 files changed, 60 insertions(+), 33 deletions(-) diff --git a/src/core/cm/launcher/balancer.cpp b/src/core/cm/launcher/balancer.cpp index 8f8b4e3f5..6c1d15138 100644 --- a/src/core/cm/launcher/balancer.cpp +++ b/src/core/cm/launcher/balancer.cpp @@ -24,8 +24,7 @@ void Balancer::Init(InstanceManager& instanceManager, ImageInfoProvider& imageIn mRunner = &runner; } -Error Balancer::RunInstances(UniqueLock& lock, Array>& instances, bool rebalancing, - const OverrideEnvVarsRequest& overrideEnvVars) +Error Balancer::RunInstances(UniqueLock& lock, Array>& instances, bool rebalancing) { if (auto err = PrepareForBalancing(rebalancing); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -48,7 +47,7 @@ Error Balancer::RunInstances(UniqueLock& lock, Array> } if (auto err = mNodeManager->SendScheduledInstances( - lock, mInstanceManager->GetActiveInstances(), mInstanceManager->GetRunningInstances(), overrideEnvVars); + lock, mInstanceManager->GetActiveInstances(), mInstanceManager->GetRunningInstances()); !err.IsNone()) { return AOS_ERROR_WRAP(err); } diff --git a/src/core/cm/launcher/balancer.hpp b/src/core/cm/launcher/balancer.hpp index b3b9de83a..9d0846b49 100644 --- a/src/core/cm/launcher/balancer.hpp +++ b/src/core/cm/launcher/balancer.hpp @@ -43,11 +43,9 @@ class Balancer { * * @param lock lock on the balancing mutex. * @param rebalancing flag indicating rebalancing. - * @param overrideEnvVars override environment variables applied to scheduled instances. * @return Error. */ - Error RunInstances(UniqueLock& lock, Array>& instances, bool rebalancing, - const OverrideEnvVarsRequest& overrideEnvVars); + Error RunInstances(UniqueLock& lock, Array>& instances, bool rebalancing); /** * Loads Service Manager (SM) data for active instances that were loaded from storage. diff --git a/src/core/cm/launcher/launcher.cpp b/src/core/cm/launcher/launcher.cpp index 632899fbf..6f9ed9d14 100644 --- a/src/core/cm/launcher/launcher.cpp +++ b/src/core/cm/launcher/launcher.cpp @@ -62,7 +62,7 @@ Error Launcher::Init(const Config& config, nodeinfoprovider::NodeInfoProviderItf mImageInfoProvider.Init(itemInfoProvider, ociSpec); mRunRequestsLoader.Init(storage, mInstanceManager, mImageInfoProvider); - mNodeManager.Init(*mNodeInfoProvider, *mNodeConfigProvider, *mRunner); + mNodeManager.Init(*mNodeInfoProvider, *mNodeConfigProvider, *mRunner, mOverrideEnvVarsProcessor); mBalancer.Init(mInstanceManager, mImageInfoProvider, mNodeManager, *mMonitorProvider, *mRunner); if (err = mOverrideEnvVarsProcessor.Init(config, storage, sender, *this); !err.IsNone()) { @@ -391,7 +391,7 @@ Error Launcher::BalanceInstances(UniqueLock& lock, bool rebalance) auto instances = MakeUnique, cMaxNumInstances>>(&mAllocator); mRunRequestsLoader.CreateInstances(mNodeManager.GetNodes(), *instances); - auto runErr = mBalancer.RunInstances(lock, *instances, rebalance, mOverrideEnvVarsProcessor.GetOverrideEnvVars()); + auto runErr = mBalancer.RunInstances(lock, *instances, rebalance); FailActivatingInstances(); UpdateInstanceStatuses(); @@ -468,8 +468,7 @@ void Launcher::ProcessUpdate() if (!mUpdatedNodes.IsEmpty()) { if (!doRebalance) { err = mNodeManager.ResendInstances(updateLock, mUpdatedNodes, mInstanceManager.GetActiveInstances(), - mInstanceManager.GetRunningInstances(), mOverrideEnvVarsProcessor.GetOverrideEnvVars(), - forceRestart); + mInstanceManager.GetRunningInstances(), forceRestart); if (!err.IsNone()) { LOG_ERR() << "Failed to resend instances" << Log::Field(AOS_ERROR_WRAP(err)); } diff --git a/src/core/cm/launcher/nodemanager.cpp b/src/core/cm/launcher/nodemanager.cpp index 499077918..40b5a1f1a 100644 --- a/src/core/cm/launcher/nodemanager.cpp +++ b/src/core/cm/launcher/nodemanager.cpp @@ -24,11 +24,13 @@ auto FilterActiveNodes(Array& array) **********************************************************************************************************************/ void NodeManager::Init(nodeinfoprovider::NodeInfoProviderItf& nodeInfoProvider, - unitconfig::NodeConfigProviderItf& nodeConfigProvider, InstanceRunnerItf& runner) + unitconfig::NodeConfigProviderItf& nodeConfigProvider, InstanceRunnerItf& runner, + OverrideEnvVarsProcessor& overrideEnvVarsProcessor) { - mNodeInfoProvider = &nodeInfoProvider; - mNodeConfigProvider = &nodeConfigProvider; - mRunner = &runner; + mNodeInfoProvider = &nodeInfoProvider; + mNodeConfigProvider = &nodeConfigProvider; + mRunner = &runner; + mOverrideEnvVarsProcessor = &overrideEnvVarsProcessor; } Error NodeManager::Start() @@ -197,13 +199,14 @@ Array& NodeManager::GetNodes() return mNodes; } -Error NodeManager::ApplyOverrideEnvVars( - const Array>& instances, const OverrideEnvVarsRequest& overrideEnvVars) +Error NodeManager::ApplyOverrideEnvVars(const Array>& instances) { Error firstErr = ErrorEnum::eNone; + auto overrideEnvVars = mOverrideEnvVarsProcessor->GetOverrideEnvVars(); + for (auto& instance : instances) { - if (auto [changed, err] = instance->OverrideEnvVars(overrideEnvVars); !err.IsNone()) { + if (auto [changed, err] = instance->OverrideEnvVars(*overrideEnvVars); !err.IsNone()) { LOG_ERR() << "Can't override env vars" << Log::Field("instance", instance->GetInfo().mInstanceIdent) << Log::Field(err); @@ -217,11 +220,11 @@ Error NodeManager::ApplyOverrideEnvVars( } Error NodeManager::SendScheduledInstances(UniqueLock& lock, const Array>& scheduledInstances, - const Array& runningInstances, const OverrideEnvVarsRequest& overrideEnvVars) + const Array& runningInstances) { Error firstErr = ErrorEnum::eNone; - if (auto err = ApplyOverrideEnvVars(scheduledInstances, overrideEnvVars); !err.IsNone()) { + if (auto err = ApplyOverrideEnvVars(scheduledInstances); !err.IsNone()) { return err; } @@ -260,12 +263,11 @@ Error NodeManager::SendScheduledInstances(UniqueLock& lock, const Array& lock, const Array>& updatedNodes, - const Array>& activeInstances, const Array& runningInstances, - const OverrideEnvVarsRequest& overrideEnvVars, bool forceRestart) + const Array>& activeInstances, const Array& runningInstances, bool forceRestart) { Error firstErr = ErrorEnum::eNone; - if (auto err = ApplyOverrideEnvVars(activeInstances, overrideEnvVars); !err.IsNone()) { + if (auto err = ApplyOverrideEnvVars(activeInstances); !err.IsNone()) { return err; } diff --git a/src/core/cm/launcher/nodemanager.hpp b/src/core/cm/launcher/nodemanager.hpp index 1fc0162c4..baf7367d5 100644 --- a/src/core/cm/launcher/nodemanager.hpp +++ b/src/core/cm/launcher/nodemanager.hpp @@ -13,6 +13,7 @@ #include #include "node.hpp" +#include "overrideenvvarsprocessor.hpp" namespace aos::cm::launcher { @@ -30,11 +31,12 @@ class NodeManager { * * @param nodeInfoProvider node info provider. * @param nodeConfigProvider node config provider. - * @param storageState storage state interface. * @param runner instance runner interface. + * @param overrideEnvVarsProcessor override env vars processor. */ void Init(nodeinfoprovider::NodeInfoProviderItf& nodeInfoProvider, - unitconfig::NodeConfigProviderItf& nodeConfigProvider, InstanceRunnerItf& runner); + unitconfig::NodeConfigProviderItf& nodeConfigProvider, InstanceRunnerItf& runner, + OverrideEnvVarsProcessor& overrideEnvVarsProcessor); /** * Starts node manager. @@ -120,11 +122,10 @@ class NodeManager { * @param lock mutex lock. * @param scheduledInstances scheduled instances. * @param runningInstances running instances. - * @param overrideEnvVars override environment variables applied to scheduled instances. * @return Error. */ Error SendScheduledInstances(UniqueLock& lock, const Array>& scheduledInstances, - const Array& runningInstances, const OverrideEnvVarsRequest& overrideEnvVars); + const Array& runningInstances); /** * Resends instances to nodes and waits for instance statuses from them. @@ -133,13 +134,12 @@ class NodeManager { * @param updatedNodes updated nodes. * @param activeInstances active instances. * @param runningInstances running instances. - * @param overrideEnvVars override environment variables applied to active instances. * @param forceRestart force restart instances. * @return Error. */ Error ResendInstances(UniqueLock& lock, const Array>& updatedNodes, const Array>& activeInstances, const Array& runningInstances, - const OverrideEnvVarsRequest& overrideEnvVars, bool forceRestart = false); + bool forceRestart = false); private: static constexpr auto cStatusUpdateTimeout = Time::cMinutes * 10; @@ -152,12 +152,12 @@ class NodeManager { Error FindImageDescriptor(const String& itemID, const String& version, const String& manifestDigest, ImageInfoProvider& imageInfoProvider, oci::IndexContentDescriptor& imageDescriptor); - Error ApplyOverrideEnvVars( - const Array>& instances, const OverrideEnvVarsRequest& overrideEnvVars); + Error ApplyOverrideEnvVars(const Array>& instances); nodeinfoprovider::NodeInfoProviderItf* mNodeInfoProvider {}; unitconfig::NodeConfigProviderItf* mNodeConfigProvider {}; InstanceRunnerItf* mRunner {}; + OverrideEnvVarsProcessor* mOverrideEnvVarsProcessor {}; StaticAllocator mAllocator; StaticAllocator mNodeAllocator; diff --git a/src/core/cm/launcher/overrideenvvarsprocessor.hpp b/src/core/cm/launcher/overrideenvvarsprocessor.hpp index 6eb5853de..1e2552d3d 100644 --- a/src/core/cm/launcher/overrideenvvarsprocessor.hpp +++ b/src/core/cm/launcher/overrideenvvarsprocessor.hpp @@ -38,6 +38,35 @@ class OverrideEnvVarsListenerItf { virtual void OnOverrideEnvVarsChanged() = 0; }; +/** + * RAII accessor that keeps the processor mutex locked while the override env vars are accessed. + */ +class OverrideEnvVarsAccessor { +public: + /** + * Locks the mutex and binds the override env vars for the accessor lifetime. + * + * @param mutex processor mutex. + * @param envVars override environment variables. + */ + OverrideEnvVarsAccessor(Mutex& mutex, const OverrideEnvVarsRequest& envVars) + : mLock(mutex) + , mEnvVars(envVars) + { + } + + /** + * Returns the locked override environment variables. + * + * @return const OverrideEnvVarsRequest&. + */ + const OverrideEnvVarsRequest& operator*() const { return mEnvVars; } + +private: + LockGuard mLock; + const OverrideEnvVarsRequest& mEnvVars; +}; + /** * Processes override environment variables requests. */ @@ -81,11 +110,11 @@ class OverrideEnvVarsProcessor { RetWithError OverrideEnvVars(const OverrideEnvVarsRequest& envVars); /** - * Returns current effective override environment variables. + * Returns an RAII accessor that keeps the mutex locked while the current override env vars are read. * - * @return const OverrideEnvVarsRequest&. + * @return OverrideEnvVarsAccessor. */ - const OverrideEnvVarsRequest& GetOverrideEnvVars() const { return mOverrideEnvVars; } + OverrideEnvVarsAccessor GetOverrideEnvVars() { return OverrideEnvVarsAccessor(mMutex, mOverrideEnvVars); } private: static void RemoveExpiredVariables(OverrideEnvVarsRequest& envVars, const Time& now); From 7af20f688d490255e5827207d4f133642df7de2f Mon Sep 17 00:00:00 2001 From: Mykola Kobets Date: Sat, 18 Jul 2026 16:17:35 +0300 Subject: [PATCH 069/112] cm: launcher: send override env vars statuses Signed-off-by: Mykola Kobets Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Oleksandr Grytsov --- src/core/cm/launcher/launcher.cpp | 4 +++ src/core/cm/launcher/nodemanager.cpp | 4 +++ .../cm/launcher/overrideenvvarsprocessor.cpp | 36 +++++++++++++++++++ .../cm/launcher/overrideenvvarsprocessor.hpp | 16 ++++++++- src/core/cm/launcher/tests/launcher.cpp | 32 +++++++++++++++++ .../cm/launcher/tests/stubs/senderstub.hpp | 28 +++++++++++++-- 6 files changed, 117 insertions(+), 3 deletions(-) diff --git a/src/core/cm/launcher/launcher.cpp b/src/core/cm/launcher/launcher.cpp index 6f9ed9d14..b2cf1dde6 100644 --- a/src/core/cm/launcher/launcher.cpp +++ b/src/core/cm/launcher/launcher.cpp @@ -559,6 +559,10 @@ Error Launcher::OnNodeInstancesStatusesReceived(const String& nodeID, const Arra firstErr = err; } + if (auto err = mOverrideEnvVarsProcessor.AddStatuses(statuses); !err.IsNone() && firstErr.IsNone()) { + firstErr = err; + } + if (!firstErr.IsNone()) { return firstErr; } diff --git a/src/core/cm/launcher/nodemanager.cpp b/src/core/cm/launcher/nodemanager.cpp index 40b5a1f1a..e35b345fa 100644 --- a/src/core/cm/launcher/nodemanager.cpp +++ b/src/core/cm/launcher/nodemanager.cpp @@ -259,6 +259,8 @@ Error NodeManager::SendScheduledInstances(UniqueLock& lock, const ArraySendStatuses(); + return ErrorEnum::eNone; } @@ -307,6 +309,8 @@ Error NodeManager::ResendInstances(UniqueLock& lock, const ArraySendStatuses(); + return ErrorEnum::eNone; } diff --git a/src/core/cm/launcher/overrideenvvarsprocessor.cpp b/src/core/cm/launcher/overrideenvvarsprocessor.cpp index 92cd50c13..49405428e 100644 --- a/src/core/cm/launcher/overrideenvvarsprocessor.cpp +++ b/src/core/cm/launcher/overrideenvvarsprocessor.cpp @@ -87,6 +87,42 @@ RetWithError OverrideEnvVarsProcessor::OverrideEnvVars(const OverrideEnvVa return {changed, err}; } +Error OverrideEnvVarsProcessor::AddStatuses(const Array& statuses) +{ + LockGuard lock {mMutex}; + + for (const auto& status : statuses) { + if (status.mEnvVarsStatuses.IsEmpty()) { + continue; + } + + if (auto err = mNewEnvVarStatuses.mStatuses.EmplaceBack(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto& item = mNewEnvVarStatuses.mStatuses.Back(); + + static_cast(item) = static_cast(status); + item.mStatuses = status.mEnvVarsStatuses; + } + + return ErrorEnum::eNone; +} + +void OverrideEnvVarsProcessor::SendStatuses() +{ + LockGuard lock {mMutex}; + + if (mNewEnvVarStatuses != mEnvVarStatuses) { + if (auto err = mEnvVarStatusSender->SendOverrideEnvsStatuses(mNewEnvVarStatuses); !err.IsNone()) { + LOG_ERR() << "Can't send override env vars statuses" << Log::Field(AOS_ERROR_WRAP(err)); + } + } + + mEnvVarStatuses = mNewEnvVarStatuses; + mNewEnvVarStatuses.mStatuses.Clear(); +} + /*********************************************************************************************************************** * Private **********************************************************************************************************************/ diff --git a/src/core/cm/launcher/overrideenvvarsprocessor.hpp b/src/core/cm/launcher/overrideenvvarsprocessor.hpp index 1e2552d3d..b48cbebe7 100644 --- a/src/core/cm/launcher/overrideenvvarsprocessor.hpp +++ b/src/core/cm/launcher/overrideenvvarsprocessor.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include "itf/sender.hpp" #include "itf/storage.hpp" @@ -116,13 +117,25 @@ class OverrideEnvVarsProcessor { */ OverrideEnvVarsAccessor GetOverrideEnvVars() { return OverrideEnvVarsAccessor(mMutex, mOverrideEnvVars); } + /** + * Adds instance env var statuses to the new instance pool. + * + * @param statuses instance statuses. + * @return Error. + */ + Error AddStatuses(const Array& statuses); + + /** + * Sends the accumulated env var statuses to the listener if they changed since the last send. + */ + void SendStatuses(); + private: static void RemoveExpiredVariables(OverrideEnvVarsRequest& envVars, const Time& now); static bool HasExpiredVariables(const OverrideEnvVarsRequest& envVars, const Time& now); void OnTTLTimerTick(); RetWithError ProcessOverrideEnvVars(const OverrideEnvVarsRequest& envVars); - Error SendStatuses(const OverrideEnvVarsRequest& envVars); Duration mCheckPeriod {}; StorageItf* mStorage {}; @@ -134,6 +147,7 @@ class OverrideEnvVarsProcessor { OverrideEnvVarsRequest mOverrideEnvVars; OverrideEnvVarsStatuses mEnvVarStatuses; + OverrideEnvVarsStatuses mNewEnvVarStatuses; }; /** @}*/ diff --git a/src/core/cm/launcher/tests/launcher.cpp b/src/core/cm/launcher/tests/launcher.cpp index 754952343..91b073655 100644 --- a/src/core/cm/launcher/tests/launcher.cpp +++ b/src/core/cm/launcher/tests/launcher.cpp @@ -566,6 +566,30 @@ EnvVar CreateEnvVar(const std::string& name, const std::string& value) return var; } +EnvVarsInstanceStatus CreateEnvVarsInstanceStatus(const InstanceIdent& ident, const std::vector& statuses) +{ + EnvVarsInstanceStatus result; + + static_cast(result) = ident; + + for (const auto& status : statuses) { + result.mStatuses.PushBack(status); + } + + return result; +} + +OverrideEnvVarsStatuses CreateOverrideEnvVarsStatuses(const std::vector& statuses) +{ + OverrideEnvVarsStatuses result; + + for (const auto& status : statuses) { + result.mStatuses.PushBack(status); + } + + return result; +} + /*********************************************************************************************************************** * Tests **********************************************************************************************************************/ @@ -2135,6 +2159,14 @@ TEST_F(CMLauncherTest, OverrideEnvVars) EXPECT_EQ(mInstanceRunner.GetRunRequests(), expectedRunRequests); + // 5) Check override env vars statuses are reported back to the sender. + ASSERT_TRUE(mSender.WaitForSendCount(1, 2s)); + + auto expectedInstanceStatus = CreateEnvVarsInstanceStatus(CreateInstanceIdent(cService1, cSubject1, 0), + {EnvVarStatus {"OVERRIDE_VAR2", ErrorEnum::eNone}, EnvVarStatus {"OVERRIDE_VAR3", ErrorEnum::eNone}}); + + EXPECT_EQ(mSender.GetOverrideEnvVarsStatuses(), CreateOverrideEnvVarsStatuses({expectedInstanceStatus})); + ASSERT_TRUE(mLauncher.Stop().IsNone()); } diff --git a/src/core/cm/launcher/tests/stubs/senderstub.hpp b/src/core/cm/launcher/tests/stubs/senderstub.hpp index 59cfaad4b..920d2f094 100644 --- a/src/core/cm/launcher/tests/stubs/senderstub.hpp +++ b/src/core/cm/launcher/tests/stubs/senderstub.hpp @@ -7,6 +7,10 @@ #ifndef AOS_CM_LAUNCHER_STUBS_SENDERSTUB_HPP_ #define AOS_CM_LAUNCHER_STUBS_SENDERSTUB_HPP_ +#include +#include +#include + #include namespace aos::cm::launcher { @@ -15,15 +19,35 @@ class SenderStub : public SenderItf { public: Error SendOverrideEnvsStatuses(const OverrideEnvVarsStatuses& statuses) override { + std::lock_guard lock {mMutex}; + mStatuses = statuses; + ++mSendCount; + + mCondVar.notify_all(); return ErrorEnum::eNone; } - const OverrideEnvVarsStatuses& GetOverrideEnvVarsStatuses() const { return mStatuses; } + OverrideEnvVarsStatuses GetOverrideEnvVarsStatuses() const + { + std::lock_guard lock {mMutex}; + + return mStatuses; + } + + bool WaitForSendCount(size_t expectedCount, std::chrono::milliseconds timeout) const + { + std::unique_lock lock {mMutex}; + + return mCondVar.wait_for(lock, timeout, [&]() { return mSendCount >= expectedCount; }); + } private: - OverrideEnvVarsStatuses mStatuses; + mutable std::mutex mMutex; + mutable std::condition_variable mCondVar; + OverrideEnvVarsStatuses mStatuses; + size_t mSendCount {}; }; } // namespace aos::cm::launcher From 23a1a05b3f75b79e1e53f6d95ad0359b05c00b79 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Wed, 29 Jul 2026 19:09:36 +0300 Subject: [PATCH 070/112] sm: launcher: implement InitInstances At startup a runtime only knows what it is currently running, while the launcher knows the full set of instances that should exist. Each runtime must reconcile its own state against that list: stop anything not in it and properly initialize already-running instances, without starting anything new yet. Group the stored instances by runtime and call each registered runtime's InitInstances with only the instances that belong to it. Add the corresponding InitInstances mock to RuntimeMock and cover the grouping behavior with a unit test. Grow the launcher allocator to fit the extra per-runtime instances array. Signed-off-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko Reviewed-by: Mykola Kobets Reviewed-by: Mykola Solianko --- src/core/sm/launcher/itf/runtime.hpp | 12 ++++ src/core/sm/launcher/launcher.cpp | 29 +++++++++ src/core/sm/launcher/launcher.hpp | 5 +- src/core/sm/launcher/tests/launcher.cpp | 59 +++++++++++++++++++ .../sm/launcher/tests/mocks/runtimemock.hpp | 1 + 5 files changed, 104 insertions(+), 2 deletions(-) diff --git a/src/core/sm/launcher/itf/runtime.hpp b/src/core/sm/launcher/itf/runtime.hpp index 97b6423bf..8b91ab602 100644 --- a/src/core/sm/launcher/itf/runtime.hpp +++ b/src/core/sm/launcher/itf/runtime.hpp @@ -48,6 +48,18 @@ class RuntimeItf : public monitoring::InstanceMonitoringProviderItf { */ virtual Error GetRuntimeInfo(RuntimeInfo& runtimeInfo) const = 0; + /** + * Initializes instances. + * + * Launcher provides list of known instances to runtime at startup. Runtime should stop all instances that are not + * in the list and properly initialize already running instances. Runtime should not start any instance at this + * stage, it should only prepare them for future start. + * + * @param instancesInfo instances info. + * @return Error. + */ + virtual Error InitInstances(const Array& instancesInfo) = 0; + /** * Start instance. * diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index edcfdfa34..a9ed66b9c 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -79,6 +79,8 @@ Error Launcher::Start() return AOS_ERROR_WRAP(err); } + InitInstances(*storedInstances); + lock.Unlock(); LoadInstancesData(*storedInstances); @@ -361,6 +363,33 @@ void Launcher::OnDisconnect() StartTTLTimer(); } +void Launcher::InitInstances(const Array& instancesInfo) +{ + LOG_DBG() << "Init instances" << Log::Field("numInstances", instancesInfo.Size()); + + for (auto& it : mRuntimes) { + auto runtimeInstances = MakeUnique(&mAllocator); + + for (const auto& instanceInfo : instancesInfo) { + if (instanceInfo.mRuntimeID != it.mSecond) { + continue; + } + + if (auto err = runtimeInstances->PushBack(instanceInfo); !err.IsNone()) { + LOG_ERR() << "Failed to add instance to runtime init list" << Log::Field("instance", instanceInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + + break; + } + } + + if (auto err = it.mFirst->InitInstances(*runtimeInstances); !err.IsNone()) { + LOG_ERR() << "Failed to init instances" << Log::Field("runtimeID", it.mSecond) + << Log::Field(AOS_ERROR_WRAP(err)); + } + } +} + void Launcher::RunRebootThread() { while (true) { diff --git a/src/core/sm/launcher/launcher.hpp b/src/core/sm/launcher/launcher.hpp index 954213834..801caafd3 100644 --- a/src/core/sm/launcher/launcher.hpp +++ b/src/core/sm/launcher/launcher.hpp @@ -173,7 +173,7 @@ class Launcher : public LauncherItf, static constexpr auto cMaxNumSubscribers = 4; static constexpr auto cAllocatorSize = 2 * sizeof(StaticArray) - + 2 * sizeof(InstanceInfoArray) + sizeof(InstanceStatusArray) + + 3 * sizeof(InstanceInfoArray) + sizeof(InstanceStatusArray) + cMaxNumConcurrentItems * (sizeof(oci::ImageConfig) + sizeof(oci::ItemConfig) + Max(sizeof(StaticString) + sizeof(oci::ImageManifest), @@ -181,7 +181,7 @@ class Launcher : public LauncherItf, + Max(sizeof(StaticArray), sizeof(StaticArray) + sizeof(StaticArray)); - static constexpr auto cMaxNumAllocations = 4 + cMaxNumConcurrentItems * 4; + static constexpr auto cMaxNumAllocations = 5 + cMaxNumConcurrentItems * 4; void OnConnect() override; void OnDisconnect() override; @@ -233,6 +233,7 @@ class Launcher : public LauncherItf, void StartTTLTimer(); void StopExpiredInstances(UniqueLock& lock); void SendNodeInstancesStatuses(); + void InitInstances(const Array& instancesInfo); StaticAllocator mAllocator; StaticArray mSubscribers; diff --git a/src/core/sm/launcher/tests/launcher.cpp b/src/core/sm/launcher/tests/launcher.cpp index 429ca0b6b..b9da0b2ae 100644 --- a/src/core/sm/launcher/tests/launcher.cpp +++ b/src/core/sm/launcher/tests/launcher.cpp @@ -162,11 +162,13 @@ class LauncherTest : public Test { EXPECT_CALL(mRuntime0, Stop).WillRepeatedly(Return(ErrorEnum::eNone)); EXPECT_CALL(mRuntime0, GetRuntimeInfo) .WillRepeatedly(DoAll(SetArgReferee<0>(CreateRuntimeInfo("runtime0")), Return(ErrorEnum::eNone))); + EXPECT_CALL(mRuntime0, InitInstances).WillRepeatedly(Return(ErrorEnum::eNone)); EXPECT_CALL(mRuntime1, Start).WillRepeatedly(Return(ErrorEnum::eNone)); EXPECT_CALL(mRuntime1, Stop).WillRepeatedly(Return(ErrorEnum::eNone)); EXPECT_CALL(mRuntime1, GetRuntimeInfo) .WillRepeatedly(DoAll(SetArgReferee<0>(CreateRuntimeInfo("runtime1")), Return(ErrorEnum::eNone))); + EXPECT_CALL(mRuntime1, InitInstances).WillRepeatedly(Return(ErrorEnum::eNone)); mImageManifest.mItemConfig.EmplaceValue(); @@ -246,6 +248,63 @@ TEST_F(LauncherTest, NoStoredInstancesOnModuleStart) ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); } +TEST_F(LauncherTest, InitInstances) +{ + const std::vector cStoredInfos = { + CreateInstanceInfo("item0", 0, "1.0.0", "runtime0"), + CreateInstanceInfo("item1", 1, "1.0.0", "runtime1"), + CreateInstanceInfo("item2", 2, "1.0.0", "runtime0"), + }; + + const std::vector cRuntime0Infos = {cStoredInfos[0], cStoredInfos[2]}; + const std::vector cRuntime1Infos = {cStoredInfos[1]}; + + mStorage.Init(cStoredInfos); + + auto err = mLauncher.Init(GetRuntimesArray(), mImageManager, mSender, mStorage, mOCISpec, mItemInfoProvider, + mCloudConnection, mNetworkManager, mInstanceIDProvider, mResourceInfoProvider); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_CALL(mRuntime0, InitInstances(Array(&cRuntime0Infos.front(), cRuntime0Infos.size()))) + .WillOnce(Return(ErrorEnum::eNone)); + EXPECT_CALL(mRuntime1, InitInstances(Array(&cRuntime1Infos.front(), cRuntime1Infos.size()))) + .WillOnce(Return(ErrorEnum::eNone)); + + EXPECT_CALL(mRuntime0, StartInstance) + .WillRepeatedly(Invoke([](const InstanceInfo& instance, InstanceStatus& status) { + SetInstanceStatus(instance, InstanceStateEnum::eActive, status); + + return ErrorEnum::eNone; + })); + + EXPECT_CALL(mRuntime1, StartInstance) + .WillRepeatedly(Invoke([](const InstanceInfo& instance, InstanceStatus& status) { + SetInstanceStatus(instance, InstanceStateEnum::eActive, status); + + return ErrorEnum::eNone; + })); + + err = mLauncher.Start(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_CALL(mRuntime0, StopInstance) + .WillRepeatedly(Invoke([](const InstanceIdent& instance, InstanceStatus& status) { + SetInstanceStatus(instance, InstanceStateEnum::eInactive, status); + + return ErrorEnum::eNone; + })); + + EXPECT_CALL(mRuntime1, StopInstance) + .WillRepeatedly(Invoke([](const InstanceIdent& instance, InstanceStatus& status) { + SetInstanceStatus(instance, InstanceStateEnum::eInactive, status); + + return ErrorEnum::eNone; + })); + + err = mLauncher.Stop(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + TEST_F(LauncherTest, SendActiveComponentNodeInstancesStatusOnModuleStart) { const std::vector cRuntime0Components = { diff --git a/src/core/sm/launcher/tests/mocks/runtimemock.hpp b/src/core/sm/launcher/tests/mocks/runtimemock.hpp index 8654e1acb..c3a4bd4a7 100644 --- a/src/core/sm/launcher/tests/mocks/runtimemock.hpp +++ b/src/core/sm/launcher/tests/mocks/runtimemock.hpp @@ -21,6 +21,7 @@ class RuntimeMock : public RuntimeItf { MOCK_METHOD(Error, Start, (), (override)); MOCK_METHOD(Error, Stop, (), (override)); MOCK_METHOD(Error, GetRuntimeInfo, (RuntimeInfo&), (const, override)); + MOCK_METHOD(Error, InitInstances, (const Array&), (override)); MOCK_METHOD(Error, StartInstance, (const InstanceInfo&, InstanceStatus&), (override)); MOCK_METHOD(Error, StopInstance, (const InstanceIdent&, InstanceStatus&), (override)); MOCK_METHOD(Error, Reboot, (), (override)); From 030c73bec320cf3bc3b315da4c3888603fedc309 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 30 Jul 2026 16:16:10 +0300 Subject: [PATCH 071/112] sm: networkmanager: add link and netns existence queries InterfaceManagerItf exposed only mutating operations, so there was no way to tell an interface that must be created from one that is already on the system and can be adopted as is. Same for network namespaces. Add InterfaceManagerItf::GetLink returning link kind, master, vlan ID and admin state (eNotFound when the link is absent) and NamespaceManagerItf::IsNetworkNamespaceExist. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../networkmanager/itf/interfacemanager.hpp | 40 +++++++++++++++++++ .../networkmanager/itf/namespacemanager.hpp | 8 ++++ .../tests/mocks/interfacemanagermock.hpp | 1 + .../tests/mocks/namespacemanagermock.hpp | 1 + 4 files changed, 50 insertions(+) diff --git a/src/core/sm/networkmanager/itf/interfacemanager.hpp b/src/core/sm/networkmanager/itf/interfacemanager.hpp index e552eb3f8..2f1ee9418 100644 --- a/src/core/sm/networkmanager/itf/interfacemanager.hpp +++ b/src/core/sm/networkmanager/itf/interfacemanager.hpp @@ -7,7 +7,9 @@ #ifndef AOS_CORE_SM_NETWORKMANAGER_ITF_INTERFACEMANAGER_HPP_ #define AOS_CORE_SM_NETWORKMANAGER_ITF_INTERFACEMANAGER_HPP_ +#include #include +#include namespace aos::sm::networkmanager { @@ -15,6 +17,35 @@ namespace aos::sm::networkmanager { * @{ */ +/** + * Link kind type. + */ +class LinkKindType { +public: + enum class Enum { eUnknown, eBridge, eVlan, eVeth }; + + static const Array GetStrings() + { + static const char* const sLinkKindStrings[] = {"unknown", "bridge", "vlan", "veth"}; + + return Array(sLinkKindStrings, ArraySize(sLinkKindStrings)); + }; +}; + +using LinkKindEnum = LinkKindType::Enum; +using LinkKind = EnumStringer; + +/** + * Network link attributes as seen on the system. + */ +struct LinkInfo { + StaticString mName; + LinkKind mKind; + StaticString mMaster; + uint64_t mVlanID {}; + bool mUp {}; +}; + /** * Network interface manager interface. */ @@ -25,6 +56,15 @@ class InterfaceManagerItf { */ virtual ~InterfaceManagerItf() = default; + /** + * Returns link attributes as they are on the system. + * + * @param ifname interface name. + * @param[out] info link attributes. + * @return Error, eNotFound if the link doesn't exist. + */ + virtual Error GetLink(const String& ifname, LinkInfo& info) const = 0; + /** * Removes interface. * diff --git a/src/core/sm/networkmanager/itf/namespacemanager.hpp b/src/core/sm/networkmanager/itf/namespacemanager.hpp index d14744afa..e5bf90b26 100644 --- a/src/core/sm/networkmanager/itf/namespacemanager.hpp +++ b/src/core/sm/networkmanager/itf/namespacemanager.hpp @@ -33,6 +33,14 @@ class NamespaceManagerItf { */ virtual Error CreateNetworkNamespace(const String& ns) = 0; + /** + * Checks whether network namespace exists on the system. + * + * @param ns network namespace name. + * @return RetWithError. + */ + virtual RetWithError IsNetworkNamespaceExist(const String& ns) const = 0; + /** * Returns network namespace path. * diff --git a/src/core/sm/networkmanager/tests/mocks/interfacemanagermock.hpp b/src/core/sm/networkmanager/tests/mocks/interfacemanagermock.hpp index 9084ea755..36296083c 100644 --- a/src/core/sm/networkmanager/tests/mocks/interfacemanagermock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/interfacemanagermock.hpp @@ -15,6 +15,7 @@ namespace aos::sm::networkmanager { class InterfaceManagerMock : public InterfaceManagerItf { public: + MOCK_METHOD(Error, GetLink, (const String&, LinkInfo&), (const, override)); MOCK_METHOD(Error, DeleteLink, (const String&), (override)); MOCK_METHOD(Error, SetupLink, (const String&, const String&), (override)); MOCK_METHOD(Error, SetMasterLink, (const String&, const String&), (override)); diff --git a/src/core/sm/networkmanager/tests/mocks/namespacemanagermock.hpp b/src/core/sm/networkmanager/tests/mocks/namespacemanagermock.hpp index 30be8bf71..4565a4523 100644 --- a/src/core/sm/networkmanager/tests/mocks/namespacemanagermock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/namespacemanagermock.hpp @@ -16,6 +16,7 @@ namespace aos::sm::networkmanager { class NamespaceManagerMock : public NamespaceManagerItf { public: MOCK_METHOD(Error, CreateNetworkNamespace, (const String&), (override)); + MOCK_METHOD(RetWithError, IsNetworkNamespaceExist, (const String&), (const, override)); MOCK_METHOD(RetWithError>, GetNetworkNamespacePath, (const String&), (const, override)); MOCK_METHOD(Error, DeleteNetworkNamespace, (const String&), (override)); }; From fbb8c59dde790f050f2cf4f3e2aff6cb1f7f4b51 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 30 Jul 2026 16:41:15 +0300 Subject: [PATCH 072/112] sm: networkmanager: adopt existing bridge and vlan links After a crash SM restarts with an empty mPhysicalNetworks, so the first StartInstanceNetwork ran CreateNetwork over links that are still up. Recreating them is not a no-op: rtnl_link_add is issued without NLM_F_EXCL, so the kernel treats it as a modify request and CreateVlan pushes a freshly generated MAC onto the live vlan, breaking traffic of the instances still running on it. Probe each link with the new InterfaceManagerItf::GetLink and create only what is missing. Rollback deletes a link only when this call created it, so a failure midway no longer tears down an adopted one. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 65 +++++++-- src/core/sm/networkmanager/networkmanager.hpp | 7 +- .../networkmanager/tests/networkmanager.cpp | 126 ++++++++++++++++++ 3 files changed, 185 insertions(+), 13 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 11e293632..e8c9993a1 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -1367,6 +1367,21 @@ Error NetworkManager::PrepareDNSServerParams(const NetworkInfo& network, DNSServ return ErrorEnum::eNone; } +RetWithError NetworkManager::IsLinkExist(const String& ifName) const +{ + LinkInfo link; + + if (auto err = mNetIf->GetLink(ifName, link); !err.IsNone()) { + if (err.Is(ErrorEnum::eNotFound)) { + return {false, ErrorEnum::eNone}; + } + + return {false, AOS_ERROR_WRAP(err)}; + } + + return {true, ErrorEnum::eNone}; +} + Error NetworkManager::CreateNetwork(const NetworkInfo& network) { LOG_DBG() << "Create network" << Log::Field("networkID", network.mNetworkID) @@ -1376,24 +1391,54 @@ Error NetworkManager::CreateNetwork(const NetworkInfo& network) Error err; - if (err = mNetIfFactory->CreateBridge(network.mBridgeIfName, network.mIP, network.mSubnet); !err.IsNone()) { - return AOS_ERROR_WRAP(err); + // A link may already be there when SM crashed without running its teardown. + // Recreating it is not a no-op: the kernel takes RTM_NEWLINK without + // NLM_F_EXCL as a modify request, so CreateVlan would push a freshly + // generated MAC onto the live vlan and break the traffic of the instances + // still running on it. Adopt what exists and create only what is missing. + bool bridgeExists = false; + + if (Tie(bridgeExists, err) = IsLinkExist(network.mBridgeIfName); !err.IsNone()) { + return err; } - auto cleanupBridge = DeferRelease(&network, [this, &err](const NetworkInfo* network) { - if (!err.IsNone()) { + bool bridgeCreated = false; + + if (!bridgeExists) { + if (err = mNetIfFactory->CreateBridge(network.mBridgeIfName, network.mIP, network.mSubnet); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + bridgeCreated = true; + } + + auto cleanupBridge = DeferRelease(&network, [this, &err, bridgeCreated](const NetworkInfo* network) { + if (!err.IsNone() && bridgeCreated) { mNetIf->DeleteLink(network->mBridgeIfName); } }); - // Create the vlan already enslaved to the bridge (master) in one operation, - // avoiding a separate SetMasterLink round-trip. - if (err = mNetIfFactory->CreateVlan(network.mVlanIfName, network.mVlanID, network.mBridgeIfName); !err.IsNone()) { - return AOS_ERROR_WRAP(err); + bool vlanExists = false; + + if (Tie(vlanExists, err) = IsLinkExist(network.mVlanIfName); !err.IsNone()) { + return err; } - auto cleanupVlan = DeferRelease(&network, [this, &err](const NetworkInfo* network) { - if (!err.IsNone()) { + bool vlanCreated = false; + + if (!vlanExists) { + // Create the vlan already enslaved to the bridge (master) in one operation, + // avoiding a separate SetMasterLink round-trip. + if (err = mNetIfFactory->CreateVlan(network.mVlanIfName, network.mVlanID, network.mBridgeIfName); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + vlanCreated = true; + } + + auto cleanupVlan = DeferRelease(&network, [this, &err, vlanCreated](const NetworkInfo* network) { + if (!err.IsNone() && vlanCreated) { mNetIf->DeleteLink(network->mVlanIfName); } }); diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index 49b6229cc..665dcac6c 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -250,9 +250,10 @@ class NetworkManager : public NetworkManagerItf { Error IsHostnameExist(const InstanceCache& instanceCache, const Array>& hosts) const; Error PushHostWithDomain( const String& host, const String& networkID, Array>& hosts) const; - Error CreateNetwork(const NetworkInfo& network); - Error DeleteInstanceNetworkConfig(const String& instanceID, const String& networkID); - Error GenerateIfName(String& ifName, const String& ifPrefix); + RetWithError IsLinkExist(const String& ifName) const; + Error CreateNetwork(const NetworkInfo& network); + Error DeleteInstanceNetworkConfig(const String& instanceID, const String& networkID); + Error GenerateIfName(String& ifName, const String& ifPrefix); template Error GenerateUniqueIfName(String& ifName, const String& ifPrefix, P&& isUnique) diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index 55859a460..984ce5fd1 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -56,6 +56,8 @@ class NetworkManagerTest : public Test { EXPECT_CALL(mDNSName, RemoveServer(_)).Times(AnyNumber()).WillRepeatedly(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mDNSServer, RemoveHost(_)).Times(AnyNumber()).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetIf, GetLink(_, _)).Times(AnyNumber()).WillRepeatedly(Return(aos::ErrorEnum::eNotFound)); + // Masquerade is a per-network rule installed/removed by CreateNetwork / // ClearNetwork; leave it lenient so per-test sequences need not assert it. EXPECT_CALL(mFirewall, AddMasquerade(_, _)).Times(AnyNumber()).WillRepeatedly(Return(aos::ErrorEnum::eNone)); @@ -196,6 +198,55 @@ class NetworkManagerTest : public Test { EXPECT_CALL(mStorage, UpdateInstanceNetworkInfo(_)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); } + NetworkInfo CreateTestNetworkInfo() + { + NetworkInfo network; + network.mNetworkID = "network1"; + network.mIP = "192.168.1.1"; + network.mSubnet = "192.168.1.0/24"; + network.mVlanID = 100ULL; + network.mVlanIfName = "vlan-1234abcd"; + network.mBridgeIfName = "br-ef567890"; + + return network; + } + + void InitWithStoredNetwork(const NetworkInfo& network) + { + mNetworkInfos.PushBack(network); + + EXPECT_CALL(mStorage, GetNetworksInfo(_)) + .WillOnce(DoAll(SetArgReferee<0>(mNetworkInfos), Return(aos::ErrorEnum::eNone))); + EXPECT_CALL(mStorage, GetInstanceNetworksInfo(_)) + .WillOnce(DoAll(SetArgReferee<0>(mInstanceNetworkInfos), Return(aos::ErrorEnum::eNone))); + + mNetManager = std::make_unique(); + + ASSERT_EQ(mNetManager->Init(mStorage, mBridgeNetwork, mFirewall, mBandwidth, mDNSName, mTrafficMonitor, mNetns, + mNetIf, mRandom, mNetIfFactory, mNetworkProvider, "test-node"), + aos::ErrorEnum::eNone); + } + + void ExpectLinkExists(const aos::String& ifName, LinkKind kind) + { + LinkInfo link; + link.mName = ifName; + link.mKind = kind; + + EXPECT_CALL(mNetIf, GetLink(ifName, _)) + .WillRepeatedly(DoAll(SetArgReferee<1>(link), Return(aos::ErrorEnum::eNone))); + } + + void ExpectStartInstanceOnStoredNetwork(const aos::String& instanceID, const aos::String& networkID) + { + EXPECT_CALL(mNetworkProvider, AllocateInstanceNetwork(_, networkID, aos::String("test-node"), _, _)) + .WillOnce(DoAll(SetArgReferee<4>(CreateTestAllocatedParams()), Return(aos::ErrorEnum::eNone))); + EXPECT_CALL(mStorage, AddInstanceNetworkInfo(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID, networkID, CreateTestInstanceNetworkConfig()), + aos::ErrorEnum::eNone); + } + void ExpectDeleteInstanceCalls(int times = 1) { EXPECT_CALL(mDNSServer, RemoveHost(_)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); @@ -1029,6 +1080,81 @@ TEST_F(NetworkManagerTest, InitWithExistingNetworks) ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, "network1"), aos::ErrorEnum::eNone); } +TEST_F(NetworkManagerTest, CreateNetwork_AdoptsExistingBridgeAndVlan) +{ + const auto network = CreateTestNetworkInfo(); + const aos::String instanceID = "test-instance"; + + InitWithStoredNetwork(network); + + ExpectLinkExists(network.mBridgeIfName, LinkKindEnum::eBridge); + ExpectLinkExists(network.mVlanIfName, LinkKindEnum::eVlan); + + EXPECT_CALL(mNetIfFactory, CreateBridge(_, _, _)).Times(0); + EXPECT_CALL(mNetIfFactory, CreateVlan(_, _, _)).Times(0); + EXPECT_CALL(mDNSName, CreateServer(_, _)) + .WillOnce(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); + + ExpectStartInstanceOnStoredNetwork(instanceID, network.mNetworkID); + + ExpectAddInstanceCalls(); + ExpectPersistInstanceCalls(); + EXPECT_CALL(mNetns, CreateNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) + .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); + EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(_, _, _, _)).WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, network.mNetworkID), aos::ErrorEnum::eNone); +} + +TEST_F(NetworkManagerTest, CreateNetwork_CreatesOnlyMissingVlan) +{ + const auto network = CreateTestNetworkInfo(); + const aos::String instanceID = "test-instance"; + + InitWithStoredNetwork(network); + + ExpectLinkExists(network.mBridgeIfName, LinkKindEnum::eBridge); + + EXPECT_CALL(mNetIfFactory, CreateBridge(_, _, _)).Times(0); + EXPECT_CALL(mNetIfFactory, CreateVlan(network.mVlanIfName, network.mVlanID, network.mBridgeIfName)) + .WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mDNSName, CreateServer(_, _)) + .WillOnce(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); + + ExpectStartInstanceOnStoredNetwork(instanceID, network.mNetworkID); + + ExpectAddInstanceCalls(); + ExpectPersistInstanceCalls(); + EXPECT_CALL(mNetns, CreateNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) + .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); + EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(_, _, _, _)).WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, network.mNetworkID), aos::ErrorEnum::eNone); +} + +TEST_F(NetworkManagerTest, CreateNetwork_KeepsAdoptedBridgeWhenVlanCreationFails) +{ + const auto network = CreateTestNetworkInfo(); + const aos::String instanceID = "test-instance"; + + InitWithStoredNetwork(network); + + ExpectLinkExists(network.mBridgeIfName, LinkKindEnum::eBridge); + + EXPECT_CALL(mNetIfFactory, CreateBridge(_, _, _)).Times(0); + EXPECT_CALL(mNetIfFactory, CreateVlan(_, _, _)).WillOnce(Return(aos::ErrorEnum::eFailed)); + EXPECT_CALL(mNetIf, DeleteLink(_)).Times(0); + + ExpectStartInstanceOnStoredNetwork(instanceID, network.mNetworkID); + + EXPECT_FALSE(mNetManager->StartInstanceNetwork(instanceID, network.mNetworkID).IsNone()); + + Mock::VerifyAndClearExpectations(&mNetIf); + Mock::VerifyAndClearExpectations(&mNetIfFactory); +} + TEST_F(NetworkManagerTest, CreateInstanceNetwork_VerifyUpdateItemNetworkParams) { const aos::String networkID = "test-network"; From 1a8dae33f3a13188c3e52f4944568e445bbd70bb Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 30 Jul 2026 19:41:19 +0300 Subject: [PATCH 073/112] sm: networkmanager: keep running instances across SM restart CleanupLeftoverInstances tore down every instance recorded in storage on Start: it detached the host veth and deleted the network namespace even when the instance was alive and correctly wired, so an SM crash cut the network of containers that kept running. Replace it with ReconcileInstances, which checks the system before acting. An instance whose host veth is up, is a veth and is enslaved to its own bridge, and whose network namespace still exists, is adopted: nothing on the system is touched, only the runtime cache is restored. Everything else keeps the previous teardown path. Adopting into the runtime cache also makes the launcher restart flow work: StartInstanceNetwork now short-circuits with eAlreadyExist, which the launcher already tolerates. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 131 +++++++++-- src/core/sm/networkmanager/networkmanager.hpp | 20 +- .../networkmanager/tests/networkmanager.cpp | 212 ++++++++++++------ 3 files changed, 275 insertions(+), 88 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index e8c9993a1..dc834c3d2 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -109,7 +109,7 @@ Error NetworkManager::Start() return err; } - if (err = CleanupLeftoverInstances(); !err.IsNone()) { + if (err = ReconcileInstances(); !err.IsNone()) { return err; } @@ -977,13 +977,93 @@ Error NetworkManager::UpdateInstanceNetworkCache( return ErrorEnum::eNone; } -Error NetworkManager::CleanupLeftoverInstances() +RetWithError NetworkManager::IsInstanceInterfaceAlive( + const String& instanceID, const String& hostIfName, const String& bridgeIfName) const { - LOG_DBG() << "Cleanup leftover instances"; + if (bridgeIfName.IsEmpty()) { + return {false, ErrorEnum::eNone}; + } + + LinkInfo link; + + if (auto err = mNetIf->GetLink(hostIfName, link); !err.IsNone()) { + if (err.Is(ErrorEnum::eNotFound)) { + return {false, ErrorEnum::eNone}; + } + + return {false, AOS_ERROR_WRAP(err)}; + } + + if (link.mKind != LinkKindEnum::eVeth || link.mMaster != bridgeIfName) { + return {false, ErrorEnum::eNone}; + } + + bool nsExists = false; + Error err; + + if (Tie(nsExists, err) = mNetns->IsNetworkNamespaceExist(instanceID); !err.IsNone()) { + return {false, AOS_ERROR_WRAP(err)}; + } + + return {nsExists, ErrorEnum::eNone}; +} + +Error NetworkManager::InitInstance(const String& instanceID, const String& networkID) +{ + LOG_DBG() << "Adopt running instance" << Log::Field("instanceID", instanceID) << Log::Field("networkID", networkID); + + if (auto errCache = AddInstanceToCache(instanceID, networkID); !errCache.IsNone()) { + return errCache; + } + + Error err; + + auto cleanupCache = DeferRelease(&instanceID, [this, &networkID, &err](const String* id) { + if (!err.IsNone()) { + if (auto errRemove = RemoveInstanceFromCache(*id, networkID); !errRemove.IsNone()) { + LOG_ERR() << "Failed to remove instance from cache" << Log::Field("instanceID", *id) + << Log::Field("networkID", networkID) << Log::Field(errRemove); + } + } + }); + + auto config = MakeUnique(&mAllocator); + + { + LockGuard lock {mMutex}; + + auto it = mInstanceNetworkInfos.Find(instanceID); + if (it == mInstanceNetworkInfos.end()) { + err = AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance network info not found")); + + return err; + } + + *config = it->mSecond.mNetworkConfig; + } + + auto hosts = MakeUnique(&mAllocator); + + if (err = PrepareHosts(instanceID, networkID, *config, *hosts); !err.IsNone()) { + return err; + } + + if (err = UpdateInstanceNetworkCache(instanceID, networkID, *hosts); !err.IsNone()) { + return err; + } + + return ErrorEnum::eNone; +} + +Error NetworkManager::ReconcileInstances() +{ + LOG_DBG() << "Reconcile instances"; struct Entry { - StaticString mInstanceID; - StaticString mNetworkID; + StaticString mInstanceID; + StaticString mNetworkID; + StaticString mHostIfName; + StaticString mBridgeIfName; }; auto entries = MakeUnique>(&mAllocator); @@ -992,25 +1072,44 @@ Error NetworkManager::CleanupLeftoverInstances() LockGuard lock {mMutex}; for (const auto& item : mInstanceNetworkInfos) { - if (auto err = entries->PushBack({item.mFirst, item.mSecond.mNetworkID}); !err.IsNone()) { + StaticString bridgeIfName; + + if (auto it = mNetworkProviders.Find(item.mSecond.mNetworkID); it != mNetworkProviders.end()) { + bridgeIfName = it->mSecond.mBridgeIfName; + } + + if (auto err + = entries->PushBack({item.mFirst, item.mSecond.mNetworkID, item.mSecond.mHostIfName, bridgeIfName}); + !err.IsNone()) { return AOS_ERROR_WRAP(err); } } } - // Adopt a DNS handle for each network with leftover instances, so the - // RemoveHost call inside DeleteInstanceNetworkConfig has a backend to - // talk to (CreateInstance is idempotent — it adopts a surviving dnsmasq - // for this networkID or respawns a fresh one). for (const auto& entry : *entries) { - if (auto err = AdoptDNSServer(entry.mNetworkID); !err.IsNone()) { - LOG_WRN() << "Failed to adopt DNS server for leftover cleanup" << Log::Field("networkID", entry.mNetworkID) - << Log::Field(err); + if (entry.mHostIfName.IsEmpty()) { + continue; } - } - for (const auto& entry : *entries) { - if (auto err = DeleteInstanceNetworkConfig(entry.mInstanceID, entry.mNetworkID); !err.IsNone()) { + bool alive = false; + Error err; + + if (Tie(alive, err) = IsInstanceInterfaceAlive(entry.mInstanceID, entry.mHostIfName, entry.mBridgeIfName); + !err.IsNone()) { + LOG_WRN() << "Failed to check leftover instance interface" << Log::Field("instanceID", entry.mInstanceID) + << Log::Field("hostIfName", entry.mHostIfName) << Log::Field(err); + } + + if (alive) { + if (err = InitInstance(entry.mInstanceID, entry.mNetworkID); err.IsNone()) { + continue; + } else { + LOG_WRN() << "Failed to adopt leftover instance, falling back to cleanup" + << Log::Field("instanceID", entry.mInstanceID) << Log::Field(err); + } + } + + if (err = DeleteInstanceNetworkConfig(entry.mInstanceID, entry.mNetworkID); !err.IsNone()) { LOG_WRN() << "Failed to delete leftover instance network config" << Log::Field("instanceID", entry.mInstanceID) << Log::Field("networkID", entry.mNetworkID) << Log::Field(err); diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index 665dcac6c..e364ded38 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -202,12 +202,15 @@ class NetworkManager : public NetworkManagerItf { // Start()/OnConnect() run once, outside the concurrent instance-operation hot path, so their // allocations are added rather than multiplied by cMaxNumConcurrentItems: RemoveDNSOrphans' known - // networks list, CleanupLeftoverInstances' leftover instance/network ID pairs plus the - // InstanceNetworkInfo DeleteInstanceNetworkConfig allocates while clearing a host interface, and - // OnConnect's state sync snapshot. + // networks list, ReconcileInstances' leftover entries (instance/network IDs plus host/bridge + // interface names, so four ID-sized strings per instance) plus, alive at the same time, either the + // InstanceNetworkInfo DeleteInstanceNetworkConfig allocates while clearing a host interface or the + // config and hosts AdoptInstance allocates while restoring the runtime cache, and OnConnect's state + // sync snapshot. static constexpr auto cAllocatorSize = cMaxOperationAllocatorSize * cMaxNumConcurrentItems + sizeof(StaticArray, cMaxNumOwners>) - + sizeof(StaticArray, cMaxNumInstances>) * 2 + sizeof(InstanceNetworkInfo) + + sizeof(StaticArray, cMaxNumInstances>) * 4 + sizeof(InstanceNetworkInfo) + + sizeof(InstanceNetworkConfig) + sizeof(InstanceHosts) + sizeof(StaticArray); static constexpr auto cNumAllocations = 8 * cMaxNumConcurrentItems; @@ -226,9 +229,12 @@ class NetworkManager : public NetworkManagerItf { static constexpr auto cVlanIfPrefix = "vlan-"; static constexpr auto cResolvConfLineLen = AOS_CONFIG_NETWORKMANAGER_RESOLV_CONF_LINE_LEN; - Error IsInstanceInNetwork(const String& instanceID, const String& networkID) const; - Error AddInstanceToCache(const String& instanceID, const String& networkID); - Error CleanupLeftoverInstances(); + Error IsInstanceInNetwork(const String& instanceID, const String& networkID) const; + Error AddInstanceToCache(const String& instanceID, const String& networkID); + RetWithError IsInstanceInterfaceAlive( + const String& instanceID, const String& hostIfName, const String& bridgeIfName) const; + Error InitInstance(const String& instanceID, const String& networkID); + Error ReconcileInstances(); Error RemoveDNSOrphans(); Error AdoptDNSServer(const String& networkID); Error PrepareBridgeParams( diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index 984ce5fd1..d3a124ac4 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -227,16 +227,84 @@ class NetworkManagerTest : public Test { aos::ErrorEnum::eNone); } - void ExpectLinkExists(const aos::String& ifName, LinkKind kind) + void ExpectLinkExists(const aos::String& ifName, LinkKind kind, const aos::String& master = "") { LinkInfo link; - link.mName = ifName; - link.mKind = kind; + link.mName = ifName; + link.mKind = kind; + link.mMaster = master; EXPECT_CALL(mNetIf, GetLink(ifName, _)) .WillRepeatedly(DoAll(SetArgReferee<1>(link), Return(aos::ErrorEnum::eNone))); } + void RestartWithStoredState(const aos::Array& networks, + const aos::Array& instances) + { + EXPECT_CALL(mTrafficMonitor, Stop()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, Stop()).WillOnce(Return(aos::ErrorEnum::eNone)); + ASSERT_EQ(mNetManager->Stop(), aos::ErrorEnum::eNone); + + EXPECT_CALL(mNetIf, DeleteLink(_)).Times(AnyNumber()).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + mNetManager.reset(); + + EXPECT_CALL(mFirewall, Start()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, Start()).WillOnce(Return(aos::ErrorEnum::eNone)); + + EXPECT_CALL(mStorage, GetNetworksInfo(_)) + .WillOnce(Invoke([&networks](aos::Array& out) { + out = networks; + return aos::ErrorEnum::eNone; + })); + EXPECT_CALL(mStorage, GetInstanceNetworksInfo(_)) + .WillOnce(Invoke([&instances](aos::Array& out) { + out = instances; + return aos::ErrorEnum::eNone; + })); + + mNetManager = std::make_unique(); + + ASSERT_EQ(mNetManager->Init(mStorage, mBridgeNetwork, mFirewall, mBandwidth, mDNSName, mTrafficMonitor, mNetns, + mNetIf, mRandom, mNetIfFactory, mNetworkProvider, "test-node"), + aos::ErrorEnum::eNone); + } + + aos::sm::networkmanager::InstanceNetworkInfo CreateLeftoverInstance( + const aos::sm::networkmanager::NetworkInfo& network) + { + aos::sm::networkmanager::InstanceNetworkInfo leftover; + leftover.mInstanceID = "leftover-instance"; + leftover.mNetworkID = network.mNetworkID; + leftover.mNetworkConfig.mHostname = "leftover-host"; + leftover.mAllocatedParams.mIP = "192.168.1.5"; + leftover.mAllocatedParams.mSubnet = network.mSubnet; + leftover.mHostIfName = "veth-leftover"; + + return leftover; + } + + void ExpectLeftoverInstanceCleaned() + { + EXPECT_CALL(mDNSName, CreateServer(_, _)).Times(0); + EXPECT_CALL(mDNSServer, RemoveHost(_)).Times(0); + EXPECT_CALL(mBandwidth, Clear(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, RemoveInstance(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mBridgeNetwork, Detach(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mStorage, UpdateInstanceNetworkInfo(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + } + + void ExpectLeftoverInstanceUntouched() + { + EXPECT_CALL(mDNSName, CreateServer(_, _)).Times(0); + EXPECT_CALL(mDNSServer, RemoveHost(_)).Times(0); + EXPECT_CALL(mBandwidth, Clear(_)).Times(0); + EXPECT_CALL(mFirewall, RemoveInstance(_)).Times(0); + EXPECT_CALL(mBridgeNetwork, Detach(_, _)).Times(0); + EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).Times(0); + EXPECT_CALL(mStorage, UpdateInstanceNetworkInfo(_)).Times(0); + } + void ExpectStartInstanceOnStoredNetwork(const aos::String& instanceID, const aos::String& networkID) { EXPECT_CALL(mNetworkProvider, AllocateInstanceNetwork(_, networkID, aos::String("test-node"), _, _)) @@ -1318,84 +1386,98 @@ TEST_F(NetworkManagerTest, OnConnect_SyncsNetworkStateWithCM) mNetManager->OnConnect(); } -TEST_F(NetworkManagerTest, Start_AdoptsDNSForLeftoverInstancesAndCleansHosts) +TEST_F(NetworkManagerTest, Start_CleansLeftoverInstanceWithMissingInterface) { - // A leftover instance from a previous SM lifetime: its network is still - // in storage. On Start, NetworkManager should reap DNS orphans (with the - // known network in the list), then adopt the DNS handle for the leftover - // network and call RemoveHost while cleaning up the leftover instance. - aos::sm::networkmanager::NetworkInfo existingNetwork; - existingNetwork.mNetworkID = "leftover-net"; - existingNetwork.mIP = "192.168.7.1"; - existingNetwork.mSubnet = "192.168.7.0/24"; - existingNetwork.mVlanID = 700ULL; - existingNetwork.mVlanIfName = "vlan-leftover"; - existingNetwork.mBridgeIfName = "br-leftover"; - - aos::sm::networkmanager::InstanceNetworkInfo leftover; - leftover.mInstanceID = "leftover-instance"; - leftover.mNetworkID = existingNetwork.mNetworkID; - leftover.mAllocatedParams.mIP = "192.168.7.5"; - leftover.mAllocatedParams.mSubnet = existingNetwork.mSubnet; - leftover.mHostIfName = "veth-leftover"; + const auto network = CreateTestNetworkInfo(); + const auto leftover = CreateLeftoverInstance(network); aos::StaticArray networks; aos::StaticArray instances; - networks.PushBack(existingNetwork); + networks.PushBack(network); instances.PushBack(leftover); - EXPECT_CALL(mFirewall, Start()).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mTrafficMonitor, Start()).WillOnce(Return(aos::ErrorEnum::eNone)); - - EXPECT_CALL(mStorage, GetNetworksInfo(_)) - .WillOnce(Invoke([&](aos::Array& out) { - out = networks; - return aos::ErrorEnum::eNone; - })); - EXPECT_CALL(mStorage, GetInstanceNetworksInfo(_)) - .WillOnce(Invoke([&](aos::Array& out) { - out = instances; - return aos::ErrorEnum::eNone; - })); - - // Stop the fixture instance — we drive Init/Start manually below. - EXPECT_CALL(mTrafficMonitor, Stop()).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mFirewall, Stop()).WillOnce(Return(aos::ErrorEnum::eNone)); - ASSERT_EQ(mNetManager->Stop(), aos::ErrorEnum::eNone); - EXPECT_CALL(mNetIf, DeleteLink(_)).Times(AnyNumber()).WillRepeatedly(Return(aos::ErrorEnum::eNone)); - mNetManager.reset(); - - mNetManager = std::make_unique(); - - ASSERT_EQ(mNetManager->Init(mStorage, mBridgeNetwork, mFirewall, mBandwidth, mDNSName, mTrafficMonitor, mNetns, - mNetIf, mRandom, mNetIfFactory, mNetworkProvider, "test-node"), - aos::ErrorEnum::eNone); + RestartWithStoredState(networks, instances); - // RemoveOrphans must receive the known networkID from storage. EXPECT_CALL(mDNSName, RemoveOrphans(_)) .WillOnce(Invoke([&](const aos::Array>& known) { EXPECT_EQ(known.Size(), 1U); if (known.Size() == 1) { - EXPECT_EQ(known[0], existingNetwork.mNetworkID); + EXPECT_EQ(known[0], network.mNetworkID); } return aos::ErrorEnum::eNone; })); - // Pre-adopt: CreateInstance with the leftover network's bridge IP / ifname. - EXPECT_CALL(mDNSName, CreateServer(existingNetwork.mNetworkID, _)) - .WillOnce(Invoke([&](const aos::String&, const DNSServerParams& params) { - EXPECT_EQ(params.mBridgeIP, existingNetwork.mIP); - EXPECT_EQ(params.mBridgeIfName, existingNetwork.mBridgeIfName); - return aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone}; - })); + ExpectLeftoverInstanceCleaned(); - // Leftover instance cleanup goes through the adopted handle. - EXPECT_CALL(mDNSServer, RemoveHost(aos::String("leftover-instance"))).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mBandwidth, Clear(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mFirewall, RemoveInstance(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mBridgeNetwork, Detach(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mStorage, UpdateInstanceNetworkInfo(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + ASSERT_EQ(mNetManager->Start(), aos::ErrorEnum::eNone); +} + +TEST_F(NetworkManagerTest, Start_KeepsLeftoverInstanceWithLiveInterface) +{ + const auto network = CreateTestNetworkInfo(); + const auto leftover = CreateLeftoverInstance(network); + + aos::StaticArray networks; + aos::StaticArray instances; + networks.PushBack(network); + instances.PushBack(leftover); + + RestartWithStoredState(networks, instances); + + ExpectLinkExists(leftover.mHostIfName, LinkKindEnum::eVeth, network.mBridgeIfName); + EXPECT_CALL(mNetns, IsNetworkNamespaceExist(leftover.mInstanceID)) + .WillRepeatedly(Return(aos::RetWithError {true, aos::ErrorEnum::eNone})); + + EXPECT_CALL(mDNSName, RemoveOrphans(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + + ExpectLeftoverInstanceUntouched(); + + ASSERT_EQ(mNetManager->Start(), aos::ErrorEnum::eNone); + + EXPECT_TRUE( + mNetManager->StartInstanceNetwork(leftover.mInstanceID, network.mNetworkID).Is(aos::ErrorEnum::eAlreadyExist)); +} + +TEST_F(NetworkManagerTest, Start_CleansLeftoverInstanceWhenNamespaceMissing) +{ + const auto network = CreateTestNetworkInfo(); + const auto leftover = CreateLeftoverInstance(network); + + aos::StaticArray networks; + aos::StaticArray instances; + networks.PushBack(network); + instances.PushBack(leftover); + + RestartWithStoredState(networks, instances); + + ExpectLinkExists(leftover.mHostIfName, LinkKindEnum::eVeth, network.mBridgeIfName); + EXPECT_CALL(mNetns, IsNetworkNamespaceExist(leftover.mInstanceID)) + .WillRepeatedly(Return(aos::RetWithError {false, aos::ErrorEnum::eNone})); + + EXPECT_CALL(mDNSName, RemoveOrphans(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + + ExpectLeftoverInstanceCleaned(); + + ASSERT_EQ(mNetManager->Start(), aos::ErrorEnum::eNone); +} + +TEST_F(NetworkManagerTest, Start_CleansLeftoverInstanceAttachedToForeignBridge) +{ + const auto network = CreateTestNetworkInfo(); + const auto leftover = CreateLeftoverInstance(network); + + aos::StaticArray networks; + aos::StaticArray instances; + networks.PushBack(network); + instances.PushBack(leftover); + + RestartWithStoredState(networks, instances); + + ExpectLinkExists(leftover.mHostIfName, LinkKindEnum::eVeth, "br-someoneelse"); + + EXPECT_CALL(mDNSName, RemoveOrphans(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + + ExpectLeftoverInstanceCleaned(); ASSERT_EQ(mNetManager->Start(), aos::ErrorEnum::eNone); } From 09ff827e47711bdca7344de265b3c1887f27fa8e Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 30 Jul 2026 21:02:24 +0300 Subject: [PATCH 074/112] sm: networkmanager: restore DNS handle before cleaning leftovers DeleteInstanceNetworkConfig removes the DNS record through the handle in mDNSServers, which is runtime state and is empty after a restart. Without a handle a dead leftover instance kept its addnhosts record. Adopt the handle before the cleanup again, now that DNSServer::Init loads the existing records instead of truncating them and so no longer drops the records of instances adopted on the same network. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 5 +++++ src/core/sm/networkmanager/tests/networkmanager.cpp | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index dc834c3d2..cfe992d48 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -1109,6 +1109,11 @@ Error NetworkManager::ReconcileInstances() } } + if (err = AdoptDNSServer(entry.mNetworkID); !err.IsNone()) { + LOG_WRN() << "Failed to adopt DNS server for leftover cleanup" << Log::Field("networkID", entry.mNetworkID) + << Log::Field(err); + } + if (err = DeleteInstanceNetworkConfig(entry.mInstanceID, entry.mNetworkID); !err.IsNone()) { LOG_WRN() << "Failed to delete leftover instance network config" << Log::Field("instanceID", entry.mInstanceID) << Log::Field("networkID", entry.mNetworkID) diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index d3a124ac4..c1588ca7f 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -285,8 +285,9 @@ class NetworkManagerTest : public Test { void ExpectLeftoverInstanceCleaned() { - EXPECT_CALL(mDNSName, CreateServer(_, _)).Times(0); - EXPECT_CALL(mDNSServer, RemoveHost(_)).Times(0); + EXPECT_CALL(mDNSName, CreateServer(_, _)) + .WillOnce(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); + EXPECT_CALL(mDNSServer, RemoveHost(aos::String("leftover-instance"))).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mBandwidth, Clear(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mFirewall, RemoveInstance(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mBridgeNetwork, Detach(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); From 28ae87bc94033c4a698af7df2e3be921353d84c8 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 30 Jul 2026 21:23:47 +0300 Subject: [PATCH 075/112] sm: networkmanager: reap orphan firewall artifacts on start FirewallItf gains RemoveOrphans so that the firewall no longer has to wipe its whole table on start to get rid of what a crashed SM left behind. Call it from Start with the instances and networks known from storage, so artifacts of anything gone are removed while the rules protecting the instances that kept running stay in place. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/itf/firewall.hpp | 26 ++++++++++++++- src/core/sm/networkmanager/networkmanager.cpp | 32 +++++++++++++++++++ src/core/sm/networkmanager/networkmanager.hpp | 14 ++++---- .../tests/mocks/firewallmock.hpp | 1 + .../networkmanager/tests/networkmanager.cpp | 2 ++ 5 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/core/sm/networkmanager/itf/firewall.hpp b/src/core/sm/networkmanager/itf/firewall.hpp index 1520b25a5..3d6af4d28 100644 --- a/src/core/sm/networkmanager/itf/firewall.hpp +++ b/src/core/sm/networkmanager/itf/firewall.hpp @@ -53,6 +53,14 @@ struct InstanceFirewallParams { StaticArray mOutput; }; +/** + * Masquerade rule identity. + */ +struct MasqueradeParams { + StaticString mSubnet; + StaticString mOutIfName; +}; + /** * Firewall interface. * @@ -69,7 +77,8 @@ class FirewallItf { /** * Starts the firewall: creates the `inet aos` table, base chains and - * netfilter hooks (forward filter, postrouting nat). + * netfilter hooks (forward filter, postrouting nat) when they are absent. + * Rules that are already there are left alone. * * @return Error. */ @@ -82,6 +91,21 @@ class FirewallItf { */ virtual Error Stop() = 0; + /** + * Removes the instance chains and masquerade rules that no longer belong + * to anything known, keeping the rest in place. + * + * Called on start so that artifacts left by a crashed SM are reaped + * without touching the rules of the instances that kept running. + * + * @param knownInstanceIDs instance ids whose chains must be kept. + * @param knownMasquerades masquerade rules that must be kept. + * @return Error. + */ + virtual Error RemoveOrphans( + const Array>& knownInstanceIDs, const Array& knownMasquerades) + = 0; + /** * Adds a per-instance chain with the given input/output access rules. * diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index cfe992d48..201ad7863 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -105,6 +105,10 @@ Error NetworkManager::Start() } }); + if (err = RemoveFirewallOrphans(); !err.IsNone()) { + return err; + } + if (err = RemoveDNSOrphans(); !err.IsNone()) { return err; } @@ -1124,6 +1128,34 @@ Error NetworkManager::ReconcileInstances() return ErrorEnum::eNone; } +Error NetworkManager::RemoveFirewallOrphans() +{ + auto knownInstanceIDs = MakeUnique, cMaxNumInstances>>(&mAllocator); + auto knownMasquerades = MakeUnique>(&mAllocator); + + { + LockGuard lock {mMutex}; + + for (const auto& [instanceID, _] : mInstanceNetworkInfos) { + if (auto err = knownInstanceIDs->PushBack(instanceID); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + + for (const auto& [_, network] : mNetworkProviders) { + if (auto err = knownMasquerades->PushBack({network.mSubnet, network.mBridgeIfName}); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + } + } + + if (auto err = mFirewall->RemoveOrphans(*knownInstanceIDs, *knownMasquerades); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + Error NetworkManager::RemoveDNSOrphans() { auto known = MakeUnique, cMaxNumOwners>>(&mAllocator); diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index e364ded38..0c12f6500 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -202,14 +202,15 @@ class NetworkManager : public NetworkManagerItf { // Start()/OnConnect() run once, outside the concurrent instance-operation hot path, so their // allocations are added rather than multiplied by cMaxNumConcurrentItems: RemoveDNSOrphans' known - // networks list, ReconcileInstances' leftover entries (instance/network IDs plus host/bridge - // interface names, so four ID-sized strings per instance) plus, alive at the same time, either the - // InstanceNetworkInfo DeleteInstanceNetworkConfig allocates while clearing a host interface or the - // config and hosts AdoptInstance allocates while restoring the runtime cache, and OnConnect's state - // sync snapshot. + // networks list, RemoveFirewallOrphans' known instance and masquerade lists, ReconcileInstances' + // leftover entries (instance/network IDs plus host/bridge interface names, so four ID-sized strings + // per instance) plus, alive at the same time, either the InstanceNetworkInfo + // DeleteInstanceNetworkConfig allocates while clearing a host interface or the config and hosts + // InitInstance allocates while restoring the runtime cache, and OnConnect's state sync snapshot. static constexpr auto cAllocatorSize = cMaxOperationAllocatorSize * cMaxNumConcurrentItems + sizeof(StaticArray, cMaxNumOwners>) - + sizeof(StaticArray, cMaxNumInstances>) * 4 + sizeof(InstanceNetworkInfo) + + sizeof(StaticArray, cMaxNumInstances>) * 5 + + sizeof(StaticArray) + sizeof(InstanceNetworkInfo) + sizeof(InstanceNetworkConfig) + sizeof(InstanceHosts) + sizeof(StaticArray); static constexpr auto cNumAllocations = 8 * cMaxNumConcurrentItems; @@ -235,6 +236,7 @@ class NetworkManager : public NetworkManagerItf { const String& instanceID, const String& hostIfName, const String& bridgeIfName) const; Error InitInstance(const String& instanceID, const String& networkID); Error ReconcileInstances(); + Error RemoveFirewallOrphans(); Error RemoveDNSOrphans(); Error AdoptDNSServer(const String& networkID); Error PrepareBridgeParams( diff --git a/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp b/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp index 3f66c52cc..db9f668d4 100644 --- a/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp @@ -17,6 +17,7 @@ class FirewallMock : public FirewallItf { public: MOCK_METHOD(Error, Start, (), (override)); MOCK_METHOD(Error, Stop, (), (override)); + MOCK_METHOD(Error, RemoveOrphans, (const Array>&, const Array&), (override)); MOCK_METHOD(Error, AddInstance, (const String&, const InstanceFirewallParams&), (override)); MOCK_METHOD(Error, RemoveInstance, (const String&), (override)); MOCK_METHOD(Error, UpdateInstance, (const String&, const InstanceFirewallParams&), (override)); diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index c1588ca7f..db6fdc061 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -42,6 +42,7 @@ class NetworkManagerTest : public Test { std::filesystem::create_directories(mWorkingDir.CStr()); EXPECT_CALL(mFirewall, Start()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, RemoveOrphans(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mTrafficMonitor, Start()).WillOnce(Return(aos::ErrorEnum::eNone)); // NetworkManager::Start reaps DNS orphans from a previous SM lifetime @@ -249,6 +250,7 @@ class NetworkManagerTest : public Test { mNetManager.reset(); EXPECT_CALL(mFirewall, Start()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, RemoveOrphans(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mTrafficMonitor, Start()).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mStorage, GetNetworksInfo(_)) From 20eb0a8fe6a89f017d746444ba628ca45796a14f Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Fri, 31 Jul 2026 09:21:44 +0300 Subject: [PATCH 076/112] sm: networkmanager: restore traffic accounting for adopted instances Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 21 +++++++++++++++++-- .../networkmanager/tests/networkmanager.cpp | 19 +++++++++++------ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 201ad7863..f9863b38a 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -1031,7 +1031,8 @@ Error NetworkManager::InitInstance(const String& instanceID, const String& netwo } }); - auto config = MakeUnique(&mAllocator); + auto config = MakeUnique(&mAllocator); + StaticString instanceIP; { LockGuard lock {mMutex}; @@ -1043,7 +1044,8 @@ Error NetworkManager::InitInstance(const String& instanceID, const String& netwo return err; } - *config = it->mSecond.mNetworkConfig; + *config = it->mSecond.mNetworkConfig; + instanceIP = it->mSecond.mAllocatedParams.mIP; } auto hosts = MakeUnique(&mAllocator); @@ -1052,6 +1054,21 @@ Error NetworkManager::InitInstance(const String& instanceID, const String& netwo return err; } + if (err + = mNetMonitor->StartInstanceMonitoring(instanceID, instanceIP, config->mDownloadLimit, config->mUploadLimit); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto cleanupMonitoring = DeferRelease(&instanceID, [this, &err](const String* id) { + if (!err.IsNone()) { + if (auto errStop = mNetMonitor->StopInstanceMonitoring(*id); !errStop.IsNone()) { + LOG_ERR() << "Failed to stop instance monitoring on rollback" << Log::Field("instanceID", *id) + << Log::Field(errStop); + } + } + }); + if (err = UpdateInstanceNetworkCache(instanceID, networkID, *hosts); !err.IsNone()) { return err; } diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index db6fdc061..382f351f5 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -275,12 +275,14 @@ class NetworkManagerTest : public Test { const aos::sm::networkmanager::NetworkInfo& network) { aos::sm::networkmanager::InstanceNetworkInfo leftover; - leftover.mInstanceID = "leftover-instance"; - leftover.mNetworkID = network.mNetworkID; - leftover.mNetworkConfig.mHostname = "leftover-host"; - leftover.mAllocatedParams.mIP = "192.168.1.5"; - leftover.mAllocatedParams.mSubnet = network.mSubnet; - leftover.mHostIfName = "veth-leftover"; + leftover.mInstanceID = "leftover-instance"; + leftover.mNetworkID = network.mNetworkID; + leftover.mNetworkConfig.mHostname = "leftover-host"; + leftover.mNetworkConfig.mDownloadLimit = 4096; + leftover.mNetworkConfig.mUploadLimit = 2048; + leftover.mAllocatedParams.mIP = "192.168.1.5"; + leftover.mAllocatedParams.mSubnet = network.mSubnet; + leftover.mHostIfName = "veth-leftover"; return leftover; } @@ -1435,6 +1437,11 @@ TEST_F(NetworkManagerTest, Start_KeepsLeftoverInstanceWithLiveInterface) ExpectLeftoverInstanceUntouched(); + EXPECT_CALL(mTrafficMonitor, + StartInstanceMonitoring(leftover.mInstanceID, leftover.mAllocatedParams.mIP, + leftover.mNetworkConfig.mDownloadLimit, leftover.mNetworkConfig.mUploadLimit)) + .WillOnce(Return(aos::ErrorEnum::eNone)); + ASSERT_EQ(mNetManager->Start(), aos::ErrorEnum::eNone); EXPECT_TRUE( From 703379dd99455bb23e199a6e04b3de5ce0751e24 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 2 Jul 2026 16:13:13 +0300 Subject: [PATCH 077/112] sm: networkmanager: reap instance veth via namespace teardown, not sync detach Stop no longer deletes the host veth synchronously. DeleteNetworkNamespace drops the instance netns (lazy MNT_DETACH umount); the kernel then reaps the peer veth - and the host end with it, since they die as a pair - asynchronously via cleanup_net, off the critical stop path. The synchronous per-instance delete blocked on a kernel RCU grace period for every instance, serialized under rtnl_lock (O(N) on mass teardown): measured ~2.5s to delete 200 host veths one-by-one vs ~0.25s for the lazy namespace teardown (which also lets cleanup_net batch the unregister). No leak: the interfaces are gone once the namespace is reaped. Detach is kept for the AddInstanceToNetwork rollback path, where the namespace may not yet own the peer. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 20 +++++++------------ .../networkmanager/tests/networkmanager.cpp | 9 ++++----- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index f9863b38a..552f22612 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -846,19 +846,12 @@ Error NetworkManager::EnsureNodeNetworkPhysical(const String& networkID) Error NetworkManager::DeleteInstanceNetworkConfig(const String& instanceID, const String& networkID) { - StaticString bridgeIfName; StaticString hostIfName; DNSServerItf* dnsServer = nullptr; { LockGuard lock {mMutex}; - if (auto it = mNetworkProviders.Find(networkID); it != mNetworkProviders.end()) { - bridgeIfName = it->mSecond.mBridgeIfName; - } else { - LOG_WRN() << "Network provider not found for cleanup" << Log::Field("networkID", networkID); - } - if (auto it = mDNSServers.Find(networkID); it != mDNSServers.end()) { dnsServer = it->mSecond; } @@ -890,12 +883,13 @@ Error NetworkManager::DeleteInstanceNetworkConfig(const String& instanceID, cons err = AOS_ERROR_WRAP(errRemove); } - if (!bridgeIfName.IsEmpty()) { - if (auto errDetach = mBridgeNetwork->Detach(instanceID, bridgeIfName); - !errDetach.IsNone() && err.IsNone()) { - err = AOS_ERROR_WRAP(errDetach); - } - } + // The host veth is intentionally NOT detached here. DeleteNetworkNamespace + // below drops the instance netns (lazy umount); the kernel then reaps the + // peer veth - and with it the host end, since they die as a pair - + // asynchronously via cleanup_net, off the critical stop path. A synchronous + // delete here would block on a per-device RCU grace period for every + // instance (O(N) rtnl_lock serialization on mass teardown) for no benefit, + // as the namespace teardown already removes the interface. } else { LOG_DBG() << "Instance was never started, skipping itf cleanup" << Log::Field("instanceID", instanceID); } diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index 382f351f5..61296ef37 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -294,7 +294,8 @@ class NetworkManagerTest : public Test { EXPECT_CALL(mDNSServer, RemoveHost(aos::String("leftover-instance"))).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mBandwidth, Clear(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mFirewall, RemoveInstance(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mBridgeNetwork, Detach(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); + // The host veth is no longer detached synchronously on cleanup; the instance + // netns teardown reaps it asynchronously. EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mStorage, UpdateInstanceNetworkInfo(_)).WillOnce(Return(aos::ErrorEnum::eNone)); } @@ -325,7 +326,6 @@ class NetworkManagerTest : public Test { EXPECT_CALL(mDNSServer, RemoveHost(_)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mBandwidth, Clear(_)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mFirewall, RemoveInstance(_)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mBridgeNetwork, Detach(_, _)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); } StrictMock mStorage; @@ -896,7 +896,7 @@ TEST_F(NetworkManagerTest, StopReleaseAndRecreateInstance) ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); } -TEST_F(NetworkManagerTest, StopInstanceNetwork_FailOnDetachError) +TEST_F(NetworkManagerTest, StopInstanceNetwork_FailOnFirewallRemoveError) { const aos::String instanceID = "test-instance"; const aos::String networkID = "test-network"; @@ -929,8 +929,7 @@ TEST_F(NetworkManagerTest, StopInstanceNetwork_FailOnDetachError) EXPECT_CALL(mTrafficMonitor, StopInstanceMonitoring(instanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mDNSServer, RemoveHost(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mBandwidth, Clear(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mFirewall, RemoveInstance(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mBridgeNetwork, Detach(_, _)).WillOnce(Return(aos::ErrorEnum::eRuntime)); + EXPECT_CALL(mFirewall, RemoveInstance(_)).WillOnce(Return(aos::ErrorEnum::eRuntime)); EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mNetIf, DeleteLink(_)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); From f17c78359f73c5a56c540dd221dcfffa3b6d09ea Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 2 Jul 2026 20:24:39 +0300 Subject: [PATCH 078/112] sm: networkmanager: skip bandwidth clear when no shaping was applied DeleteInstanceNetworkConfig only calls Bandwidth::Clear when the instance config actually had a non-zero ingress/egress limit. For unlimited instances (the common case) nothing was installed, so the previous unconditional Clear just wasted three tc round-trips (serialized on rtnl_lock) per instance on a mass teardown. The decision uses the config already held in the network manager, so no extra state is kept in the bandwidth backend. Tests to be updated separately. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 552f22612..941b57bb3 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -720,13 +720,16 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin return AOS_ERROR_WRAP(err); } - auto cleanupBandwidth = DeferRelease(&attachResult.mHostIfName, [this, &err](const String* ifName) { - if (!err.IsNone()) { - if (auto errClear = mBandwidth->Clear(*ifName); !errClear.IsNone()) { - LOG_ERR() << "Failed to clear bandwidth" << Log::Field("ifName", *ifName) << Log::Field(errClear); - } - } - }); + const bool bandwidthApplied = bandwidthParams->mIngressRate > 0 || bandwidthParams->mEgressRate > 0; + + auto cleanupBandwidth + = DeferRelease(&attachResult.mHostIfName, [this, &err, bandwidthApplied](const String* ifName) { + if (!err.IsNone() && bandwidthApplied) { + if (auto errClear = mBandwidth->Clear(*ifName); !errClear.IsNone()) { + LOG_ERR() << "Failed to clear bandwidth" << Log::Field("ifName", *ifName) << Log::Field(errClear); + } + } + }); DNSServerItf* dnsServer = nullptr; @@ -847,7 +850,8 @@ Error NetworkManager::EnsureNodeNetworkPhysical(const String& networkID) Error NetworkManager::DeleteInstanceNetworkConfig(const String& instanceID, const String& networkID) { StaticString hostIfName; - DNSServerItf* dnsServer = nullptr; + DNSServerItf* dnsServer = nullptr; + bool hasBandwidth = false; { LockGuard lock {mMutex}; @@ -857,7 +861,8 @@ Error NetworkManager::DeleteInstanceNetworkConfig(const String& instanceID, cons } if (auto it = mInstanceNetworkInfos.Find(instanceID); it != mInstanceNetworkInfos.end()) { - hostIfName = it->mSecond.mHostIfName; + hostIfName = it->mSecond.mHostIfName; + hasBandwidth = it->mSecond.mNetworkConfig.mIngressKbit > 0 || it->mSecond.mNetworkConfig.mEgressKbit > 0; } else { LOG_WRN() << "Instance network info not found for cleanup" << Log::Field("instanceID", instanceID); } @@ -875,8 +880,10 @@ Error NetworkManager::DeleteInstanceNetworkConfig(const String& instanceID, cons << Log::Field("networkID", networkID); } - if (auto errClear = mBandwidth->Clear(hostIfName); !errClear.IsNone() && err.IsNone()) { - err = AOS_ERROR_WRAP(errClear); + if (hasBandwidth) { + if (auto errClear = mBandwidth->Clear(hostIfName); !errClear.IsNone() && err.IsNone()) { + err = AOS_ERROR_WRAP(errClear); + } } if (auto errRemove = mFirewall->RemoveInstance(instanceID); !errRemove.IsNone() && err.IsNone()) { From e86a000bd063e6d8c9219223aef825475546fde9 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 10:05:52 +0300 Subject: [PATCH 079/112] sm: networkmanager: update tests for conditional bandwidth clear Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/tests/networkmanager.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index 61296ef37..df32bb9a4 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -292,8 +292,8 @@ class NetworkManagerTest : public Test { EXPECT_CALL(mDNSName, CreateServer(_, _)) .WillOnce(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); EXPECT_CALL(mDNSServer, RemoveHost(aos::String("leftover-instance"))).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mBandwidth, Clear(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mFirewall, RemoveInstance(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + // The leftover instance has no bandwidth shaping, so no bandwidth clear is expected. // The host veth is no longer detached synchronously on cleanup; the instance // netns teardown reaps it asynchronously. EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); @@ -324,7 +324,6 @@ class NetworkManagerTest : public Test { void ExpectDeleteInstanceCalls(int times = 1) { EXPECT_CALL(mDNSServer, RemoveHost(_)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mBandwidth, Clear(_)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mFirewall, RemoveInstance(_)).Times(times).WillRepeatedly(Return(aos::ErrorEnum::eNone)); } @@ -682,6 +681,7 @@ TEST_F(NetworkManagerTest, StartInstanceNetwork_FailOnTrafficMonitorError) EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mBridgeNetwork, Detach(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); ExpectDeleteInstanceCalls(); EXPECT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eRuntime); @@ -928,7 +928,6 @@ TEST_F(NetworkManagerTest, StopInstanceNetwork_FailOnFirewallRemoveError) EXPECT_CALL(mTrafficMonitor, StopInstanceMonitoring(instanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mDNSServer, RemoveHost(_)).WillOnce(Return(aos::ErrorEnum::eNone)); - EXPECT_CALL(mBandwidth, Clear(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mFirewall, RemoveInstance(_)).WillOnce(Return(aos::ErrorEnum::eRuntime)); EXPECT_CALL(mNetns, DeleteNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mNetIf, DeleteLink(_)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); From 001167c80d5ae7d3bfc97480385a5e2348625ca0 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 10:29:43 +0300 Subject: [PATCH 080/112] sm: networkmanager: add storage transaction interface Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/itf/storage.hpp | 21 +++++++++++++++++++++ src/core/sm/tests/mocks/storagemock.hpp | 3 +++ 2 files changed, 24 insertions(+) diff --git a/src/core/sm/networkmanager/itf/storage.hpp b/src/core/sm/networkmanager/itf/storage.hpp index 08882dfb0..c49654258 100644 --- a/src/core/sm/networkmanager/itf/storage.hpp +++ b/src/core/sm/networkmanager/itf/storage.hpp @@ -202,6 +202,27 @@ class StorageItf { */ virtual Error RemoveTrafficMonitorData(const String& chain) = 0; + /** + * Begins a storage transaction; subsequent writes are staged until commit. + * + * @return Error. + */ + virtual Error BeginTransaction() = 0; + + /** + * Commits the current storage transaction. + * + * @return Error. + */ + virtual Error CommitTransaction() = 0; + + /** + * Rolls back the current storage transaction, discarding staged writes. + * + * @return Error. + */ + virtual Error RollbackTransaction() = 0; + /** * Destroys storage interface. */ diff --git a/src/core/sm/tests/mocks/storagemock.hpp b/src/core/sm/tests/mocks/storagemock.hpp index 9ee8195ab..eadbcc5a3 100644 --- a/src/core/sm/tests/mocks/storagemock.hpp +++ b/src/core/sm/tests/mocks/storagemock.hpp @@ -25,6 +25,9 @@ class StorageMock : public StorageItf { MOCK_METHOD(Error, SetTrafficMonitorData, (const String&, const Time&, uint64_t), (override)); MOCK_METHOD(Error, GetTrafficMonitorData, (const String&, Time&, uint64_t&), (const, override)); MOCK_METHOD(Error, RemoveTrafficMonitorData, (const String&), (override)); + MOCK_METHOD(Error, BeginTransaction, (), (override)); + MOCK_METHOD(Error, CommitTransaction, (), (override)); + MOCK_METHOD(Error, RollbackTransaction, (), (override)); }; } // namespace aos::sm::networkmanager From a7b0ee8405ea079f21509a0c940345286782de59 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 10:30:39 +0300 Subject: [PATCH 081/112] sm: networkmanager: add firewall and traffic batch interface Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/itf/firewall.hpp | 21 +++++++++++++++++++ .../sm/networkmanager/itf/trafficmonitor.hpp | 21 +++++++++++++++++++ .../tests/mocks/firewallmock.hpp | 3 +++ .../tests/mocks/trafficmonitormock.hpp | 3 +++ 4 files changed, 48 insertions(+) diff --git a/src/core/sm/networkmanager/itf/firewall.hpp b/src/core/sm/networkmanager/itf/firewall.hpp index 3d6af4d28..a3dddca1e 100644 --- a/src/core/sm/networkmanager/itf/firewall.hpp +++ b/src/core/sm/networkmanager/itf/firewall.hpp @@ -158,6 +158,27 @@ class FirewallItf { * @return Error. */ virtual Error RemoveMasquerade(const String& subnet, const String& outIf) = 0; + + /** + * Opens a batch; AddInstance/RemoveInstance calls are staged until flush. + * + * @return Error. + */ + virtual Error BeginBatch() = 0; + + /** + * Flushes the staged batch atomically in a single nft transaction. + * + * @return Error. + */ + virtual Error FlushBatch() = 0; + + /** + * Reverts the flushed batch, deleting everything it applied by handle. + * + * @return Error. + */ + virtual Error Revert() = 0; }; /** @}*/ diff --git a/src/core/sm/networkmanager/itf/trafficmonitor.hpp b/src/core/sm/networkmanager/itf/trafficmonitor.hpp index 488e9c1c5..d71ac7951 100644 --- a/src/core/sm/networkmanager/itf/trafficmonitor.hpp +++ b/src/core/sm/networkmanager/itf/trafficmonitor.hpp @@ -104,6 +104,27 @@ class TrafficMonitorItf { */ virtual Error GetInstanceTraffic(const String& instanceID, uint64_t& inputTraffic, uint64_t& outputTraffic) const = 0; + + /** + * Opens a batch; StartInstanceMonitoring/StopInstanceMonitoring calls are staged until flush. + * + * @return Error. + */ + virtual Error BeginBatch() = 0; + + /** + * Flushes the staged batch atomically in a single nft transaction. + * + * @return Error. + */ + virtual Error FlushBatch() = 0; + + /** + * Reverts the flushed batch, deleting everything it applied by handle. + * + * @return Error. + */ + virtual Error Revert() = 0; }; /** @}*/ diff --git a/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp b/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp index db9f668d4..37d70861a 100644 --- a/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp @@ -23,6 +23,9 @@ class FirewallMock : public FirewallItf { MOCK_METHOD(Error, UpdateInstance, (const String&, const InstanceFirewallParams&), (override)); MOCK_METHOD(Error, AddMasquerade, (const String&, const String&), (override)); MOCK_METHOD(Error, RemoveMasquerade, (const String&, const String&), (override)); + MOCK_METHOD(Error, BeginBatch, (), (override)); + MOCK_METHOD(Error, FlushBatch, (), (override)); + MOCK_METHOD(Error, Revert, (), (override)); }; } // namespace aos::sm::networkmanager diff --git a/src/core/sm/networkmanager/tests/mocks/trafficmonitormock.hpp b/src/core/sm/networkmanager/tests/mocks/trafficmonitormock.hpp index af96da377..7dcc873a4 100644 --- a/src/core/sm/networkmanager/tests/mocks/trafficmonitormock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/trafficmonitormock.hpp @@ -22,6 +22,9 @@ class TrafficMonitorMock : public TrafficMonitorItf { MOCK_METHOD(Error, StopInstanceMonitoring, (const String&), (override)); MOCK_METHOD(Error, GetSystemTraffic, (uint64_t&, uint64_t&), (const, override)); MOCK_METHOD(Error, GetInstanceTraffic, (const String&, uint64_t&, uint64_t&), (const, override)); + MOCK_METHOD(Error, BeginBatch, (), (override)); + MOCK_METHOD(Error, FlushBatch, (), (override)); + MOCK_METHOD(Error, Revert, (), (override)); }; } // namespace aos::sm::networkmanager From 7708f44c048b63a2abb24642e6e20f7b5e7d402c Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 10:37:03 +0300 Subject: [PATCH 082/112] sm: networkmanager: add batch interface to network manager Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/itf/networkmanager.hpp | 15 +++++++++++++++ src/core/sm/networkmanager/networkmanager.cpp | 12 ++++++++++++ src/core/sm/networkmanager/networkmanager.hpp | 15 +++++++++++++++ src/core/sm/tests/mocks/networkmanagermock.hpp | 2 ++ 4 files changed, 44 insertions(+) diff --git a/src/core/sm/networkmanager/itf/networkmanager.hpp b/src/core/sm/networkmanager/itf/networkmanager.hpp index 6c6333194..e753cb14a 100644 --- a/src/core/sm/networkmanager/itf/networkmanager.hpp +++ b/src/core/sm/networkmanager/itf/networkmanager.hpp @@ -128,6 +128,21 @@ class NetworkManagerItf : public SystemTrafficProviderItf, * @return Error. */ virtual Error ReleaseInstanceNetwork(const String& instanceID, const String& networkID) = 0; + + /** + * Opens a batch; start/stop operations are staged and applied on flush. + * + * @return Error. + */ + virtual Error BeginBatch() = 0; + + /** + * Flushes the staged batch atomically across firewall, traffic and storage. + * + * @param[out] failedInstanceIDs instances that were not applied. + * @return Error. + */ + virtual Error FlushBatch(Array>& failedInstanceIDs) = 0; }; /** @}*/ diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 941b57bb3..e6123de02 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -516,6 +516,18 @@ Error NetworkManager::ReleaseInstanceNetwork(const String& instanceID, const Str return ErrorEnum::eNone; } +Error NetworkManager::BeginBatch() +{ + return ErrorEnum::eNone; +} + +Error NetworkManager::FlushBatch(Array>& failedInstanceIDs) +{ + failedInstanceIDs.Clear(); + + return ErrorEnum::eNone; +} + Error NetworkManager::PrepareUpdateItemNetworkParams( const InstanceNetworkConfig& params, const String& networkID, UpdateItemNetworkParams& serviceData) const { diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index 0c12f6500..e8ab55c2d 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -165,6 +165,21 @@ class NetworkManager : public NetworkManagerItf { */ Error ReleaseInstanceNetwork(const String& instanceID, const String& networkID) override; + /** + * Opens a batch for start/stop operations. + * + * @return Error. + */ + Error BeginBatch() override; + + /** + * Flushes the staged batch. + * + * @param[out] failedInstanceIDs instances that were not applied. + * @return Error. + */ + Error FlushBatch(Array>& failedInstanceIDs) override; + /** * Called when pending firewall rules are resolved for an instance. * diff --git a/src/core/sm/tests/mocks/networkmanagermock.hpp b/src/core/sm/tests/mocks/networkmanagermock.hpp index 352deab33..ee9726132 100644 --- a/src/core/sm/tests/mocks/networkmanagermock.hpp +++ b/src/core/sm/tests/mocks/networkmanagermock.hpp @@ -28,6 +28,8 @@ class NetworkManagerMock : public NetworkManagerItf { MOCK_METHOD(Error, GetHosts, (const String& instanceID, Array& hosts), (const, override)); MOCK_METHOD(Error, StopInstanceNetwork, (const String& instanceID, const String& networkID), (override)); MOCK_METHOD(Error, ReleaseInstanceNetwork, (const String& instanceID, const String& networkID), (override)); + MOCK_METHOD(Error, BeginBatch, (), (override)); + MOCK_METHOD(Error, FlushBatch, (Array> & failedInstanceIDs), (override)); MOCK_METHOD(void, OnPendingFirewallUpdate, (const String& nodeID, const aos::networkmanager::PendingFirewallUpdate& update), (override)); MOCK_METHOD(void, OnConnect, (), (override)); From a101f668cc7e89f1d9949bbb61dfd73ce8be7ceb Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 12:31:07 +0300 Subject: [PATCH 083/112] sm: networkmanager: orchestrate network batch flush Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 107 ++++++++++++++++++ src/core/sm/networkmanager/networkmanager.hpp | 24 ++-- 2 files changed, 123 insertions(+), 8 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index e6123de02..ee31c8f63 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -285,6 +285,17 @@ Error NetworkManager::StartInstanceNetwork(const String& instanceID, const Strin err = AddInstanceToNetwork(instanceID, networkID, cachedInfo->mNetworkConfig, cachedInfo->mAllocatedParams); + if (err.IsNone()) { + LockGuard lock {mMutex}; + + if (mBatchMode) { + if (auto errBatch = mBatchEntries.PushBack({instanceID, networkID, BatchOp::eAdd}); !errBatch.IsNone()) { + LOG_ERR() << "Failed to register batch entry" << Log::Field("instanceID", instanceID) + << Log::Field(errBatch); + } + } + } + return err; } @@ -415,6 +426,17 @@ Error NetworkManager::StopInstanceNetwork(const String& instanceID, const String } } + { + LockGuard lock {mMutex}; + + if (mBatchMode) { + if (auto errBatch = mBatchEntries.PushBack({instanceID, networkID, BatchOp::eRemove}); !errBatch.IsNone()) { + LOG_ERR() << "Failed to register batch entry" << Log::Field("instanceID", instanceID) + << Log::Field(errBatch); + } + } + } + if (auto errRemove = RemoveInstanceFromCache(instanceID, networkID); !errRemove.IsNone() && err.IsNone()) { err = errRemove; } @@ -518,6 +540,47 @@ Error NetworkManager::ReleaseInstanceNetwork(const String& instanceID, const Str Error NetworkManager::BeginBatch() { + Error err; + + { + LockGuard lock {mMutex}; + + mBatchEntries.Clear(); + mBatchMode = true; + } + + auto cleanupBatchMode = DeferRelease(this, [&err](NetworkManager* self) { + if (!err.IsNone()) { + LockGuard lock {self->mMutex}; + + self->mBatchMode = false; + } + }); + + if (err = mStorage->BeginTransaction(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto cleanupStorage = DeferRelease(this, [&err](NetworkManager* self) { + if (!err.IsNone()) { + self->mStorage->RollbackTransaction(); + } + }); + + if (err = mFirewall->BeginBatch(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto cleanupFirewall = DeferRelease(this, [&err](NetworkManager* self) { + if (!err.IsNone()) { + self->mFirewall->Revert(); + } + }); + + if (err = mNetMonitor->BeginBatch(); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + return ErrorEnum::eNone; } @@ -525,6 +588,50 @@ Error NetworkManager::FlushBatch(Array>& failedInstanceIDs) { failedInstanceIDs.Clear(); + Error err; + bool firewallApplied = false; + bool trafficApplied = false; + + auto cleanupFirewall = DeferRelease(this, [&err, &firewallApplied](NetworkManager* self) { + if (!err.IsNone() && firewallApplied) { + self->mFirewall->Revert(); + } + }); + + auto cleanupTraffic = DeferRelease(this, [&err, &trafficApplied](NetworkManager* self) { + if (!err.IsNone() && trafficApplied) { + self->mNetMonitor->Revert(); + } + }); + + if (err = mFirewall->FlushBatch(); err.IsNone()) { + firewallApplied = true; + + if (err = mNetMonitor->FlushBatch(); err.IsNone()) { + trafficApplied = true; + + err = mStorage->CommitTransaction(); + } + } + + if (!err.IsNone()) { + // nft flush failed: staged DB writes must not be committed. + if (!firewallApplied || !trafficApplied) { + mStorage->RollbackTransaction(); + } + + for (const auto& entry : mBatchEntries) { + failedInstanceIDs.PushBack(entry.mInstanceID); + } + } + + { + LockGuard lock {mMutex}; + + mBatchMode = false; + mBatchEntries.Clear(); + } + return ErrorEnum::eNone; } diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index e8ab55c2d..8e33e4735 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -195,18 +195,18 @@ class NetworkManager : public NetworkManagerItf { void OnConnect() override; private: - Error EnsureNodeNetwork(const String& networkID); - Error EnsureNodeNetworkPhysical(const String& networkID); - Error UpdateInstanceFirewall(const String& instanceID, const String& networkID, - const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); - - Error AddInstanceToNetwork(const String& instanceID, const String& networkID, - const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); - using InstanceHosts = StaticArray, cMaxNumHosts>; using InstanceCache = StaticMap, InstanceHosts, cMaxNumInstances>; using NetworkCache = StaticMap, InstanceCache, cMaxNumOwners>; + enum class BatchOp { eAdd, eRemove }; + + struct BatchEntry { + StaticString mInstanceID; + StaticString mNetworkID; + BatchOp mOp; + }; + // StartInstanceNetwork keeps its cached InstanceNetworkInfo alive across the nested call to // AddInstanceToNetwork, which in turn allocates hosts, bridge/firewall/bandwidth/DNS params and // its own InstanceNetworkInfo before returning. That is the largest concurrent footprint of any @@ -252,6 +252,12 @@ class NetworkManager : public NetworkManagerItf { Error InitInstance(const String& instanceID, const String& networkID); Error ReconcileInstances(); Error RemoveFirewallOrphans(); + Error EnsureNodeNetwork(const String& networkID); + Error EnsureNodeNetworkPhysical(const String& networkID); + Error UpdateInstanceFirewall(const String& instanceID, const String& networkID, + const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); + Error AddInstanceToNetwork(const String& instanceID, const String& networkID, + const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); Error RemoveDNSOrphans(); Error AdoptDNSServer(const String& networkID); Error PrepareBridgeParams( @@ -311,6 +317,8 @@ class NetworkManager : public NetworkManagerItf { StaticMap, DNSServerItf*, cMaxNumOwners> mDNSServers; StaticMap, InstanceNetworkInfo, cMaxNumInstances * cMaxNumOwners> mInstanceNetworkInfos; StaticArray, cMaxNumOwners> mPhysicalNetworks; + bool mBatchMode {false}; + StaticArray mBatchEntries; StaticAllocator)> mNetworkInfosAllocator; StaticAllocator)> mInstanceNetworkInfosAllocator; From 9f7e5dfdc8c4892632a1eb39855e90ba9cf07970 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 12:42:59 +0300 Subject: [PATCH 084/112] sm: networkmanager: fall back to per-instance apply on batch flush failure Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 124 +++++++++++++++--- src/core/sm/networkmanager/networkmanager.hpp | 1 + 2 files changed, 104 insertions(+), 21 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index ee31c8f63..02632026a 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -592,36 +592,45 @@ Error NetworkManager::FlushBatch(Array>& failedInstanceIDs) bool firewallApplied = false; bool trafficApplied = false; - auto cleanupFirewall = DeferRelease(this, [&err, &firewallApplied](NetworkManager* self) { - if (!err.IsNone() && firewallApplied) { - self->mFirewall->Revert(); - } - }); + { + auto cleanupFirewall = DeferRelease(this, [&err, &firewallApplied](NetworkManager* self) { + if (!err.IsNone() && firewallApplied) { + self->mFirewall->Revert(); + } + }); - auto cleanupTraffic = DeferRelease(this, [&err, &trafficApplied](NetworkManager* self) { - if (!err.IsNone() && trafficApplied) { - self->mNetMonitor->Revert(); - } - }); + auto cleanupTraffic = DeferRelease(this, [&err, &trafficApplied](NetworkManager* self) { + if (!err.IsNone() && trafficApplied) { + self->mNetMonitor->Revert(); + } + }); - if (err = mFirewall->FlushBatch(); err.IsNone()) { - firewallApplied = true; + if (err = mFirewall->FlushBatch(); err.IsNone()) { + firewallApplied = true; - if (err = mNetMonitor->FlushBatch(); err.IsNone()) { - trafficApplied = true; + if (err = mNetMonitor->FlushBatch(); err.IsNone()) { + trafficApplied = true; - err = mStorage->CommitTransaction(); + err = mStorage->CommitTransaction(); + } } - } - if (!err.IsNone()) { - // nft flush failed: staged DB writes must not be committed. - if (!firewallApplied || !trafficApplied) { + if (!err.IsNone() && (!firewallApplied || !trafficApplied)) { mStorage->RollbackTransaction(); } + } - for (const auto& entry : mBatchEntries) { - failedInstanceIDs.PushBack(entry.mInstanceID); + if (!err.IsNone()) { + if (firewallApplied && trafficApplied) { + for (const auto& entry : mBatchEntries) { + failedInstanceIDs.PushBack(entry.mInstanceID); + } + } else { + for (const auto& entry : mBatchEntries) { + if (auto errReapply = ReapplyInstancePolicy(entry); !errReapply.IsNone()) { + failedInstanceIDs.PushBack(entry.mInstanceID); + } + } } } @@ -635,6 +644,79 @@ Error NetworkManager::FlushBatch(Array>& failedInstanceIDs) return ErrorEnum::eNone; } +Error NetworkManager::ReapplyInstancePolicy(const BatchEntry& entry) +{ + if (entry.mOp == BatchOp::eRemove) { + Error err; + + if (auto errFW = mFirewall->RemoveInstance(entry.mInstanceID); !errFW.IsNone()) { + err = errFW; + } + + if (auto errTR = mNetMonitor->StopInstanceMonitoring(entry.mInstanceID); !errTR.IsNone() && err.IsNone()) { + err = errTR; + } + + return err; + } + + auto info = MakeUnique(&mAllocator); + + { + LockGuard lock {mMutex}; + + auto it = mInstanceNetworkInfos.Find(entry.mInstanceID); + if (it == mInstanceNetworkInfos.end()) { + return AOS_ERROR_WRAP(Error(ErrorEnum::eNotFound, "instance network info not found")); + } + + *info = it->mSecond; + } + + auto firewallParams = MakeUnique(&mAllocator); + + if (auto err = PrepareInstanceFirewallParams(info->mNetworkConfig, info->mAllocatedParams, *firewallParams); + !err.IsNone()) { + return err; + } + + Error err; + + if (err = mFirewall->AddInstance(entry.mInstanceID, *firewallParams); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto cleanupFirewall = DeferRelease(this, [&err, &entry](NetworkManager* self) { + if (!err.IsNone()) { + if (auto errRemove = self->mFirewall->RemoveInstance(entry.mInstanceID); !errRemove.IsNone()) { + LOG_ERR() << "Failed to remove firewall instance on rollback" + << Log::Field("instanceID", entry.mInstanceID) << Log::Field(errRemove); + } + } + }); + + if (err = mNetMonitor->StartInstanceMonitoring(entry.mInstanceID, info->mAllocatedParams.mIP, + info->mNetworkConfig.mDownloadLimit, info->mNetworkConfig.mUploadLimit); + !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + auto cleanupMonitoring = DeferRelease(this, [&err, &entry](NetworkManager* self) { + if (!err.IsNone()) { + if (auto errStop = self->mNetMonitor->StopInstanceMonitoring(entry.mInstanceID); !errStop.IsNone()) { + LOG_ERR() << "Failed to stop instance monitoring on rollback" + << Log::Field("instanceID", entry.mInstanceID) << Log::Field(errStop); + } + } + }); + + if (err = mStorage->UpdateInstanceNetworkInfo(*info); !err.IsNone()) { + return AOS_ERROR_WRAP(err); + } + + return ErrorEnum::eNone; +} + Error NetworkManager::PrepareUpdateItemNetworkParams( const InstanceNetworkConfig& params, const String& networkID, UpdateItemNetworkParams& serviceData) const { diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index 8e33e4735..0e14ea9f7 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -258,6 +258,7 @@ class NetworkManager : public NetworkManagerItf { const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); Error AddInstanceToNetwork(const String& instanceID, const String& networkID, const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); + Error ReapplyInstancePolicy(const BatchEntry& entry); Error RemoveDNSOrphans(); Error AdoptDNSServer(const String& networkID); Error PrepareBridgeParams( From 20818fcf43dd7d0781c872794e469e941d0eccd9 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 14:04:37 +0300 Subject: [PATCH 085/112] sm: launcher: wrap network start phase in batch and fail unapplied instances Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/launcher/launcher.cpp | 38 +++++++++++++++++++++++++++++++ src/core/sm/launcher/launcher.hpp | 1 + 2 files changed, 39 insertions(+) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index a9ed66b9c..6bfea00bc 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -900,6 +900,11 @@ void Launcher::StartNetworks(const Array& startInstances) { LOG_INF() << "[profiling] Start networks begin" << Log::Field("count", startInstances.Size()); + auto errBegin = mNetworkManager->BeginBatch(); + if (!errBegin.IsNone()) { + LOG_ERR() << "Failed to begin network batch" << Log::Field(AOS_ERROR_WRAP(errBegin)); + } + for (const auto& instance : startInstances) { auto instanceData = FindInstanceData(instance); if (!instanceData) { @@ -932,6 +937,29 @@ void Launcher::StartNetworks(const Array& startInstances) LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); } + if (errBegin.IsNone()) { + auto failedIDs = MakeUnique, cMaxNumInstances>>(&mAllocator); + + mNetworkManager->FlushBatch(*failedIDs); + + for (const auto& failedID : *failedIDs) { + auto instanceData = FindInstanceDataByID(failedID); + if (!instanceData) { + continue; + } + + SetInstanceState(*instanceData, InstanceStateEnum::eFailed, + AOS_ERROR_WRAP(Error(ErrorEnum::eFailed, "network batch apply failed"))); + + if (auto err + = mNetworkManager->StopInstanceNetwork(instanceData->mInstanceID, instanceData->mInfo.mOwnerID); + !err.IsNone() && !err.Is(ErrorEnum::eNotFound)) { + LOG_ERR() << "Failed to stop network" << Log::Field("instance", instanceData->mInfo) + << Log::Field(AOS_ERROR_WRAP(err)); + } + } + } + LOG_INF() << "[profiling] Start networks end"; } @@ -1127,6 +1155,16 @@ Launcher::InstanceData* Launcher::FindInstanceData(const InstanceIdent& instance return const_cast(this)->FindInstanceData(instanceIdent); } +Launcher::InstanceData* Launcher::FindInstanceDataByID(const String& instanceID) +{ + auto it = mInstances.FindIf([&instanceID](const auto& instance) { return instance.mInstanceID == instanceID; }); + if (it != mInstances.end()) { + return it; + } + + return nullptr; +} + RuntimeItf* Launcher::FindInstanceRuntime(const String& runtimeID) { auto it = mRuntimes.FindIf([&runtimeID](const auto& it) { return it.mSecond == runtimeID; }); diff --git a/src/core/sm/launcher/launcher.hpp b/src/core/sm/launcher/launcher.hpp index 801caafd3..f6c17a9a1 100644 --- a/src/core/sm/launcher/launcher.hpp +++ b/src/core/sm/launcher/launcher.hpp @@ -225,6 +225,7 @@ class Launcher : public LauncherItf, InstanceData* FindInstanceData(const InstanceIdent& instanceIdent); InstanceData* FindInstanceData(const InstanceIdent& instanceIdent) const; + InstanceData* FindInstanceDataByID(const String& instanceID); RuntimeItf* FindInstanceRuntime(const String& runtimeID); RuntimeItf* FindInstanceRuntime(const String& runtimeID) const; RuntimeItf* FindInstanceRuntime(const InstanceIdent& instanceIdent); From e3a5fbb73659744114c45a8971892ad3b01ee042 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 14:13:50 +0300 Subject: [PATCH 086/112] sm: launcher: wrap network stop phases in batch Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/launcher/launcher.cpp | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index 6bfea00bc..d2c4fef42 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -792,6 +792,11 @@ void Launcher::StopAllNetworks() { LOG_INF() << "[profiling] Stop all networks begin" << Log::Field("count", mInstances.Size()); + auto errBegin = mNetworkManager->BeginBatch(); + if (!errBegin.IsNone()) { + LOG_ERR() << "Failed to begin network batch" << Log::Field(AOS_ERROR_WRAP(errBegin)); + } + for (auto& instance : mInstances) { if (instance.mInfo.mType != UpdateItemTypeEnum::eService) { continue; @@ -804,6 +809,20 @@ void Launcher::StopAllNetworks() } } + if (auto err = mLaunchPool.Wait(); !err.IsNone()) { + LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); + } + + if (errBegin.IsNone()) { + auto failedIDs = MakeUnique, cMaxNumInstances>>(&mAllocator); + + mNetworkManager->FlushBatch(*failedIDs); + + if (!failedIDs->IsEmpty()) { + LOG_WRN() << "Network stop batch partially failed" << Log::Field("count", failedIDs->Size()); + } + } + LOG_INF() << "[profiling] Stop all networks end"; } @@ -985,6 +1004,11 @@ void Launcher::StopNetworks(const Array& stopInstances) { LOG_INF() << "[profiling] Stop networks begin" << Log::Field("count", stopInstances.Size()); + auto errBegin = mNetworkManager->BeginBatch(); + if (!errBegin.IsNone()) { + LOG_ERR() << "Failed to begin network batch" << Log::Field(AOS_ERROR_WRAP(errBegin)); + } + for (const auto& instance : stopInstances) { auto instanceData = FindInstanceData(instance); if (!instanceData) { @@ -1009,6 +1033,16 @@ void Launcher::StopNetworks(const Array& stopInstances) LOG_ERR() << "Thread pool wait failed" << Log::Field(AOS_ERROR_WRAP(err)); } + if (errBegin.IsNone()) { + auto failedIDs = MakeUnique, cMaxNumInstances>>(&mAllocator); + + mNetworkManager->FlushBatch(*failedIDs); + + if (!failedIDs->IsEmpty()) { + LOG_WRN() << "Network stop batch partially failed" << Log::Field("count", failedIDs->Size()); + } + } + LOG_INF() << "[profiling] Stop networks end"; } From 50dc4a48eb84e795ea5b59630c4291cac96e943a Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 15:27:48 +0300 Subject: [PATCH 087/112] sm: networkmanager: test batch flush success, revert and per-instance fallback Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../networkmanager/tests/networkmanager.cpp | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index df32bb9a4..a045d00bc 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -625,6 +625,168 @@ TEST_F(NetworkManagerTest, CreateAndStartInstanceNetwork_VerifyResolvConfFile) } } +TEST_F(NetworkManagerTest, BeginFlushBatch_ForwardToBackends) +{ + EXPECT_CALL(mStorage, BeginTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->BeginBatch(), aos::ErrorEnum::eNone); + + EXPECT_CALL(mFirewall, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mStorage, CommitTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + + aos::StaticArray, aos::cMaxNumInstances> failed; + + EXPECT_EQ(mNetManager->FlushBatch(failed), aos::ErrorEnum::eNone); + EXPECT_TRUE(failed.IsEmpty()); +} + +TEST_F(NetworkManagerTest, BeginBatch_PropagatesBackendError) +{ + EXPECT_CALL(mStorage, BeginTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eRuntime)); + EXPECT_CALL(mStorage, RollbackTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + + EXPECT_NE(mNetManager->BeginBatch(), aos::ErrorEnum::eNone); +} + +TEST_F(NetworkManagerTest, FlushBatch_NftFailure_RevertsAndFallsBack) +{ + const aos::String instanceID1 = "test-instance-1"; + const aos::String instanceID2 = "test-instance-2"; + const aos::String networkID = "test-network"; + auto params = CreateTestInstanceNetworkConfig(); + auto allocatedParams = CreateTestAllocatedParams(); + + SetupEnsureNodeNetworkCreateMocks(networkID, allocatedParams.mSubnet, "192.168.1.1", 100ULL); + + EXPECT_CALL(mNetworkProvider, AllocateInstanceNetwork(_, networkID, aos::String("test-node"), _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgReferee<4>(allocatedParams), Return(aos::ErrorEnum::eNone))); + EXPECT_CALL(mStorage, AddInstanceNetworkInfo(_)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID1, networkID, params), aos::ErrorEnum::eNone); + + params.mHosts.Clear(); + params.mAliases.Clear(); + params.mHosts.PushBack(aos::Host {"10.0.0.3", "host3.example.com"}); + params.mAliases.PushBack("alias3"); + params.mHostname = "test-host-3"; + params.mInstanceIdent.mInstance = 1; + + ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID2, networkID, params), aos::ErrorEnum::eNone); + + EXPECT_CALL(mStorage, BeginTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->BeginBatch(), aos::ErrorEnum::eNone); + + SetupEnsureNodeNetworkPhysicalMocks("192.168.1.1", allocatedParams.mSubnet, 100ULL); + + BridgeAttachResult attachResult; + attachResult.mHostIfName = "veth-test"; + attachResult.mContainerIfName = "eth0"; + + EXPECT_CALL(mBridgeNetwork, Attach(_, _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgReferee<2>(attachResult), Return(aos::ErrorEnum::eNone))); + EXPECT_CALL(mBandwidth, Apply(_, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mDNSServer, AddHost(_, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, CreateNetworkNamespace(_)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) + .Times(2) + .WillRepeatedly(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); + + EXPECT_CALL(mFirewall, AddInstance(instanceID1, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, AddInstance(instanceID2, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, RemoveInstance(instanceID2)).WillOnce(Return(aos::ErrorEnum::eNone)); + + EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(instanceID1, _, _, _)) + .Times(2) + .WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(instanceID2, _, _, _)) + .WillOnce(Return(aos::ErrorEnum::eNone)) + .WillOnce(Return(aos::ErrorEnum::eRuntime)); + + EXPECT_CALL(mStorage, UpdateInstanceNetworkInfo(_)).Times(3).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID1, networkID), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID2, networkID), aos::ErrorEnum::eNone); + + EXPECT_CALL(mFirewall, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eRuntime)); + EXPECT_CALL(mStorage, RollbackTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + + aos::StaticArray, aos::cMaxNumInstances> failed; + + EXPECT_EQ(mNetManager->FlushBatch(failed), aos::ErrorEnum::eNone); + ASSERT_EQ(failed.Size(), 1U); + EXPECT_EQ(failed[0], instanceID2); +} + +TEST_F(NetworkManagerTest, FlushBatch_CommitFailure_MarksAllFailed) +{ + const aos::String instanceID1 = "test-instance-1"; + const aos::String instanceID2 = "test-instance-2"; + const aos::String networkID = "test-network"; + auto params = CreateTestInstanceNetworkConfig(); + auto allocatedParams = CreateTestAllocatedParams(); + + SetupEnsureNodeNetworkCreateMocks(networkID, allocatedParams.mSubnet, "192.168.1.1", 100ULL); + + EXPECT_CALL(mNetworkProvider, AllocateInstanceNetwork(_, networkID, aos::String("test-node"), _, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgReferee<4>(allocatedParams), Return(aos::ErrorEnum::eNone))); + EXPECT_CALL(mStorage, AddInstanceNetworkInfo(_)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID1, networkID, params), aos::ErrorEnum::eNone); + + params.mHosts.Clear(); + params.mAliases.Clear(); + params.mHosts.PushBack(aos::Host {"10.0.0.3", "host3.example.com"}); + params.mAliases.PushBack("alias3"); + params.mHostname = "test-host-3"; + params.mInstanceIdent.mInstance = 1; + + ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID2, networkID, params), aos::ErrorEnum::eNone); + + EXPECT_CALL(mStorage, BeginTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->BeginBatch(), aos::ErrorEnum::eNone); + + SetupEnsureNodeNetworkPhysicalMocks("192.168.1.1", allocatedParams.mSubnet, 100ULL); + + ExpectAddInstanceCalls(2); + ExpectPersistInstanceCalls(2); + EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(_, _, _, _)) + .Times(2) + .WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, CreateNetworkNamespace(_)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) + .Times(2) + .WillRepeatedly(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); + + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID1, networkID), aos::ErrorEnum::eNone); + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID2, networkID), aos::ErrorEnum::eNone); + + EXPECT_CALL(mFirewall, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mStorage, CommitTransaction()).WillOnce(Return(aos::ErrorEnum::eRuntime)); + EXPECT_CALL(mFirewall, Revert()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, Revert()).WillOnce(Return(aos::ErrorEnum::eNone)); + + aos::StaticArray, aos::cMaxNumInstances> failed; + + EXPECT_EQ(mNetManager->FlushBatch(failed), aos::ErrorEnum::eNone); + ASSERT_EQ(failed.Size(), 2U); + EXPECT_EQ(failed[0], instanceID1); + EXPECT_EQ(failed[1], instanceID2); +} + TEST_F(NetworkManagerTest, StartInstanceNetwork_FailOnAttachError) { const aos::String instanceID = "test-instance"; From d372f2dffd1f989a0d629b82ebac3a1b1433f87b Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Wed, 22 Jul 2026 15:42:38 +0300 Subject: [PATCH 088/112] sm: launcher: expect network batch calls and cover start flush failure Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/launcher/tests/launcher.cpp | 61 +++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/core/sm/launcher/tests/launcher.cpp b/src/core/sm/launcher/tests/launcher.cpp index b9da0b2ae..2eaf05cd7 100644 --- a/src/core/sm/launcher/tests/launcher.cpp +++ b/src/core/sm/launcher/tests/launcher.cpp @@ -194,6 +194,8 @@ class LauncherTest : public Test { EXPECT_CALL(mNetworkManager, StartInstanceNetwork).WillRepeatedly(Return(ErrorEnum::eNone)); EXPECT_CALL(mNetworkManager, StopInstanceNetwork).WillRepeatedly(Return(ErrorEnum::eNone)); EXPECT_CALL(mNetworkManager, ReleaseInstanceNetwork).WillRepeatedly(Return(ErrorEnum::eNone)); + EXPECT_CALL(mNetworkManager, BeginBatch()).WillRepeatedly(Return(ErrorEnum::eNone)); + EXPECT_CALL(mNetworkManager, FlushBatch(_)).WillRepeatedly(Return(ErrorEnum::eNone)); } StaticArray GetRuntimesArray() @@ -462,6 +464,65 @@ TEST_F(LauncherTest, LauncherStartsStoredInstancesOnModuleStart) ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); } +TEST_F(LauncherTest, StartNetworks_FlushFailure_FailsInstance) +{ + const std::vector cStoredInfos = { + CreateInstanceInfo("item0", 0, "1.0.0", "runtime0"), + CreateInstanceInfo("item1", 1, "1.0.0", "runtime1"), + }; + + mStorage.Init(cStoredInfos); + + auto err = mLauncher.Init(GetRuntimesArray(), mImageManager, mSender, mStorage, mOCISpec, mItemInfoProvider, + mCloudConnection, mNetworkManager, mInstanceIDProvider, mResourceInfoProvider); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + EXPECT_CALL(mInstanceIDProvider, GetInstanceID) + .WillRepeatedly(Invoke([](const InstanceIdent& instance, String& instanceID) { + instanceID = instance.mItemID; + + return ErrorEnum::eNone; + })); + + EXPECT_CALL(mNetworkManager, FlushBatch(_)) + .WillRepeatedly(DoAll(WithArg<0>([](auto& failedInstanceIDs) { failedInstanceIDs.PushBack("item0"); }), + Return(ErrorEnum::eNone))); + + EXPECT_CALL(mNetworkManager, StopInstanceNetwork(String("item0"), _)) + .Times(AtLeast(1)) + .WillRepeatedly(Return(ErrorEnum::eNone)); + + EXPECT_CALL(mRuntime1, StartInstance).WillOnce(Invoke([](const InstanceInfo& instance, InstanceStatus& status) { + SetInstanceStatus(instance, InstanceStateEnum::eActive, status); + + return ErrorEnum::eNone; + })); + + err = mLauncher.Start(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + err = mLauncher.GetInstancesStatuses(mReceivedStatuses); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); + + ASSERT_EQ(mReceivedStatuses.Size(), cStoredInfos.size()); + + for (const auto& status : mReceivedStatuses) { + if (status.mItemID == "item0") { + EXPECT_EQ(status.mState, InstanceStateEnum::eFailed); + } else { + EXPECT_EQ(status.mState, InstanceStateEnum::eActive); + } + } + + EXPECT_CALL(mRuntime0, StopInstance(static_cast(cStoredInfos[0]), _)) + .WillOnce(Return(ErrorEnum::eNone)); + EXPECT_CALL(mRuntime1, StopInstance(static_cast(cStoredInfos[1]), _)) + .WillOnce(Return(ErrorEnum::eNone)); + + err = mLauncher.Stop(); + ASSERT_TRUE(err.IsNone()) << tests::utils::ErrorToStr(err); +} + TEST_F(LauncherTest, StopInstancesWithExpiredOfflineTTL) { const std::vector cStoredInfos = { From 9ae3821470a3215f77797d515840aab17b6a8805 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Thu, 23 Jul 2026 14:05:11 +0300 Subject: [PATCH 089/112] sm: networkmanager: add AbortBatch to firewall and traffic monitor Discarding a staged batch without applying it is not expressible with FlushBatch/Revert: FlushBatch commits and Revert undoes an already applied batch. Add AbortBatch to both backend interfaces and mocks. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/itf/firewall.hpp | 8 ++++++++ src/core/sm/networkmanager/itf/trafficmonitor.hpp | 8 ++++++++ src/core/sm/networkmanager/tests/mocks/firewallmock.hpp | 1 + .../sm/networkmanager/tests/mocks/trafficmonitormock.hpp | 1 + 4 files changed, 18 insertions(+) diff --git a/src/core/sm/networkmanager/itf/firewall.hpp b/src/core/sm/networkmanager/itf/firewall.hpp index a3dddca1e..37aa47f58 100644 --- a/src/core/sm/networkmanager/itf/firewall.hpp +++ b/src/core/sm/networkmanager/itf/firewall.hpp @@ -173,6 +173,14 @@ class FirewallItf { */ virtual Error FlushBatch() = 0; + /** + * Discards the staged batch and leaves batch mode without applying anything to the kernel + * (unlike FlushBatch which commits, or Revert which undoes an already-applied batch). + * + * @return Error. + */ + virtual Error AbortBatch() = 0; + /** * Reverts the flushed batch, deleting everything it applied by handle. * diff --git a/src/core/sm/networkmanager/itf/trafficmonitor.hpp b/src/core/sm/networkmanager/itf/trafficmonitor.hpp index d71ac7951..f5566c532 100644 --- a/src/core/sm/networkmanager/itf/trafficmonitor.hpp +++ b/src/core/sm/networkmanager/itf/trafficmonitor.hpp @@ -119,6 +119,14 @@ class TrafficMonitorItf { */ virtual Error FlushBatch() = 0; + /** + * Discards the staged batch and leaves batch mode without applying anything to the kernel + * (unlike FlushBatch which commits, or Revert which undoes an already-applied batch). + * + * @return Error. + */ + virtual Error AbortBatch() = 0; + /** * Reverts the flushed batch, deleting everything it applied by handle. * diff --git a/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp b/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp index 37d70861a..325d83283 100644 --- a/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/firewallmock.hpp @@ -25,6 +25,7 @@ class FirewallMock : public FirewallItf { MOCK_METHOD(Error, RemoveMasquerade, (const String&, const String&), (override)); MOCK_METHOD(Error, BeginBatch, (), (override)); MOCK_METHOD(Error, FlushBatch, (), (override)); + MOCK_METHOD(Error, AbortBatch, (), (override)); MOCK_METHOD(Error, Revert, (), (override)); }; diff --git a/src/core/sm/networkmanager/tests/mocks/trafficmonitormock.hpp b/src/core/sm/networkmanager/tests/mocks/trafficmonitormock.hpp index 7dcc873a4..0e6256cf8 100644 --- a/src/core/sm/networkmanager/tests/mocks/trafficmonitormock.hpp +++ b/src/core/sm/networkmanager/tests/mocks/trafficmonitormock.hpp @@ -24,6 +24,7 @@ class TrafficMonitorMock : public TrafficMonitorItf { MOCK_METHOD(Error, GetInstanceTraffic, (const String&, uint64_t&, uint64_t&), (const, override)); MOCK_METHOD(Error, BeginBatch, (), (override)); MOCK_METHOD(Error, FlushBatch, (), (override)); + MOCK_METHOD(Error, AbortBatch, (), (override)); MOCK_METHOD(Error, Revert, (), (override)); }; From ff13fb83864e45e104524a769ec42e927a0af940 Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Fri, 24 Jul 2026 12:54:42 +0300 Subject: [PATCH 090/112] sm: networkmanager: recover each backend on batch flush failure FlushBatch commits the firewall, traffic-monitor and storage backends in sequence and gives each failure its own recovery so a partial batch never leaves one backend applied while another is not: a firewall-flush failure aborts the still-staged traffic batch, a traffic-flush failure reverts the already-flushed firewall, and a storage-commit failure reverts both. The storage transaction is rolled back on every failure path, and the batch entries are re-applied per instance so one bad instance is isolated instead of failing the whole flush. Cover the firewall-, traffic- and commit-failure paths with tests. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 87 ++++++++++--------- src/core/sm/networkmanager/networkmanager.hpp | 2 + .../networkmanager/tests/networkmanager.cpp | 66 ++++++++++++++ 3 files changed, 114 insertions(+), 41 deletions(-) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index 02632026a..b2f619ac0 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -573,7 +573,7 @@ Error NetworkManager::BeginBatch() auto cleanupFirewall = DeferRelease(this, [&err](NetworkManager* self) { if (!err.IsNone()) { - self->mFirewall->Revert(); + self->mFirewall->AbortBatch(); } }); @@ -588,60 +588,65 @@ Error NetworkManager::FlushBatch(Array>& failedInstanceIDs) { failedInstanceIDs.Clear(); - Error err; - bool firewallApplied = false; - bool trafficApplied = false; + if (auto err = mFirewall->FlushBatch(); !err.IsNone()) { + LOG_ERR() << "Failed to flush firewall batch" << Log::Field(err); - { - auto cleanupFirewall = DeferRelease(this, [&err, &firewallApplied](NetworkManager* self) { - if (!err.IsNone() && firewallApplied) { - self->mFirewall->Revert(); - } - }); + mNetMonitor->AbortBatch(); + mStorage->RollbackTransaction(); - auto cleanupTraffic = DeferRelease(this, [&err, &trafficApplied](NetworkManager* self) { - if (!err.IsNone() && trafficApplied) { - self->mNetMonitor->Revert(); - } - }); + ReapplyBatchEntries(failedInstanceIDs); + ClearBatchState(); - if (err = mFirewall->FlushBatch(); err.IsNone()) { - firewallApplied = true; + return ErrorEnum::eNone; + } - if (err = mNetMonitor->FlushBatch(); err.IsNone()) { - trafficApplied = true; + if (auto err = mNetMonitor->FlushBatch(); !err.IsNone()) { + LOG_ERR() << "Failed to flush traffic monitor batch" << Log::Field(err); - err = mStorage->CommitTransaction(); - } - } + mFirewall->Revert(); + mStorage->RollbackTransaction(); - if (!err.IsNone() && (!firewallApplied || !trafficApplied)) { - mStorage->RollbackTransaction(); - } + ReapplyBatchEntries(failedInstanceIDs); + ClearBatchState(); + + return ErrorEnum::eNone; } - if (!err.IsNone()) { - if (firewallApplied && trafficApplied) { - for (const auto& entry : mBatchEntries) { - failedInstanceIDs.PushBack(entry.mInstanceID); - } - } else { - for (const auto& entry : mBatchEntries) { - if (auto errReapply = ReapplyInstancePolicy(entry); !errReapply.IsNone()) { - failedInstanceIDs.PushBack(entry.mInstanceID); - } - } + if (auto err = mStorage->CommitTransaction(); !err.IsNone()) { + LOG_ERR() << "Failed to commit batch transaction" << Log::Field(err); + + mFirewall->Revert(); + mNetMonitor->Revert(); + mStorage->RollbackTransaction(); + + for (const auto& entry : mBatchEntries) { + failedInstanceIDs.PushBack(entry.mInstanceID); } } - { - LockGuard lock {mMutex}; + ClearBatchState(); - mBatchMode = false; - mBatchEntries.Clear(); + return ErrorEnum::eNone; +} + +void NetworkManager::ReapplyBatchEntries(Array>& failedInstanceIDs) +{ + for (const auto& entry : mBatchEntries) { + if (auto err = ReapplyInstancePolicy(entry); !err.IsNone()) { + LOG_ERR() << "Failed to reapply instance policy" << Log::Field("instanceID", entry.mInstanceID) + << Log::Field(err); + + failedInstanceIDs.PushBack(entry.mInstanceID); + } } +} - return ErrorEnum::eNone; +void NetworkManager::ClearBatchState() +{ + LockGuard lock {mMutex}; + + mBatchMode = false; + mBatchEntries.Clear(); } Error NetworkManager::ReapplyInstancePolicy(const BatchEntry& entry) diff --git a/src/core/sm/networkmanager/networkmanager.hpp b/src/core/sm/networkmanager/networkmanager.hpp index 0e14ea9f7..eb58deb5e 100644 --- a/src/core/sm/networkmanager/networkmanager.hpp +++ b/src/core/sm/networkmanager/networkmanager.hpp @@ -259,6 +259,8 @@ class NetworkManager : public NetworkManagerItf { Error AddInstanceToNetwork(const String& instanceID, const String& networkID, const InstanceNetworkConfig& networkConfig, const aos::InstanceNetworkAllocation& networkParams); Error ReapplyInstancePolicy(const BatchEntry& entry); + void ReapplyBatchEntries(Array>& failedInstanceIDs); + void ClearBatchState(); Error RemoveDNSOrphans(); Error AdoptDNSServer(const String& networkID); Error PrepareBridgeParams( diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index a045d00bc..87d52f3bd 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -636,6 +636,7 @@ TEST_F(NetworkManagerTest, BeginFlushBatch_ForwardToBackends) EXPECT_CALL(mFirewall, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mTrafficMonitor, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mStorage, CommitTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, AbortBatch()).Times(0); aos::StaticArray, aos::cMaxNumInstances> failed; @@ -717,6 +718,8 @@ TEST_F(NetworkManagerTest, FlushBatch_NftFailure_RevertsAndFallsBack) ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID2, networkID), aos::ErrorEnum::eNone); EXPECT_CALL(mFirewall, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eRuntime)); + EXPECT_CALL(mTrafficMonitor, AbortBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, FlushBatch()).Times(0); EXPECT_CALL(mStorage, RollbackTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); aos::StaticArray, aos::cMaxNumInstances> failed; @@ -726,6 +729,67 @@ TEST_F(NetworkManagerTest, FlushBatch_NftFailure_RevertsAndFallsBack) EXPECT_EQ(failed[0], instanceID2); } +TEST_F(NetworkManagerTest, FlushBatch_NftFailure_AbortsTrafficBatchBeforeReapply) +{ + const aos::String instanceID = "test-instance"; + const aos::String networkID = "test-network"; + auto params = CreateTestInstanceNetworkConfig(); + auto allocatedParams = CreateTestAllocatedParams(); + + SetupEnsureNodeNetworkCreateMocks(networkID, allocatedParams.mSubnet, "192.168.1.1", 100ULL); + + EXPECT_CALL(mNetworkProvider, AllocateInstanceNetwork(_, networkID, aos::String("test-node"), _, _)) + .WillOnce(DoAll(SetArgReferee<4>(allocatedParams), Return(aos::ErrorEnum::eNone))); + EXPECT_CALL(mStorage, AddInstanceNetworkInfo(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->CreateInstanceNetwork(instanceID, networkID, params), aos::ErrorEnum::eNone); + + EXPECT_CALL(mStorage, BeginTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, BeginBatch()).WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->BeginBatch(), aos::ErrorEnum::eNone); + + SetupEnsureNodeNetworkPhysicalMocks("192.168.1.1", allocatedParams.mSubnet, 100ULL); + + BridgeAttachResult attachResult; + attachResult.mHostIfName = "veth-test"; + attachResult.mContainerIfName = "eth0"; + + EXPECT_CALL(mBridgeNetwork, Attach(_, _, _)) + .WillOnce(DoAll(SetArgReferee<2>(attachResult), Return(aos::ErrorEnum::eNone))); + EXPECT_CALL(mBandwidth, Apply(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mDNSServer, AddHost(_, _)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, CreateNetworkNamespace(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, GetNetworkNamespacePath(_)) + .WillOnce(Return(aos::RetWithError> {{}, aos::ErrorEnum::eNone})); + EXPECT_CALL(mFirewall, AddInstance(instanceID, _)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mStorage, UpdateInstanceNetworkInfo(_)).Times(2).WillRepeatedly(Return(aos::ErrorEnum::eNone)); + + // The batch is dead once the firewall flush fails, so the traffic batch must be dropped before + // the per-instance fallback re-applies monitoring, otherwise the re-apply is staged and lost. + Sequence trafficSeq; + + EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(instanceID, _, _, _)) + .InSequence(trafficSeq) + .WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, AbortBatch()).InSequence(trafficSeq).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, StartInstanceMonitoring(instanceID, _, _, _)) + .InSequence(trafficSeq) + .WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->StartInstanceNetwork(instanceID, networkID), aos::ErrorEnum::eNone); + + EXPECT_CALL(mFirewall, FlushBatch()).WillOnce(Return(aos::ErrorEnum::eRuntime)); + EXPECT_CALL(mTrafficMonitor, FlushBatch()).Times(0); + EXPECT_CALL(mStorage, RollbackTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + + aos::StaticArray, aos::cMaxNumInstances> failed; + + EXPECT_EQ(mNetManager->FlushBatch(failed), aos::ErrorEnum::eNone); + EXPECT_TRUE(failed.IsEmpty()); +} + TEST_F(NetworkManagerTest, FlushBatch_CommitFailure_MarksAllFailed) { const aos::String instanceID1 = "test-instance-1"; @@ -778,6 +842,8 @@ TEST_F(NetworkManagerTest, FlushBatch_CommitFailure_MarksAllFailed) EXPECT_CALL(mStorage, CommitTransaction()).WillOnce(Return(aos::ErrorEnum::eRuntime)); EXPECT_CALL(mFirewall, Revert()).WillOnce(Return(aos::ErrorEnum::eNone)); EXPECT_CALL(mTrafficMonitor, Revert()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mStorage, RollbackTransaction()).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mTrafficMonitor, AbortBatch()).Times(0); aos::StaticArray, aos::cMaxNumInstances> failed; From 50d2dd3668c4000752e862de8d4dec6f207f60ee Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Mon, 3 Aug 2026 17:17:52 +0300 Subject: [PATCH 091/112] sm: networkmanager: adopt DNS server for running instances on reconcile ReconcileInstances registered the network DNS server (AdoptDNSServer) only on the leftover-cleanup path, not for instances adopted as running via InitInstance. DeleteInstanceNetworkConfig then could not find the DNS server ("DNS server not found for cleanup") and left stale addnhosts entries when such an instance was later removed. Adopt the DNS server before continuing the running-instance branch too. AdoptDNSServer is idempotent, so multiple instances on one network are safe; a failure is logged and does not abort adoption. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- src/core/sm/networkmanager/networkmanager.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index b2f619ac0..b4242480a 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -1330,6 +1330,11 @@ Error NetworkManager::ReconcileInstances() if (alive) { if (err = InitInstance(entry.mInstanceID, entry.mNetworkID); err.IsNone()) { + if (auto dnsErr = AdoptDNSServer(entry.mNetworkID); !dnsErr.IsNone()) { + LOG_WRN() << "Failed to adopt DNS server for running instance" + << Log::Field("networkID", entry.mNetworkID) << Log::Field(dnsErr); + } + continue; } else { LOG_WRN() << "Failed to adopt leftover instance, falling back to cleanup" From 9208a0aeb1717b9253297b4c1b4e53b4fdf9221c Mon Sep 17 00:00:00 2001 From: Mykola Solianko Date: Mon, 3 Aug 2026 17:36:45 +0300 Subject: [PATCH 092/112] sm: networkmanager: test DNS server adoption for running instances Adopt a running leftover instance on restart, assert the network DNS server is registered (CreateServer), and that a subsequent StopInstanceNetwork drops the instance host entry (RemoveHost). Without the reconcile fix neither call happens, so the test fails. Update Start_KeepsLeftoverInstanceWithLiveInterface to expect the DNS server adoption too. Signed-off-by: Mykola Solianko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov Reviewed-by: Mykhailo Lohvynenko --- .../networkmanager/tests/networkmanager.cpp | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/core/sm/networkmanager/tests/networkmanager.cpp b/src/core/sm/networkmanager/tests/networkmanager.cpp index 87d52f3bd..8432aaa45 100644 --- a/src/core/sm/networkmanager/tests/networkmanager.cpp +++ b/src/core/sm/networkmanager/tests/networkmanager.cpp @@ -302,7 +302,9 @@ class NetworkManagerTest : public Test { void ExpectLeftoverInstanceUntouched() { - EXPECT_CALL(mDNSName, CreateServer(_, _)).Times(0); + // Adopting a running instance registers the network DNS server so later cleanup can reach it. + EXPECT_CALL(mDNSName, CreateServer(_, _)) + .WillOnce(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); EXPECT_CALL(mDNSServer, RemoveHost(_)).Times(0); EXPECT_CALL(mBandwidth, Clear(_)).Times(0); EXPECT_CALL(mFirewall, RemoveInstance(_)).Times(0); @@ -1674,6 +1676,44 @@ TEST_F(NetworkManagerTest, Start_KeepsLeftoverInstanceWithLiveInterface) mNetManager->StartInstanceNetwork(leftover.mInstanceID, network.mNetworkID).Is(aos::ErrorEnum::eAlreadyExist)); } +TEST_F(NetworkManagerTest, Start_AdoptsDNSServerForRunningInstanceCleanedOnStop) +{ + const auto network = CreateTestNetworkInfo(); + const auto leftover = CreateLeftoverInstance(network); + + aos::StaticArray networks; + aos::StaticArray instances; + networks.PushBack(network); + instances.PushBack(leftover); + + RestartWithStoredState(networks, instances); + + ExpectLinkExists(leftover.mHostIfName, LinkKindEnum::eVeth, network.mBridgeIfName); + EXPECT_CALL(mNetns, IsNetworkNamespaceExist(leftover.mInstanceID)) + .WillRepeatedly(Return(aos::RetWithError {true, aos::ErrorEnum::eNone})); + + EXPECT_CALL(mDNSName, RemoveOrphans(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + + // Adopting the running instance must register the network DNS server. + EXPECT_CALL(mDNSName, CreateServer(network.mNetworkID, _)) + .WillOnce(Return(aos::RetWithError {&mDNSServer, aos::ErrorEnum::eNone})); + EXPECT_CALL(mTrafficMonitor, + StartInstanceMonitoring(leftover.mInstanceID, leftover.mAllocatedParams.mIP, + leftover.mNetworkConfig.mDownloadLimit, leftover.mNetworkConfig.mUploadLimit)) + .WillOnce(Return(aos::ErrorEnum::eNone)); + + ASSERT_EQ(mNetManager->Start(), aos::ErrorEnum::eNone); + + // Stopping the adopted instance must reach the DNS server and drop its host entry. + EXPECT_CALL(mTrafficMonitor, StopInstanceMonitoring(leftover.mInstanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mDNSServer, RemoveHost(leftover.mInstanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mFirewall, RemoveInstance(leftover.mInstanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mNetns, DeleteNetworkNamespace(leftover.mInstanceID)).WillOnce(Return(aos::ErrorEnum::eNone)); + EXPECT_CALL(mStorage, UpdateInstanceNetworkInfo(_)).WillOnce(Return(aos::ErrorEnum::eNone)); + + EXPECT_EQ(mNetManager->StopInstanceNetwork(leftover.mInstanceID, network.mNetworkID), aos::ErrorEnum::eNone); +} + TEST_F(NetworkManagerTest, Start_CleansLeftoverInstanceWhenNamespaceMissing) { const auto network = CreateTestNetworkInfo(); From a901be258007bc382a09adc59c67929f5c0971c5 Mon Sep 17 00:00:00 2001 From: Mykhailo Lohvynenko Date: Mon, 3 Aug 2026 10:22:40 +0300 Subject: [PATCH 093/112] common: monitoring: decrease memory usage of monitoring Signed-off-by: Mykhailo Lohvynenko Reviewed-by: Mykola Kobets Reviewed-by: Oleksandr Grytsov --- src/core/common/monitoring/alertprocessor.cpp | 80 ++++++++++--------- src/core/common/monitoring/alertprocessor.hpp | 12 +-- src/core/common/monitoring/monitoring.cpp | 61 ++------------ src/core/common/monitoring/monitoring.hpp | 1 - .../monitoring/tests/alertprocessor.cpp | 9 +-- 5 files changed, 60 insertions(+), 103 deletions(-) diff --git a/src/core/common/monitoring/alertprocessor.cpp b/src/core/common/monitoring/alertprocessor.cpp index d37021350..4f69e9871 100644 --- a/src/core/common/monitoring/alertprocessor.cpp +++ b/src/core/common/monitoring/alertprocessor.cpp @@ -16,55 +16,55 @@ namespace { * Static **********************************************************************************************************************/ -class CreateAlertVisitor : public StaticVisitor { +class CreateAlertVisitor : public StaticVisitor { public: - CreateAlertVisitor(uint64_t currentValue, const Time& currentTime, const QuotaAlertState& state) - : mCurrentVal(currentValue) + CreateAlertVisitor( + const ResourceIdentifier& id, uint64_t currentValue, const Time& currentTime, const QuotaAlertState& state) + : mID(id) + , mCurrentVal(currentValue) , mCurrentTime(currentTime) , mState(state) { } - Res Visit(const SystemQuotaAlert& val) const + Res Visit(SystemQuotaAlert& val) const { - auto systemQuotaAlert = val; - - systemQuotaAlert.mTimestamp = mCurrentTime; - systemQuotaAlert.mValue = mCurrentVal; - systemQuotaAlert.mState = mState; - - Res result; - result.SetValue(systemQuotaAlert); - - return result; + val.mNodeID = mID.mNodeID; + val.mParameter = GetParameterName(mID); + val.mTimestamp = mCurrentTime; + val.mValue = mCurrentVal; + val.mState = mState; } - Res Visit(const InstanceQuotaAlert& val) const + Res Visit(InstanceQuotaAlert& val) const { - auto instanceQuotaAlert = val; - - instanceQuotaAlert.mTimestamp = mCurrentTime; - instanceQuotaAlert.mValue = mCurrentVal; - instanceQuotaAlert.mState = mState; - - Res result; - result.SetValue(instanceQuotaAlert); - - return result; + val.mParameter = GetParameterName(mID); + static_cast(val) = mID.mInstanceIdent.GetValue(); + val.mTimestamp = mCurrentTime; + val.mValue = mCurrentVal; + val.mState = mState; } template Res Visit(const T&) const { assert(false); - - return {}; } private: - uint64_t mCurrentVal {}; - Time mCurrentTime; - QuotaAlertState mState; + String GetParameterName(const ResourceIdentifier& id) const + { + if (id.mPartitionName.HasValue()) { + return id.mPartitionName.GetValue(); // NOSONAR cpp:S5912 - String is used as a string view. + } + + return id.mType.ToString(); // NOSONAR cpp:S5912 - String is used as a string view. + } + + const ResourceIdentifier& mID; + uint64_t mCurrentVal {}; + Time mCurrentTime; + QuotaAlertState mState; }; } // namespace @@ -73,8 +73,7 @@ class CreateAlertVisitor : public StaticVisitor { * Public **********************************************************************************************************************/ -Error AlertProcessor::Init(const ResourceIdentifier& id, const AlertRulePoints& rule, alerts::SenderItf& sender, - const AlertVariant& alertTemplate) +Error AlertProcessor::Init(const ResourceIdentifier& id, const AlertRulePoints& rule, alerts::SenderItf& sender) { mID = id; mMinTimeout = rule.mMinTimeout; @@ -84,8 +83,7 @@ Error AlertProcessor::Init(const ResourceIdentifier& id, const AlertRulePoints& LOG_DBG() << "Create alert processor" << Log::Field("id", mID) << Log::Field("minThreshold", mMinThreshold) << Log::Field("maxThreshold", mMaxThreshold) << Log::Field("minTimeout", mMinTimeout); - mAlertSender = &sender; - mAlertTemplate = alertTemplate; + mAlertSender = &sender; return ErrorEnum::eNone; } @@ -187,9 +185,19 @@ Error AlertProcessor::HandleMinThreshold(uint64_t currentValue, const Time& curr Error AlertProcessor::SendAlert(uint64_t currentValue, const Time& currentTime, const QuotaAlertState& state) { - CreateAlertVisitor visitor(currentValue, currentTime, state); + AlertVariant alert; + + if (mID.mLevel == ResourceLevelEnum::eSystem) { + alert.SetValue(); + } else if (mID.mLevel == ResourceLevelEnum::eInstance) { + alert.SetValue(); + } else { + return Error(ErrorEnum::eInvalidArgument); + } + + const CreateAlertVisitor visitor(mID, currentValue, currentTime, state); - auto alert = mAlertTemplate.ApplyVisitor(visitor); + alert.ApplyVisitor(visitor); if (auto err = mAlertSender->SendAlert(alert); !err.IsNone()) { LOG_ERR() << "Failed to send alert" << Log::Field(err); diff --git a/src/core/common/monitoring/alertprocessor.hpp b/src/core/common/monitoring/alertprocessor.hpp index a4a006b96..2ac386353 100644 --- a/src/core/common/monitoring/alertprocessor.hpp +++ b/src/core/common/monitoring/alertprocessor.hpp @@ -83,16 +83,18 @@ struct ResourceIdentifier { * @param partitionName partition name. * @param instanceIdent instance identifier. */ - ResourceIdentifier(ResourceLevel level, ResourceType type, + ResourceIdentifier(const String& nodeId, ResourceLevel level, ResourceType type, const Optional>& partitionName = {}, const Optional& instanceIdent = {}) - : mLevel(level) + : mNodeID(nodeId) + , mLevel(level) , mType(type) , mPartitionName(partitionName) , mInstanceIdent(instanceIdent) { } + StaticString mNodeID; ResourceLevel mLevel; ResourceType mType; Optional> mPartitionName; @@ -108,7 +110,7 @@ struct ResourceIdentifier { */ friend Log& operator<<(Log& log, const ResourceIdentifier& identifier) { - log << "{" << identifier.mLevel << ":" << identifier.mType; + log << "{" << identifier.mNodeID << ":" << identifier.mLevel << ":" << identifier.mType; if (identifier.mPartitionName.HasValue()) { log << ":" << identifier.mPartitionName.GetValue(); @@ -138,8 +140,7 @@ class AlertProcessor { * @param alertTemplate alert template. * @return Error. */ - Error Init(const ResourceIdentifier& id, const AlertRulePoints& rule, alerts::SenderItf& sender, - const AlertVariant& alertTemplate); + Error Init(const ResourceIdentifier& id, const AlertRulePoints& rule, alerts::SenderItf& sender); /** * Checks alert detection. If alert condition is true, sends alert. @@ -164,7 +165,6 @@ class AlertProcessor { ResourceIdentifier mID {}; alerts::SenderItf* mAlertSender {}; - AlertVariant mAlertTemplate; Duration mMinTimeout {}; uint64_t mMinThreshold {}; diff --git a/src/core/common/monitoring/monitoring.cpp b/src/core/common/monitoring/monitoring.cpp index 75aae24a4..ad3826d0f 100644 --- a/src/core/common/monitoring/monitoring.cpp +++ b/src/core/common/monitoring/monitoring.cpp @@ -36,15 +36,6 @@ Optional ToPoints(const Optional& percents, return ToPoints(*percents, totalValue); } -String GetParameterName(const ResourceIdentifier& id) -{ - if (id.mPartitionName.HasValue()) { - return id.mPartitionName.GetValue(); - } - - return id.mType.ToString(); -} - RetWithError GetCurrentUsage(const ResourceIdentifier& id, const MonitoringData& monitoringData) { switch (id.mType.GetValue()) { @@ -432,39 +423,6 @@ void Monitoring::ProcessMonitoring() } } -Error Monitoring::CreateAlertTemplate(const ResourceIdentifier& resourceIdentifier, AlertVariant& alert) const -{ - switch (resourceIdentifier.mLevel.GetValue()) { - case ResourceLevelEnum::eSystem: { - SystemQuotaAlert quotaAlert {}; - - quotaAlert.mNodeID = mNodeInfo.mNodeID; - quotaAlert.mParameter = GetParameterName(resourceIdentifier); - - alert.SetValue(quotaAlert); - - return ErrorEnum::eNone; - } - - case ResourceLevelEnum::eInstance: { - if (!resourceIdentifier.mInstanceIdent.HasValue()) { - return AOS_ERROR_WRAP(ErrorEnum::eInvalidArgument); - } - - InstanceQuotaAlert quotaAlert {}; - - static_cast(quotaAlert) = *resourceIdentifier.mInstanceIdent; - quotaAlert.mParameter = GetParameterName(resourceIdentifier); - - alert.SetValue(quotaAlert); - - return ErrorEnum::eNone; - } - } - - return AOS_ERROR_WRAP(ErrorEnum::eNotSupported); -} - Error Monitoring::AddAlertProcessor( const AlertRulePoints& rule, const ResourceIdentifier& identifier, Array& processors) { @@ -474,13 +432,7 @@ Error Monitoring::AddAlertProcessor( auto& alertProcessor = processors.Back(); - AlertVariant alertTemplate; - - if (auto err = CreateAlertTemplate(identifier, alertTemplate); !err.IsNone()) { - return AOS_ERROR_WRAP(err); - } - - if (auto err = alertProcessor.Init(identifier, rule, *mAlertSender, alertTemplate); !err.IsNone()) { + if (auto err = alertProcessor.Init(identifier, rule, *mAlertSender); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -522,7 +474,7 @@ Error Monitoring::SetAlertProcessors(const AlertRules& alertRules, const Resourc const Optional& instanceIdent, Array& processors) { if (auto cpu = ToPoints(alertRules.mCPU, mNodeInfo.mMaxDMIPS); cpu.HasValue()) { - auto id = ResourceIdentifier(level, ResourceTypeEnum::eCPU, {}, instanceIdent); + auto id = ResourceIdentifier(mNodeInfo.mNodeID, level, ResourceTypeEnum::eCPU, {}, instanceIdent); if (auto err = AddAlertProcessor(*cpu, id, processors); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -530,7 +482,7 @@ Error Monitoring::SetAlertProcessors(const AlertRules& alertRules, const Resourc } if (auto ram = ToPoints(alertRules.mRAM, mNodeInfo.mTotalRAM); ram.HasValue()) { - auto id = ResourceIdentifier(level, ResourceTypeEnum::eRAM, {}, instanceIdent); + auto id = ResourceIdentifier(mNodeInfo.mNodeID, level, ResourceTypeEnum::eRAM, {}, instanceIdent); if (auto err = AddAlertProcessor(*ram, id, processors); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -538,7 +490,7 @@ Error Monitoring::SetAlertProcessors(const AlertRules& alertRules, const Resourc } if (alertRules.mDownload.HasValue()) { - auto id = ResourceIdentifier(level, ResourceTypeEnum::eDownload, {}, instanceIdent); + auto id = ResourceIdentifier(mNodeInfo.mNodeID, level, ResourceTypeEnum::eDownload, {}, instanceIdent); if (auto err = AddAlertProcessor(*alertRules.mDownload, id, processors); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -546,7 +498,7 @@ Error Monitoring::SetAlertProcessors(const AlertRules& alertRules, const Resourc } if (alertRules.mUpload.HasValue()) { - auto id = ResourceIdentifier(level, ResourceTypeEnum::eUpload, {}, instanceIdent); + auto id = ResourceIdentifier(mNodeInfo.mNodeID, level, ResourceTypeEnum::eUpload, {}, instanceIdent); if (auto err = AddAlertProcessor(*alertRules.mUpload, id, processors); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -560,7 +512,8 @@ Error Monitoring::SetAlertProcessors(const AlertRules& alertRules, const Resourc continue; } - auto id = ResourceIdentifier(level, ResourceTypeEnum::ePartition, partition.mName, instanceIdent); + auto id = ResourceIdentifier( + mNodeInfo.mNodeID, level, ResourceTypeEnum::ePartition, partition.mName, instanceIdent); if (auto err = AddAlertProcessor(ToPoints(partition, it->mTotalSize), id, processors); !err.IsNone()) { return AOS_ERROR_WRAP(err); diff --git a/src/core/common/monitoring/monitoring.hpp b/src/core/common/monitoring/monitoring.hpp index e5d4a1b0b..aef811a21 100644 --- a/src/core/common/monitoring/monitoring.hpp +++ b/src/core/common/monitoring/monitoring.hpp @@ -92,7 +92,6 @@ class Monitoring : public MonitoringItf, void GetInstanceMonitoringData(Array& instanceMonitoringData); void ProcessAlerts(NodeMonitoringData& monitoringData); void ProcessAlerts(MonitoringData& monitoringData, AlertProcessorArray& alertProcessors); - Error CreateAlertTemplate(const ResourceIdentifier& resourceIdentifier, AlertVariant& alert) const; Error AddAlertProcessor( const AlertRulePoints& rule, const ResourceIdentifier& identifier, Array& processors); Error SetNodeAlertProcessors(const Optional& alertRules); diff --git a/src/core/common/monitoring/tests/alertprocessor.cpp b/src/core/common/monitoring/tests/alertprocessor.cpp index 5b2ae433b..2749a2442 100644 --- a/src/core/common/monitoring/tests/alertprocessor.cpp +++ b/src/core/common/monitoring/tests/alertprocessor.cpp @@ -83,15 +83,12 @@ TEST_F(AlertProcessorTest, CheckRulePointAlertDetection) { const ResourceType resourceType = ResourceTypeEnum::eDownload; const AlertRulePoints rulePoints = {Time::cSeconds, 90, 95}; - const ResourceIdentifier id = {ResourceLevelEnum::eSystem, resourceType.GetValue(), {}, {}}; + const String nodeID = "node-id"; + const ResourceIdentifier id = {nodeID, ResourceLevelEnum::eSystem, resourceType.GetValue(), {}}; AlertProcessor alertProcessor; - { - AlertVariant alertTemplate; - alertTemplate.SetValue(CreateSystemQuotaAlert("node-id", resourceType.ToString(), 0)); - ASSERT_TRUE(alertProcessor.Init(id, rulePoints, mAlertSender, alertTemplate).IsNone()); - } + ASSERT_TRUE(alertProcessor.Init(id, rulePoints, mAlertSender).IsNone()); Time currentTime = Time::Now(); From 26708dd4696d85375d9547f5be9213807ef16d67 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 30 Jul 2026 13:40:22 +0300 Subject: [PATCH 094/112] memory: check allocation before constructing objects MakeUnique/MakeShared placement-constructed objects via the Allocator-based operator new, which only guarded a failed allocation with assert(). In release builds (NDEBUG) that assert is compiled out, so an exhausted allocator returned nullptr and the constructor still ran at a null address (UB). Both factories now call Allocator::Allocate() explicitly, check the result, and return an empty pointer instead of constructing when the allocation fails. Signed-off-by: Oleksandr Grytsov --- src/core/common/tools/memory.hpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/core/common/tools/memory.hpp b/src/core/common/tools/memory.hpp index a92d02f56..6323231a7 100644 --- a/src/core/common/tools/memory.hpp +++ b/src/core/common/tools/memory.hpp @@ -8,6 +8,8 @@ #ifndef AOS_CORE_COMMON_TOOLS_MEMORY_HPP_ #define AOS_CORE_COMMON_TOOLS_MEMORY_HPP_ +#include + #include "allocator.hpp" namespace aos { @@ -526,7 +528,12 @@ inline UniquePtr MakeUnique(Allocator* allocator, Args&&... args) { assert(allocator); - return UniquePtr(new (allocator) T(args...), DefaultDeleter(allocator)); + auto data = allocator->Allocate(sizeof(T)); + if (!data) { + return UniquePtr(); + } + + return UniquePtr(new (data) T(args...), DefaultDeleter(allocator)); } /** @@ -558,7 +565,12 @@ inline SharedPtr MakeShared(Allocator* allocator, Args&&... args) { assert(allocator); - return SharedPtr(allocator, new (allocator) T(args...), SmartPtrDeleter); + auto data = allocator->Allocate(sizeof(T)); + if (!data) { + return SharedPtr(); + } + + return SharedPtr(allocator, new (data) T(args...), SmartPtrDeleter); } } // namespace aos From af4d3897aa9234fbc8a7404c827ee436f6901819 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 30 Jul 2026 16:10:33 +0300 Subject: [PATCH 095/112] tools: check MakeUnique/MakeShared allocation failures everywhere MakeUnique/MakeShared already returned an empty (falsy) pointer on allocator exhaustion, but most call sites across the codebase never checked the result before dereferencing it, so an out-of-memory condition would still crash on a null-pointer dereference instead of being reported as an error. Add a check after every such call site, propagating eNoMemory using whichever convention the enclosing function already uses (Error, RetWithError, bool, or void with a log message). Signed-off-by: Oleksandr Grytsov --- src/core/cm/alerts/alerts.cpp | 13 +++ src/core/cm/imagemanager/imagemanager.cpp | 6 +- .../cm/imagemanager/tests/imagemanager.cpp | 22 ++++ src/core/cm/launcher/balancer.cpp | 23 +++- src/core/cm/launcher/imageinfoprovider.cpp | 39 ++++++- src/core/cm/launcher/instance.cpp | 17 ++- src/core/cm/launcher/instancemanager.cpp | 19 +++ src/core/cm/launcher/launcher.cpp | 18 +++ src/core/cm/launcher/node.cpp | 21 +++- src/core/cm/launcher/nodemanager.cpp | 19 ++- src/core/cm/launcher/runrequestsloader.cpp | 12 +- src/core/cm/launcher/storagestate.cpp | 4 + .../cm/nodeinfoprovider/nodeinfoprovider.cpp | 11 ++ src/core/cm/storagestate/storagestate.cpp | 21 ++++ src/core/cm/unitconfig/unitconfig.cpp | 26 ++++- .../cm/updatemanager/desiredstatushandler.cpp | 28 ++++- .../cm/updatemanager/unitstatushandler.cpp | 6 + src/core/common/crypto/certloader.cpp | 15 ++- src/core/common/crypto/cryptohelper.cpp | 55 ++++++++- .../common/crypto/mbedtls/cryptoprovider.cpp | 13 +++ .../common/crypto/openssl/cryptoprovider.cpp | 12 ++ src/core/common/monitoring/average.cpp | 3 + src/core/common/monitoring/monitoring.cpp | 14 ++- src/core/common/pkcs11/pkcs11.cpp | 54 ++++++++- src/core/common/pkcs11/privatekey.cpp | 3 + .../common/spaceallocator/spaceallocator.hpp | 10 +- .../common/tests/stubs/spaceallocatorstub.hpp | 7 +- src/core/common/tools/fs.cpp | 8 +- src/core/common/tools/memory.hpp | 4 +- src/core/iam/certhandler/certhandler.cpp | 6 + src/core/iam/certhandler/certmodule.cpp | 35 +++++- .../certhandler/certmodules/pkcs11/pkcs11.cpp | 39 ++++++- .../iam/certhandler/tests/certhandler.cpp | 4 + src/core/iam/nodemanager/nodemanager.cpp | 20 +++- src/core/sm/launcher/launcher.cpp | 66 ++++++++++- src/core/sm/networkmanager/networkmanager.cpp | 108 +++++++++++++++++- src/core/sm/nodeconfig/nodeconfig.cpp | 6 + 37 files changed, 741 insertions(+), 46 deletions(-) diff --git a/src/core/cm/alerts/alerts.cpp b/src/core/cm/alerts/alerts.cpp index 994461f0e..a732e5d71 100644 --- a/src/core/cm/alerts/alerts.cpp +++ b/src/core/cm/alerts/alerts.cpp @@ -242,6 +242,9 @@ Error Alerts::SendAlerts() while (!mAlerts.IsEmpty()) { auto package = CreatePackage(); + if (!package) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } LOG_INF() << "Send alerts" << Log::Field("alertsCount", package->mItems.Size()); @@ -258,6 +261,11 @@ Error Alerts::SendAlerts() bool Alerts::IsDuplicated(const AlertVariant& alert) { auto alertCopy = MakeUnique(&mAllocator, alert); + if (!alertCopy) { + LOG_ERR() << "Can't allocate alert copy" << Log::Field(ErrorEnum::eNoMemory); + + return false; + } return mAlerts.FindIf([&alertCopy](const AlertVariant& item) { alertCopy->ApplyVisitor(SetTimestamp(item.ApplyVisitor(GetTimestamp()))); @@ -269,6 +277,11 @@ bool Alerts::IsDuplicated(const AlertVariant& alert) UniquePtr Alerts::CreatePackage() { auto package = MakeUnique(&mAllocator); + if (!package) { + LOG_ERR() << "Can't allocate alerts package" << Log::Field(ErrorEnum::eNoMemory); + + return package; + } const auto count = Min(cAlertItemsCount, mAlerts.Size()); diff --git a/src/core/cm/imagemanager/imagemanager.cpp b/src/core/cm/imagemanager/imagemanager.cpp index 9f2c46bb7..66dfcf1f2 100644 --- a/src/core/cm/imagemanager/imagemanager.cpp +++ b/src/core/cm/imagemanager/imagemanager.cpp @@ -1080,7 +1080,11 @@ Error ImageManager::EnsureBlob(const String& digest, const String& downloadPath, { LOG_DBG() << "Ensure blob" << Log::Field("digest", digest); - auto blobInfo = MakeUnique(&mAllocator); + auto blobInfo = MakeUnique(&mAllocator); + if (!blobInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + UniquePtr downloadingSpace; do { diff --git a/src/core/cm/imagemanager/tests/imagemanager.cpp b/src/core/cm/imagemanager/tests/imagemanager.cpp index 283890ada..1aff9c5d1 100644 --- a/src/core/cm/imagemanager/tests/imagemanager.cpp +++ b/src/core/cm/imagemanager/tests/imagemanager.cpp @@ -213,6 +213,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_Success_NewItem) EXPECT_CALL(mDownloadingSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -222,6 +224,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_Success_NewItem) EXPECT_CALL(mInstallSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -339,6 +343,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_AlreadyInstalled) EXPECT_CALL(mDownloadingSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -348,6 +354,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_AlreadyInstalled) EXPECT_CALL(mInstallSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -522,6 +530,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_MultipleItems_Success) EXPECT_CALL(mDownloadingSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -531,6 +541,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_MultipleItems_Success) EXPECT_CALL(mInstallSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -670,6 +682,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_Cancel_DownloadFailed) EXPECT_CALL(mDownloadingSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); testing::Mock::AllowLeak(space.Get()); @@ -759,6 +773,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_RemovesOldPendingVersion) EXPECT_CALL(mDownloadingSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -768,6 +784,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_RemovesOldPendingVersion) EXPECT_CALL(mInstallSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -876,6 +894,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_RemovesOldFailedVersion) EXPECT_CALL(mDownloadingSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); @@ -885,6 +905,8 @@ TEST_F(ImageManagerTest, DownloadUpdateItems_RemovesOldFailedVersion) EXPECT_CALL(mInstallSpaceAllocatorMock, AllocateSpace(_)) .WillRepeatedly(Invoke([this](size_t) -> RetWithError> { auto space = MakeUnique(&mAllocator); + EXPECT_TRUE(space); + EXPECT_CALL(*space, Accept()).Times(AtLeast(0)); EXPECT_CALL(*space, Release()).Times(AtLeast(0)); diff --git a/src/core/cm/launcher/balancer.cpp b/src/core/cm/launcher/balancer.cpp index 6c1d15138..e16f08237 100644 --- a/src/core/cm/launcher/balancer.cpp +++ b/src/core/cm/launcher/balancer.cpp @@ -92,11 +92,19 @@ Error Balancer::PerformNodeBalancing(Array>& instances) } auto imageIndex = MakeUnique(&mAllocator); + if (!imageIndex) { + LOG_ERR() << "Can't allocate image index" << Log::Field("instance", id) << Log::Field(ErrorEnum::eNoMemory); + + mInstanceManager->ScheduleInstance(instance, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)); + + continue; + } if (auto err = mImageInfoProvider->GetImageIndex(id.mItemID, info.mVersion, *imageIndex); !err.IsNone()) { LOG_ERR() << "Can't get images" << Log::Field("instance", id) << Log::Field(err); mInstanceManager->ScheduleInstance(instance, AOS_ERROR_WRAP(err)); + continue; } @@ -106,8 +114,7 @@ Error Balancer::PerformNodeBalancing(Array>& instances) LOG_DBG() << "Try to schedule instance" << Log::Field("instance", id) << Log::Field("manifest", manifest.mDigest); - scheduleErr = ScheduleInstance(instance, manifest); - if (scheduleErr.IsNone()) { + if (scheduleErr = ScheduleInstance(instance, manifest); scheduleErr.IsNone()) { LOG_DBG() << "Instance scheduled successfully" << Log::Field("nodeID", info.mNodeID); break; @@ -127,6 +134,9 @@ Error Balancer::PerformNodeBalancing(Array>& instances) Error Balancer::ScheduleInstance(SharedPtr& instance, const oci::IndexContentDescriptor& imageDescriptor) { auto nodes = MakeUnique>(&mAllocator); + if (!nodes) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto releaseConfigs = DeferRelease(reinterpret_cast(1), [&](int*) { instance->ResetConfigs(); }); @@ -197,6 +207,9 @@ void Balancer::FilterNodesByResources(Instance& instance, Array& nodes) RetWithError> Balancer::SelectRuntime(Instance& instance, const Array& nodes) { auto nodeRuntimes = MakeUnique(&mAllocator); + if (!nodeRuntimes) { + return {nullptr, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } if (auto err = CreateRuntimes(nodes, *nodeRuntimes); !err.IsNone()) { return {nullptr, AOS_ERROR_WRAP(err)}; @@ -385,6 +398,9 @@ void Balancer::FilterTopPriorityNodes(NodeRuntimes& nodes) Error Balancer::PerformPolicyBalancing(Array>& instances) { auto imageIndex = MakeUnique(&mAllocator); + if (!imageIndex) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } for (auto& instance : instances) { const auto& info = instance->GetInfo(); @@ -466,6 +482,9 @@ Error Balancer::UpdateMonitoringData(bool isInitialUpdate) const auto& nodeID = node.GetInfo().mNodeID; auto nodeMonitoring = MakeUnique(&mAllocator); + if (!nodeMonitoring) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } // Monitoring data immediately after startup is not availble. // Assign zero consumption on start. diff --git a/src/core/cm/launcher/imageinfoprovider.cpp b/src/core/cm/launcher/imageinfoprovider.cpp index 7198363e0..8233e34b3 100644 --- a/src/core/cm/launcher/imageinfoprovider.cpp +++ b/src/core/cm/launcher/imageinfoprovider.cpp @@ -20,8 +20,19 @@ void ImageInfoProvider::Init(imagemanager::ItemInfoProviderItf& itemInfoProvider Error ImageInfoProvider::GetImageConfig(const oci::IndexContentDescriptor& imageDescriptor, oci::ImageConfig& config) { auto manifestPath = MakeUnique>(&mAllocator); - auto manifest = MakeUnique(&mAllocator); - auto configPath = MakeUnique>(&mAllocator); + if (!manifestPath) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto manifest = MakeUnique(&mAllocator); + if (!manifest) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto configPath = MakeUnique>(&mAllocator); + if (!configPath) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mItemInfoProvider->GetBlobPath(imageDescriptor.mDigest, *manifestPath); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -45,8 +56,19 @@ Error ImageInfoProvider::GetImageConfig(const oci::IndexContentDescriptor& image Error ImageInfoProvider::GetItemConfig(const oci::IndexContentDescriptor& imageDescriptor, oci::ItemConfig& itemConfig) { auto manifestPath = MakeUnique>(&mAllocator); - auto manifest = MakeUnique(&mAllocator); - auto servicePath = MakeUnique>(&mAllocator); + if (!manifestPath) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto manifest = MakeUnique(&mAllocator); + if (!manifest) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto servicePath = MakeUnique>(&mAllocator); + if (!servicePath) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mItemInfoProvider->GetBlobPath(imageDescriptor.mDigest, *manifestPath); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -74,7 +96,14 @@ Error ImageInfoProvider::GetItemConfig(const oci::IndexContentDescriptor& imageD Error ImageInfoProvider::GetImageIndex(const String& itemID, const String& version, oci::ImageIndex& imageIndex) { auto indexDigest = MakeUnique>(&mAllocator); - auto indexPath = MakeUnique>(&mAllocator); + if (!indexDigest) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto indexPath = MakeUnique>(&mAllocator); + if (!indexPath) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mItemInfoProvider->GetIndexDigest(itemID, version, *indexDigest); !err.IsNone()) { return AOS_ERROR_WRAP(err); diff --git a/src/core/cm/launcher/instance.cpp b/src/core/cm/launcher/instance.cpp index 48fbf62a7..f9a7b12d7 100644 --- a/src/core/cm/launcher/instance.cpp +++ b/src/core/cm/launcher/instance.cpp @@ -37,8 +37,15 @@ Instance::Instance( Error Instance::LoadConfigs(const oci::IndexContentDescriptor& imageDescriptor) { - mItemConfig = MakeUnique(&mAllocator); + mItemConfig = MakeUnique(&mAllocator); + if (!mItemConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + mImageConfig = MakeUnique(&mAllocator); + if (!mImageConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto releaseConfigs = DeferRelease(reinterpret_cast(1), [&](int*) { ResetConfigs(); }); if (auto err = mImageInfoProvider.GetItemConfig(imageDescriptor, *mItemConfig); !err.IsNone()) { @@ -74,6 +81,11 @@ bool Instance::IsImageValid() } auto imageIndex = MakeUnique(&mAllocator); + if (!imageIndex) { + LOG_ERR() << "Can't allocate image index" << Log::Field(ErrorEnum::eNoMemory); + + return false; + } auto err = mImageInfoProvider.GetImageIndex(mInfo.mInstanceIdent.mItemID, mInfo.mVersion, *imageIndex); if (!err.IsNone()) { @@ -196,6 +208,9 @@ bool Instance::AreNodeLabelsOk(const LabelsArray& nodeLabels) RetWithError Instance::OverrideEnvVars(const OverrideEnvVarsRequest& envVars) { auto newEnvVars = MakeUnique(&mAllocator); + if (!newEnvVars) { + return {false, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } for (const auto& item : envVars.mItems) { if (!item.Match(mInfo.mInstanceIdent)) { diff --git a/src/core/cm/launcher/instancemanager.cpp b/src/core/cm/launcher/instancemanager.cpp index c4f024be2..cd9129897 100644 --- a/src/core/cm/launcher/instancemanager.cpp +++ b/src/core/cm/launcher/instancemanager.cpp @@ -218,6 +218,9 @@ RetWithError> InstanceManager::CreateInstance(const RunInsta } auto instanceInfo = CreateInfo(id, "", "", request); + if (!instanceInfo) { + return {nullptr, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } if (auto err = mStorage->AddInstance(*instanceInfo); !err.IsNone()) { return {nullptr, AOS_ERROR_WRAP(err)}; @@ -241,6 +244,9 @@ RetWithError> InstanceManager::CreateInstance(const RunInsta auto id = InstanceIdent {request.mItemID, request.mSubjectInfo.mSubjectID, index, request.mUpdateItemType}; auto instanceInfo = CreateInfo(id, nodeID, runtimeID, request); + if (!instanceInfo) { + return {nullptr, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } if (auto err = mStorage->AddInstance(*instanceInfo); !err.IsNone()) { return {nullptr, AOS_ERROR_WRAP(err)}; @@ -416,6 +422,10 @@ Error InstanceManager::LoadInstancesFromStorage() mCachedInstances.Clear(); auto instances = MakeUnique>(&mAllocator); + if (!instances) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + if (auto err = mStorage->LoadActiveInstances(*instances); !err.IsNone()) { return AOS_ERROR_WRAP(err); } @@ -561,6 +571,10 @@ RetWithError> InstanceManager::CreateInstance(const Instance return {{}, AOS_ERROR_WRAP(ErrorEnum::eNotSupported)}; } + if (!newInstance) { + return {nullptr, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } + if (auto err = newInstance->Init(); !err.IsNone()) { // Do not leave invalid instance in storage. if (auto rmErr = newInstance->Remove(); !rmErr.IsNone()) { @@ -723,6 +737,11 @@ UniquePtr InstanceManager::CreateInfo( const InstanceIdent& id, const String& nodeID, const String& runtimeID, const RunInstanceRequest& request) { auto info = MakeUnique(&mAllocator); + if (!info) { + LOG_ERR() << "Can't allocate instance info" << Log::Field(ErrorEnum::eNoMemory); + + return info; + } info->mInstanceIdent = id; info->mManifestDigest = ""; diff --git a/src/core/cm/launcher/launcher.cpp b/src/core/cm/launcher/launcher.cpp index b2cf1dde6..379c2fd76 100644 --- a/src/core/cm/launcher/launcher.cpp +++ b/src/core/cm/launcher/launcher.cpp @@ -110,6 +110,9 @@ Error Launcher::Start() // Set initial subjects list. auto subjects = MakeUnique(&mAllocator); + if (!subjects) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mIdentProvider->GetSubjects(*subjects); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -301,6 +304,12 @@ void Launcher::UpdateInstanceStatuses() // Copy old statuses. auto oldInstanceStatuses = MakeUnique>(&mAllocator); + if (!oldInstanceStatuses) { + LOG_ERR() << "Failed to allocate old instance statuses" << Log::Field(AOS_ERROR_WRAP(ErrorEnum::eNoMemory)); + + return; + } + if (auto err = oldInstanceStatuses->Assign(mInstanceStatuses); !err.IsNone()) { LOG_ERR() << "Failed to copy old instance statuses" << Log::Field(AOS_ERROR_WRAP(err)); @@ -341,6 +350,11 @@ void Launcher::UpdateInstanceStatuses() // Find new statuses. auto changedStatuses = MakeUnique>(&mAllocator); + if (!changedStatuses) { + LOG_ERR() << "Failed to allocate changed statuses" << Log::Field(AOS_ERROR_WRAP(ErrorEnum::eNoMemory)); + + return; + } for (size_t i = 0; i < mInstanceStatuses.Size(); ++i) { auto newStatus = !oldInstanceStatuses->Contains(mInstanceStatuses[i]); @@ -389,6 +403,10 @@ Error Launcher::BalanceInstances(UniqueLock& lock, bool rebalance) // Create instances from run requests. auto instances = MakeUnique, cMaxNumInstances>>(&mAllocator); + if (!instances) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + mRunRequestsLoader.CreateInstances(mNodeManager.GetNodes(), *instances); auto runErr = mBalancer.RunInstances(lock, *instances, rebalance); diff --git a/src/core/cm/launcher/node.cpp b/src/core/cm/launcher/node.cpp index 4eb880a0d..84001942d 100644 --- a/src/core/cm/launcher/node.cpp +++ b/src/core/cm/launcher/node.cpp @@ -268,8 +268,15 @@ Error Node::ReserveResources(const InstanceIdent& instanceIdent, const String& r Error Node::SendScheduledInstances( const Array>& scheduledInstances, const Array& runningInstances) { - auto stopInstances = MakeUnique>(mAllocator); + auto stopInstances = MakeUnique>(mAllocator); + if (!stopInstances) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + auto startInstances = MakeUnique>(mAllocator); + if (!startInstances) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } for (const auto& status : FilterActiveNodeInstances(runningInstances, mInfo.mNodeID)) { // Check if the instance is scheduled on this node (ident, runtime, node, and service version must match). @@ -318,8 +325,16 @@ Error Node::SendScheduledInstances( RetWithError Node::ResendInstances( const Array>& activeInstances, const Array& runningInstances, bool forceRestart) { - auto stopInstances = MakeUnique>(mAllocator); - auto startInstances = MakeUnique>(mAllocator); + auto stopInstances = MakeUnique>(mAllocator); + if (!stopInstances) { + return {false, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } + + auto startInstances = MakeUnique>(mAllocator); + if (!startInstances) { + return {false, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } + size_t runningNodeInstances = 0; for (const auto& status : FilterActiveNodeInstances(runningInstances, mInfo.mNodeID)) { diff --git a/src/core/cm/launcher/nodemanager.cpp b/src/core/cm/launcher/nodemanager.cpp index e35b345fa..6aecc3ed1 100644 --- a/src/core/cm/launcher/nodemanager.cpp +++ b/src/core/cm/launcher/nodemanager.cpp @@ -36,6 +36,9 @@ void NodeManager::Init(nodeinfoprovider::NodeInfoProviderItf& nodeInfoProvider, Error NodeManager::Start() { auto nodes = MakeUnique, cMaxNumNodes>>(&mAllocator); + if (!nodes) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mNodeInfoProvider->GetAllNodeIDs(*nodes); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -44,6 +47,9 @@ Error NodeManager::Start() LOG_DBG() << "Start node manager" << Log::Field("nodes", nodes->Size()); auto nodeInfo = MakeUnique(&mAllocator); + if (!nodeInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } for (const auto& nodeID : *nodes) { if (auto err = mNodeInfoProvider->GetNodeInfo(nodeID, *nodeInfo); !err.IsNone()) { @@ -105,11 +111,19 @@ Error NodeManager::LoadSMDataForActiveInstances( if (node == nullptr) { LOG_ERR() << "Can't find node" << Log::Field("instanceID", instanceID) << Log::Field("nodeID", nodeID) << Log::Field(AOS_ERROR_WRAP(ErrorEnum::eNotFound)); + continue; } auto imageDescriptor = MakeUnique(&mAllocator); - auto findDescErr = FindImageDescriptor( + if (!imageDescriptor) { + LOG_ERR() << "Can't allocate image descriptor" << Log::Field("instanceID", instanceID) + << Log::Field(AOS_ERROR_WRAP(ErrorEnum::eNoMemory)); + + continue; + } + + auto findDescErr = FindImageDescriptor( instanceID.mItemID, instance->GetInfo().mVersion, manifestDigest, imageInfoProvider, *imageDescriptor); if (!findDescErr.IsNone()) { LOG_ERR() << "Can't find image descriptor" << Log::Field("instanceID", instanceID) @@ -352,6 +366,9 @@ Error NodeManager::FindImageDescriptor(const String& itemID, const String& versi ImageInfoProvider& imageInfoProvider, oci::IndexContentDescriptor& imageDescriptor) { auto imageIndex = MakeUnique(&mAllocator); + if (!imageIndex) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = imageInfoProvider.GetImageIndex(itemID, version, *imageIndex); !err.IsNone()) { return AOS_ERROR_WRAP(err); diff --git a/src/core/cm/launcher/runrequestsloader.cpp b/src/core/cm/launcher/runrequestsloader.cpp index cf865001c..11b3e6ed3 100644 --- a/src/core/cm/launcher/runrequestsloader.cpp +++ b/src/core/cm/launcher/runrequestsloader.cpp @@ -89,13 +89,23 @@ Error RunRequestsLoader::GenerateInstances( const RunInstanceRequest& request, const Array& nodes, Array>& instances) { auto imageIndex = MakeUnique(&mAllocator); + if (!imageIndex) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mImageInfoProvider->GetImageIndex(request.mItemID, request.mVersion, *imageIndex); !err.IsNone()) { return AOS_ERROR_WRAP(err); } auto combinedRuntimes = MakeUnique(&mAllocator); - auto itemConfig = MakeUnique(&mAllocator); + if (!combinedRuntimes) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto itemConfig = MakeUnique(&mAllocator); + if (!itemConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = CombinedRuntimes(*imageIndex, *combinedRuntimes, *itemConfig); !err.IsNone()) { return AOS_ERROR_WRAP(err); diff --git a/src/core/cm/launcher/storagestate.cpp b/src/core/cm/launcher/storagestate.cpp index a3f1f259f..4f4f1e72f 100644 --- a/src/core/cm/launcher/storagestate.cpp +++ b/src/core/cm/launcher/storagestate.cpp @@ -44,6 +44,10 @@ Error StorageState::PrepareForBalancing() mAvailableStorage = MakeShared(&mAllocator, 0); } + if (!mAvailableState || !mAvailableStorage) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + const auto& [stateSize, stateErr] = mStorageStateManager->GetTotalStateSize(); if (!stateErr.IsNone()) { return AOS_ERROR_WRAP(stateErr); diff --git a/src/core/cm/nodeinfoprovider/nodeinfoprovider.cpp b/src/core/cm/nodeinfoprovider/nodeinfoprovider.cpp index e29e2cd6d..f48136fa3 100644 --- a/src/core/cm/nodeinfoprovider/nodeinfoprovider.cpp +++ b/src/core/cm/nodeinfoprovider/nodeinfoprovider.cpp @@ -36,6 +36,9 @@ Error NodeInfoProvider::Start() } auto ids = MakeUnique, cMaxNumNodes>>(&mAllocator); + if (!ids) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mNodeInfoProvider->GetAllNodeIDs(*ids); !err.IsNone()) { return err; @@ -43,6 +46,9 @@ Error NodeInfoProvider::Start() for (const auto& id : *ids) { auto nodeInfo = MakeUnique(&mAllocator); + if (!nodeInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mNodeInfoProvider->GetNodeInfo(id, *nodeInfo); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -250,6 +256,11 @@ NodeInfoCache* NodeInfoProvider::AddOrGetCacheItem(const String& nodeID) void NodeInfoProvider::NotifyListeners(const NodeInfoCache& info) { auto unitNodeInfo = MakeUnique(&mAllocator); + if (!unitNodeInfo) { + LOG_ERR() << "Can't allocate unit node info" << Log::Field(ErrorEnum::eNoMemory); + + return; + } info.GetUnitNodeInfo(*unitNodeInfo); diff --git a/src/core/cm/storagestate/storagestate.cpp b/src/core/cm/storagestate/storagestate.cpp index 4f1bf36e8..f6f513dbf 100644 --- a/src/core/cm/storagestate/storagestate.cpp +++ b/src/core/cm/storagestate/storagestate.cpp @@ -137,6 +137,9 @@ Error StorageState::UpdateState(const aos::UpdateState& state) } auto storageStateInfo = MakeUnique(&mAllocator); + if (!storageStateInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mStorage->GetStorageStateInfo(state, *storageStateInfo); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -189,6 +192,9 @@ Error StorageState::AcceptState(const StateAcceptance& state) } auto storageStateInfo = MakeUnique(&mAllocator); + if (!storageStateInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mStorage->GetStorageStateInfo(state, *storageStateInfo); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -213,10 +219,16 @@ Error StorageState::Setup( LOG_DBG() << "Setup storage and state" << setupParams; auto storageData = MakeUnique(&mAllocator); + if (!storageData) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = mStorage->GetStorageStateInfo(instanceIdent, *storageData); if (err.Is(ErrorEnum::eNotFound)) { storageData = MakeUnique(&mAllocator); + if (!storageData) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } storageData->mInstanceIdent = instanceIdent; @@ -353,6 +365,9 @@ Error StorageState::InitStateWatching() LOG_DBG() << "Init state watching"; auto infos = MakeUnique(&mAllocator); + if (!infos) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mStorage->GetAllStorageStateInfo(*infos); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -474,6 +489,9 @@ Error StorageState::CheckChecksumAndSendUpdateRequest(const State& state) LOG_DBG() << "Check checksum and send update request" << state; auto stateContent = MakeUnique>(&mAllocator); + if (!stateContent) { + return ErrorEnum::eNoMemory; + } if (auto err = fs::ReadFileToString(state.mFilePath, *stateContent); !err.IsNone()) { return err; @@ -575,6 +593,9 @@ Error StorageState::SetQuotas(const SetupParams& setupParams) Error StorageState::SendNewStateIfFileChanged(State& state) { auto newState = MakeUnique(&mAllocator); + if (!newState) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } static_cast(*newState) = state.mInstanceIdent; diff --git a/src/core/cm/unitconfig/unitconfig.cpp b/src/core/cm/unitconfig/unitconfig.cpp index 98aee4419..12aa9b298 100644 --- a/src/core/cm/unitconfig/unitconfig.cpp +++ b/src/core/cm/unitconfig/unitconfig.cpp @@ -84,6 +84,9 @@ Error UnitConfig::CheckUnitConfig(const aos::UnitConfig& config) for (const auto& id : nodeIds) { auto nodeInfo = MakeUnique(&mAllocator); + if (!nodeInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mNodeInfoProvider->GetNodeInfo(id, *nodeInfo); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -103,6 +106,9 @@ Error UnitConfig::CheckUnitConfig(const aos::UnitConfig& config) if (nodeConfigStatus.mVersion != config.mVersion || !nodeConfigStatus.mError.IsNone()) { auto nodeConfig = MakeUnique(&mAllocator); + if (!nodeConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = FindNodeConfig(nodeInfo->mNodeID, nodeInfo->mNodeType, config, *nodeConfig); !err.IsNone()) { return err; @@ -144,6 +150,9 @@ Error UnitConfig::UpdateUnitConfig(const aos::UnitConfig& unitConfig) mUnitConfig = unitConfig; auto unitConfigJSON = MakeUnique>(&mAllocator); + if (!unitConfigJSON) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mJSONProvider->UnitConfigToJSON(unitConfig, *unitConfigJSON); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -164,7 +173,14 @@ Error UnitConfig::UpdateUnitConfig(const aos::UnitConfig& unitConfig) for (const auto& id : nodeIds) { auto nodeConfig = MakeUnique(&mAllocator); - auto nodeInfo = MakeUnique(&mAllocator); + if (!nodeConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto nodeInfo = MakeUnique(&mAllocator); + if (!nodeInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mNodeInfoProvider->GetNodeInfo(id, *nodeInfo); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -228,6 +244,11 @@ void UnitConfig::OnNodeInfoChanged(const UnitNodeInfo& info) } auto nodeConfig = MakeUnique(&mAllocator); + if (!nodeConfig) { + LOG_ERR() << "Can't allocate node config" << Log::Field(ErrorEnum::eNoMemory); + + return; + } if (auto err = FindNodeConfig(info.mNodeID, info.mNodeType, mUnitConfig, *nodeConfig); !err.IsNone()) { LOG_ERR() << "Error finding node config" << Log::Field(err); @@ -251,6 +272,9 @@ Error UnitConfig::LoadConfig() LOG_DBG() << "Load config"; auto unitConfig = MakeUnique>(&mAllocator); + if (!unitConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = fs::ReadFileToString(mUnitConfigFile, *unitConfig); if (!err.IsNone()) { diff --git a/src/core/cm/updatemanager/desiredstatushandler.cpp b/src/core/cm/updatemanager/desiredstatushandler.cpp index 7c8ce5f33..f05a832b1 100644 --- a/src/core/cm/updatemanager/desiredstatushandler.cpp +++ b/src/core/cm/updatemanager/desiredstatushandler.cpp @@ -332,6 +332,9 @@ void DesiredStatusHandler::SetState(UpdateState state) Error DesiredStatusHandler::DownloadUpdateItems() { auto itemsStatuses = MakeUnique>(&mAllocator); + if (!itemsStatuses) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } LOG_DBG() << "Download update items" << Log::Field("count", mCurrentDesiredStatus.mUpdateItems.Size()); @@ -402,8 +405,15 @@ Error DesiredStatusHandler::InstallDesiredStatus() Error DesiredStatusHandler::LaunchInstances() { - auto runRequest = MakeUnique>(&mAllocator); + auto runRequest = MakeUnique>(&mAllocator); + if (!runRequest) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + auto instancesStatuses = MakeUnique>(&mAllocator); + if (!instancesStatuses) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } LOG_DBG() << "Launch instances" << Log::Field("count", mCurrentDesiredStatus.mInstances.Size()); @@ -459,6 +469,9 @@ Error DesiredStatusHandler::LaunchInstances() Error DesiredStatusHandler::WaitInstancesActive() { auto instancesStatuses = MakeUnique>(&mAllocator); + if (!instancesStatuses) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } while (mIsRunning) { if (auto err = mLauncher->GetInstancesStatuses(*instancesStatuses); !err.IsNone()) { @@ -492,6 +505,9 @@ Error DesiredStatusHandler::WaitInstancesActive() Error DesiredStatusHandler::FinalizeUpdate() { auto itemsStatuses = MakeUnique>(&mAllocator); + if (!itemsStatuses) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } LOG_DBG() << "Install update items" << Log::Field("count", mCurrentDesiredStatus.mUpdateItems.Size()); @@ -535,6 +551,11 @@ bool DesiredStatusHandler::IsUpdateRequired(const DesiredStatus& desiredStatus) bool DesiredStatusHandler::IsUpdateItemsRequired(const DesiredStatus& desiredStatus) const { auto itemsStatuses = MakeUnique>(&mAllocator); + if (!itemsStatuses) { + LOG_ERR() << "Failed to allocate update items statuses" << Log::Field(ErrorEnum::eNoMemory); + + return true; + } if (auto err = mImageManager->GetUpdateItemsStatuses(*itemsStatuses); !err.IsNone()) { LOG_ERR() << "Failed to get update items statuses" << Log::Field(err); @@ -595,6 +616,11 @@ bool DesiredStatusHandler::IsSameUpdate(const DesiredStatus& desiredStatus) cons bool DesiredStatusHandler::IsUpdateInstancesRequired(const DesiredStatus& desiredStatus) const { auto instancesStatuses = MakeUnique>(&mAllocator); + if (!instancesStatuses) { + LOG_ERR() << "Failed to allocate instances statuses" << Log::Field(ErrorEnum::eNoMemory); + + return true; + } if (auto err = mLauncher->GetInstancesStatuses(*instancesStatuses); !err.IsNone()) { LOG_ERR() << "Failed to get instances statuses" << Log::Field(err); diff --git a/src/core/cm/updatemanager/unitstatushandler.cpp b/src/core/cm/updatemanager/unitstatushandler.cpp index 14c0a61d0..c4939b98b 100644 --- a/src/core/cm/updatemanager/unitstatushandler.cpp +++ b/src/core/cm/updatemanager/unitstatushandler.cpp @@ -444,6 +444,9 @@ Error UnitStatusHandler::SetNodesInfo() Error UnitStatusHandler::SetUpdateItemsStatus() { auto itemsStatuses = MakeUnique(&mAllocator); + if (!itemsStatuses) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } mItemStatusProvider->GetUpdateItemsStatuses(*itemsStatuses); @@ -468,6 +471,9 @@ Error UnitStatusHandler::SetInstancesStatus() mUnitInstancesStatuses.Clear(); auto instancesStatuses = MakeUnique>(&mAllocator); + if (!instancesStatuses) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mInstanceStatusProvider->GetInstancesStatuses(*instancesStatuses); !err.IsNone()) { return AOS_ERROR_WRAP(err); diff --git a/src/core/common/crypto/certloader.cpp b/src/core/common/crypto/certloader.cpp index c64e32d37..146b2bd36 100644 --- a/src/core/common/crypto/certloader.cpp +++ b/src/core/common/crypto/certloader.cpp @@ -165,7 +165,11 @@ RetWithError> CertLoader::OpenSession( RetWithError CertLoader::FindToken(const pkcs11::LibraryContext& library, const String& token) { StaticArray slotList; - auto tokenInfo = MakeUnique(&mAllocator); + + auto tokenInfo = MakeUnique(&mAllocator); + if (!tokenInfo) { + return {0, ErrorEnum::eNoMemory}; + } auto err = library.GetSlotList(true, slotList); if (!err.IsNone()) { @@ -191,6 +195,9 @@ RetWithError> CertLoader::LoadCertsFromFile(co LOG_DBG() << "Load certs chain from file: fileName=" << fileName; auto buff = MakeUnique(&mAllocator); + if (!buff) { + return {nullptr, ErrorEnum::eNoMemory}; + } auto err = fs::ReadFileToString(fileName, *buff); if (!err.IsNone()) { @@ -198,6 +205,9 @@ RetWithError> CertLoader::LoadCertsFromFile(co } auto certificates = MakeShared(&mAllocator); + if (!certificates) { + return {nullptr, ErrorEnum::eNoMemory}; + } err = mCryptoProvider->PEMToX509Certs(*buff, *certificates); @@ -209,6 +219,9 @@ RetWithError> CertLoader::LoadPrivKeyFromFile(const Str LOG_DBG() << "Load private key from file: fileName=" << fileName; auto buff = MakeUnique>(&mAllocator); + if (!buff) { + return {nullptr, ErrorEnum::eNoMemory}; + } auto err = fs::ReadFileToString(fileName, *buff); if (!err.IsNone()) { diff --git a/src/core/common/crypto/cryptohelper.cpp b/src/core/common/crypto/cryptohelper.cpp index 0c1ccccd2..e3115173d 100644 --- a/src/core/common/crypto/cryptohelper.cpp +++ b/src/core/common/crypto/cryptohelper.cpp @@ -30,6 +30,9 @@ Error CryptoHelper::Init(iamclient::CertProviderItf& certProvider, CryptoProvide mServiceDiscoveryURL = serviceDiscoveryURL; auto caCertsPEM = MakeUnique>(&mAllocator); + if (!caCertsPEM) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = fs::ReadFileToString(caCert, *caCertsPEM); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -111,6 +114,9 @@ Error CryptoHelper::ValidateSigns(const String& decryptedPath, const SignInfo& s LockGuard lock {mSemaphore}; auto signCtx = MakeUnique(&mAllocator); + if (!signCtx) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = AddCertificates(certs, *signCtx); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -132,7 +138,14 @@ Error CryptoHelper::DecryptMetadata(const Array& input, Array& LockGuard lock {mSemaphore}; auto contentInfo = MakeUnique(&mAllocator); - auto symKey = MakeUnique>(&mAllocator); + if (!contentInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto symKey = MakeUnique>(&mAllocator); + if (!symKey) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = UnmarshalCMS(input, *contentInfo); if (!err.IsNone()) { @@ -165,6 +178,10 @@ Error CryptoHelper::DecryptMetadata(const Array& input, Array& RetWithError> CryptoHelper::GetOnlineCert() { auto certInfo = MakeUnique(&mAllocator); + if (!certInfo) { + return {{}, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } + if (auto err = mCertProvider->GetCert(cOnlineCert, {}, {}, *certInfo); !err.IsNone()) { return {{}, AOS_ERROR_WRAP(err)}; } @@ -206,6 +223,9 @@ Error CryptoHelper::GetServiceDiscoveryFromOrganization( const x509::Certificate& cert, Array>& urls) { auto subject = MakeUnique>(&mAllocator); + if (!subject) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mCryptoProvider->ASN1DecodeDN(cert.mSubject, *subject); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -222,7 +242,14 @@ Error CryptoHelper::GetServiceDiscoveryFromOrganization( auto [valueEnd, _] = subject->FindSubstr(valueStart, ","); auto orgName = MakeUnique>(&mAllocator); - auto url = MakeUnique>(&mAllocator); + if (!orgName) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto url = MakeUnique>(&mAllocator); + if (!url) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto assignErr = orgName->Insert(orgName->begin(), subject->begin() + valueStart, subject->begin() + valueEnd); if (!assignErr.IsNone()) { @@ -331,8 +358,15 @@ Error CryptoHelper::CheckSessionKey( Error CryptoHelper::DecodeFile(const String& encryptedFile, const String& decryptedFile, AESCipherItf& decoder) { - auto inBlock = MakeUnique>(&mAllocator); + auto inBlock = MakeUnique>(&mAllocator); + if (!inBlock) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + auto outBlock = MakeUnique>(&mAllocator); + if (!outBlock) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } fs::File inputFile, outputFile; @@ -411,6 +445,9 @@ Error CryptoHelper::AddCertificates(const Array& certs, SignCon } auto cert = MakeUnique(&mAllocator); + if (!cert) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mCryptoProvider->DERToX509Cert(certInfo.mCertificate, *cert); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -476,6 +513,9 @@ Error CryptoHelper::VerifySigns(const String& file, const SignInfo& signs, SignC // Verify sign auto hashSum = MakeUnique>(&mAllocator); + if (!hashSum) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = CalculateFileHash(file, hash, *mCryptoProvider, *hashSum); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -501,6 +541,9 @@ Error CryptoHelper::VerifySigns(const String& file, const SignInfo& signs, SignC // Verify certs auto intermCertPool = MakeUnique>(&mAllocator); + if (!intermCertPool) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = CreateIntermCertPool(signCtx, *chain, *intermCertPool); !err.IsNone()) { return err; @@ -857,6 +900,9 @@ Error CryptoHelper::ParseEncryptedContentInfo(const Array& data, Encryp Error CryptoHelper::GetKeyForEnvelope(const TransRecipientInfo& info, Array& symmetricKey) { auto certInfo = MakeUnique(&mAllocator); + if (!certInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = mCertProvider->GetCert(cOfflineCert, info.mRID.mIssuer, info.mRID.mSerial, *certInfo); if (!err.IsNone()) { @@ -921,6 +967,9 @@ Error CryptoHelper::DecryptMessage( Error CryptoHelper::DecodeMessage(AESCipherItf& decoder, const Array& input, Array& message) { auto outBlock = MakeUnique>(&mAllocator); + if (!outBlock) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (input.Size() % AESCipherItf::cBlockSize != 0) { return AOS_ERROR_WRAP(Error(ErrorEnum::eInvalidArgument, "message should be a multiple of CBC block size")); diff --git a/src/core/common/crypto/mbedtls/cryptoprovider.cpp b/src/core/common/crypto/mbedtls/cryptoprovider.cpp index ec3980795..addb2d326 100644 --- a/src/core/common/crypto/mbedtls/cryptoprovider.cpp +++ b/src/core/common/crypto/mbedtls/cryptoprovider.cpp @@ -828,6 +828,9 @@ RetWithError> MbedTLSCryptoProvider::PEMToX509PrivKey(c LOG_ERR() << "Create private key from PEM"; auto res = MakeShared(&mAllocator); + if (!res) { + return {{}, ErrorEnum::eNoMemory}; + } auto err = res->Init(pemBlob); if (!err.IsNone()) { @@ -942,6 +945,10 @@ RetWithError> MbedTLSCryptoProvider::CreateHash(Hash algorith } auto hasher = MakeUnique(&mAllocator, alg); + if (!hasher) { + return {nullptr, ErrorEnum::eNoMemory}; + } + if (auto err = hasher->Init(); !err.IsNone()) { return {nullptr, AOS_ERROR_WRAP(err)}; } @@ -1056,6 +1063,9 @@ RetWithError> MbedTLSCryptoProvider::CreateAESEncoder( } auto cipher = MakeUnique(&mAllocator); + if (!cipher) { + return {{}, ErrorEnum::eNoMemory}; + } auto err = cipher->Init(key, iv, true); if (!err.IsNone()) { @@ -1073,6 +1083,9 @@ RetWithError> MbedTLSCryptoProvider::CreateAESDecoder( } auto cipher = MakeUnique(&mAllocator); + if (!cipher) { + return {{}, ErrorEnum::eNoMemory}; + } auto err = cipher->Init(key, iv, false); if (!err.IsNone()) { diff --git a/src/core/common/crypto/openssl/cryptoprovider.cpp b/src/core/common/crypto/openssl/cryptoprovider.cpp index 5fa63dd78..11db12e9a 100644 --- a/src/core/common/crypto/openssl/cryptoprovider.cpp +++ b/src/core/common/crypto/openssl/cryptoprovider.cpp @@ -1494,6 +1494,9 @@ RetWithError> OpenSSLCryptoProvider::PEMToX509PrivKey(c auto type = EVP_PKEY_base_id(pkey.Get()); if (type == EVP_PKEY_RSA) { auto res = MakeShared(&mAllocator); + if (!res) { + return {{}, ErrorEnum::eNoMemory}; + } auto err = res->Init(pkey.Get()); if (!err.IsNone()) { @@ -1769,6 +1772,9 @@ RetWithError> OpenSSLCryptoProvider::CreateHash(Hash algorith } auto hasher = MakeUnique(&mAllocator); + if (!hasher) { + return {{}, ErrorEnum::eNoMemory}; + } auto err = hasher->Init(mLibCtx, algorithm.ToString().CStr()); if (!err.IsNone()) { @@ -1856,6 +1862,9 @@ RetWithError> OpenSSLCryptoProvider::CreateAESEncoder( } auto cipher = MakeUnique(&mAllocator); + if (!cipher) { + return {{}, ErrorEnum::eNoMemory}; + } auto err = cipher->Init(mLibCtx, key, iv, true); if (!err.IsNone()) { @@ -1873,6 +1882,9 @@ RetWithError> OpenSSLCryptoProvider::CreateAESDecoder( } auto cipher = MakeUnique(&mAllocator); + if (!cipher) { + return {{}, ErrorEnum::eNoMemory}; + } auto err = cipher->Init(mLibCtx, key, iv, false); if (!err.IsNone()) { diff --git a/src/core/common/monitoring/average.cpp b/src/core/common/monitoring/average.cpp index f5b6d6dc3..cacc41366 100644 --- a/src/core/common/monitoring/average.cpp +++ b/src/core/common/monitoring/average.cpp @@ -125,6 +125,9 @@ Error Average::StartInstanceMonitoring(const InstanceIdent& instanceIdent) } auto averageData = MakeUnique(&mAllocator); + if (!averageData) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mAverageInstancesData.Set(instanceIdent, *averageData); !err.IsNone()) { return AOS_ERROR_WRAP(err); diff --git a/src/core/common/monitoring/monitoring.cpp b/src/core/common/monitoring/monitoring.cpp index ad3826d0f..760c20d2c 100644 --- a/src/core/common/monitoring/monitoring.cpp +++ b/src/core/common/monitoring/monitoring.cpp @@ -144,6 +144,9 @@ Error Monitoring::Start() if (mInstanceInfoProvider) { auto statuses = MakeUnique(&mAllocator); + if (!statuses) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mInstanceInfoProvider->GetInstancesStatuses(*statuses); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -164,6 +167,9 @@ Error Monitoring::Start() { auto nodeConfig = MakeUnique(&mAllocator); + if (!nodeConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mNodeConfigProvider->GetNodeConfig(*nodeConfig); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -401,7 +407,13 @@ void Monitoring::ProcessMonitoring() { UniqueLock lock {mMutex}; - auto nodeMonitoringData = MakeUnique(&mAllocator); + auto nodeMonitoringData = MakeUnique(&mAllocator); + if (!nodeMonitoringData) { + LOG_ERR() << "Can't allocate node monitoring data" << Log::Field(ErrorEnum::eNoMemory); + + return; + } + nodeMonitoringData->mMonitoringData.mTimestamp = Time::Now(); GetInstanceMonitoringData(nodeMonitoringData->mInstances); diff --git a/src/core/common/pkcs11/pkcs11.cpp b/src/core/common/pkcs11/pkcs11.cpp index 9b4c7ee51..c204f8b19 100644 --- a/src/core/common/pkcs11/pkcs11.cpp +++ b/src/core/common/pkcs11/pkcs11.cpp @@ -551,6 +551,9 @@ RetWithError> LibraryContext::PKCS11OpenSession(SlotID } auto session = MakeShared(&mAllocator, handle, mFunctionList); + if (!session) { + return {nullptr, ErrorEnum::eNoMemory}; + } return {session, ErrorEnum::eNone}; } @@ -1266,7 +1269,11 @@ RetWithError> Utils::FindCertificateCh } SharedPtr certificate; - auto chain = MakeShared(&mAllocator); + + auto chain = MakeShared(&mAllocator); + if (!chain) { + return {nullptr, ErrorEnum::eNoMemory}; + } Tie(certificate, err) = GetCertificate(certHandles[0]); if (!err.IsNone()) { @@ -1331,7 +1338,14 @@ RetWithError Utils::ExportPrivateKey( attrTypes.PushBack(CKA_PUBLIC_EXPONENT); auto n = MakeUnique>(&mAllocator); + if (!n) { + return {{}, ErrorEnum::eNoMemory}; + } + auto e = MakeUnique>(&mAllocator); + if (!e) { + return {{}, ErrorEnum::eNoMemory}; + } attrValues.PushBack(*n); attrValues.PushBack(*e); @@ -1341,8 +1355,15 @@ RetWithError Utils::ExportPrivateKey( return {{}, err}; } - auto pubKey = MakeUnique(&mAllocator, attrValues[0], attrValues[1]); + auto pubKey = MakeUnique(&mAllocator, attrValues[0], attrValues[1]); + if (!pubKey) { + return {{}, ErrorEnum::eNoMemory}; + } + auto cryptoKey = MakeShared(&mAllocator, mSession, privKeyHandle, *pubKey); + if (!cryptoKey) { + return {{}, ErrorEnum::eNoMemory}; + } PrivateKey pkcsKey = {privKeyHandle, pubKeyHandle, cryptoKey}; @@ -1357,7 +1378,14 @@ RetWithError Utils::ExportPrivateKey( attrTypes.PushBack(CKA_EC_POINT); auto derEncodedParams = MakeUnique>(&mAllocator); - auto derEncodedPoint = MakeUnique>(&mAllocator); + if (!derEncodedParams) { + return {{}, ErrorEnum::eNoMemory}; + } + + auto derEncodedPoint = MakeUnique>(&mAllocator); + if (!derEncodedPoint) { + return {{}, ErrorEnum::eNoMemory}; + } attrValues.PushBack(*derEncodedParams); attrValues.PushBack(*derEncodedPoint); @@ -1368,7 +1396,14 @@ RetWithError Utils::ExportPrivateKey( } auto params = MakeUnique>(&mAllocator); - auto point = MakeUnique>(&mAllocator); + if (!params) { + return {{}, ErrorEnum::eNoMemory}; + } + + auto point = MakeUnique>(&mAllocator); + if (!point) { + return {{}, ErrorEnum::eNoMemory}; + } err = mCryptoProvider.ASN1DecodeOID(attrValues[0], *params); if (!err.IsNone()) { @@ -1381,8 +1416,15 @@ RetWithError Utils::ExportPrivateKey( } auto pubKey = MakeUnique(&mAllocator, *params, *point); + if (!pubKey) { + return {{}, ErrorEnum::eNoMemory}; + } + auto cryptoKey = MakeShared(&mAllocator, mSession, mCryptoProvider, privKeyHandle, *pubKey); + if (!cryptoKey) { + return {{}, ErrorEnum::eNoMemory}; + } PrivateKey pkcsKey = {privKeyHandle, pubKeyHandle, cryptoKey}; @@ -1483,6 +1525,10 @@ RetWithError> Utils::FindCertificateByKeyID RetWithError> Utils::GetCertificate(ObjectHandle handle) { auto certificate = MakeShared(&mAllocator); + if (!certificate) { + return {nullptr, ErrorEnum::eNoMemory}; + } + StaticArray, cObjectAttributesCount> attrValues; StaticArray attrTypes; diff --git a/src/core/common/pkcs11/privatekey.cpp b/src/core/common/pkcs11/privatekey.cpp index 863ae10fb..4cbb61a0e 100644 --- a/src/core/common/pkcs11/privatekey.cpp +++ b/src/core/common/pkcs11/privatekey.cpp @@ -39,6 +39,9 @@ Error PKCS11RSAPrivateKey::Sign( const Array& digest, const crypto::SignOptions& options, Array& signature) const { auto t = MakeUnique>(&mAllocator); + if (!t) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } t->Append(GetPrefix(options.mHash)); t->Append(digest); diff --git a/src/core/common/spaceallocator/spaceallocator.hpp b/src/core/common/spaceallocator/spaceallocator.hpp index f64f00e6a..982cbfd9f 100644 --- a/src/core/common/spaceallocator/spaceallocator.hpp +++ b/src/core/common/spaceallocator/spaceallocator.hpp @@ -461,7 +461,15 @@ class SpaceAllocator : public SpaceAllocatorItf, public SpaceAllocatorStorage { return {nullptr, err}; } - return UniquePtr(MakeUnique(&mAllocator, size, this)); + auto space = MakeUnique(&mAllocator, size, this); + if (!space) { + mPartition->Free(size); + Free(size); + + return {nullptr, ErrorEnum::eNoMemory}; + } + + return UniquePtr(Move(space)); }; /** diff --git a/src/core/common/tests/stubs/spaceallocatorstub.hpp b/src/core/common/tests/stubs/spaceallocatorstub.hpp index b40a35a66..0ff70fd19 100644 --- a/src/core/common/tests/stubs/spaceallocatorstub.hpp +++ b/src/core/common/tests/stubs/spaceallocatorstub.hpp @@ -78,7 +78,12 @@ class SpaceAllocatorStub : public SpaceAllocatorItf { */ RetWithError> AllocateSpace(size_t size) override { - return {UniquePtr(MakeUnique(&mAllocator, size))}; + auto space = MakeUnique(&mAllocator, size); + if (!space) { + return {nullptr, ErrorEnum::eNoMemory}; + } + + return UniquePtr(Move(space)); } /** diff --git a/src/core/common/tools/fs.cpp b/src/core/common/tools/fs.cpp index d957625f1..7e0b00a29 100644 --- a/src/core/common/tools/fs.cpp +++ b/src/core/common/tools/fs.cpp @@ -525,8 +525,12 @@ RetWithError CalculateSize(const String& path) return {static_cast(st.st_size)}; } - size_t size = 0; - auto dirIterators = MakeUnique(&sCalculateSizeAllocator); + size_t size = 0; + + auto dirIterators = MakeUnique(&sCalculateSizeAllocator); + if (!dirIterators) { + return {0, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } if (auto err = dirIterators->EmplaceBack(path); !err.IsNone()) { return {0, AOS_ERROR_WRAP(err)}; diff --git a/src/core/common/tools/memory.hpp b/src/core/common/tools/memory.hpp index 6323231a7..8289da43b 100644 --- a/src/core/common/tools/memory.hpp +++ b/src/core/common/tools/memory.hpp @@ -521,7 +521,7 @@ class SharedPtr : public SmartPtr { * @tparam Args holding object constructor parameters types. * @param allocator allocator. * @param args holding object constructor parameters. - * @return UniquePtr constructed unique ptr. + * @return UniquePtr constructed unique ptr, empty if allocation failed. */ template inline UniquePtr MakeUnique(Allocator* allocator, Args&&... args) @@ -558,7 +558,7 @@ inline UniquePtr DeferRelease(T* ptr, Deleter&& deleter) * @tparam Args holding object constructor parameters types. * @param allocator allocator. * @param args holding object constructor parameters. - * @return SharedPtr constructed shared ptr. + * @return SharedPtr constructed shared ptr, empty if allocation failed. */ template inline SharedPtr MakeShared(Allocator* allocator, Args&&... args) diff --git a/src/core/iam/certhandler/certhandler.cpp b/src/core/iam/certhandler/certhandler.cpp index 88f8c57ef..5a0c6044f 100644 --- a/src/core/iam/certhandler/certhandler.cpp +++ b/src/core/iam/certhandler/certhandler.cpp @@ -166,6 +166,9 @@ Error CertHandler::SubscribeListener(const String& certType, iamclient::CertList } auto certInfo = MakeUnique(&mAllocator); + if (!certInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = module->GetCertificate(Array(), Array(), *certInfo); if (!err.IsNone()) { @@ -244,6 +247,9 @@ CertModule* CertHandler::FindModule(const String& certType) const Error CertHandler::UpdateCerts(CertModule& certModule) { auto certInfo = MakeUnique(&mAllocator); + if (!certInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = certModule.GetCertificate(Array(), Array(), *certInfo); if (!err.IsNone()) { diff --git a/src/core/iam/certhandler/certmodule.cpp b/src/core/iam/certhandler/certmodule.cpp index dd6c8e049..c85cdf3fc 100644 --- a/src/core/iam/certhandler/certmodule.cpp +++ b/src/core/iam/certhandler/certmodule.cpp @@ -36,6 +36,9 @@ Error CertModule::Init(const String& certType, const ModuleConfig& config, crypt } auto validCerts = MakeUnique(&mAllocator); + if (!validCerts) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mHSM->ValidateCertificates(mInvalidCerts, mInvalidKeys, *validCerts); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -47,6 +50,9 @@ Error CertModule::Init(const String& certType, const ModuleConfig& config, crypt Error CertModule::GetCertificate(const Array& issuer, const Array& serial, CertInfo& resCert) { auto certsInStorage = MakeUnique(&mAllocator); + if (!certsInStorage) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (serial.IsEmpty()) { auto err = mStorage->GetCertsInfo(GetCertType(), *certsInStorage); @@ -58,7 +64,7 @@ Error CertModule::GetCertificate(const Array& issuer, const Array(&mAllocator); + resCert = CertInfo(); for (const auto& item : *certsInStorage) { if (resCert.mNotAfter.IsZero() || resCert.mNotAfter < item.mNotAfter) { @@ -121,7 +127,11 @@ RetWithError> CertModule::CreateKey(const Strin Error CertModule::CreateCSR(const String& subjectCommonName, const crypto::PrivateKeyItf& privKey, String& pemCSR) { - auto templ = MakeUnique(&mAllocator); + auto templ = MakeUnique(&mAllocator); + if (!templ) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + StaticString subject; templ->mDNSNames = mModuleConfig.mAlternativeNames; @@ -182,6 +192,9 @@ Error CertModule::CreateCSR(const String& subjectCommonName, const crypto::Priva Error CertModule::ApplyCert(const String& pemCert, CertInfo& info) { auto certificates = MakeUnique(&mAllocator); + if (!certificates) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = mX509Provider->PEMToX509Certs(pemCert, *certificates); if (!err.IsNone()) { @@ -221,7 +234,11 @@ Error CertModule::CreateSelfSignedCert(const String& password) } const uint64_t serial = Time::Now().UnixNano(); - auto templ = MakeUnique(&mAllocator); + + auto templ = MakeUnique(&mAllocator); + if (!templ) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } templ->mSerial = Array(reinterpret_cast(&serial), sizeof(serial)); templ->mNotBefore = Time::Now(); @@ -238,6 +255,9 @@ Error CertModule::CreateSelfSignedCert(const String& password) } auto pemCert = MakeUnique(&mAllocator); + if (!pemCert) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } err = mX509Provider->CreateCertificate(*templ, *templ, *key.mValue, *pemCert); if (!err.IsNone()) { @@ -245,6 +265,9 @@ Error CertModule::CreateSelfSignedCert(const String& password) } auto certInfo = MakeUnique(&mAllocator); + if (!certInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } return ApplyCert(*pemCert, *certInfo); } @@ -313,6 +336,9 @@ Error CertModule::RemoveInvalidKeys(const String& password) Error CertModule::TrimCerts(const String& password) { auto certsInStorage = MakeUnique(&mAllocator); + if (!certsInStorage) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = mStorage->GetCertsInfo(GetCertType(), *certsInStorage); if (!err.IsNone() && err != ErrorEnum::eNotFound) { @@ -405,6 +431,9 @@ Error CertModule::CheckCertChain(const Array& chain) Error CertModule::SyncValidCerts(const Array& validCerts) { auto certsInStorage = MakeUnique(&mAllocator); + if (!certsInStorage) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = mStorage->GetCertsInfo(GetCertType(), *certsInStorage); if (!err.IsNone() && err != ErrorEnum::eNotFound) { diff --git a/src/core/iam/certhandler/certmodules/pkcs11/pkcs11.cpp b/src/core/iam/certhandler/certmodules/pkcs11/pkcs11.cpp index e5508e9ee..238b74140 100644 --- a/src/core/iam/certhandler/certmodules/pkcs11/pkcs11.cpp +++ b/src/core/iam/certhandler/certmodules/pkcs11/pkcs11.cpp @@ -166,7 +166,14 @@ Error PKCS11Module::Clear() // certs, privKeys, pubKeys auto objects = MakeUnique>(&mTmpObjAllocator); - auto filter = MakeUnique(&mTmpObjAllocator); + if (!objects) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto filter = MakeUnique(&mTmpObjAllocator); + if (!filter) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } err = FindObject(*session, *filter, *objects); if (err.IsNone()) { @@ -485,6 +492,9 @@ RetWithError PKCS11Module::GetSlotID() if ((slotInfo.mFlags & CKF_TOKEN_PRESENT) != 0) { auto tokenInfo = MakeUnique(&mTmpObjAllocator); + if (!tokenInfo) { + return {0, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } err = mPKCS11->GetTokenInfo(slotID, *tokenInfo); if (!err.IsNone()) { @@ -513,6 +523,9 @@ RetWithError PKCS11Module::GetSlotID() RetWithError PKCS11Module::IsOwned() const { auto tokenInfo = MakeUnique(&mTmpObjAllocator); + if (!tokenInfo) { + return {false, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } auto err = mPKCS11->GetTokenInfo(mSlotID, *tokenInfo); if (!err.IsNone()) { @@ -633,6 +646,9 @@ RetWithError> PKCS11Module::CreateSession(bool LOG_DBG() << "Create session: session=" << mSession->GetHandle() << ", slotID=" << mSlotID; auto sessionInfo = MakeShared(&mTmpObjAllocator); + if (!sessionInfo) { + return {nullptr, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; + } err = mSession->GetSessionInfo(*sessionInfo); if (!err.IsNone()) { @@ -819,7 +835,14 @@ Error PKCS11Module::CreateURL(const String& label, const Array& id, Str }; auto opaque = MakeUnique>(&mTmpObjAllocator); - auto query = MakeUnique>(&mTmpObjAllocator); + if (!opaque) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + + auto query = MakeUnique>(&mTmpObjAllocator); + if (!query) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } // create opaque part of url AddParam("token", mTokenLabel.CStr(), true, *opaque); @@ -894,8 +917,15 @@ Error PKCS11Module::GetValidInfo(const pkcs11::SessionContext& session, Array(&mAllocator); + ASSERT_TRUE(mCertHandler); RegisterPKCS11Module("iam"); // Check Storage is restored. @@ -547,6 +549,7 @@ TEST_F(CerthandlerTest, RemoveInvalidPKCS11Objects) // reinit certhandler to sync certificates/keys with PKCS11 storage mCertHandler = MakeShared(&mAllocator); + ASSERT_TRUE(mCertHandler); RegisterPKCS11Module("iam"); // create key, because certmodule updates PKCS11 storage after that only @@ -583,6 +586,7 @@ TEST_F(CerthandlerTest, RenewCertificate) // reinit certhandler to sync certificates/keys with PKCS11 storage mCertHandler = MakeShared(&mAllocator); + ASSERT_TRUE(mCertHandler); RegisterPKCS11Module("iam"); RegisterPKCS11Module("sm"); diff --git a/src/core/iam/nodemanager/nodemanager.cpp b/src/core/iam/nodemanager/nodemanager.cpp index b5a52a47f..62fccf5f4 100644 --- a/src/core/iam/nodemanager/nodemanager.cpp +++ b/src/core/iam/nodemanager/nodemanager.cpp @@ -24,6 +24,9 @@ Error NodeManager::Init(StorageItf& storage) mStorage = &storage; auto nodeIDs = MakeUnique, cMaxNumNodes>>(&mAllocator); + if (!nodeIDs) { + return ErrorEnum::eNoMemory; + } auto err = storage.GetAllNodeIDs(*nodeIDs); if (!err.IsNone()) { @@ -32,6 +35,9 @@ Error NodeManager::Init(StorageItf& storage) for (const auto& nodeID : *nodeIDs) { auto nodeInfo = MakeUnique(&mAllocator); + if (!nodeInfo) { + return ErrorEnum::eNoMemory; + } err = storage.GetNodeInfo(nodeID, *nodeInfo); if (!err.IsNone()) { @@ -77,6 +83,9 @@ Error NodeManager::SetNodeState(const String& nodeID, const NodeState& state) } auto nodeInfo = MakeUnique(&mAllocator); + if (!nodeInfo) { + return ErrorEnum::eNoMemory; + } *nodeInfo = *cachedInfo; nodeInfo->mState = state; @@ -104,6 +113,9 @@ Error NodeManager::SetNodeConnected(const String& nodeID, bool isConnected) } auto nodeInfo = MakeUnique(&mAllocator); + if (!nodeInfo) { + return ErrorEnum::eNoMemory; + } *nodeInfo = *cachedInfo; nodeInfo->mIsConnected = isConnected; @@ -216,8 +228,12 @@ Error NodeManager::UpdateCache(const NodeInfo& nodeInfo) Error NodeManager::UpdateStorage(const NodeInfo& info) { - auto storageInfo = MakeUnique(&mAllocator); - const auto* cachedInfo = GetNodeFromCache(info.mNodeID); + auto storageInfo = MakeUnique(&mAllocator); + if (!storageInfo) { + return ErrorEnum::eNoMemory; + } + + const auto* cachedInfo = GetNodeFromCache(info.mNodeID); *storageInfo = info; diff --git a/src/core/sm/launcher/launcher.cpp b/src/core/sm/launcher/launcher.cpp index d2c4fef42..af97278c6 100644 --- a/src/core/sm/launcher/launcher.cpp +++ b/src/core/sm/launcher/launcher.cpp @@ -74,6 +74,9 @@ Error Launcher::Start() } auto storedInstances = MakeUnique(&mAllocator); + if (!storedInstances) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mStorage->GetAllInstancesInfos(*storedInstances); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -159,8 +162,15 @@ Error Launcher::UpdateInstances(const Array& stopInstances, const // Wait in case previous request is not yet finished mThread.Join(); - auto stop = MakeShared>(&mAllocator, stopInstances); + auto stop = MakeShared>(&mAllocator, stopInstances); + if (!stop) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + auto start = MakeShared(&mAllocator, startInstances); + if (!start) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mThread.Run([this, stop, start](void*) { UpdateInstancesImpl(*stop, *start); @@ -540,6 +550,11 @@ void Launcher::SendNodeInstancesStatuses() LOG_INF() << "Send node instances statuses" << Log::Field("count", mInstances.Size()); auto statuses = MakeUnique(&mAllocator); + if (!statuses) { + LOG_ERR() << "Failed to allocate instance statuses" << Log::Field(ErrorEnum::eNoMemory); + + return; + } for (const auto& instance : mInstances) { LOG_INF() << "Node instance status" << Log::Field("instance", instance.mInfo) @@ -578,6 +593,9 @@ Error Launcher::HandleComponentStatus(const aos::InstanceStatus& status) } auto instanceInfo = MakeUnique(&mAllocator); + if (!instanceInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } static_cast(*instanceInfo) = status; instanceInfo->mRuntimeID = status.mRuntimeID; @@ -594,8 +612,15 @@ Error Launcher::HandleComponentStatus(const aos::InstanceStatus& status) Error Launcher::LoadInstanceData(InstanceData& instanceData) { - auto itemConfig = MakeUnique(&mAllocator); + auto itemConfig = MakeUnique(&mAllocator); + if (!itemConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + auto imageConfig = MakeUnique(&mAllocator); + if (!imageConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = GetInstanceConfigs(instanceData.mInfo, *itemConfig, *imageConfig); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -674,6 +699,11 @@ void Launcher::UpdateInstancesImpl(Array& stopInstances, const Ar } auto removeItems = MakeUnique>(&mAllocator); + if (!removeItems) { + LOG_ERR() << "Failed to allocate remove update items" << Log::Field(ErrorEnum::eNoMemory); + + return; + } if (!mFirstStart) { GetRemoveUpdateItems(stopInstances, startInstances, *removeItems); @@ -838,8 +868,15 @@ Error Launcher::PrepareInstance(InstanceData& instanceData) return ErrorEnum::eNone; } - auto itemConfig = MakeUnique(&mAllocator); + auto itemConfig = MakeUnique(&mAllocator); + if (!itemConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } + auto imageConfig = MakeUnique(&mAllocator); + if (!imageConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = GetInstanceConfigs(instanceData.mInfo, *itemConfig, *imageConfig); !err.IsNone()) { return err; @@ -1282,7 +1319,18 @@ void Launcher::RemoveUpdateItems(const Array& removeItems) void Launcher::InstallUpdateItems(const Array& startInstances) { auto currentItems = MakeUnique>(&mAllocator); + if (!currentItems) { + LOG_ERR() << "Failed to allocate current items" << Log::Field(ErrorEnum::eNoMemory); + + return; + } + auto installItems = MakeUnique>(&mAllocator); + if (!installItems) { + LOG_ERR() << "Failed to allocate install items" << Log::Field(ErrorEnum::eNoMemory); + + return; + } if (auto err = mImageManager->GetAllInstalledItems(*currentItems); !err.IsNone()) { LOG_ERR() << "Get update items statuses failed" << Log::Field(AOS_ERROR_WRAP(err)); @@ -1427,12 +1475,18 @@ Error Launcher::GetInstanceConfigs( LOG_DBG() << "Get instance configs" << Log::Field("instance", instance); auto path = MakeUnique>(&mAllocator); + if (!path) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mItemInfoProvider->GetBlobPath(instance.mManifestDigest, *path); !err.IsNone()) { return AOS_ERROR_WRAP(err); } auto manifest = MakeUnique(&mAllocator); + if (!manifest) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mOCISpec->LoadImageManifest(*path, *manifest); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -1467,6 +1521,9 @@ Error Launcher::GetInstanceNetworkConfig(const InstanceInfo& instance, const oci networkConfig.mInstanceIdent = static_cast(instance); auto resourceInfo = MakeUnique(&mAllocator); + if (!resourceInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } for (const auto& resource : itemConfig.mResources) { @@ -1516,6 +1573,9 @@ Error Launcher::CreateNetwork( const InstanceData& instanceData, const oci::ItemConfig& itemConfig, const oci::ImageConfig& imageConfig) { auto networkConfig = MakeUnique(&mAllocator); + if (!networkConfig) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = GetInstanceNetworkConfig(instanceData.mInfo, itemConfig, imageConfig, *networkConfig); !err.IsNone()) { diff --git a/src/core/sm/networkmanager/networkmanager.cpp b/src/core/sm/networkmanager/networkmanager.cpp index b4242480a..41f77c981 100644 --- a/src/core/sm/networkmanager/networkmanager.cpp +++ b/src/core/sm/networkmanager/networkmanager.cpp @@ -37,6 +37,9 @@ Error NetworkManager::Init(StorageItf& storage, BridgeNetworkItf& bridgeNet, Fir auto instanceNetworkInfos = MakeUnique>(&mInstanceNetworkInfosAllocator); + if (!instanceNetworkInfos) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mStorage->GetInstanceNetworksInfo(*instanceNetworkInfos); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -47,6 +50,9 @@ Error NetworkManager::Init(StorageItf& storage, BridgeNetworkItf& bridgeNet, Fir } auto networkInfos = MakeUnique>(&mNetworkInfosAllocator); + if (!networkInfos) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mStorage->GetNetworksInfo(*networkInfos); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -204,12 +210,22 @@ Error NetworkManager::CreateInstanceNetwork( } auto serviceData = MakeUnique(&mAllocator); + if (!serviceData) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } if (err = PrepareUpdateItemNetworkParams(instanceNetworkParameters, networkID, *serviceData); !err.IsNone()) { return err; } auto allocatedParams = MakeUnique(&mAllocator); + if (!allocatedParams) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } if (err = mNetworkProvider->AllocateInstanceNetwork( instanceNetworkParameters.mInstanceIdent, networkID, mNodeID, *serviceData, *allocatedParams); @@ -229,6 +245,11 @@ Error NetworkManager::CreateInstanceNetwork( auto info = MakeUnique( &mAllocator, instanceID, networkID, instanceNetworkParameters, *allocatedParams); + if (!info) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } if (err = mStorage->AddInstanceNetworkInfo(*info); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -248,6 +269,9 @@ Error NetworkManager::StartInstanceNetwork(const String& instanceID, const Strin LOG_DBG() << "Start instance network" << Log::Field("instanceID", instanceID) << Log::Field("networkID", networkID); auto cachedInfo = MakeUnique(&mAllocator); + if (!cachedInfo) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } { LockGuard lock {mMutex}; @@ -303,7 +327,10 @@ Error NetworkManager::GetResolvServers(const String& instanceID, Array networkID; StaticString bridgeIP; - auto dns = MakeUnique, cMaxNumDNSServers>>(&mResolvHostsAllocator); + auto dns = MakeUnique, cMaxNumDNSServers>>(&mResolvHostsAllocator); + if (!dns) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } { LockGuard lock {mMutex}; @@ -350,7 +377,10 @@ Error NetworkManager::GetHosts(const String& instanceID, Array& hosts) con StaticString networkID; StaticString instanceIP; StaticString hostname; - auto customHosts = MakeUnique>(&mResolvHostsAllocator); + auto customHosts = MakeUnique>(&mResolvHostsAllocator); + if (!customHosts) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } { LockGuard lock {mMutex}; @@ -863,6 +893,11 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin }); auto hosts = MakeUnique, cMaxNumHosts>>(&mAllocator); + if (!hosts) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } if (err = PrepareHosts(instanceID, networkID, networkConfig, *hosts); !err.IsNone()) { return err; @@ -876,6 +911,11 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin } auto bridgeParams = MakeUnique(&mAllocator); + if (!bridgeParams) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } if (err = PrepareBridgeParams(networkID, networkParams, *bridgeParams); !err.IsNone()) { return err; @@ -898,6 +938,11 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin }); auto firewallParams = MakeUnique(&mAllocator); + if (!firewallParams) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } if (err = PrepareInstanceFirewallParams(networkConfig, networkParams, *firewallParams); !err.IsNone()) { return err; @@ -917,6 +962,11 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin }); auto bandwidthParams = MakeUnique(&mAllocator); + if (!bandwidthParams) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } if (err = PrepareBandwidthParams(networkConfig, *bandwidthParams); !err.IsNone()) { return err; @@ -953,6 +1003,11 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin } auto dnsParams = MakeUnique(&mAllocator); + if (!dnsParams) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } if (err = PrepareDNSAliasesParams(networkParams, *hosts, *dnsParams); !err.IsNone()) { return err; @@ -994,6 +1049,11 @@ Error NetworkManager::AddInstanceToNetwork(const String& instanceID, const Strin } auto info = MakeUnique(&mAllocator); + if (!info) { + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } { LockGuard lock {mMutex}; @@ -1112,7 +1172,16 @@ Error NetworkManager::DeleteInstanceNetworkConfig(const String& instanceID, cons } if (err.IsNone() && !hostIfName.IsEmpty()) { - auto info = MakeUnique(&mAllocator); + auto info = MakeUnique(&mAllocator); + if (!info) { + LOG_ERR() << "Failed to allocate instance network info" << Log::Field("instanceID", instanceID) + << Log::Field(ErrorEnum::eNoMemory); + + err = AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + + return err; + } + bool needPersist = false; { @@ -1295,6 +1364,9 @@ Error NetworkManager::ReconcileInstances() }; auto entries = MakeUnique>(&mAllocator); + if (!entries) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } { LockGuard lock {mMutex}; @@ -1388,6 +1460,9 @@ Error NetworkManager::RemoveFirewallOrphans() Error NetworkManager::RemoveDNSOrphans() { auto known = MakeUnique, cMaxNumOwners>>(&mAllocator); + if (!known) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } { LockGuard lock {mMutex}; @@ -1889,8 +1964,19 @@ void NetworkManager::OnPendingFirewallUpdate( StaticString networkID; bool isRunning = false; - auto networkConfig = MakeUnique(&mAllocator); + auto networkConfig = MakeUnique(&mAllocator); + if (!networkConfig) { + LOG_ERR() << "Failed to allocate network config" << Log::Field(ErrorEnum::eNoMemory); + + return; + } + auto allocatedParams = MakeUnique(&mAllocator); + if (!allocatedParams) { + LOG_ERR() << "Failed to allocate network allocation params" << Log::Field(ErrorEnum::eNoMemory); + + return; + } StaticString hostIfName; @@ -1925,6 +2011,12 @@ void NetworkManager::OnPendingFirewallUpdate( auto info = MakeUnique( &mInstanceNetworkInfosAllocator, instanceID, networkID, *networkConfig, *allocatedParams, hostIfName); + if (!info) { + LOG_ERR() << "Failed to allocate instance network info" << Log::Field("instanceID", instanceID) + << Log::Field(ErrorEnum::eNoMemory); + + return; + } if (auto err = mStorage->UpdateInstanceNetworkInfo(*info); !err.IsNone()) { LOG_ERR() << "Failed to update instance network info" << Log::Field("instanceID", instanceID) @@ -1951,6 +2043,11 @@ void NetworkManager::OnConnect() LOG_DBG() << "SM connected to CM, synchronizing network state"; auto instances = MakeUnique>(&mAllocator); + if (!instances) { + LOG_ERR() << "Failed to allocate instances sync state" << Log::Field(ErrorEnum::eNoMemory); + + return; + } { LockGuard lock {mMutex}; @@ -1984,6 +2081,9 @@ Error NetworkManager::UpdateInstanceFirewall(const String& instanceID, const Str << Log::Field("networkID", networkID); auto firewallParams = MakeUnique(&mAllocator); + if (!firewallParams) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = PrepareInstanceFirewallParams(networkConfig, networkParams, *firewallParams); !err.IsNone()) { return err; diff --git a/src/core/sm/nodeconfig/nodeconfig.cpp b/src/core/sm/nodeconfig/nodeconfig.cpp index 3ad412b29..7ce950bae 100644 --- a/src/core/sm/nodeconfig/nodeconfig.cpp +++ b/src/core/sm/nodeconfig/nodeconfig.cpp @@ -67,6 +67,9 @@ Error NodeConfig::UpdateNodeConfig(const aos::NodeConfig& config) mNodeConfig = config; auto nodeConfigJSON = MakeUnique>(&mAllocator); + if (!nodeConfigJSON) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } if (auto err = mJSONProvider->NodeConfigToJSON(config, *nodeConfigJSON); !err.IsNone()) { return AOS_ERROR_WRAP(err); @@ -146,6 +149,9 @@ Error NodeConfig::LoadConfig() LOG_DBG() << "Load config"; auto nodeConfigJSON = MakeUnique>(&mAllocator); + if (!nodeConfigJSON) { + return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); + } auto err = fs::ReadFileToString(mNodeConfigFile, *nodeConfigJSON); if (!err.IsNone()) { From b6857b7b7ec26d2af39e91f6f2af5ae41c50bf39 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 30 Jul 2026 20:26:30 +0300 Subject: [PATCH 096/112] tools: replace static allocator with AllocatorItf/HeapAllocator Rework the memory allocation abstraction in memory.hpp: - Rename Allocator to AllocatorItf, keeping only Allocate/Free and a virtual destructor. This decouples callers from any particular allocation strategy. - Remove StaticAllocator, BufferAllocator and the custom placement new/delete operator overloads (allocator.hpp is deleted). Sizing a static arena correctly, especially for multithreaded usage, was error prone and required extra bookkeeping. - Add HeapAllocator (malloc/free backed) for Linux and test usage. Safety-critical targets can provide their own AllocatorItf implementation. - Rework SharedPtr to use intrusive control blocks (SharedControlBlock/SharedObjectControlBlock/SharedAdoptControlBlock) instead of allocator-external ref-counting, so it works uniformly over any AllocatorItf implementation. - MakeUnique/MakeShared now check the allocation result before constructing the object, returning a null pointer on failure instead of relying on assert(), which is stripped in release builds. Signed-off-by: Oleksandr Grytsov --- src/core/common/tools/CMakeLists.txt | 2 +- src/core/common/tools/allocator.hpp | 356 --------------- src/core/common/tools/heapallocator.hpp | 43 ++ src/core/common/tools/memory.hpp | 502 ++++++++++++++-------- src/core/common/tools/tests/allocator.cpp | 94 ++-- src/core/common/tools/tests/memory.cpp | 105 ++--- 6 files changed, 436 insertions(+), 666 deletions(-) delete mode 100644 src/core/common/tools/allocator.hpp create mode 100644 src/core/common/tools/heapallocator.hpp diff --git a/src/core/common/tools/CMakeLists.txt b/src/core/common/tools/CMakeLists.txt index a62b5212f..c9f65fd3b 100644 --- a/src/core/common/tools/CMakeLists.txt +++ b/src/core/common/tools/CMakeLists.txt @@ -22,7 +22,6 @@ set(SOURCES fs.cpp semver.cpp time.cpp timer.cpp uuid.cpp) set(HEADERS algorithm.hpp - allocator.hpp array.hpp buffer.hpp config.hpp @@ -30,6 +29,7 @@ set(HEADERS error.hpp fs.hpp function.hpp + heapallocator.hpp identifierpool.hpp list.hpp log.hpp diff --git a/src/core/common/tools/allocator.hpp b/src/core/common/tools/allocator.hpp deleted file mode 100644 index 5e63d1660..000000000 --- a/src/core/common/tools/allocator.hpp +++ /dev/null @@ -1,356 +0,0 @@ -/* - * Copyright (C) 2023 Renesas Electronics Corporation. - * Copyright (C) 2023 EPAM Systems, Inc. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef AOS_CORE_COMMON_TOOLS_ALLOCATOR_HPP_ -#define AOS_CORE_COMMON_TOOLS_ALLOCATOR_HPP_ - -#include -#include - -#include "buffer.hpp" -#include "list.hpp" -#include "noncopyable.hpp" -#include "thread.hpp" - -namespace aos { - -/** - * Allocator instance. - */ -class Allocator { -public: - /** - * Allocation instance. - */ - class Allocation { - public: - /** - * Creates allocation. - */ - Allocation() = default; - - /** - * Creates allocation. - * - * @param data pointer to allocated data. - * @param size allocated size. - */ - Allocation(uint8_t* data, size_t size) - : mData(data) - , mSize(size) - , mSharedCount(0) - { - } - - /** - * Returns pointer to allocated data. - * - * @return uint8_t* pointer to allocated data. - */ - uint8_t* Data() const { return mData; } - - /** - * Returns allocated size. - * - * @return size_t allocated size. - */ - size_t Size() const { return mSize; } - - /** - * Increases shared count. - * - * @param mutex mutex to lock context. - * @return size_t chared count value; - */ - size_t Take(Mutex& mutex) - { - LockGuard lock(mutex); - - return ++mSharedCount; - } - - /** - * Decreases shared count. - * - * @param mutex mutex to lock context. - * @return size_t chared count value; - */ - size_t Give(Mutex& mutex) - { - LockGuard lock(mutex); - - return --mSharedCount; - } - - private: - uint8_t* mData = nullptr; - size_t mSize = 0; - size_t mSharedCount = 0; - }; - - /** - * Clears allocator. - */ - void Clear() - { - LockGuard lock {mMutex}; - - mAllocations->Clear(); - } - - /** - * Allocates data with specified size. - * - * @param size allocate size. - * @return void* pointer to allocated data. - */ - void* Allocate(size_t size) - { - LockGuard lock {mMutex}; - - if (mAllocations->IsFull() || GetAllocatedSize() + size > mMaxSize) { - assert(!mAllocations->IsFull()); - assert(GetAllocatedSize() + size <= mMaxSize); - - return nullptr; - } - - auto* pos = mBuffer; - auto it = mAllocations->begin(); - - for (; it != mAllocations->end(); ++it) { - size_t availableSize = it->Data() - pos; - - if (availableSize >= size) { - return Allocate(it, pos, size); - } - - pos = it->Data() + it->Size(); - } - - if (pos + size <= mBuffer + mMaxSize) { - return Allocate(mAllocations->end(), pos, size); - } - - assert(false); - - return nullptr; - } - - /** - * Frees previously allocated data. - * - * @param data allocated data to free. - */ - void Free(void* data) - { - LockGuard lock {mMutex}; - - [[maybe_unused]] auto curSize = mAllocations->Size(); - mAllocations->RemoveIf([data](const Allocation& allocation) { return allocation.Data() == data; }); - [[maybe_unused]] auto newSize = mAllocations->Size(); - - assert(curSize != newSize); - } - - /** - * Finds allocation by data. - * - * @param data allocated data. - * @return List::Iterator. - */ - RetWithError::Iterator> FindAllocation(const void* data) - { - LockGuard lock {mMutex}; - - return mAllocations->FindIf([data](const Allocation& allocation) { return allocation.Data() == data; }); - } - - /** - * Increases allocation shared count. - * - * @param it allocation to increase shared count. - * @return size_t allocation shared count. - */ - size_t TakeAllocation(List::Iterator it) { return it->Take(mMutex); } - - /** - * Decreases allocation shared count. - * - * @param it allocation to increase shared count. - * @return size_t allocation shared count. - */ - size_t GiveAllocation(List::Iterator it) { return it->Give(mMutex); } - - /** - * Returns allocator free size. - * - * @return size_t free size. - */ - size_t FreeSize() const - { - LockGuard lock {mMutex}; - - return mMaxSize - GetAllocatedSize(); - } - - /** - * Return allocator max size. - * - * @return size_t max size. - */ - size_t MaxSize() const - { - LockGuard lock {mMutex}; - - return mMaxSize; - } - - /** - * Return max allocated size. - * - * @return size_t max allocated size. - */ - size_t MaxAllocatedSize() const - { - LockGuard lock {mMutex}; - - return mMaxAllocatedSize; - } - - /** - * Resets max allocated size. - */ - void ResetMaxAllocatedSize() - { - LockGuard lock {mMutex}; - - mMaxAllocatedSize = 0; - } - -protected: - void SetBuffer(const Buffer& buffer, List& allocations) - { - mBuffer = static_cast(buffer.Get()); - mMaxSize = buffer.Size(); - mAllocations = &allocations; - mAllocations->Clear(); - } - -private: - // cppcheck-suppress passedByValue - void* Allocate(List::ConstIterator it, uint8_t* data, size_t size) - { - [[maybe_unused]] auto err = mAllocations->Emplace(it, Allocation(data, size)); - assert(err.IsNone()); - - if (GetAllocatedSize() > mMaxAllocatedSize) { - mMaxAllocatedSize = GetAllocatedSize(); - } - - return data; - } - - size_t GetAllocatedSize() const - { - size_t allocatedSize = 0; - - for (const auto& allocation : *mAllocations) { - allocatedSize += allocation.Size(); - } - - return allocatedSize; - } - - uint8_t* mBuffer = {}; - List* mAllocations = {}; - size_t mMaxSize = {}; - size_t mMaxAllocatedSize = {}; - mutable Mutex mMutex; -}; - -/** - * Buffer allocator instance. - */ -template -class BufferAllocator : public Allocator { -public: - /** - * Creates buffer allocator instance. - */ - explicit BufferAllocator(const Buffer& buffer) { SetBuffer(buffer, mAllocations); } - -private: - StaticArray mAllocations; -}; - -/** - * Static allocator instance. - */ -template -class StaticAllocator : public Allocator { -public: - /** - * Creates static allocator instance. - */ - StaticAllocator() { SetBuffer(mBuffer, mAllocations); } - -private: - StaticBuffer mBuffer; - StaticList mAllocations; -}; - -} // namespace aos - -/** - * Overloads placement new operator to allocate object on Aos buffer. - * - * @param size allocate size. - * @param allocator allocator. - * @return void* allocated space. - */ -inline void* operator new(size_t size, aos::Allocator* allocator) -{ - assert(allocator); - - auto data = allocator->Allocate(size); - assert(data); - - return data; -} - -/** - * Overloads placement sized new operator to allocate object on Aos buffer. - * - * @param size allocate size. - * @param allocator allocator. - * @return void* allocated space. - */ -inline void* operator new[](size_t size, aos::Allocator* allocator) -{ - assert(allocator); - - auto data = allocator->Allocate(size); - assert(data); - - return data; -} - -/** - * Overloads placement delete operator to release object on Aos buffer. - * - * @param data pointer to allocated data. - * @param allocator allocator. - */ -inline void operator delete(void* data, aos::Allocator* allocator) -{ - assert(allocator); - - allocator->Free(data); -} - -#endif diff --git a/src/core/common/tools/heapallocator.hpp b/src/core/common/tools/heapallocator.hpp new file mode 100644 index 000000000..17e17ebb6 --- /dev/null +++ b/src/core/common/tools/heapallocator.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2023 Renesas Electronics Corporation. + * Copyright (C) 2023 EPAM Systems, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef AOS_CORE_COMMON_TOOLS_HEAPALLOCATOR_HPP_ +#define AOS_CORE_COMMON_TOOLS_HEAPALLOCATOR_HPP_ + +#include + +#include "memory.hpp" + +namespace aos { + +/** + * Heap allocator instance. Backs the AllocatorItf interface with the standard heap + * (malloc/free). Intended for hosted targets (e.g. Linux) where dynamic memory allocation + * is acceptable, and for unit tests; safety critical/embedded targets should provide their + * own custom AllocatorItf implementation. + */ +class HeapAllocator : public AllocatorItf { +public: + /** + * Allocates data with specified size. + * + * @param size allocate size. + * @return void* pointer to allocated data, nullptr if allocation failed. + */ + void* Allocate(size_t size) override { return std::malloc(size); } + + /** + * Frees previously allocated data. + * + * @param data allocated data to free. + */ + void Free(void* data) override { std::free(data); } +}; + +} // namespace aos + +#endif diff --git a/src/core/common/tools/memory.hpp b/src/core/common/tools/memory.hpp index 8289da43b..8b825cd65 100644 --- a/src/core/common/tools/memory.hpp +++ b/src/core/common/tools/memory.hpp @@ -8,12 +8,45 @@ #ifndef AOS_CORE_COMMON_TOOLS_MEMORY_HPP_ #define AOS_CORE_COMMON_TOOLS_MEMORY_HPP_ +#include #include +#include -#include "allocator.hpp" +#include "noncopyable.hpp" +#include "utils.hpp" namespace aos { +/** + * Allocator interface. + * + * Any implementation only has to provide Allocate/Free semantics, so it can be backed by a + * heap (see HeapAllocator) or by any custom allocation strategy required by a specific target + * (e.g. a static/embedded arena for safety critical domains). + */ +class AllocatorItf { +public: + /** + * Allocates data with specified size. + * + * @param size allocate size. + * @return void* pointer to allocated data, nullptr if allocation failed. + */ + virtual void* Allocate(size_t size) = 0; + + /** + * Frees previously allocated data. + * + * @param data allocated data to free. + */ + virtual void Free(void* data) = 0; + + /** + * Destroys allocator instance. + */ + virtual ~AllocatorItf() = default; +}; + /** * Default deleter invokes delete operator for the given pointer. */ @@ -25,7 +58,7 @@ class DefaultDeleter : public NonCopyable { * * @param allocator input allocator. */ - explicit DefaultDeleter(Allocator* allocator = nullptr) + explicit DefaultDeleter(AllocatorItf* allocator = nullptr) : mAllocator(allocator) { } @@ -53,9 +86,9 @@ class DefaultDeleter : public NonCopyable { /** * Returns allocator. * - * @return Allocator*. + * @return AllocatorItf*. */ - Allocator* GetAllocator() const { return mAllocator; } + AllocatorItf* GetAllocator() const { return mAllocator; } /** * Destroys object & deallocates memory. @@ -66,149 +99,29 @@ class DefaultDeleter : public NonCopyable { { if (mAllocator) { ptr->~T(); - operator delete(ptr, mAllocator); + mAllocator->Free(ptr); } } private: - Allocator* mAllocator; + AllocatorItf* mAllocator; }; /** - * Deleter function for shared pointer. + * Deleter function for objects adopted by a shared pointer. */ template -inline void SmartPtrDeleter(void* ptr, Allocator* allocator) +inline void SmartPtrDeleter(void* ptr, AllocatorItf* allocator) { if (ptr) { static_cast(ptr)->~T(); if (allocator) { - operator delete(ptr, allocator); + allocator->Free(ptr); } } } -/** - * Smart pointer instance. - * - * @tparam T holding object type. - */ -template -class SmartPtr { -public: - /** - * Deleter. - */ - using Deleter = void (*)(void*, Allocator*); - - // cppcheck-suppress noExplicitConstructor - /** - * Creates smart pointer. - * - * @param allocator allocator. - * @param object object to make smart pointer. - */ - SmartPtr(Allocator* allocator = nullptr, T* object = nullptr, Deleter deleter = SmartPtrDeleter) - : mAllocator(allocator) - , mObject(object) - , mDeleter(deleter) - { - assert(!(object && !allocator && !deleter)); - } - - /** - * Deletes holding object and release smart pointer. - * - * @param allocator new allocator. - * @param object new object. - */ - void Reset(Allocator* allocator = nullptr, T* object = nullptr, Deleter deleter = SmartPtrDeleter) - { - if (mAllocator && mObject && mDeleter) { - mDeleter(const_cast*>(mObject), mAllocator); - } - - assert(!(object && !allocator)); - - Release(allocator, object, deleter); - } - - /** - * Releases smart pointer. - * - * @param allocator new allocator. - * @param object new object. - * @return T* pointer to holding object. - */ - T* Release(Allocator* allocator = nullptr, T* object = nullptr, Deleter deleter = SmartPtrDeleter) - { - auto curObject = mObject; - - mObject = object; - mAllocator = allocator; - mDeleter = deleter; - - return curObject; - } - - /** - * Returns holding object. - * - * @return T* holding object. - */ - T* Get() const { return mObject; } - - /** - * Returns holding allocator. - * - * @return Allocator* holding allocator. - */ - Allocator* GetAllocator() const { return mAllocator; } - - /** - * Returns holding deleter. - * - * @return Deleter. - */ - Deleter GetDeleter() const { return mDeleter; } - - /** - * Checks if pointer holds object. - * - * @return bool. - */ - explicit operator bool() const { return mObject != nullptr; } - - /** - * Compares two smart pointers. - * - * @param ptr1 first smart pointer. - * @param ptr2 second smart pointer. - * @return bool. - */ - friend bool operator==(const SmartPtr& ptr1, const SmartPtr& ptr2) { return ptr1.mObject == ptr2.mObject; } - - /** - * Provides access to holding object fields. - * - * @return T* holding object pointer. - */ - T* operator->() const { return mObject; } - - /** - * Dereferences holding object. - * - * @return T& holding object value. - */ - T& operator*() const { return *(mObject); } - -private: - Allocator* mAllocator {}; - T* mObject {}; - Deleter mDeleter {}; -}; - /** * Unique pointer instance. * @@ -238,7 +151,7 @@ class UniquePtr : private NonCopyable { * @param allocator allocator that object was allocated with. * */ - UniquePtr(T* ptr = nullptr, Allocator* allocator = nullptr) + UniquePtr(T* ptr = nullptr, AllocatorItf* allocator = nullptr) : mObject(ptr) , mDeleter(DefaultDeleter(allocator)) { @@ -389,30 +302,189 @@ class UniquePtr : private NonCopyable { Deleter mDeleter; }; +/** + * Defers object destruction till the end of the current scope. + * + * @tparam T type of the object to be destroyed. + * @tparam Deleter type of the deleter. + * @param ptr pointer to the object to be destroyed. + * @param deleter functor object to be deferred. + * @return UniquePtr. + */ +template +inline UniquePtr DeferRelease(T* ptr, Deleter&& deleter) +{ + return UniquePtr(ptr, Move(deleter)); +} + +/** + * Base class for shared pointer control blocks. + * + * Owns the reference count and knows how to dispose of itself (and whatever it holds) through + * the same AllocatorItf it was created with. This keeps SharedPtr's ref-counting independent + * from any allocator-specific bookkeeping, so it works the same way for any AllocatorItf + * implementation (heap based or static/embedded). + */ +class SharedControlBlock : private NonCopyable { +public: + /** + * Creates control block instance. + * + * @param allocator allocator the control block itself was allocated from. + */ + explicit SharedControlBlock(AllocatorItf& allocator) + : mAllocator(allocator) + { + } + + /** + * Increases shared count. + * + * @return size_t shared count value. + */ + size_t Take() { return ++mRefCount; } + + /** + * Decreases shared count. Disposes the control block once the count reaches zero. + * + * @return size_t shared count value. + */ + size_t Give() + { + auto count = --mRefCount; + + if (count == 0) { + Dispose(); + } + + return count; + } + + /** + * Destroys control block instance. + */ + virtual ~SharedControlBlock() = default; + +protected: + /** + * Allocator the control block itself was allocated from. + */ + AllocatorItf& mAllocator; + +private: + virtual void Dispose() = 0; + + size_t mRefCount = 1; +}; + +/** + * Control block that owns an object of type T directly (single allocation). Used by MakeShared. + * + * @tparam T holding object type. + */ +template +class SharedObjectControlBlock : public SharedControlBlock { +public: + /** + * Creates control block instance constructing the held object in place. + * + * @param allocator allocator the control block is allocated from. + * @param args holding object constructor parameters. + */ + template + explicit SharedObjectControlBlock(AllocatorItf& allocator, Args&&... args) + : SharedControlBlock(allocator) + , mObject(args...) + { + } + + /** + * Returns pointer to the held object. + * + * @return T*. + */ + T* GetObject() { return &mObject; } + +private: + void Dispose() override + { + auto& allocator = mAllocator; + + this->~SharedObjectControlBlock(); + allocator.Free(this); + } + + T mObject; +}; + +/** + * Control block that adopts an already constructed, separately allocated object. + * + * @tparam T holding object type. + */ +template +class SharedAdoptControlBlock : public SharedControlBlock { +public: + /** + * Deleter. + */ + using Deleter = void (*)(void*, AllocatorItf*); + + /** + * Creates control block instance. + * + * @param allocator allocator the control block itself is allocated from. + * @param object object to adopt. + * @param deleter functor destroying the adopted object. + */ + SharedAdoptControlBlock(AllocatorItf& allocator, T* object, Deleter deleter) + : SharedControlBlock(allocator) + , mObject(object) + , mDeleter(deleter) + { + } + +private: + void Dispose() override + { + auto& allocator = mAllocator; + + if (mDeleter) { + mDeleter(mObject, &allocator); + } + + this->~SharedAdoptControlBlock(); + allocator.Free(this); + } + + T* mObject; + Deleter mDeleter; +}; + /** * Shared pointer instance. * * @tparam T holding object type. */ template -class SharedPtr : public SmartPtr { +class SharedPtr { public: /** * Deleter. */ - using typename SmartPtr::Deleter; + using Deleter = void (*)(void*, AllocatorItf*); // cppcheck-suppress noExplicitConstructor /** - * Creates shared pointer. + * Creates shared pointer adopting an already allocated object. + * + * @param allocator allocator object was allocated with. + * @param object object to adopt. + * @param deleter functor destroying the object. */ - SharedPtr(Allocator* allocator = nullptr, T* object = nullptr, Deleter deleter = SmartPtrDeleter) - : SmartPtr(allocator, object, deleter) + SharedPtr(AllocatorItf* allocator = nullptr, T* object = nullptr, Deleter deleter = SmartPtrDeleter) { - if (allocator && object) { - mAllocation = allocator->FindAllocation(object).mValue; - SmartPtr::GetAllocator()->TakeAllocation(mAllocation); - } + Adopt(allocator, object, deleter); } /** @@ -421,11 +493,11 @@ class SharedPtr : public SmartPtr { * @param ptr pointer to create from. */ SharedPtr(const SharedPtr& ptr) - : SmartPtr(ptr.GetAllocator(), ptr.Get(), ptr.GetDeleter()) - , mAllocation(ptr.mAllocation) + : mObject(ptr.mObject) + , mControlBlock(ptr.mControlBlock) { - if (SmartPtr::GetAllocator()) { - SmartPtr::GetAllocator()->TakeAllocation(mAllocation); + if (mControlBlock) { + mControlBlock->Take(); } } @@ -436,11 +508,17 @@ class SharedPtr : public SmartPtr { */ SharedPtr& operator=(const SharedPtr& ptr) { - SmartPtr::Release(ptr.GetAllocator(), ptr.Get(), ptr.GetDeleter()); - mAllocation = ptr.mAllocation; + if (this == &ptr) { + return *this; + } - if (SmartPtr::GetAllocator()) { - SmartPtr::GetAllocator()->TakeAllocation(mAllocation); + Reset(); + + mObject = ptr.mObject; + mControlBlock = ptr.mControlBlock; + + if (mControlBlock) { + mControlBlock->Take(); } return *this; @@ -454,11 +532,11 @@ class SharedPtr : public SmartPtr { template // cppcheck-suppress noExplicitConstructor SharedPtr(const SharedPtr

& ptr) - : SmartPtr(ptr.GetAllocator(), ptr.Get(), ptr.GetDeleter()) - , mAllocation(ptr.mAllocation) + : mObject(ptr.mObject) + , mControlBlock(ptr.mControlBlock) { - if (SmartPtr::GetAllocator()) { - SmartPtr::GetAllocator()->TakeAllocation(mAllocation); + if (mControlBlock) { + mControlBlock->Take(); } } @@ -470,40 +548,76 @@ class SharedPtr : public SmartPtr { template SharedPtr& operator=(const SharedPtr

& ptr) { - SmartPtr::Release(ptr.GetAllocator(), ptr.Get(), ptr.GetDeleter()); - mAllocation = ptr.mAllocation; + Reset(); + + mObject = ptr.mObject; + mControlBlock = ptr.mControlBlock; - if (SmartPtr::GetAllocator()) { - SmartPtr::GetAllocator()->TakeAllocation(mAllocation); + if (mControlBlock) { + mControlBlock->Take(); } return *this; } - // cppcheck-suppress duplInheritedMember /** * Resets shared pointer. * * @param allocator new allocator. * @param object new object. + * @param deleter functor destroying the object. */ - void Reset(Allocator* allocator = nullptr, T* object = nullptr, Deleter deleter = SmartPtrDeleter) + void Reset(AllocatorItf* allocator = nullptr, T* object = nullptr, Deleter deleter = SmartPtrDeleter) { - if (SmartPtr::GetAllocator() && SmartPtr::GetAllocator()->GiveAllocation(mAllocation) == 0) { - SmartPtr::Reset(allocator, object, deleter); + if (mControlBlock) { + mControlBlock->Give(); } - SmartPtr::Release(allocator, object, deleter); - mAllocation = {}; + mObject = nullptr; + mControlBlock = nullptr; - if (allocator && object) { - mAllocation = allocator->FindAllocation(object).mValue; - SmartPtr::GetAllocator()->TakeAllocation(mAllocation); - } + Adopt(allocator, object, deleter); } /** - * Destroys unique pointer. + * Returns holding object. + * + * @return T* holding object. + */ + T* Get() const { return mObject; } + + /** + * Checks if pointer holds object. + * + * @return bool. + */ + explicit operator bool() const { return mObject != nullptr; } + + /** + * Compares two shared pointers. + * + * @param ptr1 first shared pointer. + * @param ptr2 second shared pointer. + * @return bool. + */ + friend bool operator==(const SharedPtr& ptr1, const SharedPtr& ptr2) { return ptr1.mObject == ptr2.mObject; } + + /** + * Provides access to holding object fields. + * + * @return T* holding object pointer. + */ + T* operator->() const { return mObject; } + + /** + * Dereferences holding object. + * + * @return T& holding object value. + */ + T& operator*() const { return *(mObject); } + + /** + * Destroys shared pointer. */ ~SharedPtr() { Reset(); } @@ -511,7 +625,36 @@ class SharedPtr : public SmartPtr { template friend class SharedPtr; - List::Iterator mAllocation; + template + friend SharedPtr MakeShared(AllocatorItf* allocator, Args&&... args); + + SharedPtr(SharedControlBlock* controlBlock, T* object) + : mObject(object) + , mControlBlock(controlBlock) + { + } + + void Adopt(AllocatorItf* allocator, T* object, Deleter deleter) + { + if (!allocator || !object) { + return; + } + + auto data = allocator->Allocate(sizeof(SharedAdoptControlBlock)); + if (!data) { + if (deleter) { + deleter(object, allocator); + } + + return; + } + + mControlBlock = new (data) SharedAdoptControlBlock(*allocator, object, deleter); + mObject = object; + } + + T* mObject {}; + SharedControlBlock* mControlBlock {}; }; /** @@ -524,7 +667,7 @@ class SharedPtr : public SmartPtr { * @return UniquePtr constructed unique ptr, empty if allocation failed. */ template -inline UniquePtr MakeUnique(Allocator* allocator, Args&&... args) +inline UniquePtr MakeUnique(AllocatorItf* allocator, Args&&... args) { assert(allocator); @@ -536,21 +679,6 @@ inline UniquePtr MakeUnique(Allocator* allocator, Args&&... args) return UniquePtr(new (data) T(args...), DefaultDeleter(allocator)); } -/** - * Defers object destruction till the end of the current scope. - * - * @tparam T type of the object to be destroyed. - * @tparam Deleter type of the deleter. - * @param ptr pointer to the object to be destroyed. - * @param deleter functor object to be deferred. - * @return UniquePtr. - */ -template -inline UniquePtr DeferRelease(T* ptr, Deleter&& deleter) -{ - return UniquePtr(ptr, Move(deleter)); -} - /** * Constructs shared pointer. * @@ -561,16 +689,18 @@ inline UniquePtr DeferRelease(T* ptr, Deleter&& deleter) * @return SharedPtr constructed shared ptr, empty if allocation failed. */ template -inline SharedPtr MakeShared(Allocator* allocator, Args&&... args) +inline SharedPtr MakeShared(AllocatorItf* allocator, Args&&... args) { assert(allocator); - auto data = allocator->Allocate(sizeof(T)); + auto data = allocator->Allocate(sizeof(SharedObjectControlBlock)); if (!data) { return SharedPtr(); } - return SharedPtr(allocator, new (data) T(args...), SmartPtrDeleter); + auto* controlBlock = new (data) SharedObjectControlBlock(*allocator, args...); + + return SharedPtr(controlBlock, controlBlock->GetObject()); } } // namespace aos diff --git a/src/core/common/tools/tests/allocator.cpp b/src/core/common/tools/tests/allocator.cpp index 1fa84c9a0..602f4306a 100644 --- a/src/core/common/tools/tests/allocator.cpp +++ b/src/core/common/tools/tests/allocator.cpp @@ -7,80 +7,46 @@ #include -#include +#include +#include using namespace aos; -TEST(AllocatorTest, Allocator) +TEST(AllocatorTest, HeapAllocator) { - StaticAllocator<256> allocator; + HeapAllocator allocator; - EXPECT_EQ(allocator.MaxSize(), 256); - EXPECT_EQ(allocator.FreeSize(), 256); + auto* data = allocator.Allocate(128); + ASSERT_NE(data, nullptr); - struct TestItem { - void* mData; - size_t mSize; - }; - - TestItem testData[] = {{nullptr, 32}, {nullptr, 64}, {nullptr, 128}}; - - auto freeSize = allocator.MaxSize(); - - for (auto& item : testData) { - item.mData = allocator.Allocate(item.mSize); - freeSize -= item.mSize; - EXPECT_EQ(allocator.FreeSize(), freeSize); - } + allocator.Free(data); - for (size_t i = 0; i < ArraySize(testData); i++) { - freeSize += testData[i].mSize; - allocator.Free(testData[i].mData); - - EXPECT_EQ(allocator.FreeSize(), freeSize); - } - - allocator.Allocate(32); - - allocator.Clear(); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); -} + struct TestStruct { + TestStruct(int a, int b) + : mA(a) + , mB(b) + { + } -TEST(AllocatorTest, New) -{ - StaticAllocator<256> allocator; - - auto freeSize = allocator.MaxSize(); - - auto val1 = new (&allocator) uint8_t(); - freeSize -= sizeof(uint8_t); - EXPECT_EQ(allocator.FreeSize(), freeSize); - - auto val2 = new (&allocator) uint16_t(); - freeSize -= sizeof(uint16_t); - EXPECT_EQ(allocator.FreeSize(), freeSize); - - auto val3 = new (&allocator) uint32_t(); - freeSize -= sizeof(uint32_t); - EXPECT_EQ(allocator.FreeSize(), freeSize); - - auto val4 = new (&allocator) uint64_t(); - freeSize -= sizeof(uint64_t); - EXPECT_EQ(allocator.FreeSize(), freeSize); + int mA; + int mB; + }; - operator delete(val4, &allocator); - freeSize += sizeof(uint64_t); - EXPECT_EQ(allocator.FreeSize(), freeSize); + auto uPtr = MakeUnique(&allocator, 1, 2); + ASSERT_TRUE(uPtr); + EXPECT_EQ(uPtr->mA, 1); + EXPECT_EQ(uPtr->mB, 2); - operator delete(val3, &allocator); - freeSize += sizeof(uint32_t); - EXPECT_EQ(allocator.FreeSize(), freeSize); + auto shPtr = MakeShared(&allocator, 3, 4); + ASSERT_TRUE(shPtr); + EXPECT_EQ(shPtr->mA, 3); + EXPECT_EQ(shPtr->mB, 4); - operator delete(val2, &allocator); - freeSize += sizeof(uint16_t); - EXPECT_EQ(allocator.FreeSize(), freeSize); + auto shPtr2 = shPtr; + EXPECT_EQ(shPtr2->mA, 3); - operator delete(val1, &allocator); - freeSize += sizeof(uint8_t); - EXPECT_EQ(allocator.FreeSize(), freeSize); + shPtr.Reset(); + EXPECT_FALSE(shPtr); + EXPECT_TRUE(shPtr2); + EXPECT_EQ(shPtr2->mB, 4); } diff --git a/src/core/common/tools/tests/memory.cpp b/src/core/common/tools/tests/memory.cpp index 70ea4a780..2b23628f7 100644 --- a/src/core/common/tools/tests/memory.cpp +++ b/src/core/common/tools/tests/memory.cpp @@ -7,6 +7,7 @@ #include +#include #include using namespace aos; @@ -38,35 +39,36 @@ class NewClass : public BaseClass { TEST(MemoryTest, UniquePtr) { - StaticAllocator<256> allocator; + HeapAllocator allocator; // Basic test { UniquePtr uPtr = MakeUnique(&allocator, 0); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(uint32_t)); + EXPECT_TRUE(uPtr); + EXPECT_EQ(*uPtr, 0U); } - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); - // Construct with allocator { - UniquePtr uPtr(new (&allocator) uint32_t(), &allocator); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(uint32_t)); - } + auto* raw = static_cast(allocator.Allocate(sizeof(uint32_t))); + ASSERT_NE(raw, nullptr); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + UniquePtr uPtr(new (raw) uint32_t(), &allocator); + EXPECT_TRUE(uPtr); + } // Construct with deleter { auto deleter = [&allocator](uint32_t* ptr) { allocator.Free(ptr); }; - UniquePtr uPtr(new (&allocator) uint32_t(), Move(deleter)); - } + auto* raw = static_cast(allocator.Allocate(sizeof(uint32_t))); + ASSERT_NE(raw, nullptr); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + UniquePtr uPtr(new (raw) uint32_t(), Move(deleter)); + } // Move ownership @@ -80,37 +82,36 @@ TEST(MemoryTest, UniquePtr) uPtr = MakeUnique(&allocator); } - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(uint32_t)); + EXPECT_TRUE(uPtr); OwnUniquePtr(Move(uPtr)); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + EXPECT_FALSE(uPtr); // Make unique auto uPtr2 = MakeUnique(&allocator); - - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(uint32_t)); + EXPECT_TRUE(uPtr2); // Check reset uPtr2.Reset(); - - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + EXPECT_FALSE(uPtr2); } TEST(MemoryTest, SharedPtr) { - StaticAllocator<256> allocator; + HeapAllocator allocator; // Basic test { - SharedPtr shPtr(&allocator, new (&allocator) uint32_t()); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(uint32_t)); - } + auto* raw = static_cast(allocator.Allocate(sizeof(uint32_t))); + ASSERT_NE(raw, nullptr); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + SharedPtr shPtr(&allocator, new (raw) uint32_t()); + EXPECT_TRUE(shPtr); + } // Test share @@ -122,69 +123,58 @@ TEST(MemoryTest, SharedPtr) EXPECT_TRUE(nullptr == shPtr); { - shPtr = SharedPtr(&allocator, new (&allocator) uint32_t()); + auto* raw = static_cast(allocator.Allocate(sizeof(uint32_t))); + ASSERT_NE(raw, nullptr); + + shPtr = SharedPtr(&allocator, new (raw) uint32_t()); } - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(uint32_t)); + EXPECT_TRUE(shPtr); TakeSharedPtr(shPtr); } - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); - // Make shared auto shPtr2 = MakeShared(&allocator); - - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(uint32_t)); + EXPECT_TRUE(shPtr2); // Check reset shPtr2.Reset(); - - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + EXPECT_FALSE(shPtr2); } TEST(MemoryTest, UniquePtrDerivedClass) { - StaticAllocator<256> allocator; - - { - UniquePtr basePtr; + HeapAllocator allocator; - { - auto newPtr = MakeUnique(&allocator); + UniquePtr basePtr; - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(NewClass)); - - basePtr = Move(newPtr); - } + { + auto newPtr = MakeUnique(&allocator); + EXPECT_TRUE(newPtr); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(NewClass)); + basePtr = Move(newPtr); } - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + EXPECT_TRUE(basePtr); } TEST(MemoryTest, SharedPtrDerivedClass) { - StaticAllocator<256> allocator; + HeapAllocator allocator; - { - SharedPtr basePtr; - - { - auto newPtr = MakeShared(&allocator); - - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(NewClass)); + SharedPtr basePtr; - basePtr = newPtr; - } + { + auto newPtr = MakeShared(&allocator); + EXPECT_TRUE(newPtr); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(NewClass)); + basePtr = newPtr; } - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + EXPECT_TRUE(basePtr); } TEST(MemoryTest, DeferRelease) @@ -268,16 +258,13 @@ TEST(MemoryTest, SharedPtrDerivedValueClass) MockFunction* mFunc; }; - StaticAllocator<256> allocator; - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); + HeapAllocator allocator; // Check NewClass destructor is called EXPECT_CALL(callback, Call()).Times(1); { SharedPtr basePtr = MakeShared(&allocator, &callback); - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize() - sizeof(NewClass)); + EXPECT_TRUE(basePtr); } - - EXPECT_EQ(allocator.FreeSize(), allocator.MaxSize()); } From 37b8124163393f8755c8318cfc1d06e7d7fa3096 Mon Sep 17 00:00:00 2001 From: Oleksandr Grytsov Date: Thu, 30 Jul 2026 20:29:26 +0300 Subject: [PATCH 097/112] common: migrate to injected AllocatorItf Convert common/ classes that previously owned a private static allocator to receive an AllocatorItf reference/pointer instead, following the two-phase construct-then-Init() pattern used across the codebase (allocator as the first Init()/constructor parameter): - crypto: CertLoader, CryptoHelper, mbedtls/openssl CryptoProviderItf implementations, pkcs11::Utils, PKCS11RSAPrivateKey. - pkcs11: LibraryContext, PKCS11Manager. - monitoring: Average, Monitoring. - spaceallocator: SpaceAllocator; also renamed its own "space" allocator members (OutdatedItem::mSpaceAllocator, the nested Space class's mSpaceAllocator) to avoid confusion with the new memory AllocatorItf member. - fs: CalculateSize takes an AllocatorItf parameter (first), and FileInfoProvider forwards it internally; dropped the shared static allocator and its guarding mutex, since callers now own their allocator's thread-safety. Multiple per-class named allocators are consolidated into a single AllocatorItf pointer where heap allocation removes the need for separate statically-sized pools. Unit tests construct a HeapAllocator and pass it in, always declared before any member that may allocate from it, since C++ destroys members in reverse declaration order. Signed-off-by: Oleksandr Grytsov --- src/core/common/crypto/certloader.cpp | 15 ++--- src/core/common/crypto/certloader.hpp | 13 +---- src/core/common/crypto/cryptohelper.cpp | 36 ++++++------ src/core/common/crypto/cryptohelper.hpp | 15 ++--- src/core/common/crypto/cryptoutils.cpp | 1 + .../common/crypto/mbedtls/cryptoprovider.cpp | 12 ++-- .../common/crypto/mbedtls/cryptoprovider.hpp | 10 +--- .../common/crypto/openssl/cryptoprovider.cpp | 12 ++-- .../common/crypto/openssl/cryptoprovider.hpp | 11 +--- src/core/common/crypto/tests/certloader.cpp | 24 ++++---- src/core/common/crypto/tests/cryptohelper.cpp | 14 +++-- .../common/crypto/tests/cryptoprovider.cpp | 7 ++- src/core/common/crypto/tests/cryptoutils.cpp | 7 ++- src/core/common/monitoring/average.cpp | 6 +- src/core/common/monitoring/average.hpp | 7 +-- src/core/common/monitoring/monitoring.cpp | 12 ++-- src/core/common/monitoring/monitoring.hpp | 9 ++- .../common/monitoring/tests/monitoring.cpp | 25 ++++---- src/core/common/pkcs11/pkcs11.cpp | 22 +++++-- src/core/common/pkcs11/pkcs11.hpp | 29 +++++++--- src/core/common/pkcs11/privatekey.cpp | 7 ++- src/core/common/pkcs11/privatekey.hpp | 12 ++-- src/core/common/pkcs11/tests/pkcs11.cpp | 52 ++++++++--------- .../common/spaceallocator/spaceallocator.hpp | 58 ++++++++++--------- .../spaceallocator/tests/spaceallocator.cpp | 21 ++++--- .../crypto/providers/cryptofactoryitf.hpp | 3 +- .../tests/crypto/providers/mbedtlsfactory.cpp | 4 +- .../tests/crypto/providers/mbedtlsfactory.hpp | 3 +- .../tests/crypto/providers/opensslfactory.cpp | 4 +- .../tests/crypto/providers/opensslfactory.hpp | 3 +- src/core/common/tests/crypto/softhsmenv.cpp | 8 ++- src/core/common/tests/crypto/softhsmenv.hpp | 5 +- .../common/tests/stubs/spaceallocatorstub.hpp | 3 +- src/core/common/tools/fs.cpp | 22 ++----- src/core/common/tools/fs.hpp | 8 ++- src/core/common/tools/tests/fs.cpp | 14 +++-- 36 files changed, 277 insertions(+), 237 deletions(-) diff --git a/src/core/common/crypto/certloader.cpp b/src/core/common/crypto/certloader.cpp index 146b2bd36..bcfe966fe 100644 --- a/src/core/common/crypto/certloader.cpp +++ b/src/core/common/crypto/certloader.cpp @@ -24,10 +24,11 @@ constexpr auto cSchemeMaxLength = Max(sizeof(cSchemeFile), sizeof(cSchemePKCS11) * CertLoader **********************************************************************************************************************/ -Error CertLoader::Init(x509::ProviderItf& cryptoProvider, pkcs11::PKCS11Manager& pkcs11Manager) +Error CertLoader::Init(AllocatorItf& allocator, x509::ProviderItf& cryptoProvider, pkcs11::PKCS11Manager& pkcs11Manager) { LOG_DBG() << "Init cert loader"; + mAllocator = &allocator; mCryptoProvider = &cryptoProvider; mPKCS11 = &pkcs11Manager; @@ -73,7 +74,7 @@ RetWithError> CertLoader::LoadCertsChainByURL( return {nullptr, err}; } - return pkcs11::Utils(session, *mCryptoProvider, mAllocator).FindCertificateChain(id, label); + return pkcs11::Utils(*mAllocator, session, *mCryptoProvider).FindCertificateChain(id, label); } return {nullptr, ErrorEnum::eInvalidArgument}; @@ -118,7 +119,7 @@ RetWithError> CertLoader::LoadPrivKeyByURL(const String return {nullptr, err}; } - auto key = pkcs11::Utils(session, *mCryptoProvider, mAllocator).FindPrivateKey(id, label); + auto key = pkcs11::Utils(*mAllocator, session, *mCryptoProvider).FindPrivateKey(id, label); return {key.mValue.GetPrivKey(), key.mError}; } @@ -166,7 +167,7 @@ RetWithError CertLoader::FindToken(const pkcs11::LibraryContext& { StaticArray slotList; - auto tokenInfo = MakeUnique(&mAllocator); + auto tokenInfo = MakeUnique(mAllocator); if (!tokenInfo) { return {0, ErrorEnum::eNoMemory}; } @@ -194,7 +195,7 @@ RetWithError> CertLoader::LoadCertsFromFile(co { LOG_DBG() << "Load certs chain from file: fileName=" << fileName; - auto buff = MakeUnique(&mAllocator); + auto buff = MakeUnique(mAllocator); if (!buff) { return {nullptr, ErrorEnum::eNoMemory}; } @@ -204,7 +205,7 @@ RetWithError> CertLoader::LoadCertsFromFile(co return {nullptr, err}; } - auto certificates = MakeShared(&mAllocator); + auto certificates = MakeShared(mAllocator); if (!certificates) { return {nullptr, ErrorEnum::eNoMemory}; } @@ -218,7 +219,7 @@ RetWithError> CertLoader::LoadPrivKeyFromFile(const Str { LOG_DBG() << "Load private key from file: fileName=" << fileName; - auto buff = MakeUnique>(&mAllocator); + auto buff = MakeUnique>(mAllocator); if (!buff) { return {nullptr, ErrorEnum::eNoMemory}; } diff --git a/src/core/common/crypto/certloader.hpp b/src/core/common/crypto/certloader.hpp index fa63febe6..cd2a58d4a 100644 --- a/src/core/common/crypto/certloader.hpp +++ b/src/core/common/crypto/certloader.hpp @@ -24,11 +24,12 @@ class CertLoader : public CertLoaderItf { /** * Initializes object instance. * + * @param allocator allocator to use for certificates/keys. * @param cryptoProvider crypto provider interface. * @param pkcs11Manager PKCS11 library manager. * @return Error. */ - Error Init(x509::ProviderItf& cryptoProvider, pkcs11::PKCS11Manager& pkcs11Manager); + Error Init(AllocatorItf& allocator, x509::ProviderItf& cryptoProvider, pkcs11::PKCS11Manager& pkcs11Manager); /** * Loads certificate chain by URL. @@ -49,12 +50,6 @@ class CertLoader : public CertLoaderItf { private: using PEMCertChainBlob = StaticString; - static constexpr auto cCertAllocatorSize - = cCertChainsCount * cCertChainSize * sizeof(x509::Certificate) + sizeof(PEMCertChainBlob); - static constexpr auto cKeyAllocatorSize - = AOS_CONFIG_CRYPTO_PRIV_KEYS_COUNT * pkcs11::cPrivateKeyMaxSize + sizeof(cPrivKeyPEMLen); - static constexpr auto cNumAllocation = AOS_CONFIG_CRYPTO_NUM_ALLOCATIONS; - static constexpr auto cDefaultPKCS11Library = AOS_CONFIG_CRYPTO_DEFAULT_PKCS11_LIB; RetWithError> OpenSession( @@ -66,9 +61,7 @@ class CertLoader : public CertLoaderItf { x509::ProviderItf* mCryptoProvider = nullptr; pkcs11::PKCS11Manager* mPKCS11 = nullptr; - - StaticAllocator - mAllocator; + AllocatorItf* mAllocator {}; }; } // namespace aos::crypto diff --git a/src/core/common/crypto/cryptohelper.cpp b/src/core/common/crypto/cryptohelper.cpp index e3115173d..0be0cc313 100644 --- a/src/core/common/crypto/cryptohelper.cpp +++ b/src/core/common/crypto/cryptohelper.cpp @@ -21,15 +21,17 @@ CryptoHelper::CryptoHelper() { } -Error CryptoHelper::Init(iamclient::CertProviderItf& certProvider, CryptoProviderItf& cryptoProvider, - CertLoaderItf& certLoader, const String& serviceDiscoveryURL, const String& caCert) +Error CryptoHelper::Init(AllocatorItf& allocator, iamclient::CertProviderItf& certProvider, + CryptoProviderItf& cryptoProvider, CertLoaderItf& certLoader, const String& serviceDiscoveryURL, + const String& caCert) { + mAllocator = &allocator; mCertProvider = &certProvider; mCryptoProvider = &cryptoProvider; mCertLoader = &certLoader; mServiceDiscoveryURL = serviceDiscoveryURL; - auto caCertsPEM = MakeUnique>(&mAllocator); + auto caCertsPEM = MakeUnique>(mAllocator); if (!caCertsPEM) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -113,7 +115,7 @@ Error CryptoHelper::ValidateSigns(const String& decryptedPath, const SignInfo& s { LockGuard lock {mSemaphore}; - auto signCtx = MakeUnique(&mAllocator); + auto signCtx = MakeUnique(mAllocator); if (!signCtx) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -137,12 +139,12 @@ Error CryptoHelper::DecryptMetadata(const Array& input, Array& { LockGuard lock {mSemaphore}; - auto contentInfo = MakeUnique(&mAllocator); + auto contentInfo = MakeUnique(mAllocator); if (!contentInfo) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } - auto symKey = MakeUnique>(&mAllocator); + auto symKey = MakeUnique>(mAllocator); if (!symKey) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -177,7 +179,7 @@ Error CryptoHelper::DecryptMetadata(const Array& input, Array& RetWithError> CryptoHelper::GetOnlineCert() { - auto certInfo = MakeUnique(&mAllocator); + auto certInfo = MakeUnique(mAllocator); if (!certInfo) { return {{}, AOS_ERROR_WRAP(ErrorEnum::eNoMemory)}; } @@ -222,7 +224,7 @@ Error CryptoHelper::GetServiceDiscoveryFromExtensions(const x509::Certificate& c Error CryptoHelper::GetServiceDiscoveryFromOrganization( const x509::Certificate& cert, Array>& urls) { - auto subject = MakeUnique>(&mAllocator); + auto subject = MakeUnique>(mAllocator); if (!subject) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -241,12 +243,12 @@ Error CryptoHelper::GetServiceDiscoveryFromOrganization( auto valueStart = orgPos + orgKey.Size(); auto [valueEnd, _] = subject->FindSubstr(valueStart, ","); - auto orgName = MakeUnique>(&mAllocator); + auto orgName = MakeUnique>(mAllocator); if (!orgName) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } - auto url = MakeUnique>(&mAllocator); + auto url = MakeUnique>(mAllocator); if (!url) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -358,12 +360,12 @@ Error CryptoHelper::CheckSessionKey( Error CryptoHelper::DecodeFile(const String& encryptedFile, const String& decryptedFile, AESCipherItf& decoder) { - auto inBlock = MakeUnique>(&mAllocator); + auto inBlock = MakeUnique>(mAllocator); if (!inBlock) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } - auto outBlock = MakeUnique>(&mAllocator); + auto outBlock = MakeUnique>(mAllocator); if (!outBlock) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -444,7 +446,7 @@ Error CryptoHelper::AddCertificates(const Array& certs, SignCon continue; } - auto cert = MakeUnique(&mAllocator); + auto cert = MakeUnique(mAllocator); if (!cert) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -512,7 +514,7 @@ Error CryptoHelper::VerifySigns(const String& file, const SignInfo& signs, SignC } // Verify sign - auto hashSum = MakeUnique>(&mAllocator); + auto hashSum = MakeUnique>(mAllocator); if (!hashSum) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -540,7 +542,7 @@ Error CryptoHelper::VerifySigns(const String& file, const SignInfo& signs, SignC } // Verify certs - auto intermCertPool = MakeUnique>(&mAllocator); + auto intermCertPool = MakeUnique>(mAllocator); if (!intermCertPool) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -899,7 +901,7 @@ Error CryptoHelper::ParseEncryptedContentInfo(const Array& data, Encryp Error CryptoHelper::GetKeyForEnvelope(const TransRecipientInfo& info, Array& symmetricKey) { - auto certInfo = MakeUnique(&mAllocator); + auto certInfo = MakeUnique(mAllocator); if (!certInfo) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } @@ -966,7 +968,7 @@ Error CryptoHelper::DecryptMessage( Error CryptoHelper::DecodeMessage(AESCipherItf& decoder, const Array& input, Array& message) { - auto outBlock = MakeUnique>(&mAllocator); + auto outBlock = MakeUnique>(mAllocator); if (!outBlock) { return AOS_ERROR_WRAP(ErrorEnum::eNoMemory); } diff --git a/src/core/common/crypto/cryptohelper.hpp b/src/core/common/crypto/cryptohelper.hpp index a569e6f5b..df8e6b4ae 100644 --- a/src/core/common/crypto/cryptohelper.hpp +++ b/src/core/common/crypto/cryptohelper.hpp @@ -8,6 +8,7 @@ #define AOS_AOS_COMMON_CRYPTO_CRYPTOHELPER_HPP_ #include +#include #include "itf/certloader.hpp" #include "itf/crypto.hpp" @@ -97,6 +98,7 @@ class CryptoHelper : public CryptoHelperItf { /** * Initializes crypto helper. * + * @param allocator allocator to use for temporary objects. * @param certProvider certificate provider interface. * @param cryptoProvider cryptographic provider interface. * @param certLoader certificate loader interface. @@ -104,8 +106,8 @@ class CryptoHelper : public CryptoHelperItf { * @param caCert root certificate path. * @return Error. */ - Error Init(iamclient::CertProviderItf& certProvider, CryptoProviderItf& cryptoProvider, CertLoaderItf& certLoader, - const String& serviceDiscoveryURL, const String& caCert); + Error Init(AllocatorItf& allocator, iamclient::CertProviderItf& certProvider, CryptoProviderItf& cryptoProvider, + CertLoaderItf& certLoader, const String& serviceDiscoveryURL, const String& caCert); /** * Retrieves available service discovery URLs. @@ -157,11 +159,6 @@ class CryptoHelper : public CryptoHelperItf { static constexpr auto cRSAEncryptionOid = "1.2.840.113549.1.1.1"; static constexpr auto cAES256CBCOid = "2.16.840.1.101.3.4.1.42"; - static constexpr auto cThreadHeapUsage = 2 * sizeof(CertInfo) + sizeof(StaticString) - + sizeof(StaticArray) + sizeof(SignContext) + sizeof(x509::Certificate) - + sizeof(StaticArray) + sizeof(StaticArray) - + sizeof(StaticArray) * 2 + sizeof(StaticString) * 2; - RetWithError> GetOnlineCert(); Error SetDefaultServiceDiscoveryURL(Array>& urls); Error GetServiceDiscoveryFromExtensions(const x509::Certificate& cert, Array>& urls); @@ -201,8 +198,8 @@ class CryptoHelper : public CryptoHelperItf { StaticString mServiceDiscoveryURL; x509::CertificateChain mCACerts; - Semaphore mSemaphore; - StaticAllocator mAllocator; + Semaphore mSemaphore; + AllocatorItf* mAllocator {}; }; } // namespace aos::crypto diff --git a/src/core/common/crypto/cryptoutils.cpp b/src/core/common/crypto/cryptoutils.cpp index be1ee173b..243f89c31 100644 --- a/src/core/common/crypto/cryptoutils.cpp +++ b/src/core/common/crypto/cryptoutils.cpp @@ -6,6 +6,7 @@ #include #include +#include #include "cryptoutils.hpp" diff --git a/src/core/common/crypto/mbedtls/cryptoprovider.cpp b/src/core/common/crypto/mbedtls/cryptoprovider.cpp index addb2d326..3bd3e92f3 100644 --- a/src/core/common/crypto/mbedtls/cryptoprovider.cpp +++ b/src/core/common/crypto/mbedtls/cryptoprovider.cpp @@ -574,10 +574,12 @@ Error VerifyECDSASignature(const ECDSAPublicKey& pubKey, const Array& d * Public **********************************************************************************************************************/ -Error MbedTLSCryptoProvider::Init() +Error MbedTLSCryptoProvider::Init(AllocatorItf& allocator) { LOG_DBG() << "Init mbedTLS crypto provider"; + mAllocator = &allocator; + auto ret = psa_crypto_init(); return ret != PSA_SUCCESS ? AOS_ERROR_WRAP(ret) : ErrorEnum::eNone; @@ -827,7 +829,7 @@ RetWithError> MbedTLSCryptoProvider::PEMToX509PrivKey(c { LOG_ERR() << "Create private key from PEM"; - auto res = MakeShared(&mAllocator); + auto res = MakeShared(mAllocator); if (!res) { return {{}, ErrorEnum::eNoMemory}; } @@ -944,7 +946,7 @@ RetWithError> MbedTLSCryptoProvider::CreateHash(Hash algorith return {nullptr, ErrorEnum::eNotSupported}; } - auto hasher = MakeUnique(&mAllocator, alg); + auto hasher = MakeUnique(mAllocator, alg); if (!hasher) { return {nullptr, ErrorEnum::eNoMemory}; } @@ -1062,7 +1064,7 @@ RetWithError> MbedTLSCryptoProvider::CreateAESEncoder( return {{}, AOS_ERROR_WRAP(ErrorEnum::eNotSupported)}; } - auto cipher = MakeUnique(&mAllocator); + auto cipher = MakeUnique(mAllocator); if (!cipher) { return {{}, ErrorEnum::eNoMemory}; } @@ -1082,7 +1084,7 @@ RetWithError> MbedTLSCryptoProvider::CreateAESDecoder( return {{}, AOS_ERROR_WRAP(ErrorEnum::eNotSupported)}; } - auto cipher = MakeUnique(&mAllocator); + auto cipher = MakeUnique(mAllocator); if (!cipher) { return {{}, ErrorEnum::eNoMemory}; } diff --git a/src/core/common/crypto/mbedtls/cryptoprovider.hpp b/src/core/common/crypto/mbedtls/cryptoprovider.hpp index 88ec9d269..7b1f43b61 100644 --- a/src/core/common/crypto/mbedtls/cryptoprovider.hpp +++ b/src/core/common/crypto/mbedtls/cryptoprovider.hpp @@ -27,9 +27,10 @@ class MbedTLSCryptoProvider : public CryptoProviderItf { /** * Initializes the object. * + * @param allocator allocator to use for temporary and key objects. * @result Error. */ - Error Init(); + Error Init(AllocatorItf& allocator); /** * Creates a new certificate based on a template. @@ -396,11 +397,6 @@ class MbedTLSCryptoProvider : public CryptoProviderItf { mutable mbedtls_pk_context mPrivKey; }; - static constexpr auto cAllocatorSize - = AOS_CONFIG_CRYPTO_PUB_KEYS_COUNT * Max(sizeof(RSAPublicKey), sizeof(ECDSAPublicKey)) - + AOS_CONFIG_CRYPTO_HASHER_COUNT * sizeof(MBedTLSHash) - + AOS_CONFIG_CRYPTO_PRIV_KEYS_COUNT * sizeof(MbedTLSRSAPrivKey); - static int VerifyTime(void* data, mbedtls_x509_crt* crt, int depth, uint32_t* flags); static RetWithError