Make previous identity endorsement fetching robust to the ledger gaps

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 [7.0.5]: https://github.com/microsoft/CCF/releases/tag/ccf-7.0.5
 
+### Changed
+
+- Missing previous-identity endorsement ledger chunks no longer block node startup: the network identity subsystem now exposes a `Partial` fetch status, serves the validated chain prefix, and offers a `trigger_extension()` call so callers can request another fetch attempt. Chain-integrity violations still fail-hard (#7913).
+
 ### Deprecated
 
 - Accessing ledger-signature names (table names, exception classes) via `ccf.ledger` now emits a `DeprecationWarning`; import them from `ccf.signatures` instead (#7904).

CMakeLists.txt

@@ -771,6 +771,15 @@ if(BUILD_TESTS)
       historical_queries_test
       PRIVATE http_parser ccf_kv ccf_endpoints ccf_tasks
     )
+
+    add_unit_test(
+      network_identity_subsystem_test
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/node/test/network_identity_subsystem.cpp
+    )
+    target_link_libraries(
+      network_identity_subsystem_test
+      PRIVATE ccf_kv ccf_endpoints ccf_tasks
+    )
     add_unit_test(
       indexing_test
       ${CMAKE_CURRENT_SOURCE_DIR}/src/indexing/test/indexing.cpp

include/ccf/network_identity_interface.h

@@ -6,10 +6,8 @@
 #include "ccf/node_subsystem_interface.h"
 #include "ccf/tx_id.h"
 
-#include <exception>
 #include <map>
 #include <optional>
-#include <string>
 #include <vector>
 
 namespace ccf
@@ -24,29 +22,19 @@ namespace ccf
   /// Status of the network identity endorsement fetching process.
   enum class FetchStatus : uint8_t
   {
-    Retry, ///< Fetching should be retried
-    Done, ///< Fetching completed successfully
-    Failed ///< Fetching failed
+    Done, ///< Fetching trusted identities completed successfully
+    Partial, ///< Chain is still being built or fetching attempts were
+             ///< exhausted (e.g. ledger files missing). Readers see the
+             ///< validated subset; @ref
+             ///< NetworkIdentitySubsystemInterface::trigger_extension
+             ///< can request more.
+    Failed ///< Fetching failed with error and cannot be resumed
   };
 
   /// Map from sequence number to EC public key, representing the trusted
   /// network identity keys over the history of the service.
   using TrustedKeys = std::map<ccf::SeqNo, ccf::crypto::ECPublicKeyPtr>;
 
-  /// Exception thrown when identity data is requested before the
-  /// asynchronous identity-history-fetching process has completed.
-  struct IdentityHistoryNotFetched : public std::exception
-  {
-    std::string msg;
-
-    IdentityHistoryNotFetched(std::string msg) : msg(std::move(msg)) {}
-
-    [[nodiscard]] const char* what() const noexcept override
-    {
-      return msg.c_str();
-    }
-  };
-
   /// Interface for accessing the network identity subsystem, which manages
   /// the service's cryptographic identity and its historical trusted keys.
   class NetworkIdentitySubsystemInterface : public ccf::AbstractNodeSubSystem
@@ -62,34 +50,36 @@ namespace ccf
     /// Returns a reference to the current network identity.
     virtual const std::unique_ptr<NetworkIdentity>& get() = 0;
 
-    /// Returns the current status of endorsement fetching.
+    /// Returns the current status of endorsement fetching. Callers
+    /// should check this before acting on a nullopt/nullptr/empty
+    /// reader result: in @ref FetchStatus::Partial more data may
+    /// arrive via @ref NetworkIdentitySubsystemInterface::trigger_extension;
+    /// in @ref FetchStatus::Failed the fetch is unrecoverable.
     [[nodiscard]] virtual FetchStatus endorsements_fetching_status() const = 0;
 
+    /// Schedule a fresh attempt to fetch the next missing predecessor
+    /// endorsement. No-op outside @ref FetchStatus::Partial. Thread-safe
+    /// and idempotent: concurrent callers trigger at most one cycle.
+    virtual void trigger_extension() = 0;
+
     /// Returns the COSE endorsements chain for the given sequence number,
-    /// or std::nullopt if the chain is not available for the given sequence
-    /// number.
-    ///
-    /// @throws IdentityHistoryNotFetched if identity history fetching has not
-    /// completed.
+    /// or std::nullopt if the chain does not yet reach back to the
+    /// requested seqno (see @ref
+    /// NetworkIdentitySubsystemInterface::trigger_extension).
     [[nodiscard]] virtual std::optional<CoseEndorsementsChain>
     get_cose_endorsements_chain(ccf::SeqNo seqno) const = 0;
 
     /// Returns the trusted EC public key that was active at the given
-    /// sequence number, or nullptr if the sequence number precedes the
-    /// earliest known trusted key.
-    ///
-    /// @throws IdentityHistoryNotFetched if identity history fetching has not
-    /// completed.
-    /// @throws std::logic_error if no trusted keys have been fetched, or if
-    /// internal key resolution is inconsistent.
+    /// sequence number, or nullptr if the sequence number predates the
+    /// earliest known trusted key (see @ref
+    /// NetworkIdentitySubsystemInterface::trigger_extension).
     [[nodiscard]] virtual ccf::crypto::ECPublicKeyPtr get_trusted_identity_for(
       ccf::SeqNo seqno) const = 0;
 
     /// Returns all trusted network identity keys as a map from sequence
-    /// number to EC public key.
-    ///
-    /// @throws IdentityHistoryNotFetched if identity history fetching has not
-    /// completed.
+    /// number to EC public key. In @ref FetchStatus::Partial older epochs
+    /// may be missing -- see @ref
+    /// NetworkIdentitySubsystemInterface::trigger_extension.
     [[nodiscard]] virtual TrustedKeys get_trusted_keys() const = 0;
   };
 }

src/node/historical_queries_utils.cpp

@@ -222,25 +222,30 @@ namespace ccf
 
       const auto fetching =
         network_identity_subsystem->endorsements_fetching_status();
-      if (fetching == FetchStatus::Retry)
-      {
-        return false;
-      }
       if (fetching == FetchStatus::Failed)
       {
         throw std::runtime_error(fmt::format(
           "The service identity endorsement for the receipt at seqno {} "
           "cannot be fetched",
           state->transaction_id.seqno));
       }
-      if (fetching != FetchStatus::Done)
-      {
-        throw std::logic_error("Unexpected endorsements fetching status");
-      }
 
       auto cose_endorsements =
         network_identity_subsystem->get_cose_endorsements_chain(
           state->transaction_id.seqno);
+      if (!cose_endorsements.has_value())
+      {
+        // The chain does not yet reach back to the requested seqno
+        // (either the initial walk hasn't gotten there or a previous
+        // extension cycle exhausted its retries). Trigger a fresh
+        // extension cycle (no-op if not in Partial or one is already
+        // running) and signal the caller to retry.
+        if (fetching == FetchStatus::Partial)
+        {
+          network_identity_subsystem->trigger_extension();
+        }
+        return false;
+      }
       state->receipt->cose_endorsements = cose_endorsements;
       return true;
     }

src/node/rpc/network_identity_accessors.h

@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/tx_id.h"
+#include "service/tables/previous_service_identity.h"
+#include "tasks/task_system.h"
+
+#include <chrono>
+#include <functional>
+#include <optional>
+
+namespace ccf
+{
+  struct INodeStateAccessor
+  {
+    virtual ~INodeStateAccessor() = default;
+
+    [[nodiscard]] virtual bool is_part_of_network() const = 0;
+
+    // Current service's create-txid, or nullopt if not yet available.
+    virtual std::optional<TxID> read_current_service_from() = 0;
+
+    // Topmost previous-identity endorsement entry, or nullopt if none.
+    virtual std::optional<CoseEndorsement> read_topmost_endorsement() = 0;
+  };
+
+  struct IHistoricalStateAccessor
+  {
+    virtual ~IHistoricalStateAccessor() = default;
+
+    // Endorsement entry at the given historical kv version, or nullopt
+    // if the historical state is not yet loaded. Implementations may
+    // throw on hard errors.
+    virtual std::optional<CoseEndorsement> get_endorsement_at(SeqNo) = 0;
+  };
+
+  struct TaskScheduler
+  {
+    virtual ~TaskScheduler() = default;
+
+    virtual void add_task(std::function<void()> fn) = 0;
+
+    virtual void add_delayed_task(
+      std::function<void()> fn, std::chrono::milliseconds delay) = 0;
+  };
+}

src/node/rpc/network_identity_accessors_impl.h

@@ -0,0 +1,130 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/service/tables/service.h"
+#include "node/historical_queries.h"
+#include "node/rpc/network_identity_accessors.h"
+#include "node/rpc/node_interface.h"
+#include "service/internal_tables_access.h"
+#include "tasks/basic_task.h"
+#include "tasks/task_system.h"
+
+#include <fmt/format.h>
+#include <memory>
+#include <stdexcept>
+#include <utility>
+
+namespace ccf
+{
+  class NodeStateAccessor : public INodeStateAccessor
+  {
+  protected:
+    AbstractNodeState& node_state;
+
+  public:
+    NodeStateAccessor(AbstractNodeState& node_state_) : node_state(node_state_)
+    {}
+
+    [[nodiscard]] bool is_part_of_network() const override
+    {
+      return node_state.is_part_of_network();
+    }
+
+    std::optional<TxID> read_current_service_from() override
+    {
+      auto store = node_state.get_store();
+      auto tx = store->create_read_only_tx();
+      auto* service_info_handle =
+        tx.template ro<ccf::Service>(ccf::Tables::SERVICE);
+      auto service_info = service_info_handle->get();
+      if (
+        !service_info || !service_info->current_service_create_txid.has_value())
+      {
+        return std::nullopt;
+      }
+      if (service_info->status != ServiceStatus::OPEN)
+      {
+        // It can happen that node advances its internal state machine to
+        // part-of-network, but the service opening tx has not been
+        // replicated yet. This will cause the first fetched endorsement
+        // to be obsolete, but waiting for ServiceStatus::OPEN is
+        // sufficient, as it's supposed to arrive in the same TX that
+        // the previous identity endorsement.
+        return std::nullopt;
+      }
+      return service_info->current_service_create_txid;
+    }
+
+    std::optional<CoseEndorsement> read_topmost_endorsement() override
+    {
+      auto store = node_state.get_store();
+      auto tx = store->create_read_only_tx();
+      return tx
+        .template ro<ccf::PreviousServiceIdentityEndorsement>(
+          ccf::Tables::PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT)
+        ->get();
+    }
+  };
+
+  class HistoricalStateAccessor : public IHistoricalStateAccessor
+  {
+  protected:
+    std::shared_ptr<historical::StateCacheImpl> historical_cache;
+
+  public:
+    HistoricalStateAccessor(
+      std::shared_ptr<historical::StateCacheImpl> historical_cache_) :
+      historical_cache(std::move(historical_cache_))
+    {}
+
+    std::optional<CoseEndorsement> get_endorsement_at(SeqNo seq) override
+    {
+      auto state = historical_cache->get_state_at(
+        ccf::historical::CompoundHandle{
+          ccf::historical::RequestNamespace::System, seq},
+        seq);
+      if (!state)
+      {
+        return std::nullopt;
+      }
+      if (!state->store)
+      {
+        throw std::runtime_error(fmt::format(
+          "Historical state with seqno {} is loaded but its store is "
+          "missing",
+          seq));
+      }
+      auto htx = state->store->create_read_only_tx();
+      auto endorsement =
+        htx
+          .template ro<ccf::PreviousServiceIdentityEndorsement>(
+            ccf::Tables::PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT)
+          ->get();
+      if (!endorsement.has_value())
+      {
+        throw std::runtime_error(fmt::format(
+          "COSE endorsement entry for seqno {} is missing from its "
+          "historical state",
+          seq));
+      }
+      return endorsement;
+    }
+  };
+
+  class TaskSchedulerImpl : public TaskScheduler
+  {
+  public:
+    void add_task(std::function<void()> fn) override
+    {
+      ccf::tasks::add_task(ccf::tasks::make_basic_task(std::move(fn)));
+    }
+
+    void add_delayed_task(
+      std::function<void()> fn, std::chrono::milliseconds delay) override
+    {
+      auto task = ccf::tasks::make_basic_task(std::move(fn));
+      ccf::tasks::add_delayed_task(task, delay);
+    }
+  };
+}