From 0e80bc48cd72d294132164500cf5867a3af63c3f Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Wed, 10 Jun 2026 10:08:08 +0200
Subject: [PATCH 01/28] release: record SSE-KMS Secrets Cache backport metadata

---
 .../019eb08c-159f-7a78-b689-5d37a7d343d0.md   | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 release-management/backports/019eb08c-159f-7a78-b689-5d37a7d343d0.md

diff --git a/release-management/backports/019eb08c-159f-7a78-b689-5d37a7d343d0.md b/release-management/backports/019eb08c-159f-7a78-b689-5d37a7d343d0.md
new file mode 100644
index 000000000000..03299d70ba38
--- /dev/null
+++ b/release-management/backports/019eb08c-159f-7a78-b689-5d37a7d343d0.md
@@ -0,0 +1,49 @@
+---
+id: 019eb08c-159f-7a78-b689-5d37a7d343d0
+title: KMS Cache
+component: rgw
+category: feature
+provenance:
+  type: upstream_pr
+  upstream_prs:
+  - https://github.com/ceph/ceph/pull/61256
+  upstream_pr_state: merged-main
+  upstream_commits:
+  - 56253c4ae02ecffcf878f5046839991f3e7b0d0e
+  upstream_trackers:
+  - tracker.ceph.com/issues/68524
+justification:
+  type: customer
+  refs:
+  - cobaltcore-dev/cloud-storage#463
+  - https://github.com/cobaltcore-dev/cloud-storage/issues/125
+risk:
+  blast: data-loss
+  conflict: substantive
+  coverage: partial
+---
+
+## Justification (public)
+
+Adds a secure in-memory cache for SSE-KMS encryption keys retrieved from Barbican (or other KMS backends), eliminating redundant round-trips on repeated access to the same encrypted objects. Uses a new SIEVE-based eviction cache (WebCache) and stores decrypted secrets in the Linux kernel keyring.
+Please refer to the Ceph documentation on how to configure this feature.
+
+## Justification (internal)
+
+Customer roadmap feature. 
+
+## Risk notes
+
+- TTLs mean that a deleted or rotated Barbican key won't be immediately reflected.
+- New async primitives (async::call_once) and the WebCache are significant new library code shipping for the first time.
+- Cache size tuning and system wide keyring quota adjustment needed for production workloads (refer to the docs).
+
+## Behavior change
+
+Repeated SSE-KMS access no longer hits the KMS backend on every
+request; decrypted secrets are cached in-memory and in the Linux
+kernel keyring. KMS-cache is ON out of the box.
+
+## Upgrade notes
+
+Feature flag: rgw_crypt_s3_kms_cache_enabled. Set false to disable caching entirely and revert to per-request KMS lookups.

From 1a65d4a7f638fa160ce96be648d19998b533a4ca Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 24 Mar 2025 12:50:16 -0400
Subject: [PATCH 02/28] common/async: yield_waiter overloads for unique_lock

if async_wait() can race with complete() across threads, the
yield_waiter's handler_state needs to be protected by a mutex. add
an async_wait() overload for unique_lock that behaves like
condition_variable::wait(): the lock is released immediately before
suspending, and reacquired immediately before calling its completion
handler

Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit 604fad5692c61d23a22e0512b2b83c1e3253b2aa)
---
 src/common/async/yield_waiter.h            | 106 ++++++++++++++--
 src/test/common/test_async_yield_waiter.cc | 135 +++++++++++++++++++++
 2 files changed, 230 insertions(+), 11 deletions(-)

diff --git a/src/common/async/yield_waiter.h b/src/common/async/yield_waiter.h
index 9c14d9bafe4b..401e189c7e75 100644
--- a/src/common/async/yield_waiter.h
+++ b/src/common/async/yield_waiter.h
@@ -16,6 +16,7 @@
 #pragma once
 
 #include <exception>
+#include <mutex>
 #include <optional>
 #include <boost/asio/append.hpp>
 #include <boost/asio/associated_cancellation_slot.hpp>
@@ -25,6 +26,29 @@
 
 namespace ceph::async {
 
+namespace detail {
+
+// handler wrapper that reacquires a lock immediately before completion
+template <typename Handler, typename BasicLockable>
+struct lock_handler {
+  Handler handler;
+  BasicLockable* lock = nullptr;
+
+  template <typename ...Args>
+  void operator()(Args&& ...args) {
+    if (lock) {
+      lock->lock();
+    }
+    std::move(handler)(std::forward<Args>(args)...);
+  }
+};
+
+// deduction guide required by windows?
+template <typename Handler, typename BasicLockable>
+lock_handler(Handler&&, BasicLockable*) -> lock_handler<Handler, BasicLockable>;
+
+} // namespace detail
+
 /// Captures a yield_context handler for deferred completion or cancellation.
 template <typename Ret>
 class yield_waiter {
@@ -52,7 +76,25 @@ class yield_waiter {
           if (slot.is_connected()) {
             slot.template emplace<op_cancellation>(this);
           }
-          state.emplace(std::move(h));
+          constexpr std::unique_lock<std::mutex>* lock = nullptr;
+          state.emplace(std::move(h), lock);
+        }, token);
+  }
+
+  /// Suspends the given yield_context until the captured handler is invoked
+  /// via complete() or cancel(). The given lock is released immediately before
+  /// it suspends and reacquired immediately after it resumes.
+  template <typename CompletionToken>
+  auto async_wait(std::unique_lock<std::mutex>& lock, CompletionToken&& token)
+  {
+    return boost::asio::async_initiate<CompletionToken, Signature>(
+        [this, &lock] (handler_type h) {
+          auto slot = get_associated_cancellation_slot(h);
+          if (slot.is_connected()) {
+            slot.template emplace<op_cancellation>(this);
+          }
+          state.emplace(std::move(h), &lock);
+          lock.unlock(); // unlock before suspend
         }, token);
   }
 
@@ -61,8 +103,10 @@ class yield_waiter {
   {
     auto s = std::move(*state);
     state.reset();
-    auto h = boost::asio::append(std::move(s.handler), ec, std::move(value));
-    boost::asio::dispatch(std::move(h));
+    boost::asio::dispatch(
+        boost::asio::append(
+            detail::lock_handler{std::move(s.handler), s.lock},
+            ec, std::move(value)));
   }
 
   /// Destroy the completion handler.
@@ -80,10 +124,10 @@ class yield_waiter {
   struct handler_state {
     handler_type handler;
     work_guard work;
+    std::unique_lock<std::mutex>* lock = nullptr;
 
-    explicit handler_state(handler_type&& h)
-      : handler(std::move(h)),
-        work(make_work_guard(handler))
+    handler_state(handler_type&& h, std::unique_lock<std::mutex>* lock)
+      : handler(std::move(h)), work(make_work_guard(handler)), lock(lock)
     {}
   };
   std::optional<handler_state> state;
@@ -134,7 +178,25 @@ class yield_waiter<void> {
           if (slot.is_connected()) {
             slot.template emplace<op_cancellation>(this);
           }
-          state.emplace(std::move(h));
+          constexpr std::unique_lock<std::mutex>* lock = nullptr;
+          state.emplace(std::move(h), lock);
+        }, token);
+  }
+
+  /// Suspends the given yield_context until the captured handler is invoked
+  /// via complete() or cancel(). The given lock is released immediately before
+  /// it suspends and reacquired immediately after it resumes.
+  template <typename CompletionToken>
+  auto async_wait(std::unique_lock<std::mutex>& lock, CompletionToken&& token)
+  {
+    return boost::asio::async_initiate<CompletionToken, Signature>(
+        [this, &lock] (handler_type h) {
+          auto slot = get_associated_cancellation_slot(h);
+          if (slot.is_connected()) {
+            slot.template emplace<op_cancellation>(this);
+          }
+          state.emplace(std::move(h), &lock);
+          lock.unlock(); // unlock before suspend
         }, token);
   }
 
@@ -143,7 +205,9 @@ class yield_waiter<void> {
   {
     auto s = std::move(*state);
     state.reset();
-    boost::asio::dispatch(boost::asio::append(std::move(s.handler), ec));
+    boost::asio::dispatch(
+        boost::asio::append(
+            detail::lock_handler{std::move(s.handler), s.lock}, ec));
   }
 
   /// Destroy the completion handler.
@@ -161,10 +225,10 @@ class yield_waiter<void> {
   struct handler_state {
     handler_type handler;
     work_guard work;
+    std::unique_lock<std::mutex>* lock = nullptr;
 
-    explicit handler_state(handler_type&& h)
-      : handler(std::move(h)),
-        work(make_work_guard(handler))
+    handler_state(handler_type&& h, std::unique_lock<std::mutex>* lock)
+      : handler(std::move(h)), work(make_work_guard(handler)), lock(lock)
     {}
   };
   std::optional<handler_state> state;
@@ -189,3 +253,23 @@ class yield_waiter<void> {
 };
 
 } // namespace ceph::async
+
+namespace boost::asio {
+
+// forward the handler's associated executor, allocator, cancellation slot, etc
+template <template <typename, typename> class Associator,
+          typename Handler, typename BasicLockable, typename DefaultCandidate>
+struct associator<Associator,
+    ceph::async::detail::lock_handler<Handler, BasicLockable>, DefaultCandidate>
+  : Associator<Handler, DefaultCandidate>
+{
+  static auto get(const ceph::async::detail::lock_handler<Handler, BasicLockable>& h) noexcept {
+    return Associator<Handler, DefaultCandidate>::get(h.handler);
+  }
+  static auto get(const ceph::async::detail::lock_handler<Handler, BasicLockable>& h,
+                  const DefaultCandidate& c) noexcept {
+    return Associator<Handler, DefaultCandidate>::get(h.handler, c);
+  }
+};
+
+} // namespace boost::asio
diff --git a/src/test/common/test_async_yield_waiter.cc b/src/test/common/test_async_yield_waiter.cc
index cd74ffc526e7..6826a4774c33 100644
--- a/src/test/common/test_async_yield_waiter.cc
+++ b/src/test/common/test_async_yield_waiter.cc
@@ -101,6 +101,71 @@ TEST(YieldWaiterVoid, wait_error)
   }
 }
 
+TEST(YieldWaiterVoid, wait_locked_complete)
+{
+  asio::io_context ctx;
+  yield_waiter<void> waiter;
+  std::optional<std::exception_ptr> eptr;
+
+  std::mutex mutex;
+  auto lock = std::unique_lock{mutex};
+
+  asio::spawn(ctx, [&waiter, &lock] (asio::yield_context yield) {
+        ASSERT_TRUE(lock.owns_lock());
+        waiter.async_wait(lock, yield);
+        EXPECT_TRUE(lock.owns_lock());
+      }, capture(eptr));
+
+  ctx.poll();
+  ASSERT_FALSE(ctx.stopped());
+
+  ASSERT_FALSE(lock.owns_lock()); // unlocked while suspended
+  ASSERT_TRUE(waiter);
+  waiter.complete(error_code{});
+  EXPECT_FALSE(waiter);
+
+  ctx.poll();
+  ASSERT_TRUE(ctx.stopped());
+  ASSERT_TRUE(lock.owns_lock());
+}
+
+TEST(YieldWaiterVoid, wait_locked_error)
+{
+  asio::io_context ctx;
+  yield_waiter<void> waiter;
+  std::optional<std::exception_ptr> eptr;
+
+  std::mutex mutex;
+  auto lock = std::unique_lock{mutex};
+
+  asio::spawn(ctx, [&waiter, &lock] (asio::yield_context yield) {
+        ASSERT_TRUE(lock.owns_lock());
+        waiter.async_wait(lock, yield);
+        EXPECT_TRUE(lock.owns_lock());
+      }, capture(eptr));
+
+  ctx.poll();
+  ASSERT_FALSE(ctx.stopped());
+
+  ASSERT_FALSE(lock.owns_lock()); // unlocked while suspended
+  ASSERT_TRUE(waiter);
+  waiter.complete(make_error_code(asio::error::operation_aborted));
+  EXPECT_FALSE(waiter);
+
+  ctx.poll();
+  ASSERT_TRUE(ctx.stopped());
+  ASSERT_TRUE(lock.owns_lock());
+  ASSERT_TRUE(eptr);
+  ASSERT_TRUE(*eptr);
+  try {
+    std::rethrow_exception(*eptr);
+  } catch (const boost::system::system_error& e) {
+    EXPECT_EQ(e.code(), asio::error::operation_aborted);
+  } catch (const std::exception&) {
+    EXPECT_THROW(throw, boost::system::system_error);
+  }
+}
+
 
 TEST(YieldWaiterInt, wait_shutdown)
 {
@@ -241,6 +306,76 @@ TEST(YieldWaiterPtr, wait_error)
   }
 }
 
+TEST(YieldWaiterPtr, wait_locked_complete)
+{
+  asio::io_context ctx;
+  yield_waiter<std::unique_ptr<int>> waiter;
+  std::optional<std::unique_ptr<int>> result;
+
+  std::mutex mutex;
+  auto lock = std::unique_lock{mutex};
+
+  asio::spawn(ctx, [&waiter, &lock, &result] (asio::yield_context yield) {
+        ASSERT_TRUE(lock.owns_lock());
+        result = waiter.async_wait(lock, yield);
+        EXPECT_TRUE(lock.owns_lock());
+      }, rethrow);
+
+  ctx.poll();
+  ASSERT_FALSE(ctx.stopped());
+  ASSERT_FALSE(lock.owns_lock()); // unlocked while suspended
+
+  ASSERT_TRUE(waiter);
+  waiter.complete(error_code{}, std::make_unique<int>(42));
+  EXPECT_FALSE(waiter);
+
+  ctx.poll();
+  EXPECT_TRUE(ctx.stopped());
+  ASSERT_TRUE(lock.owns_lock());
+  ASSERT_TRUE(result);
+  ASSERT_TRUE(*result);
+  EXPECT_EQ(42, **result);
+}
+
+TEST(YieldWaiterPtr, wait_locked_error)
+{
+  asio::io_context ctx;
+  yield_waiter<std::unique_ptr<int>> waiter;
+  std::optional<std::unique_ptr<int>> result;
+  std::optional<std::exception_ptr> eptr;
+
+  std::mutex mutex;
+  auto lock = std::unique_lock{mutex};
+
+  asio::spawn(ctx, [&waiter, &lock, &result] (asio::yield_context yield) {
+        ASSERT_TRUE(lock.owns_lock());
+        result = waiter.async_wait(lock, yield);
+        EXPECT_TRUE(lock.owns_lock());
+      }, capture(eptr));
+
+  ctx.poll();
+  ASSERT_FALSE(ctx.stopped());
+  EXPECT_FALSE(lock.owns_lock()); // unlocked while suspended
+
+  ASSERT_TRUE(waiter);
+  waiter.complete(make_error_code(std::errc::no_such_file_or_directory), nullptr);
+  EXPECT_FALSE(waiter);
+
+  ctx.poll();
+  ASSERT_TRUE(ctx.stopped());
+  ASSERT_TRUE(lock.owns_lock());
+  EXPECT_FALSE(result);
+  ASSERT_TRUE(eptr);
+  ASSERT_TRUE(*eptr);
+  try {
+    std::rethrow_exception(*eptr);
+  } catch (const boost::system::system_error& e) {
+    EXPECT_EQ(e.code(), std::errc::no_such_file_or_directory);
+  } catch (const std::exception&) {
+    EXPECT_THROW(throw, boost::system::system_error);
+  }
+}
+
 void invoke_callback(int expected_reply, std::function<void(int)> cb) {
   auto t = std::thread([cb, expected_reply] {
       cb(expected_reply);

From dc67eeb79909db49225941fa786a84ba3f257b3d Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Tue, 25 Mar 2025 18:06:36 -0400
Subject: [PATCH 03/28] common/async: yield_waiter can return the associated
 executor

also adds an empty() function so it's easier to specify its precondition

Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit 165f2ec18188c8a6e81a2a1960a6d27e57a77dd3)
---
 src/common/async/yield_waiter.h | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/common/async/yield_waiter.h b/src/common/async/yield_waiter.h
index 401e189c7e75..667dbbbb2521 100644
--- a/src/common/async/yield_waiter.h
+++ b/src/common/async/yield_waiter.h
@@ -20,6 +20,7 @@
 #include <optional>
 #include <boost/asio/append.hpp>
 #include <boost/asio/associated_cancellation_slot.hpp>
+#include <boost/asio/associated_executor.hpp>
 #include <boost/asio/async_result.hpp>
 #include <boost/asio/dispatch.hpp>
 #include <boost/asio/spawn.hpp>
@@ -65,6 +66,18 @@ class yield_waiter {
   /// Returns true if there's a handler awaiting completion.
   operator bool() const { return state.has_value(); }
 
+  /// Returns false if there's a handler awaiting completion.
+  bool empty() const { return !state.has_value(); }
+
+  /// Returns the captured completion handler's executor.
+  /// Precondition: !empty()
+  using executor_type = boost::asio::any_io_executor;
+  executor_type get_executor() const
+  {
+    return boost::asio::get_associated_executor(
+        state->handler, state->work.get_executor());
+  }
+
   /// Suspends the given yield_context until the captured handler is invoked
   /// via complete() or cancel().
   template <typename CompletionToken>
@@ -167,6 +180,18 @@ class yield_waiter<void> {
   /// Returns true if there's a handler awaiting completion.
   operator bool() const { return state.has_value(); }
 
+  /// Returns false if there's a handler awaiting completion.
+  bool empty() const { return !state.has_value(); }
+
+  /// Returns the captured completion handler's executor.
+  /// Precondition: !empty()
+  using executor_type = boost::asio::any_io_executor;
+  executor_type get_executor() const
+  {
+    return boost::asio::get_associated_executor(
+        state->handler, state->work.get_executor());
+  }
+
   /// Suspends the given yield_context until the captured handler is invoked
   /// via complete() or cancel().
   template <typename CompletionToken>

From 417b8a76368640bc72d37faff5697ca291b445d0 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 24 Mar 2025 12:51:15 -0400
Subject: [PATCH 04/28] common/async: add call_once() algorithm for
 optional_yield

modeled after std::call_once() to guarantee that racing callers wait for
the initial caller to finish. the main differences here are

* support for coroutine callers to suspend instead of blocking while
  waiting for the initial caller, and
* the wrapped function must return a value, which is cached and returned
  to all callers

Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit 3cd3c113486121c222c9de15cf5848347db2c18a)
---
 src/common/async/call_once.h            | 113 +++++++++
 src/common/async/detail/call_once.h     | 129 +++++++++++
 src/test/common/CMakeLists.txt          |   4 +
 src/test/common/test_async_call_once.cc | 291 ++++++++++++++++++++++++
 4 files changed, 537 insertions(+)
 create mode 100644 src/common/async/call_once.h
 create mode 100644 src/common/async/detail/call_once.h
 create mode 100644 src/test/common/test_async_call_once.cc

diff --git a/src/common/async/call_once.h b/src/common/async/call_once.h
new file mode 100644
index 000000000000..8c48a2c3c6a8
--- /dev/null
+++ b/src/common/async/call_once.h
@@ -0,0 +1,113 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <concepts>
+#include <exception>
+#include <mutex>
+#include <variant>
+
+#include "include/function2.hpp"
+
+#include "yield_context.h"
+#include "detail/call_once.h"
+
+namespace ceph::async {
+
+/// Stores the result of call_once(), if any.
+///
+/// The cached Result type must be semiregular (copyable and default-
+/// constructible).
+template <std::semiregular Result>
+class once_result {
+ public:
+  /// Call f() once and cache its return value.
+  ///
+  /// If a call has already completed, return the cached result. If a call
+  /// is already in progress, wait for its completion and return the result.
+  /// If a yield context is provided in y, its coroutine will be suspended
+  /// while waiting. f() may also suspend/resume the same coroutine.
+  ///
+  /// f() must return a Result when called with no arguments. If it throws,
+  /// that exception is rethrown by all calls.
+  ///
+  /// This function is thread-safe.
+  template <typename Callable>
+  friend Result call_once(once_result& self, optional_yield y, Callable&& f)
+      requires std::convertible_to<std::invoke_result_t<Callable>, Result>
+  {
+    auto lock = std::unique_lock{self.mutex};
+    return std::visit(
+        fu2::overload(
+            [&] (const Init&) { return self.do_init(lock, f); },
+            [&] (Wait& w) { return self.do_wait(lock, y, w); },
+            [] (const Complete& c) { return do_complete(c); }),
+        self.state);
+  }
+
+ private:
+  std::mutex mutex;
+
+  // state machine: [Init] -> [Wait] -> [Complete]
+  using Init = std::monostate;
+  using Wait = detail::call_once::WaitState;
+  struct Complete {
+    Result result;
+    std::exception_ptr eptr;
+  };
+  std::variant<Init, Wait, Complete> state;
+
+  Result do_init(std::unique_lock<std::mutex>& lock, auto&& f)
+  {
+    // transition to Wait state
+    auto& wait = state.template emplace<Wait>();
+    lock.unlock();
+
+    Complete complete;
+    try {
+      complete.result = f();
+    } catch (const std::exception&) {
+      complete.eptr = std::current_exception();
+    }
+
+    lock.lock();
+    // wake any waiters
+    wait.wake_all();
+
+    // transition to Complete state and return the result
+    return do_complete(state.template emplace<Complete>(std::move(complete)));
+  }
+
+  Result do_wait(std::unique_lock<std::mutex>& lock,
+                 optional_yield y, Wait& wait)
+  {
+    // wait for the response to an outstanding request
+    wait.wait(lock, y);
+
+    // state must be Complete after wakeup
+    return do_complete(std::get<Complete>(state));
+  }
+
+  static Result do_complete(const Complete& complete)
+  {
+    if (complete.eptr) { // rethrow cached exception
+      std::rethrow_exception(complete.eptr);
+    }
+    return complete.result; // return cached result
+  }
+};
+
+} // namespace ceph::async
diff --git a/src/common/async/detail/call_once.h b/src/common/async/detail/call_once.h
new file mode 100644
index 000000000000..c7d676457934
--- /dev/null
+++ b/src/common/async/detail/call_once.h
@@ -0,0 +1,129 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <condition_variable>
+#include <mutex>
+
+#include <boost/intrusive/list.hpp>
+
+#include "common/async/service.h"
+#include "common/async/yield_context.h"
+#include "common/async/yield_waiter.h"
+
+namespace ceph::async::detail::call_once {
+
+class AsyncWaiter : public boost::intrusive::list_base_hook<> {
+  yield_waiter<void> impl;
+ public:
+  void wait(std::unique_lock<std::mutex>& lock,
+            boost::asio::yield_context yield)
+  {
+    impl.async_wait(lock, yield);
+  }
+  void wake()
+  {
+    // async_wait() may complete inline via boost::asio::dispatch(), but would
+    // immediately try to reacquire the lock we're already holding. defer the
+    // completion until after we've returned and dropped our lock
+    boost::asio::defer(impl.get_executor(), [this] {
+        impl.complete(boost::system::error_code{});
+      });
+  }
+  void destroy()
+  {
+    impl.shutdown();
+  }
+};
+
+class SyncWaiter : public boost::intrusive::list_base_hook<> {
+  std::condition_variable cond;
+  bool done = false;
+ public:
+  void wait(std::unique_lock<std::mutex>& lock)
+  {
+    cond.wait(lock, [this] { return done; });
+  }
+  void wake()
+  {
+    done = true;
+    cond.notify_one();
+  }
+};
+
+struct WaitState : service_list_base_hook {
+  service<WaitState>* svc = nullptr;
+  boost::intrusive::list<SyncWaiter> sync_waiters;
+  boost::intrusive::list<AsyncWaiter> async_waiters;
+
+  ~WaitState()
+  {
+    if (svc) {
+      svc->remove(*this);
+    }
+  }
+
+  void wait(std::unique_lock<std::mutex>& lock, optional_yield y)
+  {
+    if (y) {
+      auto yield = y.get_yield_context();
+      if (!svc) {
+        // on first async wait, register for service_shutdown() notifications
+        svc = &boost::asio::use_service<service<WaitState>>(
+            boost::asio::query(yield.get_executor(),
+                               boost::asio::execution::context));
+        svc->add(*this);
+      }
+
+      AsyncWaiter waiter;
+      async_waiters.push_back(waiter);
+
+      waiter.wait(lock, yield);
+    } else {
+      SyncWaiter waiter;
+      sync_waiters.push_back(waiter);
+
+      waiter.wait(lock);
+    }
+  }
+
+  void wake_all()
+  {
+    while (!sync_waiters.empty()) {
+      auto& waiter = sync_waiters.front();
+      sync_waiters.pop_front();
+      waiter.wake();
+    }
+    while (!async_waiters.empty()) {
+      auto& waiter = async_waiters.front();
+      async_waiters.pop_front();
+      waiter.wake();
+    }
+  }
+
+  // on shutdown, destroy async waiters that didn't complete. this breaks any
+  // ownership cycles between once_result and the completion handlers
+  void service_shutdown()
+  {
+    while (!async_waiters.empty()) {
+      auto& waiter = async_waiters.front();
+      async_waiters.pop_front();
+      waiter.destroy();
+    }
+  }
+};
+
+} // namespace ceph::async::detail::call_once
diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index 8f046eba3afb..0d8d40b98d59 100644
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -359,6 +359,10 @@ add_executable(unittest_hobject test_hobject.cc
 target_link_libraries(unittest_hobject global ceph-common)
 add_ceph_unittest(unittest_hobject)
 
+add_executable(unittest_async_call_once test_async_call_once.cc)
+add_ceph_unittest(unittest_async_call_once)
+target_link_libraries(unittest_async_call_once ceph-common Boost::context)
+
 add_executable(unittest_async_completion test_async_completion.cc)
 add_ceph_unittest(unittest_async_completion)
 target_link_libraries(unittest_async_completion ceph-common Boost::system)
diff --git a/src/test/common/test_async_call_once.cc b/src/test/common/test_async_call_once.cc
new file mode 100644
index 000000000000..c0fc76c52b8d
--- /dev/null
+++ b/src/test/common/test_async_call_once.cc
@@ -0,0 +1,291 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "common/async/call_once.h"
+
+#include <exception>
+#include <future>
+#include <latch>
+#include <optional>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <gtest/gtest.h>
+#include "common/async/yield_waiter.h"
+
+void rethrow(std::exception_ptr eptr)
+{
+  if (eptr) std::rethrow_exception(eptr);
+}
+
+auto capture(std::optional<std::exception_ptr>& out) {
+  return [&out] (std::exception_ptr eptr) {
+    out = std::move(eptr);
+  };
+}
+
+template <typename T>
+auto capture(std::optional<T>& out) {
+  return [&out] (std::exception_ptr eptr, T value) {
+    rethrow(eptr);
+    out = std::move(value);
+  };
+}
+
+namespace ceph::async {
+
+TEST(CallOnceAsync, call_wait_shutdown)
+{
+  boost::asio::io_context context;
+  // test that memory doesn't leak when both coroutines hold a shared_ptr to
+  // the once_result they're waiting on
+  auto once = std::make_shared<once_result<int>>();
+  yield_waiter<int> waiter;
+
+  boost::asio::spawn(context, [once, &waiter] (boost::asio::yield_context yield) {
+        call_once(*once, yield, [&] { return waiter.async_wait(yield); });
+      }, rethrow);
+  boost::asio::spawn(context, [once] (boost::asio::yield_context yield) {
+        call_once(*once, yield, [] { return 0; });
+      }, rethrow);
+
+  context.poll();
+  EXPECT_FALSE(context.stopped()); // blocked on waiter, never completes
+}
+
+TEST(CallOnceAsync, call_immediate_result)
+{
+  boost::asio::io_context context;
+  once_result<int> once;
+
+  std::optional<int> result;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+        return call_once(once, yield, [] { return 0; });
+      }, capture(result));
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+  ASSERT_TRUE(result);
+  EXPECT_EQ(*result, 0);
+}
+
+TEST(CallOnceAsync, call_immediate_exception)
+{
+  boost::asio::io_context context;
+  once_result<int> once;
+
+  std::optional<std::exception_ptr> eptr;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+        call_once(once, yield, [] { throw std::bad_alloc(); return 0; });
+      }, capture(eptr));
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+  ASSERT_TRUE(eptr);
+  EXPECT_THROW(std::rethrow_exception(*eptr), std::bad_alloc);
+}
+
+TEST(CallOnceAsync, call_wait_result)
+{
+  boost::asio::io_context context;
+  once_result<int> once;
+  yield_waiter<int> waiter;
+
+  std::optional<int> call_result;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+        return call_once(once, yield, [&] { return waiter.async_wait(yield); });
+      }, capture(call_result));
+
+  std::optional<int> wait_result;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+        return call_once(once, yield, [] { return 0; });
+      }, capture(wait_result));
+
+  context.poll();
+  EXPECT_FALSE(context.stopped());
+  ASSERT_FALSE(call_result);
+  ASSERT_FALSE(wait_result);
+
+  waiter.complete(boost::system::error_code{}, 42);
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+  ASSERT_TRUE(call_result);
+  EXPECT_EQ(*call_result, 42);
+  ASSERT_TRUE(wait_result);
+  EXPECT_EQ(*wait_result, 42);
+
+  std::optional<int> cached_result;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+        return call_once(once, yield, [] { return 0; });
+      }, capture(cached_result));
+
+  context.restart();
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+  ASSERT_TRUE(cached_result);
+  EXPECT_EQ(*cached_result, 42);
+}
+
+TEST(CallOnceAsync, call_wait_exception)
+{
+  boost::asio::io_context context;
+  once_result<int> once;
+  yield_waiter<void> waiter;
+
+  std::optional<std::exception_ptr> call_eptr;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+        call_once(once, yield, [&] {
+            waiter.async_wait(yield);
+            throw std::bad_alloc();
+            return 0;
+          });
+      }, capture(call_eptr));
+
+  std::optional<std::exception_ptr> wait_eptr;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+        call_once(once, yield, [] { return 0; });
+      }, capture(wait_eptr));
+
+  context.poll();
+  EXPECT_FALSE(context.stopped());
+  ASSERT_FALSE(call_eptr);
+  ASSERT_FALSE(wait_eptr);
+
+  waiter.complete(boost::system::error_code{});
+
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+  ASSERT_TRUE(call_eptr);
+  EXPECT_THROW(std::rethrow_exception(*call_eptr), std::bad_alloc);
+  ASSERT_TRUE(wait_eptr);
+  EXPECT_THROW(std::rethrow_exception(*wait_eptr), std::bad_alloc);
+
+  std::optional<std::exception_ptr> cached_eptr;
+  boost::asio::spawn(context, [&] (boost::asio::yield_context yield) {
+        call_once(once, yield, [] { return 0; });
+      }, capture(cached_eptr));
+
+  context.restart();
+  context.poll();
+  EXPECT_TRUE(context.stopped());
+  ASSERT_TRUE(cached_eptr);
+  EXPECT_THROW(std::rethrow_exception(*cached_eptr), std::bad_alloc);
+}
+
+TEST(CallOnceSync, call_immediate_result)
+{
+  once_result<int> once;
+
+  EXPECT_EQ(call_once(once, null_yield, [] { return 42; }), 42);
+}
+
+TEST(CallOnceSync, call_immediate_result_lvalue)
+{
+  once_result<int> once;
+
+  // test that call_once() properly forwards the Callable so
+  // it can be passed by lvalue-ref without copy/move
+  struct noncopyable_fn {
+   public:
+    noncopyable_fn() = default;
+    noncopyable_fn(const noncopyable_fn&) = delete;
+    noncopyable_fn& operator=(const noncopyable_fn&) = delete;
+
+    int operator()() { return 42; }
+  };
+  noncopyable_fn fn;
+
+  EXPECT_EQ(call_once(once, null_yield, fn), 42);
+}
+
+TEST(CallOnceSync, call_immediate_exception)
+{
+  once_result<int> once;
+
+  EXPECT_THROW(call_once(once, null_yield, [] { throw std::bad_alloc(); return 0; }), std::bad_alloc);
+}
+
+TEST(CallOnceSync, call_wait_result)
+{
+  once_result<int> once;
+
+  std::future<int> call_future;
+  std::future<int> wait_future;
+  std::latch call_latch{1};
+  std::latch wait_latch{2};
+
+  call_future = std::async(std::launch::async, [&] {
+      return call_once(once, null_yield, [&] {
+          wait_future = std::async(std::launch::async, [&] {
+              wait_latch.count_down();
+              return call_once(once, null_yield, [] { return 0; });
+            });
+          wait_latch.count_down();
+          call_latch.wait();
+          return 42;
+        });
+    });
+
+  wait_latch.wait();
+
+  using namespace std::chrono_literals;
+  EXPECT_NE(wait_future.wait_for(0s), std::future_status::ready);
+
+  call_latch.count_down(); // let call return
+
+  EXPECT_EQ(call_future.get(), 42);
+  EXPECT_EQ(wait_future.get(), 42);
+
+  // return cached result
+  EXPECT_EQ(call_once(once, null_yield, [] { return 0; }), 42);
+}
+
+TEST(CallOnceSync, call_wait_exception)
+{
+  once_result<int> once;
+
+  std::future<int> call_future;
+  std::future<int> wait_future;
+  std::latch call_latch{1};
+  std::latch wait_latch{2};
+
+  call_future = std::async(std::launch::async, [&] {
+      return call_once(once, null_yield, [&] {
+          wait_future = std::async(std::launch::async, [&] {
+              wait_latch.count_down();
+              return call_once(once, null_yield, [] { return 0; });
+            });
+          wait_latch.count_down();
+          call_latch.wait();
+          throw std::bad_alloc();
+          return 0;
+        });
+    });
+
+  wait_latch.wait();
+
+  using namespace std::chrono_literals;
+  EXPECT_NE(wait_future.wait_for(0s), std::future_status::ready);
+
+  call_latch.count_down(); // let call return
+
+  EXPECT_THROW(call_future.get(), std::bad_alloc);
+  EXPECT_THROW(wait_future.get(), std::bad_alloc);
+
+  // rethrow cached exception
+  EXPECT_THROW(call_once(once, null_yield, [] { return 0; }), std::bad_alloc);
+}
+
+} // namespace ceph::async

From 0a79029ba223c23bbcbf809fb6e64998779970fe Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 11 Feb 2025 13:48:22 +0100
Subject: [PATCH 05/28] common: Add WebCache

A cache data structure for values that need to be retrieved form
outside systems (e.g Key Management Systems).

Features:
- Thread safe, optimized for concurrent lookups and cache hits
- Entry TTL expiration
- Cache replacement strategy tuned to "web" workloads (SIEVE)
- Performance Counters on hit, miss, expire, size, capacity, clears

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 9618827200a5e7d686bb1e65dcf3f2bb2b1ee4df)
---
 src/common/web_cache.h            | 559 ++++++++++++++++++++++++++
 src/test/common/CMakeLists.txt    |   7 +
 src/test/common/test_web_cache.cc | 644 ++++++++++++++++++++++++++++++
 3 files changed, 1210 insertions(+)
 create mode 100644 src/common/web_cache.h
 create mode 100644 src/test/common/test_web_cache.cc

diff --git a/src/common/web_cache.h b/src/common/web_cache.h
new file mode 100644
index 000000000000..97b362d003b1
--- /dev/null
+++ b/src/common/web_cache.h
@@ -0,0 +1,559 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2025 Clyso GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <fmt/format.h>
+#include <fmt/ostream.h>
+
+#include <atomic>
+#include <boost/intrusive/list.hpp>
+#include <chrono>
+#include <iterator>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <ostream>
+#include <shared_mutex>
+#include <unordered_map>
+#include <utility>
+
+#include "common/ceph_context.h"
+#include "common/ceph_time.h"
+#include "include/ceph_assert.h"
+#include "include/common_fwd.h"
+#include "common/perf_counters_collection.h"
+
+// Web Cache
+// A cache for data living on other systems like Key Management Systems
+// Goals/Features
+// (1) Thread safe, bias towards handling highly concurrent lookups
+// (2) Expire entries by ttl
+// (3) Cache replacement tuned to "web" workloads
+//
+// Algorithm
+// The implementation is based on SIEVE [0] with additional TTL
+// expiration support
+//
+// Data Structures
+// - key lookup table: key -> Node{value ptr, metadata) (owns nodes)
+// - SIEVE FIFO + hand intrusive list on lookup's nodes + pointer
+//
+// Cache Stampedes
+//
+// Occurs when numerous threads simultaneously request the same data
+// that is not yet in the cache. This can lead to many redundant,
+// costly requests, that may overwhelm external systems. Although
+// WebCache does not have built-in stampede mitigation, the lookup_or
+// function is designed to be used with additional logic, such as
+// ceph::async::call_once / std::call_once.
+//
+// [0] Zhang, Yazhuo, et al. "{SIEVE} is Simpler than {LRU}: an Efficient
+// {Turn-Key} Eviction Algorithm for Web Caches." 21st USENIX
+// Symposium on Networked Systems Design and Implementation (NSDI 24).
+// 2024.
+
+class WebCacheTest;
+
+namespace webcache {
+
+enum class Metric {
+  metrics_start = 84000,
+  hit,
+  miss,
+  expired,
+  size,
+  capacity,
+  clear,
+  metrics_stop
+};
+
+template <typename Key, typename Value>
+class WebCache {
+ public:
+  // TODO(irq0) let the user choose the value pointer type
+  using ValuePtr = std::shared_ptr<Value>;
+
+ protected:
+  struct Node : public boost::intrusive::list_base_hook<> {
+    std::atomic_bool visited;
+    ceph::real_time expires_at;
+    Key const* key;
+    ValuePtr value;
+
+    explicit Node(ValuePtr value, ceph::timespan ttl)
+        : visited(false),
+          expires_at(ceph::real_clock::now() + ttl),
+          key(nullptr),
+          value(std::move(value)) {};
+    // for testing
+    Node(ValuePtr value, ceph::real_time expires_at)
+        : visited(false),
+          expires_at(expires_at),
+          key(nullptr),
+          value(std::move(value)) {};
+
+    Node(Node&&) = default;
+    Node& operator=(Node&&) = default;
+    Node(const Node&) = delete;
+    Node& operator=(const Node&) = delete;
+    ~Node() = default;
+
+    friend std::ostream& operator<<(
+        std::ostream& os, const WebCache<Key, Value>::Node& node) {
+      fmt::print(
+          os, "$n[{}->{} v:{} expires:{}]", fmt::ptr(node.key),
+          fmt::ptr(node.value.get()), node.visited.load(), node.expires_at);
+      return os;
+    }
+  };
+  using SieveQueue = boost::intrusive::list<Node>;
+
+ private:
+  CephContext* _cct;
+  std::string _name;
+  PerfCounters* _perf;
+  size_t _capacity;
+  ceph::timespan _ttl;
+  std::unordered_map<Key, Node> _lookup;
+  SieveQueue _sieve_queue;
+  Node* _sieve_hand;
+  mutable std::shared_mutex _cache_mutex;
+
+ protected:
+  // sieve_evict removes the next node using the SIEVE algorithm from
+  // _sieve_queue and returns a pointer to the evicted node
+  Node* sieve_evict();
+
+  // sieve_expire_erase_unmutexed removes all expired nodes from the
+  // sieve_queue in place. It writes the expired nodes to out_expired.
+  // Updates the sieve hand
+  static void sieve_expire_erase_unmutexed(
+      SieveQueue& sieve_queue, Node* sieve_hand,
+      ceph::real_time eviction_cutoff, SieveQueue& out_expired);
+
+  using SieveRemoveRet = std::pair<typename SieveQueue::iterator, Node*>;
+  // sieve_remove_unmutexed removes a node from sieve_queue and
+  // returns an iterator to next (or end()) and an updated sieve hand
+  static SieveRemoveRet sieve_remove_unmutexed(
+      SieveQueue& sieve_queue, Node* sieve_hand, const Node& node);
+
+  static PerfCounters* initialize_perf_counters(
+      CephContext* cct, const std::string& name);
+
+  std::optional<ValuePtr> lookup_unmutexed(const Key& key);
+  Node& insert_or_existing_unmutexed(const Key& key, ValuePtr value);
+
+  void perf_tinc(Metric metric, ceph::timespan elapsed) {
+    if (_perf != nullptr) {
+      _perf->tinc(static_cast<int>(metric), elapsed);
+    }
+  }
+  void perf_inc(Metric metric, uint64_t inc = 1) {
+    if (_perf != nullptr) {
+      _perf->inc(static_cast<int>(metric), inc);
+    }
+  }
+  void perf_set(Metric metric, uint64_t set = 1) {
+    if (_perf != nullptr) {
+      _perf->set(static_cast<int>(metric), set);
+    }
+  }
+
+ public:
+  // For testing, custom use
+  explicit WebCache(size_t capacity, ceph::timespan ttl);
+
+  // For system use, activates perf counters
+  WebCache(
+      CephContext* cct, const std::string& name, size_t capacity,
+      ceph::timespan ttl = ceph::timespan::zero());
+
+  // If enabled, remove perf counters (Requires perf counter subsystem
+  // running)
+  ~WebCache();
+
+  // lookup returns the stored value for a given key or not
+  std::optional<ValuePtr> lookup(const Key& key);
+
+  // add caches a key/value pair. If key already exists it does
+  // nothing. Return the stored timestamp of the k/v mapping
+  ceph::real_time add(const Key& key, ValuePtr val);
+
+  // lookup_or returns a value for key. If none is cached yet, insert
+  // new_val and return that. A common use is with
+  // ceph::async::call_once to add cache stampede mitigation
+  ValuePtr lookup_or(const Key& key, ValuePtr new_val);
+
+  // update_ttl_if updates an entry's TTL if its value matches val.
+  // Return true if we updated an entry. False otherwise.
+  bool update_ttl_if(
+      const Key& key, const ValuePtr& val_ptr, ceph::timespan new_ttl);
+
+  // remove_if removes an entry if it finds an entry where its values
+  // matches expected_val. return true if we removed an entry, false
+  // otherwise
+  bool remove_if(const Key& key, const ValuePtr& expected_val);
+
+  size_t size() const;
+  size_t clear();
+
+  // expire_erase erases all expired cache entries
+  size_t expire_erase();
+
+  const std::string& name() { return _name; }
+
+  PerfCounters* perf() { return _perf; }
+
+  friend std::ostream& operator<<(
+      std::ostream& os, const WebCache<Key, Value>& cache) {
+    std::shared_lock<std::shared_mutex> lock(cache._cache_mutex);
+    const auto now = ceph::real_clock::now();
+    os << "$" << cache._name << "[";
+
+    for (const auto& node : cache._sieve_queue) {
+      const auto ttl = node.expires_at - now;
+      fmt::print(
+          os, "\"{}\"({}){}{}", *(node.key),
+          std::chrono::duration_cast<std::chrono::seconds>(ttl),
+          (&node == cache._sieve_hand) ? "👉" : "", node.visited ? "▮" : "▯");
+      if (&node != &cache._sieve_queue.back()) {
+        os << ", ";
+      }
+    }
+    os << "]";
+    return os;
+  }
+
+  friend std::ostream& operator<<(std::ostream& os, const SieveQueue& nodes) {
+    for (const auto& node : nodes) {
+      os << node;
+      if (&node != &nodes.back()) {
+        os << ", ";
+      }
+    }
+    return os;
+  }
+
+  friend class WebCacheTest;
+  friend class WebCacheConcurrencyTest;
+  friend class WebCacheRandomizedTest;
+  friend class WebCacheTest_SieveExample_Test;
+  friend class WebCacheTest_ExpireEraseOne_Test;
+  friend class WebCacheTest_ExpireEraseAll_Test;
+  friend class WebCacheTest_ExpireEraseEmpty_Test;
+  friend class WebCacheTest_ExpireEraseUpdatedTTLs_Test;
+  friend class WebCacheTest_SieveRemoveHand_Test;
+};
+
+template <typename Key, typename Value>
+typename WebCache<Key, Value>::Node* WebCache<Key, Value>::sieve_evict() {
+  if (_sieve_queue.empty()) {
+    return nullptr;
+  }
+  if (_sieve_hand == nullptr) {
+    _sieve_hand = &_sieve_queue.back();
+  }
+  Node* result = nullptr;
+  const size_t queue_size_before = _sieve_queue.size();
+
+  auto hand = _sieve_queue.iterator_to(*_sieve_hand);
+  const auto rev_it = typename SieveQueue::reverse_iterator(std::next(hand));
+  for (auto it = rev_it; it != _sieve_queue.rend(); ++it) {
+    Node& node = (*it);
+    if (node.visited) {
+      node.visited = false;
+      --hand;
+    } else {
+      hand = std::prev(_sieve_queue.erase(std::next(it).base()));
+      result = &node;
+      break;
+    }
+  }
+  // every node was visited. we need still need to evict the tail
+  if (result == nullptr) {
+    result = &_sieve_queue.back();
+    _sieve_queue.pop_back();
+  }
+
+  ceph_assertf(
+      queue_size_before == 0 || (queue_size_before - _sieve_queue.size()) == 1,
+      "%d -> %d capacity:%d", queue_size_before, _sieve_queue.size(),
+      _capacity);
+  _sieve_hand = (hand == _sieve_queue.begin()) ? &_sieve_queue.back() : &*hand;
+  return result;
+}
+
+template <typename Key, typename Value>
+WebCache<Key, Value>::WebCache(size_t capacity, ceph::timespan ttl)
+    : _cct(nullptr),
+      _perf(nullptr),
+      _capacity(capacity),
+      _ttl(ttl),
+      _lookup(),
+      _sieve_queue(),
+      _sieve_hand(nullptr) {}
+
+template <typename Key, typename Value>
+WebCache<Key, Value>::WebCache(
+    CephContext* cct, const std::string& name, size_t capacity,
+    ceph::timespan ttl)
+    : _cct(cct),
+      _name(name),
+      _perf(initialize_perf_counters(cct, name)),
+      _capacity(capacity),
+      _ttl(ttl),
+      _lookup(),
+      _sieve_queue(),
+      _sieve_hand(nullptr) {
+  ceph_assert(cct != nullptr);
+  perf_set(Metric::capacity, capacity);
+}
+
+template <typename Key, typename Value>
+WebCache<Key, Value>::~WebCache() {
+  if (_cct != nullptr && _perf != nullptr) {
+    _cct->get_perfcounters_collection()->remove(_perf);
+  }
+}
+
+template <typename Key, typename Value>
+PerfCounters* WebCache<Key, Value>::initialize_perf_counters(
+    CephContext* cct, const std::string& name) {
+  PerfCountersBuilder pcb(
+      cct, name, static_cast<int>(Metric::metrics_start),
+      static_cast<int>(Metric::metrics_stop));
+  pcb.set_prio_default(PerfCountersBuilder::PRIO_USEFUL);
+  pcb.add_u64_counter(static_cast<int>(Metric::hit), "hit", "Cache hits");
+  pcb.add_u64_counter(static_cast<int>(Metric::miss), "miss", "Cache misses");
+  pcb.add_u64_counter(
+      static_cast<int>(Metric::expired), "expired", "Expired cache entries");
+  pcb.add_u64(
+      static_cast<int>(Metric::size), "size", "Total number of cache entries");
+  pcb.add_u64(
+      static_cast<int>(Metric::capacity), "capacity", "Maximum cache capacity");
+  pcb.add_u64_counter(
+      static_cast<int>(Metric::clear), "clear", "Total number of cache clears");
+
+  auto* perf = pcb.create_perf_counters();
+  cct->get_perfcounters_collection()->add(perf);
+  return perf;
+}
+
+template <typename Key, typename Value>
+size_t WebCache<Key, Value>::size() const {
+  std::shared_lock<std::shared_mutex> lock(_cache_mutex);
+  ceph_assert(_lookup.size() == _sieve_queue.size());
+  return _lookup.size();
+}
+
+template <typename Key, typename Value>
+size_t WebCache<Key, Value>::clear() {
+  std::lock_guard<std::shared_mutex> lock(_cache_mutex);
+  perf_inc(Metric::clear);
+  const size_t size_before = _sieve_queue.size();
+  _sieve_queue.clear();
+  _sieve_hand = nullptr;
+  _lookup.clear();
+  perf_set(Metric::size, 0);
+  perf_set(Metric::hit, 0);
+  perf_set(Metric::miss, 0);
+  perf_set(Metric::expired, 0);
+  return size_before;
+}
+
+template <typename Key, typename Value>
+ceph::real_time WebCache<Key, Value>::add(const Key& key, ValuePtr value) {
+  // cache hit - fast under read lock
+  {
+    std::shared_lock<std::shared_mutex> lock(_cache_mutex);
+    if (auto search = _lookup.find(key); search != _lookup.end()) {
+      perf_inc(Metric::hit);
+      search->second.visited = true;
+      return search->second.expires_at;
+    }
+  }
+  // miss, take unique lock
+  {
+    std::lock_guard<std::shared_mutex> lock(_cache_mutex);
+    return insert_or_existing_unmutexed(key, value).expires_at;
+  }
+}
+
+template <typename Key, typename Value>
+typename WebCache<Key, Value>::ValuePtr WebCache<Key, Value>::lookup_or(
+    const Key& key, ValuePtr new_val) {
+  {
+    std::shared_lock<std::shared_mutex> cache_lock(_cache_mutex);
+    auto maybe_value = lookup_unmutexed(key);
+    if (maybe_value.has_value()) {
+      perf_inc(Metric::hit);
+      return maybe_value.value();
+    }
+  }
+
+  // miss, take unique lock
+  {
+    std::lock_guard<std::shared_mutex> lock(_cache_mutex);
+    return insert_or_existing_unmutexed(key, new_val).value;
+  }
+}
+
+template <typename Key, typename Value>
+bool WebCache<Key, Value>::update_ttl_if(
+    const Key& key, const ValuePtr& val, ceph::timespan new_ttl) {
+  std::lock_guard<std::shared_mutex> lock(_cache_mutex);
+  if (auto search = _lookup.find(key);
+      (search != _lookup.end() && search->second.value == val)) {
+    search->second.expires_at = ceph::real_clock::now() + new_ttl;
+    return true;
+  }
+  return false;
+}
+
+template <typename Key, typename Value>
+bool WebCache<Key, Value>::remove_if(
+    const Key& key, const ValuePtr& expected_val) {
+  std::lock_guard<std::shared_mutex> lock(_cache_mutex);
+  if (auto search = _lookup.find(key);
+      (search != _lookup.end() && search->second.value == expected_val)) {
+    auto [_, hand_moved] =
+        sieve_remove_unmutexed(_sieve_queue, _sieve_hand, search->second);
+    _lookup.erase(search);
+    _sieve_hand = hand_moved;
+    return true;
+  }
+  return false;
+}
+
+template <typename Key, typename Value>
+std::optional<typename WebCache<Key, Value>::ValuePtr>
+WebCache<Key, Value>::lookup_unmutexed(const Key& key) {
+  if (auto search = _lookup.find(key); search != _lookup.end()) {  // cache hit
+    search->second.visited = true;
+    return search->second.value;
+  } else {
+    return std::nullopt;
+  }
+}
+
+template <typename Key, typename Value>
+typename WebCache<Key, Value>::Node& WebCache<Key, Value>::insert_or_existing_unmutexed(
+    const Key& key, ValuePtr value) {
+  const auto& [it, took_place] = _lookup.emplace(
+      std::piecewise_construct, std::forward_as_tuple(key),
+      std::forward_as_tuple(std::move(value), _ttl));
+
+  if (took_place) {  // cache miss
+    perf_inc(Metric::miss);
+
+    ceph_assert(_lookup.size() == _sieve_queue.size() + 1);
+    // cache full? -> evict
+    if (_sieve_queue.size() >= _capacity) {
+      const auto node = sieve_evict();
+      if (node != nullptr) {
+        _lookup.erase(*(node->key));
+      }
+    }
+    auto& [stored_key, node] = *it;
+    node.key = &stored_key;
+    _sieve_queue.push_front(node);
+    perf_set(Metric::size, _lookup.size());
+    return node;
+  } else {  // cache hit
+    perf_inc(Metric::hit);
+    it->second.visited = true;
+    return it->second;
+  }
+}
+
+template <typename Key, typename Value>
+typename WebCache<Key, Value>::SieveRemoveRet
+WebCache<Key, Value>::sieve_remove_unmutexed(
+    SieveQueue& sieve_queue, Node* sieve_hand, const Node& node) {
+  const bool was_hand = &node == sieve_hand;
+  const auto node_it = sieve_queue.iterator_to(node);
+  auto it = sieve_queue.erase(node_it);
+  if (sieve_queue.empty()) {
+    return {sieve_queue.end(), nullptr};
+  }
+  if (was_hand) {
+    return {
+        it,
+        (it == sieve_queue.begin()) ? &sieve_queue.back() : &(*std::prev(it))};
+  }
+  return {it, sieve_hand};
+}
+
+template <typename Key, typename Value>
+std::optional<typename WebCache<Key, Value>::ValuePtr>
+WebCache<Key, Value>::lookup(const Key& key) {
+  std::shared_lock<std::shared_mutex> lock(_cache_mutex);
+  auto result = lookup_unmutexed(key);
+  if (result.has_value()) {
+    perf_inc(Metric::hit);
+  } else {
+    perf_inc(Metric::miss);
+  }
+  return result;
+}
+
+template <typename Key, typename Value>
+void WebCache<Key, Value>::sieve_expire_erase_unmutexed(
+    SieveQueue& sieve_queue, Node* sieve_hand, ceph::real_time eviction_cutoff,
+    SieveQueue& out_expired) {
+  // The sieve queue is ordered by ascending insertion time which
+  // would allow for efficient epiration by finding the first not
+  // expired element. BUT, since we allow updating TTLs this property
+  // no longer holds and we have do a full sieve queue sweep.
+  for (auto it = sieve_queue.begin(); it != sieve_queue.end();) {
+    Node& node = (*it);
+    const bool expired = node.expires_at <= eviction_cutoff;
+    if (expired) {
+      auto [next_it, next_hand] =
+          sieve_remove_unmutexed(sieve_queue, sieve_hand, node);
+      ceph_assert(!node.is_linked());
+      out_expired.push_back(node);
+      it = next_it;
+      sieve_hand = next_hand;
+    } else {
+      ++it;
+    }
+  }
+}
+
+template <typename Key, typename Value>
+size_t WebCache<Key, Value>::expire_erase() {
+  std::lock_guard<std::shared_mutex> lock(_cache_mutex);
+  const auto expiration_cutoff = ceph::real_clock::now();
+  const auto lookup_size_before = _lookup.size();
+  SieveQueue expired;
+  sieve_expire_erase_unmutexed(
+      _sieve_queue, _sieve_hand, expiration_cutoff, expired);
+  const auto expired_size = expired.size();
+  expired.clear_and_dispose(
+      [&](const Node* node) { _lookup.erase(*node->key); });
+  ceph_assert((lookup_size_before - expired_size) == _lookup.size());
+  perf_inc(Metric::expired, expired_size);
+  perf_set(Metric::size, _lookup.size());
+  return expired_size;
+}
+
+}  // namespace webcache
+
+#if FMT_VERSION >= 90000
+template <typename Key, typename Value>
+struct fmt::formatter<webcache::WebCache<Key, Value>> : fmt::ostream_formatter {
+};
+#endif
diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index 0d8d40b98d59..7e212f88b081 100644
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -114,6 +114,13 @@ add_executable(unittest_shared_cache
 add_ceph_unittest(unittest_shared_cache)
 target_link_libraries(unittest_shared_cache global)
 
+# unittest_web_cache
+add_executable(unittest_web_cache
+  test_web_cache.cc
+  )
+add_ceph_unittest(unittest_web_cache)
+target_link_libraries(unittest_web_cache ceph-common)
+
 # unittest_sloppy_crc_map
 add_executable(unittest_sloppy_crc_map
   test_sloppy_crc_map.cc
diff --git a/src/test/common/test_web_cache.cc b/src/test/common/test_web_cache.cc
new file mode 100644
index 000000000000..6f1f133f0195
--- /dev/null
+++ b/src/test/common/test_web_cache.cc
@@ -0,0 +1,644 @@
+#include <gmock/gmock-matchers.h>
+
+#include <atomic>
+#include <chrono>
+#include <include/expected.hpp>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <queue>
+#include <random>
+#include <thread>
+
+#include "common/ceph_context.h"
+#include "common/ceph_time.h"
+#include "common/dout.h"
+#include "common/Formatter.h"
+#include "common/web_cache.h"
+#include "gtest/gtest.h"
+#include "include/ceph_assert.h"
+#include "include/msgr.h"
+#include "include/uuid.h"
+#include "log/Log.h"
+
+using namespace std::chrono_literals;
+
+namespace webcache {
+
+class WebCacheTest : public ::testing::Test {
+ protected:
+  static constexpr size_t SIEVE_EXAMPLE_CACHE_SIZE = 7;
+  const std::string a_key = "testkey";
+  const std::string a_value = "testvalue";
+  const std::shared_ptr<std::string> a_valptr =
+      std::make_shared<std::string>(a_value);
+
+  const std::unique_ptr<CephContext> _cct;
+  size_t _initialized_capacity;
+  std::unique_ptr<WebCache<std::string, std::string>> _uut;
+
+  WebCacheTest()
+      : _cct(new CephContext(CEPH_ENTITY_TYPE_ANY)),
+        _initialized_capacity(SIEVE_EXAMPLE_CACHE_SIZE),
+        _uut(std::make_unique<WebCache<std::string, std::string>>(
+            SIEVE_EXAMPLE_CACHE_SIZE, 42s)) {
+    _cct->_log->start();
+  }
+  void TearDown() override { lderr(_cct.get()) << "AFTER: " << *_uut << dendl; }
+
+  void reset_cache(size_t capacity) {
+    _uut = std::make_unique<WebCache<std::string, std::string>>(capacity, 42s);
+    _initialized_capacity = capacity;
+  }
+  void reset_cache_system_mode(size_t capacity) {
+    _uut = std::make_unique<WebCache<std::string, std::string>>(
+        _cct.get(), "testing", capacity);
+    _initialized_capacity = capacity;
+  }
+
+  std::vector<std::string> sieve_queue_keys() {
+    std::vector<std::string> keys;
+    std::transform(
+        _uut->_sieve_queue.begin(), _uut->_sieve_queue.end(),
+        std::back_inserter(keys), [](const auto& node) { return *node.key; });
+    return keys;
+  }
+  size_t sieve_hand_pos() {
+    if (_uut->_sieve_hand == nullptr) {
+      return _uut->_capacity -1;
+    }
+    return std::distance(
+        _uut->_sieve_queue.begin(),
+        _uut->_sieve_queue.iterator_to(*_uut->_sieve_hand));
+  }
+  std::optional<std::string> sieve_hand_key() {
+    if (_uut->size() == 0) {
+      return std::nullopt;
+    }
+    if (_uut->_sieve_hand == nullptr) {
+      return *_uut->_sieve_queue.back().key;
+    }
+    return *_uut->_sieve_hand->key;
+  }
+};
+
+TEST_F(WebCacheTest, AddReturnsGreaterZeroTs) {
+  auto expires_ts = _uut->add(a_key, a_valptr);
+  ASSERT_GT(expires_ts, ceph::real_clock::zero());
+}
+
+TEST_F(WebCacheTest, MultipleAddsToEmptyReturnFirstTs) {
+  const std::vector<ceph::real_time> inserts = {
+      _uut->add(a_key, a_valptr),
+      _uut->add(a_key, a_valptr),
+      _uut->add(a_key, a_valptr),
+      _uut->add(a_key, a_valptr),
+  };
+  ASSERT_THAT(inserts, ::testing::Each(inserts[0]));
+}
+
+TEST_F(WebCacheTest, TwoSameKeyAddsGivesSize1) {
+  const auto unused = {
+      _uut->add(a_key, a_valptr),
+      _uut->add(a_key, a_valptr),
+      _uut->add(a_key, a_valptr),
+  };
+  (void)unused;
+  ASSERT_THAT(_uut->size(), 1);
+}
+
+TEST_F(WebCacheTest, TwoDistinctKeyAddsGiveSize2) {
+  const auto unused = {
+      _uut->add("a", a_valptr),
+      _uut->add("b", a_valptr),
+  };
+  (void)unused;
+  ASSERT_THAT(_uut->size(), 2);
+}
+
+TEST_F(WebCacheTest, LookupNonExistingReturnsIsNullopt) {
+  ASSERT_FALSE(_uut->lookup("not-existing").has_value());
+}
+
+TEST_F(WebCacheTest, AddDoesNotOverwrite) {
+  _uut->add(a_key, a_valptr);
+  ASSERT_EQ(_uut->lookup(a_key).value(), a_valptr);
+
+  auto another_valptr = std::make_shared<std::string>("another-value");
+  _uut->add(a_key, another_valptr);
+  ASSERT_EQ(_uut->lookup(a_key).value(), a_valptr);
+}
+
+TEST_F(WebCacheTest, LookupReturnsWhatWasAdded) {
+  const std::string key = "testkey";
+  const std::string value = "testvalue";
+  auto addptr = std::make_shared<std::string>(value);
+  _uut->add(key, addptr);
+
+  auto maybe_val = _uut->lookup(key);
+  ASSERT_TRUE(maybe_val);
+  ASSERT_EQ(value, *maybe_val.value());
+}
+
+TEST_F(WebCacheTest, AddsOverCapacityDontGoOver) {
+  for (size_t i = 0; i < (_initialized_capacity + 10); ++i) {
+    _uut->add(std::to_string(i), a_valptr);
+  }
+  ASSERT_EQ(_initialized_capacity, _uut->size());
+}
+
+TEST_F(WebCacheTest, AddALotUnique) {
+  reset_cache(100);
+  for (size_t i = 0; i < 10000; ++i) {
+    uuid_d uuid;
+    uuid.generate_random();
+    _uut->add(uuid.to_string(), a_valptr);
+  }
+}
+
+TEST_F(WebCacheTest, CacheTakesValOwnership) {
+  const std::string a_key = "testkey";
+  const std::string a_value = "testvalue";
+  std::shared_ptr<std::string> a_valptr =
+      std::make_shared<std::string>(a_value);
+  ASSERT_EQ(1, a_valptr.use_count());
+  _uut->add(a_key, a_valptr);
+  ASSERT_EQ(2, a_valptr.use_count());
+  auto retrieved = _uut->lookup(a_key).value();
+  ASSERT_EQ(3, retrieved.use_count());
+  a_valptr.reset();
+  ASSERT_EQ(2, retrieved.use_count());
+}
+
+TEST_F(WebCacheTest, SimpleLookupOr) {
+  auto value = _uut->lookup_or(a_key, std::make_shared<std::string>("test"));
+  ASSERT_EQ("test", *value);
+}
+
+TEST_F(WebCacheTest, LookupOrAddsToCache) {
+  auto future = _uut->lookup_or(a_key, std::make_shared<std::string>("test"));
+  ASSERT_EQ("test", *_uut->lookup(a_key).value());
+}
+
+TEST_F(WebCacheTest, SieveRemoveHand) {
+  std::array<WebCache<std::string, std::string>::Node, 3> store{
+      WebCache<std::string, std::string>::Node{
+          std::make_shared<std::string>("a"), ceph::real_clock::from_time_t(1)},
+      WebCache<std::string, std::string>::Node{
+          std::make_shared<std::string>("b"), ceph::real_clock::from_time_t(2)},
+      WebCache<std::string, std::string>::Node{
+          std::make_shared<std::string>("c"), ceph::real_clock::from_time_t(3)},
+  };
+  WebCache<std::string, std::string>::SieveQueue nodes;
+  for (auto& node : store) {
+    nodes.push_back(node);
+  }
+  // SIEVE QUEUE: a b c
+  WebCache<std::string, std::string>::Node* hand = &store[2];
+  lderr(_cct.get()) << "NODES: " << &store[0] << ", " << &store[1] << ", "
+                    << &store[2] << dendl;
+  lderr(_cct.get()) << "HAND: " << hand << dendl;
+  {
+    auto [it, hand_moved] =
+        WebCache<std::string, std::string>::sieve_remove_unmutexed(
+            nodes, hand, store[2]);
+    ASSERT_EQ(&*nodes.end(), &*it);        // we removed the last element
+    ASSERT_EQ(hand_moved, &nodes.back());  // hand pointed to the last element
+    ASSERT_EQ(hand_moved, &store[1]);
+    hand = hand_moved;
+  }
+
+  {
+    auto [it, hand_moved] =
+        WebCache<std::string, std::string>::sieve_remove_unmutexed(
+            nodes, hand, store[0]);
+    ASSERT_EQ(*it->value, "b");
+    ASSERT_EQ(hand_moved, &store[1]);
+    hand = hand_moved;
+  }
+
+  {
+    auto [it, hand_moved] =
+        WebCache<std::string, std::string>::sieve_remove_unmutexed(
+            nodes, hand, store[1]);
+    ASSERT_EQ(&*nodes.end(), &*it);
+    ASSERT_EQ(hand_moved, nullptr);
+  }
+}
+
+TEST_F(WebCacheTest, ExpireEraseOne) {
+  WebCache<std::string, std::string>::Node alive_node(
+      a_valptr, ceph::real_clock::from_time_t(301));
+  WebCache<std::string, std::string>::Node expired_node(
+      a_valptr, ceph::real_clock::from_time_t(10));
+  WebCache<std::string, std::string>::SieveQueue nodes;
+  nodes.push_back(alive_node);
+  nodes.push_back(expired_node);
+  WebCache<std::string, std::string>::Node* hand = nullptr;
+
+  lderr(_cct.get()) << "NODES:         " << &alive_node << ":" << alive_node
+                    << ", " << &expired_node << ":" << expired_node << dendl;
+  lderr(_cct.get()) << "BEFORE EXPIRE: " << alive_node << ", " << expired_node
+                    << dendl;
+  EXPECT_EQ(2, nodes.size());
+  const auto expiration_cutoff = ceph::real_clock::from_time_t(300);
+  WebCache<std::string, std::string>::SieveQueue expired;
+  WebCache<std::string, std::string>::sieve_expire_erase_unmutexed(
+      nodes, hand, expiration_cutoff, expired);
+  lderr(_cct.get()) << "AFTER EXPIRE:  " << alive_node << ", " << expired_node
+                    << dendl;
+  lderr(_cct.get()) << "EXPIRED:       " << expired << dendl;
+  EXPECT_EQ(1, nodes.size());
+  EXPECT_EQ(nullptr, hand);
+  ASSERT_EQ(1, expired.size());
+  EXPECT_EQ(&*expired.begin(), &expired_node);
+}
+
+TEST_F(WebCacheTest, ExpireEraseAll) {
+  WebCache<std::string, std::string>::Node expired_node_1(
+      a_valptr, ceph::real_clock::from_time_t(23));
+  WebCache<std::string, std::string>::Node expired_node_2(
+      a_valptr, ceph::real_clock::from_time_t(42));
+  WebCache<std::string, std::string>::SieveQueue nodes;
+  nodes.push_back(expired_node_1);
+  nodes.push_back(expired_node_2);
+  WebCache<std::string, std::string>::Node* hand = nullptr;
+  lderr(_cct.get()) << "NODES:         " << &expired_node_1 << ":"
+                    << expired_node_1 << ", " << &expired_node_2 << ":"
+                    << expired_node_2 << dendl;
+  lderr(_cct.get()) << "BEFORE EXPIRE: " << expired_node_1 << ", "
+                    << expired_node_2 << dendl;
+  EXPECT_EQ(2, nodes.size());
+  const auto expiration_cutoff = ceph::real_clock::from_time_t(300);
+  WebCache<std::string, std::string>::SieveQueue expired;
+  WebCache<std::string, std::string>::sieve_expire_erase_unmutexed(
+      nodes, hand, expiration_cutoff, expired);
+  lderr(_cct.get()) << "AFTER EXPIRE:  " << expired_node_1 << ", "
+                    << expired_node_2 << dendl;
+  lderr(_cct.get()) << "EXPIRED:       " << expired << dendl;
+  EXPECT_EQ(0, nodes.size());
+  EXPECT_EQ(nullptr, hand);
+  ASSERT_EQ(2, expired.size());
+  ASSERT_EQ(&*expired.begin(), &expired_node_1);
+  ASSERT_EQ(&expired.back(), &expired_node_2);
+}
+
+TEST_F(WebCacheTest, ExpireEraseUpdatedTTLs) {
+  _uut->add("a", a_valptr);  // oldest
+  _uut->add("b", a_valptr);
+  _uut->add("c", a_valptr);
+  _uut->add("d", a_valptr);
+  _uut->add("e", a_valptr);  // newest
+
+  _uut->update_ttl_if("a", a_valptr, std::chrono::seconds(0));  // expired
+  _uut->update_ttl_if(
+      "b", a_valptr, std::chrono::seconds(100000));  // not expired
+  _uut->update_ttl_if("c", a_valptr, std::chrono::seconds(0));
+  _uut->update_ttl_if("d", a_valptr, std::chrono::seconds(100000));
+  _uut->update_ttl_if("e", a_valptr, std::chrono::seconds(0));
+
+  EXPECT_EQ(5, _uut->size()) << *_uut;
+  EXPECT_EQ(3, _uut->expire_erase());
+  EXPECT_EQ(2, _uut->size()) << *_uut;
+}
+
+TEST_F(WebCacheTest, ExpireEraseEmpty) {
+  WebCache<std::string, std::string>::SieveQueue nodes;
+  const auto expiration_cutoff = ceph::real_clock::from_time_t(42);
+  WebCache<std::string, std::string>::SieveQueue expired;
+  WebCache<std::string, std::string>::Node* hand = nullptr;
+  WebCache<std::string, std::string>::sieve_expire_erase_unmutexed(
+      nodes, hand, expiration_cutoff, expired);
+  ASSERT_EQ(0, nodes.size());
+  ASSERT_EQ(0, expired.size());
+  EXPECT_EQ(nullptr, hand);
+}
+
+TEST_F(WebCacheTest, CacheHasSizeZeroAfterClear) {
+  _uut->add("a", a_valptr);
+  _uut->add("b", a_valptr);
+  _uut->add("c", a_valptr);
+  ASSERT_EQ(3, _uut->size());
+  ASSERT_EQ(3, _uut->clear());
+  ASSERT_EQ(0, _uut->size());
+}
+
+TEST_F(WebCacheTest, SieveExample) {
+  // Example from https://cachemon.github.io/SIEVE-website/
+  _uut->add("a", a_valptr);
+  _uut->add("b", a_valptr);
+  _uut->add("c", a_valptr);
+  _uut->add("d", a_valptr);
+  _uut->add("e", a_valptr);
+  _uut->add("f", a_valptr);
+  _uut->add("g", a_valptr);
+
+  _uut->add("a", a_valptr);
+  _uut->add("b", a_valptr);
+  _uut->add("g", a_valptr);
+  ASSERT_THAT(
+      sieve_queue_keys(),
+      ::testing::ElementsAre("g", "f", "e", "d", "c", "b", "a"));
+  ASSERT_EQ(sieve_hand_pos(), 6);
+
+  _uut->add("h", a_valptr);
+  ASSERT_THAT(
+      sieve_queue_keys(),
+      ::testing::ElementsAre("h", "g", "f", "e", "d", "b", "a"));
+  ASSERT_EQ(sieve_hand_pos(), 4);
+
+  _uut->add("a", a_valptr);
+  ASSERT_THAT(
+      sieve_queue_keys(),
+      ::testing::ElementsAre("h", "g", "f", "e", "d", "b", "a"));
+  ASSERT_EQ(sieve_hand_pos(), 4);
+
+  _uut->add("d", a_valptr);
+  ASSERT_THAT(
+      sieve_queue_keys(),
+      ::testing::ElementsAre("h", "g", "f", "e", "d", "b", "a"));
+  ASSERT_EQ(sieve_hand_pos(), 4);
+
+  _uut->add("i", a_valptr);
+  ASSERT_THAT(
+      sieve_queue_keys(),
+      ::testing::ElementsAre("i", "h", "g", "f", "d", "b", "a"));
+  ASSERT_EQ(sieve_hand_pos(), 3);
+
+  _uut->add("b", a_valptr);
+  ASSERT_THAT(
+      sieve_queue_keys(),
+      ::testing::ElementsAre("i", "h", "g", "f", "d", "b", "a"));
+  ASSERT_EQ(sieve_hand_pos(), 3);
+
+  _uut->add("j", a_valptr);
+  ASSERT_THAT(
+      sieve_queue_keys(),
+      ::testing::ElementsAre("j", "i", "h", "g", "d", "b", "a"));
+  ASSERT_EQ(sieve_hand_pos(), 3);
+}
+
+TEST_F(WebCacheTest, RemoveIfNonExistingReturnsFalse) {
+  ASSERT_FALSE(_uut->remove_if("h", a_valptr));
+}
+
+TEST_F(WebCacheTest, RemoveIfExistingDifferentValueReturnsFalseDoesNothing) {
+  _uut->add("a", a_valptr);
+  ASSERT_FALSE(
+      _uut->remove_if("a", std::make_shared<std::string>("someothervalue")));
+  ASSERT_THAT(sieve_queue_keys(), ::testing::ElementsAre("a"));
+}
+
+TEST_F(WebCacheTest, RemoveIfExampleHandMovement) {
+  // Start with the example from https://cachemon.github.io/SIEVE-website/
+  _uut->add("a", a_valptr);
+  _uut->add("b", a_valptr);
+  _uut->add("c", a_valptr);
+  _uut->add("d", a_valptr);
+  _uut->add("e", a_valptr);
+  _uut->add("f", a_valptr);
+  _uut->add("g", a_valptr);
+  _uut->add("a", a_valptr);
+  _uut->add("b", a_valptr);
+  _uut->add("g", a_valptr);
+  _uut->add("h", a_valptr);
+  ASSERT_THAT(
+      sieve_queue_keys(),
+      ::testing::ElementsAre("h", "g", "f", "e", "d", "b", "a"));
+  ASSERT_EQ(sieve_hand_key(), "d");
+
+  ASSERT_TRUE(_uut->remove_if("h", a_valptr));
+  ASSERT_THAT(
+      sieve_queue_keys(), ::testing::ElementsAre("g", "f", "e", "d", "b", "a"));
+  ASSERT_EQ(sieve_hand_key(), "d");
+
+  ASSERT_TRUE(_uut->remove_if("d", a_valptr));
+  ASSERT_THAT(
+      sieve_queue_keys(), ::testing::ElementsAre("g", "f", "e", "b", "a"));
+  ASSERT_EQ(sieve_hand_key(), "e");
+
+  ASSERT_TRUE(_uut->remove_if("a", a_valptr));
+  ASSERT_THAT(sieve_queue_keys(), ::testing::ElementsAre("g", "f", "e", "b"));
+  ASSERT_EQ(sieve_hand_key(), "e");
+
+  ASSERT_TRUE(_uut->remove_if("f", a_valptr));
+  ASSERT_THAT(sieve_queue_keys(), ::testing::ElementsAre("g", "e", "b"));
+  ASSERT_EQ(sieve_hand_key(), "e");
+
+  ASSERT_TRUE(_uut->remove_if("e", a_valptr));
+  ASSERT_THAT(sieve_queue_keys(), ::testing::ElementsAre("g", "b"));
+  ASSERT_EQ(sieve_hand_key(), "g");
+
+  ASSERT_TRUE(_uut->remove_if("b", a_valptr));
+  ASSERT_THAT(sieve_queue_keys(), ::testing::ElementsAre("g"));
+  ASSERT_EQ(sieve_hand_key(), "g");
+
+  ASSERT_TRUE(_uut->remove_if("g", a_valptr));
+  ASSERT_THAT(0, _uut->size());
+  ASSERT_FALSE(sieve_hand_key().has_value());
+}
+
+TEST_F(WebCacheTest, SieveAllVisited) {
+  reset_cache(2);
+  _uut->add("a", a_valptr);
+  _uut->add("b", a_valptr);
+  _uut->add("a", a_valptr);
+  _uut->add("b", a_valptr);
+  _uut->add("c", a_valptr);
+}
+
+class WebCacheConcurrencyTest : public WebCacheTest {
+  void TearDown() override {
+    if (_uut->perf() != nullptr) {
+      JSONFormatter f(true);
+      _uut->perf()->dump_formatted(&f, false, select_labeled_t::labeled);
+      f.flush(std::cout);
+      _uut->perf()->reset();
+    }
+  }
+};
+
+TEST_F(WebCacheConcurrencyTest, BasicAddSame) {
+  reset_cache_system_mode(100);
+  const auto num_threads =
+      std::max(std::thread::hardware_concurrency() * 100, 100U);
+  std::vector<std::thread> threads;
+  for (size_t i = 0; i < num_threads; ++i) {
+    threads.emplace_back([&]() {
+      for (size_t j = 0; j < 1000; ++j) {
+        _uut->add(a_key, a_valptr);
+      }
+    });
+  }
+  for (auto& th : threads) {
+    th.join();
+  }
+
+  ASSERT_EQ(1, _uut->size());
+  ASSERT_TRUE(_uut->lookup(a_key).has_value());
+}
+
+TEST_F(WebCacheConcurrencyTest, BasicAddUnique) {
+  reset_cache_system_mode(100);
+  const auto num_threads =
+      std::max(std::thread::hardware_concurrency() * 100, 100U);
+  std::vector<std::thread> threads;
+  for (size_t i = 0; i < num_threads; ++i) {
+    threads.emplace_back([&]() {
+      for (size_t j = 0; j < 10; ++j) {
+        uuid_d uuid;
+        uuid.generate_random();
+        _uut->add(uuid.to_string(), a_valptr);
+      }
+    });
+  }
+  for (auto& th : threads) {
+    th.join();
+  }
+}
+
+// Example: Mitigate cache stampedes using std::call_once
+TEST_F(WebCacheConcurrencyTest, StampedeSyncCallOnce) {
+  struct CacheValue {
+    std::once_flag once;
+    std::string value;
+  };
+  webcache::WebCache<std::string, CacheValue> cache(
+      _cct.get(), "test_web_cache", 100);
+  std::atomic_int fetches = 0;
+  const auto num_threads =
+      std::max(std::thread::hardware_concurrency() * 100, 100U);
+  std::vector<std::thread> threads;
+  for (size_t i = 0; i < num_threads; ++i) {
+    threads.emplace_back([&]() {
+      std::shared_ptr<CacheValue> cache_value =
+          cache.lookup_or(a_key, std::make_shared<CacheValue>());
+      std::call_once(cache_value->once, [cache_value, &fetches]() {
+        fetches++;
+        std::this_thread::sleep_for(1000ms);
+        cache_value->value = "test";
+      });
+    });
+  }
+  for (auto& th : threads) {
+    th.join();
+  }
+  ASSERT_EQ(fetches.load(), 1);
+}
+
+// Example: Mitigate cache stampedes using a mutex per value
+TEST_F(WebCacheConcurrencyTest, StampedeMutex) {
+  struct CacheValue {
+    std::mutex mutex;
+    std::string value;
+    std::function<std::string()> fn;
+    CacheValue(std::function<std::string()> fn) : fn(fn) {}
+    std::string get() {
+      std::unique_lock<std::mutex> lock(mutex);
+      if (value.empty()) {
+        value = fn();
+      }
+      return value;
+    }
+  };
+  webcache::WebCache<std::string, CacheValue> cache(
+      _cct.get(), "test_web_cache", 100);
+  std::atomic_int fetches = 0;
+  const auto num_threads =
+      std::max(std::thread::hardware_concurrency() * 100, 100U);
+  std::vector<std::thread> threads;
+  for (size_t i = 0; i < num_threads; ++i) {
+    threads.emplace_back([&]() {
+      std::shared_ptr<CacheValue> cache_value =
+          cache.lookup_or(a_key, std::make_shared<CacheValue>([&fetches]() {
+                            fetches++;
+                            std::this_thread::sleep_for(1000ms);
+                            return "test";
+                          }));
+      cache_value->get();
+    });
+  }
+  for (auto& th : threads) {
+    th.join();
+  }
+  ASSERT_EQ(fetches.load(), 1);
+}
+
+class WebCacheRandomizedTest : public WebCacheTest {
+  void TearDown() override {
+    if (_uut->perf() != nullptr) {
+      JSONFormatter f(true);
+      _uut->perf()->dump_formatted(&f, false, select_labeled_t::labeled);
+      f.flush(std::cout);
+      _uut->perf()->reset();
+    }
+  }
+};
+
+TEST_F(WebCacheRandomizedTest, RandomCallMainOperations) {
+  reset_cache_system_mode(1000);
+  const size_t ops = 100000;
+  const auto num_threads = std::max(std::thread::hardware_concurrency(), 8U);
+
+  std::vector<int> base(100);
+  std::iota(base.begin(), base.end(), 0);
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_int_distribution<> dist(0, base.size() - 1);
+  std::queue<std::string> keys;
+  for (size_t i = 0; i < ops; ++i) {
+    keys.emplace(std::to_string(base[dist(gen)]));
+  }
+
+  std::discrete_distribution<> op_dist({90, 1, 9});
+
+  std::mutex mutex;
+  std::vector<std::thread> threads;
+  std::atomic_int lookups;
+  std::atomic_int clears;
+  std::atomic_int expires;
+  for (size_t i = 0; i < num_threads; ++i) {
+    threads.emplace_back([&]() {
+      for (size_t op_i = 0; op_i < ops / num_threads; ++op_i) {
+        std::string key;
+        int op = -1;
+        {
+          std::unique_lock<std::mutex> lock(mutex);
+          key = keys.front();
+          keys.pop();
+          op = op_dist(gen);
+        }
+
+        switch (op) {
+          case 0: {  // lookup_or
+            auto value = _uut->lookup_or(key, a_valptr);
+            lookups++;
+          } break;
+          case 1:  // clear cache
+            _uut->clear();
+            clears++;;
+            break;
+          case 2:  // expire
+            _uut->expire_erase();
+            expires++;
+            break;
+          default:
+            ceph_abort("should not happen");
+        }
+      }
+    });
+  }
+  for (auto& th : threads) {
+    th.join();
+  }
+  EXPECT_GT(lookups, 1);
+  EXPECT_GT(expires, 1);
+  EXPECT_GT(clears, 1);
+  std::cout << "lookups: " << lookups
+            << ", expires: " << expires
+            << ", clears: " << clears
+            << std::endl;
+}
+
+}  // namespace webcache

From 74fecd1d9453efdbfba3faa07954561b6c5a4741 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Fri, 21 Feb 2025 12:42:07 +0100
Subject: [PATCH 06/28] test: Add Cache benchmarks

Add Google benchmark [0] based micro benchmarks for Cache/LRU
implementations in the Ceph code base.

[0] https://github.com/google/benchmark

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 8354a97001baec285b930a9ef1232684fbeb5656)
---
 src/test/CMakeLists.txt |  10 +
 src/test/bench_lrus.cc  | 461 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 471 insertions(+)
 create mode 100644 src/test/bench_lrus.cc

diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index 2e756eeb5838..9e96817a908c 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -1008,3 +1008,13 @@ add_ceph_unittest(unittest_weighted_shuffle)
 add_executable(unittest_intarith test_intarith.cc)
 add_ceph_unittest(unittest_intarith)
 #make check ends here
+
+find_package(benchmark QUIET)
+if (benchmark_FOUND)
+  add_executable(bench_lrus
+  bench_lrus.cc
+  )
+  target_link_libraries(bench_lrus ceph-common global-static benchmark::benchmark Boost::context)
+else()
+  message(STATUS "The google/benchmark library was not found. Skipping micro benchmark tests")
+endif(benchmark_FOUND)
diff --git a/src/test/bench_lrus.cc b/src/test/bench_lrus.cc
new file mode 100644
index 000000000000..02155220e639
--- /dev/null
+++ b/src/test/bench_lrus.cc
@@ -0,0 +1,461 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2025 Clyso GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <benchmark/benchmark.h>
+#include <uuid/uuid.h>
+#include <xxhash.h>
+
+#include <atomic>
+#include <boost/asio/detached.hpp>
+#include <boost/asio/executor_work_guard.hpp>
+#include <initializer_list>
+#include <iterator>
+#include <numeric>
+#include <random>
+#include <thread>
+
+#include "common/ceph_argparse.h"
+#include "common/cohort_lru.h"
+#include "common/shared_cache.hpp"
+#include "common/simple_cache.hpp"
+#include "common/web_cache.h"
+#include "global/global_context.h"
+#include "global/global_init.h"
+#include "include/uuid.h"
+
+
+// Cache implementations in the Ceph codebase:
+// ✅ cohort lru
+// ✅ shared_cache.hpp SharedLRU
+// ✅ simple_cache SimpleLRU
+// ✅ web_cache
+// Not benchmarked here (reason):
+// ❌ LRUSet (not concurrent)
+// ❌ intrusive_lru: lru implementation with embedded map and list hook (not concurrent)
+// ❌ include/lru.h LRU - (not concurrent)
+
+
+// Workload Generator Helper
+//
+// (1) insert unique items > cache size
+//   - exercises cache replacement algorithm
+//   - with > 1 thread - test concurrency
+// (2) Inserts using pareto distributed keys
+//   - approximates real world workload
+
+namespace {
+
+// Config
+
+constexpr size_t SMALL_CACHE = 100;
+constexpr size_t LARGE_CACHE = 1000;
+constexpr size_t CACHE_OP_COUNT = 1000000;
+constexpr size_t THREADS_SINGLE = 1;
+constexpr size_t THREADS_LOTS = 128;
+constexpr size_t RAND_VALUE_LEN = 32;
+constexpr size_t PARETO_KEY_POOL_SIZE = 1000;
+
+std::string random_key() {
+  uuid_d uuid;
+  uuid.generate_random();
+  return uuid.to_string();
+}
+
+std::string random_value() {
+  std::string result(RAND_VALUE_LEN, '0');
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_int_distribution<uint16_t> dist(0, 0xFF);
+  for (size_t i=0; i< RAND_VALUE_LEN; ++i) {
+    result[i] = static_cast<char>(dist(gen));
+  }
+  return result;
+}
+
+std::vector<std::string> key_pool(size_t len) {
+  std::vector<std::string> result;
+  result.reserve(len);
+  for (size_t i = 0; i < len; ++i) {
+    result.push_back(random_key());
+  }
+  return result;
+}
+
+std::vector<std::string_view> workload(
+    const std::vector<std::string>& pool, size_t length) {
+  const double alpha = 1.5;
+  std::vector<double> weights(pool.size());
+  for (size_t i = 0; i < pool.size(); ++i) {
+    weights[i] = std::pow(static_cast<double>(i + 1), -alpha);
+  }
+  const double weights_sum =
+      std::accumulate(weights.begin(), weights.end(), 0.0);
+  for (auto& weight : weights) {
+    weight /= weights_sum;
+  }
+  std::vector<double> partial_sums(weights.size());
+  std::partial_sum(weights.begin(), weights.end(), partial_sums.begin());
+
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_real_distribution<double> dis(0.0, 1.0);
+
+  std::vector<std::string_view> result;
+  result.reserve(length);
+
+  for (size_t i = 0; i < length; ++i) {
+    double u = dis(gen);
+    auto it = std::ranges::lower_bound(partial_sums, u);
+    size_t idx = std::distance(partial_sums.begin(), it);
+    result.push_back(pool[idx]);
+  }
+  return result;
+}
+
+//
+// Cache Adapter (cache impl <-> benchmark)
+//
+
+struct CacheAdapter {
+  CacheAdapter() = default;
+  CacheAdapter(const CacheAdapter&) = default;
+  CacheAdapter(CacheAdapter&&) = delete;
+  CacheAdapter& operator=(const CacheAdapter&) = delete;
+  CacheAdapter& operator=(CacheAdapter&&) = delete;
+  virtual ~CacheAdapter() = default;
+
+  // Simulate common cache operation: lookup value by key, if it isn't
+  // cached add it
+  virtual void cache(const std::string& key, const std::string& value) = 0;
+
+  virtual double hit_miss_ratio() { return -23.42; };
+  virtual bool reset() { return false; };
+};
+
+// Shared LRU {{{
+struct SharedLRUAdapter : public CacheAdapter {
+  SharedLRU<std::string, std::string> _cache;
+
+  std::atomic_int hits;
+  std::atomic_int misses;
+  explicit SharedLRUAdapter(size_t size) : _cache(g_ceph_context, size) {}
+  void cache(const std::string& key, const std::string& value) override {
+    bool existed = false;
+    auto* copy = new std::string(value);
+    const auto ptr = _cache.add(key, copy, &existed);
+    if (existed) {
+      hits++;
+      delete copy;
+    } else {
+      misses++;
+    }
+  }
+
+  double hit_miss_ratio() override {
+    return static_cast<double>(hits) /
+           (static_cast<double>(hits) + static_cast<double>(misses));
+  }
+
+  bool reset() override {
+    _cache.clear();
+    hits = 0;
+    misses = 0;
+    return true;
+  }
+};
+
+// }}}
+
+// Simple LRU {{{
+struct SimpleLRUAdapter : public CacheAdapter {
+  SimpleLRU<std::string, std::string> _cache;
+
+  explicit SimpleLRUAdapter(size_t size) : _cache(size) {}
+  void cache(const std::string& key, const std::string& value) override {
+    std::string out;
+    if (!_cache.lookup(key, &out)) {
+      _cache.add(key, value);
+    }
+  }
+};
+
+// }}}
+
+// Cohort LRU {{{
+namespace cohortlru {
+class Factory;
+
+struct Object : public cohort::lru::Object {
+  std::string m_key;
+  std::string m_value;
+
+  Object(const Object&) = delete;
+  Object(Object&&) = delete;
+  Object& operator=(const Object&) = delete;
+  Object& operator=(Object&&) = delete;
+  ~Object() override = default;
+
+  Object(const std::string& key, const std::string& value) :
+    cohort::lru::Object(), m_key(key), m_value(value)
+  {}
+
+  bool reclaim(const cohort::lru::ObjectFactory* newobj_fac) override;
+
+};
+
+struct Factory : public cohort::lru::ObjectFactory {
+  std::string m_key;
+  std::string m_value;
+
+  Factory(const Factory&) = default;
+  Factory(Factory&&) = delete;
+  Factory& operator=(const Factory&) = default;
+  Factory& operator=(Factory&&) = delete;
+  ~Factory() override = default;
+
+  Factory(const std::string& key, const std::string& value) :
+    cohort::lru::ObjectFactory(), m_key(key), m_value(value)
+  {}
+
+  cohort::lru::Object* alloc() override { return new Object(m_key, m_value); }
+
+  void recycle(cohort::lru::Object* o) override {
+    auto oo = dynamic_cast<Object*>(o);
+    oo->m_key = m_key;
+    oo->m_value = m_value;
+  }
+};
+
+bool Object::reclaim(const cohort::lru::ObjectFactory* newobj_fac) {
+  const auto* factory = dynamic_cast<const Factory*>(newobj_fac);
+  return (factory != nullptr);
+}
+
+}  // namespace cohortlru
+
+struct CohortLRUAdapter : public CacheAdapter {
+  cohort::lru::LRU<std::mutex> _cache;
+
+  explicit CohortLRUAdapter(size_t size)
+      : _cache(
+            static_cast<int>(size / std::thread::hardware_concurrency()),
+            size / std::thread::hardware_concurrency()) {}
+
+  void cache(const std::string& key, const std::string& value) override {
+    cohortlru::Factory prototype(key, value);
+    uint32_t iflags{cohort::lru::FLAG_INITIAL};
+    auto o = dynamic_cast<cohortlru::Object*>(
+        _cache.insert(&prototype, cohort::lru::Edge::MRU, iflags));
+    ceph_assert(o != nullptr);
+    ceph_assert(o->m_key == key);
+    ceph_assert(o->m_value == value);
+  }
+};
+
+// }}}
+
+// Web Cache {{{
+
+struct WebCacheAdapter : public CacheAdapter {
+  using CacheValue = std::string;
+  using Cache = webcache::WebCache<std::string, CacheValue>;
+  Cache _cache;
+
+  WebCacheAdapter(const WebCacheAdapter&) = delete;
+  WebCacheAdapter(WebCacheAdapter&&) = delete;
+  WebCacheAdapter& operator=(const WebCacheAdapter&) = delete;
+  WebCacheAdapter& operator=(WebCacheAdapter&&) = delete;
+
+  explicit WebCacheAdapter(size_t size) :
+    _cache(g_ceph_context, "benchmark", size)
+  {}
+  void cache(const std::string& key, const std::string& value) override {
+    if (!_cache.lookup(key).has_value()) {
+      _cache.add(key, std::make_shared<std::string>(value));
+    }
+  }
+
+  ~WebCacheAdapter() override { _cache.perf()->reset(); }
+
+  double hit_miss_ratio() override {
+    return static_cast<double>(
+               _cache.perf()->get(static_cast<int>(webcache::Metric::hit))) /
+           (static_cast<double>(
+                _cache.perf()->get(static_cast<int>(webcache::Metric::hit))) +
+            static_cast<double>(
+                _cache.perf()->get(static_cast<int>(webcache::Metric::miss))));
+  }
+
+  bool reset() override {
+    _cache.clear();
+    return true;
+  }
+};
+
+struct WebCacheLookupOrAdapter : public CacheAdapter {
+  struct CacheValue {
+    std::once_flag once;
+    std::string value;
+  };
+  using Cache = webcache::WebCache<std::string, CacheValue>;
+  Cache _cache;
+
+  WebCacheLookupOrAdapter(const WebCacheLookupOrAdapter&) = delete;
+  WebCacheLookupOrAdapter(WebCacheLookupOrAdapter&&) = delete;
+  WebCacheLookupOrAdapter& operator=(const WebCacheLookupOrAdapter&) = delete;
+  WebCacheLookupOrAdapter& operator=(WebCacheLookupOrAdapter&&) = delete;
+
+  explicit WebCacheLookupOrAdapter(size_t size) :
+    _cache(g_ceph_context, "benchmark", size)
+  {}
+
+  ~WebCacheLookupOrAdapter() override {
+    _cache.perf()->reset();
+  }
+
+  void cache(const std::string& key, const std::string& value) override {
+    std::shared_ptr<CacheValue> cache_value =
+        _cache.lookup_or(key, std::make_shared<CacheValue>());
+
+    std::call_once(cache_value->once, [&]() { cache_value->value = value; });
+  }
+  double hit_miss_ratio() override {
+    return static_cast<double>(
+               _cache.perf()->get(static_cast<int>(webcache::Metric::hit))) /
+           (static_cast<double>(
+                _cache.perf()->get(static_cast<int>(webcache::Metric::hit))) +
+            static_cast<double>(
+                _cache.perf()->get(static_cast<int>(webcache::Metric::miss))));
+  }
+  bool reset() override {
+    _cache.clear();
+    return true;
+  }
+};
+
+/// }}}
+
+// Benchmarks {{{
+
+template <typename C>
+class CacheFixture : public benchmark::Fixture {
+ public:
+  std::unique_ptr<C> cache;
+  void SetUp(::benchmark::State& state) override {
+    if (state.thread_index() == 0) {
+      cache = std::make_unique<C>(state.range(0));
+    }
+  }
+
+  void TearDown(::benchmark::State& state) override {
+    if (state.thread_index() == 0) {
+      state.counters["hit/miss"] = cache->hit_miss_ratio();
+    }
+  }
+};
+
+template <typename C>
+class ParetoFixture : public CacheFixture<C> {
+ public:
+  std::vector<std::string> pool;
+  std::array<std::vector<std::string_view>, THREADS_LOTS> pareto_keys;
+
+  ParetoFixture() : pool(key_pool(PARETO_KEY_POOL_SIZE)) {}
+
+  void SetUp(::benchmark::State& state) override {
+    if (state.thread_index() == 0) {
+      this->cache = std::make_unique<C>(state.range(0));
+      state.counters["Key Pool"] = pool.size();
+    }
+    pareto_keys[state.thread_index()] =
+        workload(pool, state.range(1) / state.threads());
+    state.counters["Keys/Thread"] = benchmark::Counter(
+        pareto_keys[state.thread_index()].size(),
+        benchmark::Counter::kAvgThreads);
+    ceph_assert(pareto_keys[state.thread_index()].size() > 1000);
+  }
+};
+
+BENCHMARK_TEMPLATE_METHOD_F(CacheFixture, BM_UniqueAdd)(
+    benchmark::State& state) {
+  for (auto _ : state) {
+    const size_t ops = state.range(1) / state.threads();
+    for (size_t i = 0; i < ops; ++i) {
+      this->cache->cache(random_key(), random_value());
+    }
+    state.counters["KeysProcessed"] =
+        benchmark::Counter(ops, benchmark::Counter::kIsRate);
+    state.counters["KeysProcessedInv"] = benchmark::Counter(
+        ops, benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
+  }
+}
+
+BENCHMARK_TEMPLATE_METHOD_F(ParetoFixture, BM_Pareto)(benchmark::State& state) {
+  for (auto _ : state) {
+    const auto keys = this->pareto_keys[state.thread_index()];
+    for (auto key : keys) {
+      this->cache->cache(std::string(key), "some_value");
+    }
+    state.counters["KeysProcessed"] =
+        benchmark::Counter(keys.size(), benchmark::Counter::kIsRate);
+    state.counters["KeysProcessedInv"] = benchmark::Counter(
+        keys.size(), benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
+  }
+}
+
+// }}}
+
+// Benchmark Run Configuration {{{
+
+void DefaultArgs(benchmark::internal::Benchmark* bench) {
+  bench->Args({SMALL_CACHE, CACHE_OP_COUNT})
+      ->Args({LARGE_CACHE, CACHE_OP_COUNT})
+      ->Threads(THREADS_SINGLE)
+      ->Threads(static_cast<int>(std::thread::hardware_concurrency()))
+      ->Threads(THREADS_LOTS);
+}
+
+BENCHMARK_TEMPLATE_INSTANTIATE_F(CacheFixture, BM_UniqueAdd, SharedLRUAdapter)			->Name("UNIQUE shared")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(CacheFixture, BM_UniqueAdd, SimpleLRUAdapter)			->Name("UNIQUE simple")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(CacheFixture, BM_UniqueAdd, CohortLRUAdapter)			->Name("UNIQUE cohort")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(CacheFixture, BM_UniqueAdd, WebCacheAdapter)			->Name("UNIQUE web   ")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(CacheFixture, BM_UniqueAdd, WebCacheLookupOrAdapter)		->Name("UNIQUE web-O ")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(ParetoFixture, BM_Pareto, SharedLRUAdapter)			->Name("PARETO shared")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(ParetoFixture, BM_Pareto, SimpleLRUAdapter)			->Name("PARETO simple")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(ParetoFixture, BM_Pareto, CohortLRUAdapter)			->Name("PARETO cohort")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(ParetoFixture, BM_Pareto, WebCacheAdapter)			->Name("PARETO web   ")->Apply(DefaultArgs);
+BENCHMARK_TEMPLATE_INSTANTIATE_F(ParetoFixture, BM_Pareto, WebCacheLookupOrAdapter)		->Name("PARETO web-O ")->Apply(DefaultArgs);
+
+// }}}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  auto args = argv_to_vec(argc, argv);
+  auto cct = global_init(
+      nullptr, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+      CINIT_FLAG_NO_MON_CONFIG);
+  common_init_finish(g_ceph_context);
+
+  char arg0_default[] = "benchmark";
+  char* args_default = arg0_default;
+  if (argv == nullptr) {
+    argc = 1;
+    argv = &args_default;
+  }
+  ::benchmark::Initialize(&argc, argv);
+  ::benchmark::RunSpecifiedBenchmarks();
+  ::benchmark::Shutdown();
+  return 0;
+}

From bcaf58f1c4b033bd4ff12d69fd1e11ce75aa2526 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Thu, 20 Mar 2025 17:27:11 +0100
Subject: [PATCH 07/28] =?UTF-8?q?test:=20Add=20Secrets=20Store=20=C2=B5Ben?=
 =?UTF-8?q?chmarks?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Benchmark:
- Linux Kernel Key Retention Service (kernel keystore) [0]
- memfd_secret(2)
- plain memory

Tests:
- Random reads
- (keystore) Write, Read, Remove

[0] https://docs.kernel.org/security/keys/core.html

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit a4b8d7d3c995d696d0495e565321eaf9024af644)
---
 src/test/CMakeLists.txt  |   5 +
 src/test/bench_secmem.cc | 257 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 262 insertions(+)
 create mode 100644 src/test/bench_secmem.cc

diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index 9e96817a908c..0aaac04289c2 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -1015,6 +1015,11 @@ if (benchmark_FOUND)
   bench_lrus.cc
   )
   target_link_libraries(bench_lrus ceph-common global-static benchmark::benchmark Boost::context)
+
+  add_executable(bench_secmem
+  bench_secmem.cc
+  )
+  target_link_libraries(bench_secmem ceph-common global-static benchmark::benchmark keyutils)
 else()
   message(STATUS "The google/benchmark library was not found. Skipping micro benchmark tests")
 endif(benchmark_FOUND)
diff --git a/src/test/bench_secmem.cc b/src/test/bench_secmem.cc
new file mode 100644
index 000000000000..16863bf03831
--- /dev/null
+++ b/src/test/bench_secmem.cc
@@ -0,0 +1,257 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2025 Clyso GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+// Standalone build:
+// g++ --std=c++20 -O2 -o bench ../src/test/bench_secmem.cc -lbenchmark -lkeyutils
+
+#include <benchmark/benchmark.h>
+extern "C" {
+#include <keyutils.h>
+}
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#include <cstring>
+#include <format>
+#include <initializer_list>
+#include <mutex>
+#include <random>
+#include <thread>
+
+namespace {
+
+constexpr size_t MEMFD_SECRET_SIZE = 5L * 1024 * 1024;
+constexpr size_t SECRET_SIZE = 128;
+
+void
+BM_MemfdSecret_RandomRead(benchmark::State& state)
+{
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_int_distribution<> dis(0, (MEMFD_SECRET_SIZE / SECRET_SIZE) - 1);
+  static std::once_flag mem_initialized;
+  static void* mem;
+  std::call_once(mem_initialized, [&]() {
+    const int fd = static_cast<int>(syscall(SYS_memfd_secret, 0));
+    if (fd == -1) {
+      state.SkipWithError(
+          std::format(
+              "memfd_secret: {} errno:{} error:{}", fd, errno,
+              std::strerror(errno)));
+      return;
+    }
+    const int ret = ftruncate(fd, MEMFD_SECRET_SIZE);
+    if (ret == -1) {
+      state.SkipWithError(
+          std::format(
+              "ftruncate: {} errno:{} error:{}", ret, errno,
+              std::strerror(errno)));
+      return;
+    }
+    mem = ::mmap(
+        nullptr, MEMFD_SECRET_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (mem == MAP_FAILED) {
+      state.SkipWithError(
+          std::format(
+              "mmap: {} errno:{} error:{}", ret, errno, std::strerror(errno)));
+      return;
+    }
+    std::memset(mem, '$', MEMFD_SECRET_SIZE);
+  });
+
+  if (state.thread_index() == 0) {
+  }
+
+  for (auto _ : state) {
+    const size_t keys_to_read = state.range(0) / state.threads();
+    thread_local size_t bytes_read = 0;
+    for (size_t i = 0; i < keys_to_read; ++i) {
+      std::string buf(SECRET_SIZE, '_');
+      const size_t offset = dis(gen) * SECRET_SIZE;
+      std::memcpy(buf.data(), static_cast<char*>(mem) + offset, SECRET_SIZE);
+      bytes_read += buf.size();
+    }
+    state.counters["BytesRead"] = benchmark::Counter(
+        static_cast<double>(bytes_read), benchmark::Counter::kIsRate);
+    state.counters["KeysRead"] = benchmark::Counter(
+        static_cast<double>(keys_to_read), benchmark::Counter::kIsRate);
+    state.counters["KeysReadInv"] = benchmark::Counter(
+        static_cast<double>(keys_to_read),
+        benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
+  }
+  if (state.thread_index() == 0) {
+  }
+}
+
+void
+BM_PlainMem_RandomRead(benchmark::State& state)
+{
+  std::random_device rd;
+  std::mt19937 gen(rd());
+  std::uniform_int_distribution<> dis(0, (MEMFD_SECRET_SIZE / SECRET_SIZE) - 1);
+  static auto* mem = ::mmap(
+      nullptr, MEMFD_SECRET_SIZE, PROT_READ | PROT_WRITE,
+      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (state.thread_index() == 0) {
+    std::memset(mem, '$', MEMFD_SECRET_SIZE);
+  }
+  for (auto _ : state) {
+    const size_t keys_to_read = state.range(0) / state.threads();
+    thread_local size_t bytes_read = 0;
+    for (size_t i = 0; i < keys_to_read; ++i) {
+      std::string buf(SECRET_SIZE, '_');
+      const size_t offset = dis(gen) * SECRET_SIZE;
+      std::memcpy(buf.data(), static_cast<char*>(mem) + offset, SECRET_SIZE);
+      bytes_read += buf.size();
+    }
+    state.counters["BytesRead"] = benchmark::Counter(
+        static_cast<double>(bytes_read), benchmark::Counter::kIsRate);
+    state.counters["KeysRead"] = benchmark::Counter(
+        static_cast<double>(keys_to_read), benchmark::Counter::kIsRate);
+    state.counters["KeysReadInv"] = benchmark::Counter(
+        static_cast<double>(keys_to_read),
+        benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
+  }
+  if (state.thread_index() == 0) {
+  }
+}
+
+void
+BM_KeyRing_RandomRead(benchmark::State& state)
+{
+  const std::string secret(SECRET_SIZE, '$');
+  static std::vector<key_serial_t> serials;
+  static std::random_device rd;
+  static std::mt19937 gen(rd());
+  static std::uniform_int_distribution<size_t> dis(0, state.range(0) - 1);
+  if (state.thread_index() == 0) {
+    for (size_t i = 0; i < static_cast<size_t>(state.range(0)); ++i) {
+      const key_serial_t serial = ::add_key(
+          "user", std::to_string(i).c_str(), secret.c_str(), secret.size(),
+          KEY_SPEC_PROCESS_KEYRING);
+      if (serial == -1) {
+        state.SkipWithError(
+            std::format(
+                "serial:{} i:{} errno:{} err:{}", serial, i, errno,
+                std::strerror(errno)));
+        return;
+      }
+      serials.push_back(serial);
+    }
+  }
+  for (auto _ : state) {
+    const size_t keys_to_read = state.range(1) / state.threads();
+    thread_local size_t bytes_read = 0;
+    for (size_t i = 0; i < keys_to_read; ++i) {
+      const key_serial_t serial = serials[dis(gen)];
+      std::string out(secret.size(), '$');
+      const auto ret = ::keyctl_read(serial, out.data(), out.size());
+      if (ret < 0) {
+        state.SkipWithError("keyctl_read");
+        return;
+      }
+      bytes_read += ret;
+    }
+    state.counters["BytesRead"] = benchmark::Counter(
+        static_cast<double>(bytes_read), benchmark::Counter::kIsRate);
+    state.counters["KeysRead"] = benchmark::Counter(
+        static_cast<double>(keys_to_read), benchmark::Counter::kIsRate);
+    state.counters["KeysReadInv"] = benchmark::Counter(
+        static_cast<double>(keys_to_read),
+        benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
+  }
+  if (state.thread_index() == 0) {
+  }
+}
+
+void
+BM_KeyRing_WriteReadRemove(benchmark::State& state)
+{
+  const std::string key_prefix = "benchmark_secret_";
+  const std::string secret = "asjdfoilkuj3o8rijsdafo23yo8ifsdjadf";
+  if (state.thread_index() == 0) {
+  }
+  for (auto _ : state) {
+    const size_t keys_to_read = state.range(1) / state.threads();
+    thread_local size_t bytes_read = 0;
+    for (size_t i = 0; i < keys_to_read; ++i) {
+      std::string key(key_prefix);
+      key.append(std::to_string(state.thread_index()));
+      const key_serial_t serial = ::add_key(
+          "user", key.c_str(), secret.c_str(), secret.size(),
+          KEY_SPEC_PROCESS_KEYRING);
+      if (serial == -1) {
+        state.SkipWithError(
+            std::format(
+                "add_key: serial:{} i:{} errno:{} err:{}", serial, i, errno,
+                std::strerror(errno)));
+        return;
+      }
+      std::string out("$", secret.size());
+      const auto read = ::keyctl_read(serial, out.data(), out.size());
+      if (read < 0) {
+        state.SkipWithError(
+            std::format(
+                "keyctl_read(3): {} errno:{} err:{}", read, errno,
+                std::strerror(errno)));
+        return;
+      }
+      bytes_read += read;
+      assert(out == secret);
+
+      const auto inval = ::keyctl_invalidate(serial);
+      if (inval != 0) {
+        state.SkipWithError(std::format("keyctl_invalidate(3): {}", inval));
+        return;
+      }
+    }
+    state.counters["BytesProcessed"] = benchmark::Counter(
+        static_cast<double>(bytes_read), benchmark::Counter::kIsRate);
+    state.counters["KeysProcessed"] = benchmark::Counter(
+        static_cast<double>(keys_to_read), benchmark::Counter::kIsRate);
+    state.counters["KeysProcessedInv"] = benchmark::Counter(
+        static_cast<double>(keys_to_read),
+        benchmark::Counter::kIsRate | benchmark::Counter::kInvert);
+  }
+  if (state.thread_index() == 0) {
+  }
+}
+
+// note: max number of keys is subject to quota. see /proc/key-users
+// increas using /proc/sys/kernel/keys/maxkeys
+BENCHMARK(BM_KeyRing_RandomRead)
+    ->Args({1000, 1000000})
+    ->Threads(1)
+    ->Threads(static_cast<int>(std::thread::hardware_concurrency()))
+    ->Threads(128);
+BENCHMARK(BM_MemfdSecret_RandomRead)
+    ->Args({10000000})
+    ->Threads(1)
+    ->Threads(static_cast<int>(std::thread::hardware_concurrency()))
+    ->Threads(128);
+BENCHMARK(BM_PlainMem_RandomRead)
+    ->Args({10000000})
+    ->Threads(1)
+    ->Threads(static_cast<int>(std::thread::hardware_concurrency()))
+    ->Threads(128);
+BENCHMARK(BM_KeyRing_WriteReadRemove)
+    ->Args({1000, 1000000})
+    ->Threads(1)
+    ->Threads(static_cast<int>(std::thread::hardware_concurrency()))
+    ->Threads(128);
+
+} // namespace
+
+BENCHMARK_MAIN();

From 08aabb36e1b534a7d9a8b7481168515c95e2f644 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Fri, 25 Apr 2025 16:27:57 +0200
Subject: [PATCH 08/28] common: Add Linux Keyring Secret Store Wrapper

Add RAII wrapper around the Linux Key Retention Service
add_key(2), keyctl_read(3), keyctl_invalidate(3)

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 0307119cdfbdbb09285ad82544be0c4be9b69882)
---
 src/common/CMakeLists.txt       |   5 ++
 src/common/keyring.cc           | 143 ++++++++++++++++++++++++++++++++
 src/common/keyring.h            |  84 +++++++++++++++++++
 src/test/CMakeLists.txt         |   2 +-
 src/test/common/CMakeLists.txt  |   9 ++
 src/test/common/test_keyring.cc |  65 +++++++++++++++
 6 files changed, 307 insertions(+), 1 deletion(-)
 create mode 100644 src/common/keyring.cc
 create mode 100644 src/common/keyring.h
 create mode 100644 src/test/common/test_keyring.cc

diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 3da83cd9ed1c..641bebe7ca46 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -246,4 +246,9 @@ if(HAVE_KEYUTILS)
   set(parse_secret_srcs
     secret.c)
   add_library(parse_secret_objs OBJECT ${parse_secret_srcs})
+
+  set(keyring_srcs
+    keyring.cc)
+  add_library(keyring STATIC ${keyring_srcs})
+  target_link_libraries(keyring keyutils::keyutils)
 endif()
diff --git a/src/common/keyring.cc b/src/common/keyring.cc
new file mode 100644
index 000000000000..860ec91ae9f8
--- /dev/null
+++ b/src/common/keyring.cc
@@ -0,0 +1,143 @@
+
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2025 Clyso GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "keyring.h"
+
+#ifndef _WIN32
+extern "C" {
+#include <keyutils.h>
+}
+#endif
+
+namespace ceph {
+
+#ifdef _WIN32
+
+LinuxKeyringSecret::LinuxKeyringSecret(key_serial_t serial, size_t len) noexcept
+    : _len(len), _serial(serial) {}
+
+LinuxKeyringSecret::~LinuxKeyringSecret() noexcept {}
+
+tl::expected<LinuxKeyringSecret, std::error_code> LinuxKeyringSecret::add(
+    const std::string& key, const std::string& secret) noexcept {
+  return tl::unexpected(std::error_code(-ENOSYS, std::system_category()));
+}
+
+bool LinuxKeyringSecret::supported() noexcept {
+  return false;
+}
+
+void LinuxKeyringSecret::initialize_process_keyring() noexcept {
+}
+
+[[nodiscard]] std::error_code LinuxKeyringSecret::read(std::string& out) const {
+  return {-ENOSYS, std::system_category()};
+}
+
+[[nodiscard]] std::error_code LinuxKeyringSecret::remove() const {
+  return {-ENOSYS, std::system_category()};
+}
+
+[[nodiscard]] std::error_code LinuxKeyringSecret::reset() {
+  return {-ENOSYS, std::system_category()};
+}
+
+#else
+
+LinuxKeyringSecret::LinuxKeyringSecret(key_serial_t serial, size_t len) noexcept
+    : _len(len), _serial(serial) {}
+
+LinuxKeyringSecret::~LinuxKeyringSecret() noexcept {
+  reset();
+}
+
+tl::expected<LinuxKeyringSecret, std::error_code> LinuxKeyringSecret::add(
+    const std::string& key, const std::string& secret) noexcept {
+  const auto serial = add_key(
+      "user", key.c_str(), secret.c_str(), secret.size(),
+      KEY_SPEC_PROCESS_KEYRING);
+  if (serial == -1) {
+    return tl::unexpected(std::error_code(errno, std::system_category()));
+  }
+  return LinuxKeyringSecret(serial, secret.size());
+}
+
+bool LinuxKeyringSecret::supported(std::error_code* ec) noexcept {
+  auto secret = LinuxKeyringSecret::add("ceph_test_keyring_support", "ceph");
+  if (!secret) {
+    if (ec != nullptr) {
+      *ec = secret.error();
+    }
+    return false;
+  }
+  std::string out;
+  if (auto err = secret.value().read(out); err) {
+    if (ec != nullptr) {
+      *ec = err;
+    }
+    return false;
+  }
+  if (auto err = secret.value().reset(); err) {
+    if (ec != nullptr) {
+      *ec = err;
+    }
+    return false;
+  }
+  return true;
+}
+
+void LinuxKeyringSecret::initialize_process_keyring() noexcept {
+  keyctl_get_keyring_ID(KEY_SPEC_PROCESS_KEYRING, 1);
+}
+
+[[nodiscard]] std::error_code LinuxKeyringSecret::read(std::string& out) const {
+  out.clear();
+  out.resize(_len);
+  const auto ret = keyctl_read(_serial, out.data(), _len);
+  if (ret == -1) {
+    return {errno, std::system_category()};
+  } else if (static_cast<size_t>(ret) != _len) {
+    return {-EINVAL, std::generic_category()};
+  }
+  return {};
+}
+
+[[nodiscard]] std::error_code LinuxKeyringSecret::remove() const {
+  const auto ret = keyctl_invalidate(_serial);
+  if (ret != 0) {
+    return {errno, std::system_category()};
+  }
+  return {};
+}
+
+[[nodiscard]] std::error_code LinuxKeyringSecret::reset() {
+  if (_serial == -1) {
+    return {-EINVAL, std::generic_category()};
+  }
+  if (const auto ret = remove(); !ret) {
+    return ret;
+  }
+  _serial = -1;
+  _len = 0;
+  return {};
+}
+
+[[nodiscard]] bool LinuxKeyringSecret::initialized() const {
+  return _len > 0 && _serial != -1;
+}
+
+#endif
+
+}  // namespace ceph
diff --git a/src/common/keyring.h b/src/common/keyring.h
new file mode 100644
index 000000000000..63fe29d16f5c
--- /dev/null
+++ b/src/common/keyring.h
@@ -0,0 +1,84 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2025 Clyso GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <ostream>
+#include <system_error>
+#include <utility>
+
+#include "fmt/ostream.h"
+#include "include/expected.hpp"
+
+namespace ceph {
+
+/// RAII style wrapper around a secret stored in the Linux Key
+/// Retention Service.
+/// See: add_key(2), keyctl_read(3), keyctl_invalidate(3)
+class LinuxKeyringSecret {
+  size_t _len;
+  int _serial;
+
+  explicit LinuxKeyringSecret(int serial, size_t len) noexcept;
+ public:
+  LinuxKeyringSecret() = delete;
+  LinuxKeyringSecret(const LinuxKeyringSecret&) = delete;
+  LinuxKeyringSecret& operator=(const LinuxKeyringSecret&) = delete;
+  LinuxKeyringSecret(LinuxKeyringSecret&& other) noexcept
+      : _len(std::exchange(other._len, 0)),
+        _serial(std::exchange(other._serial, -1)) {};
+  LinuxKeyringSecret& operator=(LinuxKeyringSecret&& other) noexcept {
+    if (this != &other) {
+      if (this->initialized()) {
+	this->reset();
+      }
+      _len = std::exchange(other._len, 0);
+      _serial = std::exchange(other._serial, -1);
+    }
+    return *this;
+  };
+
+  ~LinuxKeyringSecret() noexcept;
+
+  // Add a secret to the kernel process keyring. Beware: calling this
+  // with with an existing key _updates_ the value and causes multiple
+  // instances to refer to the same key.
+  static tl::expected<LinuxKeyringSecret, std::error_code> add(
+      const std::string& key, const std::string& secret) noexcept;
+  static bool supported(std::error_code* ec = nullptr) noexcept;
+  // Initialize the process keyring. Do this before starting any
+  // threads that want to share possession of keys in the process
+  // keyring
+  static void initialize_process_keyring() noexcept;
+  [[nodiscard]] std::error_code read(std::string& out) const;
+  [[nodiscard]] std::error_code remove() const;
+  [[nodiscard]] bool initialized() const;
+ private:
+  std::error_code reset();
+
+  friend std::ostream& operator<<(
+      std::ostream& os, const LinuxKeyringSecret& secret) {
+    if (secret._serial == -1) {
+      return os << "LinuxKeyringSecret{}";
+    }
+    return os << "LinuxKeyringSecret{" << secret._serial << "}";
+  }
+  friend class LinuxKeyringTest_LifecycleMoveAssignResetsDestination_Test;
+};
+}  // namespace ceph
+
+#if FMT_VERSION >= 90000
+template <>
+struct fmt::formatter<ceph::LinuxKeyringSecret> : fmt::ostream_formatter {};
+#endif
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index 0aaac04289c2..7993d3e6cdac 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -1019,7 +1019,7 @@ if (benchmark_FOUND)
   add_executable(bench_secmem
   bench_secmem.cc
   )
-  target_link_libraries(bench_secmem ceph-common global-static benchmark::benchmark keyutils)
+  target_link_libraries(bench_secmem ceph-common global-static benchmark::benchmark keyutils::keyutils)
 else()
   message(STATUS "The google/benchmark library was not found. Skipping micro benchmark tests")
 endif(benchmark_FOUND)
diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index 7e212f88b081..0296690f2dee 100644
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -121,6 +121,15 @@ add_executable(unittest_web_cache
 add_ceph_unittest(unittest_web_cache)
 target_link_libraries(unittest_web_cache ceph-common)
 
+if(HAVE_KEYUTILS)
+  # unittest_web_cache
+  add_executable(unittest_keyring
+    test_keyring.cc
+    )
+  add_ceph_unittest(unittest_keyring)
+  target_link_libraries(unittest_keyring global keyring)
+endif()
+
 # unittest_sloppy_crc_map
 add_executable(unittest_sloppy_crc_map
   test_sloppy_crc_map.cc
diff --git a/src/test/common/test_keyring.cc b/src/test/common/test_keyring.cc
new file mode 100644
index 000000000000..ced857e97772
--- /dev/null
+++ b/src/test/common/test_keyring.cc
@@ -0,0 +1,65 @@
+#include <common/keyring.h>
+#include <gmock/gmock-matchers.h>
+#include <gtest/gtest.h>
+extern "C" {
+#include <keyutils.h>
+}
+
+namespace ceph {
+
+class LinuxKeyringTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    std::error_code ec;
+    if (!LinuxKeyringSecret::supported(&ec)) {
+      GTEST_SKIP() << "Linux Keyring is unsupported. " << ec
+                   << ". Skipping test";
+    }
+  }
+};
+
+TEST_F(LinuxKeyringTest, Basics) {
+  std::string secret("secret");
+  auto maybe_keyring_secret = LinuxKeyringSecret::add("testkey", secret);
+
+  ASSERT_TRUE(maybe_keyring_secret.has_value()) << maybe_keyring_secret.error();
+  auto keyring_secret = std::move(maybe_keyring_secret.value());
+
+  std::string out;
+  ASSERT_FALSE(keyring_secret.read(out));
+  ASSERT_EQ(secret, out);
+
+  ASSERT_FALSE(keyring_secret.remove());
+  ASSERT_TRUE(keyring_secret.read(out));
+}
+
+TEST_F(LinuxKeyringTest, Lifecycle) {
+  std::string secret("secret");
+  auto uut = LinuxKeyringSecret::add("testkey", secret);
+  ASSERT_TRUE(uut.has_value());
+  ASSERT_TRUE(uut->initialized());
+  auto next = std::move(uut);
+  ASSERT_FALSE(uut->initialized());
+  ASSERT_TRUE(next->initialized());
+}
+
+TEST_F(LinuxKeyringTest, LifecycleMoveAssignResetsDestination) {
+  std::string secret("secret");
+  auto dest = LinuxKeyringSecret::add("testkey", secret);
+  auto source = LinuxKeyringSecret::add("testkey2", secret);
+  ASSERT_TRUE(dest.has_value());
+  ASSERT_TRUE(source.has_value());
+
+  auto dest_serial = dest->_serial;
+  ASSERT_NE(-1, dest_serial);
+
+  dest = std::move(source);
+
+  std::array<char, 1024> buf = {0};
+  auto ret = keyctl_describe(dest_serial, buf.data(), buf.size());
+
+  EXPECT_EQ(-1, ret) << buf.data();
+  ASSERT_EQ(ENOKEY, errno);
+}
+
+}  // namespace ceph

From 8a527be09b47a21867c552f64cadc18e5f41f339 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Fri, 13 Jun 2025 16:45:41 +0200
Subject: [PATCH 09/28] rgw: Early Linux process keyring initialization

To allow RGW threads to share possession over process keyring keys the
keyring must be created before a child thread adds keys.

Since we only use the process keyring for KMS cache secrets, only
initialize the keyring if it is enabled on startup.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 000046793dab0f56bcc46aa318d9d0619bd850f8)
---
 src/rgw/CMakeLists.txt | 1 +
 src/rgw/rgw_main.cc    | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index b090b2eeeb8f..5e35b3c57fa6 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -285,6 +285,7 @@ target_link_libraries(rgw_common
     cls_version_client
     librados
     libneorados
+    keyring
     rt
     ICU::uc
     OATH::OATH
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 5536e2810f6a..560b00dbf649 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -3,6 +3,7 @@
 
 #include <boost/intrusive/list.hpp>
 #include "common/ceph_argparse.h"
+#include "common/keyring.h"
 #include "global/global_init.h"
 #include "global/signal_handler.h"
 #include "common/config.h"
@@ -107,6 +108,8 @@ int main(int argc, char *argv[])
   DoutPrefix dp(cct.get(), dout_subsys, "rgw main: ");
   rgw::AppMain main(&dp);
 
+  LinuxKeyringSecret::initialize_process_keyring();
+
   main.init_frontends1(false /* nfs */);
   main.init_numa();
 

From d585b23e63d2533688cd2b22f60a5cc0d48c2a80 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Fri, 27 Jun 2025 12:22:06 +0200
Subject: [PATCH 10/28] doc: Document RGW KMS Cache

Add caching section to the RGW Encryption docs. Add cache
settings to the RGW configuration reference.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit a88f02959bb6d7125ebe878ecf796046b6b6f559)
---
 doc/radosgw/config-ref.rst | 10 ++++++++
 doc/radosgw/encryption.rst | 48 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst
index b4c4829ae9d3..c403ba320427 100644
--- a/doc/radosgw/config-ref.rst
+++ b/doc/radosgw/config-ref.rst
@@ -271,6 +271,16 @@ SSE-S3 Settings
 .. confval:: rgw_crypt_sse_s3_vault_ssl_clientcert
 .. confval:: rgw_crypt_sse_s3_vault_ssl_clientkey
 
+KMS Secrets Cache Settings
+==========================
+
+.. confval:: rgw_crypt_s3_kms_cache_enabled
+.. confval:: rgw_crypt_s3_kms_cache_max_size
+.. confval:: rgw_crypt_s3_kms_cache_positive_ttl
+.. confval:: rgw_crypt_s3_kms_cache_transient_error_ttl
+.. confval:: rgw_crypt_s3_kms_cache_negative_ttl
+
+.. _Encryption: ../encryption
 
 QoS settings
 ------------
diff --git a/doc/radosgw/encryption.rst b/doc/radosgw/encryption.rst
index e30fe1468725..dc55a5ec0b68 100644
--- a/doc/radosgw/encryption.rst
+++ b/doc/radosgw/encryption.rst
@@ -81,7 +81,53 @@ The configuration expects a base64-encoded 256 bit key. For example::
    file is not a secure method for storing encryption keys. Keys that are
    accidentally exposed in this way should be considered compromised.
 
-
+Caching
+=======
+
+The caching feature for Key Management Service (KMS) secrets greatly
+improves the performance of server-side encryption and lessens the
+load on the KMS.
+
+Secrets are stored using the `Linux Kernel Key Retention Service`_ in
+the RGW processes' process keyring. This is subject to a global quota
+and must be set in accordance with the configured cache size.
+Depending on whether RGW runs as root, these quotas can be managed by adjusting:
+- ``/proc/sys/kernel/keys/root_maxkeys`` and ``/proc/sys/kernel/keys/root_maxbytes``
+- ``/proc/sys/kernel/keys/maxkeys`` and ``/proc/sys/kernel/keys/maxbytes``
+
+Exceeding a quota will disable the cache, fail the request with an
+internal error, and log a failure message.
+
+Three different Cache Time-to-Live (TTL) values can be set:
+- **Positive TTL**: How long a successfully retrieved secret remains
+  in the cache.
+- **Negative TTL**: How long to remember that a key does not exist,
+  preventing unnecessary requests to the KMS.
+- **Transient Error TTL**: How long to cache failures due to temporary
+  issues like KMS timeouts.
+
+Metrics
+---------
+
+The cache exports metrics under the ``kms-cache`` collection.
+- ``hit``: Hit counter
+- ``miss``:  Miss counter
+- ``expired``: Number of TTL expired entries
+- ``size``: Current cache size
+- ``capacity``: Cache maximum size
+- ``clear``: Number of cache clears. Resets ``size``, ``hit``,
+  ``miss``, ``expired``
+
+In addition the ``rgw`` collection has:
+- ``kms_fetch_lat``: Average KMS fetch latency. Also includes a
+  successful request counter. Each event results in a positive cache
+  entry.
+- ``kms_error_transient``: Transient KMS fetch error counter. Each
+  event results in a transient error cache entry.
+- ``kms_error_permanent``: Permanent KMS fetch error counter. Each
+  event results in a negative cache cache entry.
+
+.. _Linux Kernel Key Retention Service:  https://www.kernel.org/doc/html/latest/security/keys/core.html
 .. _Amazon SSE-C: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
 .. _Amazon SSE-KMS: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
 .. _Amazon SSE-S3: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html

From cefee72aa08370246c49ae79a10c00bc52228e3f Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Thu, 18 Dec 2025 19:42:07 +0100
Subject: [PATCH 11/28] common: Refactor LinuxKeyringSecret into Keyring
 Interface

Goal: Support multiple backends and faking / mocking for testing.

Add abstract classes Keyring (factory) and KeyringSecret. Add
"Unsupported" implementation for non-Linux platforms. Add a get_best
factory function that currently returns the LinuxKeyring impl on Linux
or Unsupported elsewhere.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 35f2bb4c11da480950dd653955f45f20815556e2)
---
 src/common/keyring.cc           | 58 ++++++++++++---------
 src/common/keyring.h            | 92 ++++++++++++++++++++++++++++-----
 src/test/common/test_keyring.cc | 34 +++++++-----
 3 files changed, 131 insertions(+), 53 deletions(-)

diff --git a/src/common/keyring.cc b/src/common/keyring.cc
index 860ec91ae9f8..b6823753efb8 100644
--- a/src/common/keyring.cc
+++ b/src/common/keyring.cc
@@ -15,7 +15,9 @@
 
 #include "keyring.h"
 
-#ifndef _WIN32
+#include <memory>
+
+#if defined(__linux__)
 extern "C" {
 #include <keyutils.h>
 }
@@ -23,38 +25,41 @@ extern "C" {
 
 namespace ceph {
 
-#ifdef _WIN32
-
-LinuxKeyringSecret::LinuxKeyringSecret(key_serial_t serial, size_t len) noexcept
-    : _len(len), _serial(serial) {}
-
-LinuxKeyringSecret::~LinuxKeyringSecret() noexcept {}
+std::unique_ptr<Keyring> Keyring::get_best() {
+#if defined(__linux__)
+  return std::make_unique<LinuxKeyring>();
+#else
+  return std::make_unique<UnsupportedKeyring>();
+#endif
+}
 
-tl::expected<LinuxKeyringSecret, std::error_code> LinuxKeyringSecret::add(
+tl::expected<std::unique_ptr<KeyringSecret>, std::error_code>
+UnsupportedKeyring::add(
     const std::string& key, const std::string& secret) noexcept {
   return tl::unexpected(std::error_code(-ENOSYS, std::system_category()));
 }
 
-bool LinuxKeyringSecret::supported() noexcept {
+bool UnsupportedKeyring::supported(std::error_code* ec) noexcept {
+  if (ec != nullptr) {
+    *ec = {-ENOSYS, std::system_category()};
+  }
   return false;
 }
 
-void LinuxKeyringSecret::initialize_process_keyring() noexcept {
-}
-
-[[nodiscard]] std::error_code LinuxKeyringSecret::read(std::string& out) const {
+[[nodiscard]] std::error_code UnsupportedKeyringSecret::read(
+    std::string& out) const {
   return {-ENOSYS, std::system_category()};
 }
 
-[[nodiscard]] std::error_code LinuxKeyringSecret::remove() const {
+[[nodiscard]] std::error_code UnsupportedKeyringSecret::remove() const {
   return {-ENOSYS, std::system_category()};
 }
 
-[[nodiscard]] std::error_code LinuxKeyringSecret::reset() {
-  return {-ENOSYS, std::system_category()};
+[[nodiscard]] bool UnsupportedKeyringSecret::initialized() const {
+  return false;
 }
 
-#else
+#if defined(__linux__)
 
 LinuxKeyringSecret::LinuxKeyringSecret(key_serial_t serial, size_t len) noexcept
     : _len(len), _serial(serial) {}
@@ -63,7 +68,7 @@ LinuxKeyringSecret::~LinuxKeyringSecret() noexcept {
   reset();
 }
 
-tl::expected<LinuxKeyringSecret, std::error_code> LinuxKeyringSecret::add(
+tl::expected<std::unique_ptr<KeyringSecret>, std::error_code> LinuxKeyring::add(
     const std::string& key, const std::string& secret) noexcept {
   const auto serial = add_key(
       "user", key.c_str(), secret.c_str(), secret.size(),
@@ -71,25 +76,28 @@ tl::expected<LinuxKeyringSecret, std::error_code> LinuxKeyringSecret::add(
   if (serial == -1) {
     return tl::unexpected(std::error_code(errno, std::system_category()));
   }
-  return LinuxKeyringSecret(serial, secret.size());
+  return std::unique_ptr<KeyringSecret>(
+      new LinuxKeyringSecret(serial, secret.size()));
 }
 
-bool LinuxKeyringSecret::supported(std::error_code* ec) noexcept {
-  auto secret = LinuxKeyringSecret::add("ceph_test_keyring_support", "ceph");
-  if (!secret) {
+bool LinuxKeyring::supported(std::error_code* ec) noexcept {
+  LinuxKeyring keyring;
+  auto maybe_secret = keyring.add("ceph_test_keyring_support", "ceph");
+  if (!maybe_secret) {
     if (ec != nullptr) {
-      *ec = secret.error();
+      *ec = maybe_secret.error();
     }
     return false;
   }
   std::string out;
-  if (auto err = secret.value().read(out); err) {
+  auto* secret = static_cast<LinuxKeyringSecret*>(maybe_secret.value().get());
+  if (auto err = secret->read(out); err) {
     if (ec != nullptr) {
       *ec = err;
     }
     return false;
   }
-  if (auto err = secret.value().reset(); err) {
+  if (auto err = secret->reset(); err) {
     if (ec != nullptr) {
       *ec = err;
     }
diff --git a/src/common/keyring.h b/src/common/keyring.h
index 63fe29d16f5c..48e1ee59a99c 100644
--- a/src/common/keyring.h
+++ b/src/common/keyring.h
@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include <memory>
 #include <ostream>
 #include <system_error>
 #include <utility>
@@ -23,14 +24,62 @@
 
 namespace ceph {
 
+class KeyringSecret {
+ public:
+  virtual ~KeyringSecret() noexcept = default;
+  [[nodiscard]] virtual std::error_code read(std::string& out) const = 0;
+  [[nodiscard]] virtual std::error_code remove() const = 0;
+  [[nodiscard]] virtual bool initialized() const = 0;
+  friend std::ostream& operator<<(
+      std::ostream& os, const KeyringSecret& secret) {
+    return os;
+  }
+};
+
+class Keyring {
+ public:
+  virtual ~Keyring() noexcept = default;
+  virtual tl::expected<std::unique_ptr<KeyringSecret>, std::error_code> add(
+      const std::string& key, const std::string& secret) noexcept = 0;
+  virtual bool supported(std::error_code* ec) noexcept = 0;
+  [[nodiscard]] virtual std::string_view name() const noexcept = 0;
+
+  static std::unique_ptr<Keyring> get_best();
+};
+
+class UnsupportedKeyringSecret : public KeyringSecret {
+ public:
+  ~UnsupportedKeyringSecret() noexcept override = default;
+  [[nodiscard]] std::error_code read(std::string& out) const override;
+  [[nodiscard]] std::error_code remove() const override;
+  [[nodiscard]] bool initialized() const override;
+  friend std::ostream& operator<<(
+      std::ostream& os, const UnsupportedKeyringSecret& secret) {
+    os << "UnsupportedKeyringSecret{}";
+    return os;
+  }
+};
+
+class UnsupportedKeyring : public Keyring {
+ public:
+  ~UnsupportedKeyring() noexcept override = default;
+  tl::expected<std::unique_ptr<KeyringSecret>, std::error_code> add(
+      const std::string& key, const std::string& secret) noexcept override;
+  bool supported(std::error_code* ec) noexcept override;
+  [[nodiscard]] std::string_view name() const noexcept override {
+    return "Unsupported";
+  }
+};
+
 /// RAII style wrapper around a secret stored in the Linux Key
 /// Retention Service.
 /// See: add_key(2), keyctl_read(3), keyctl_invalidate(3)
-class LinuxKeyringSecret {
+class LinuxKeyringSecret : public KeyringSecret {
   size_t _len;
   int _serial;
 
   explicit LinuxKeyringSecret(int serial, size_t len) noexcept;
+
  public:
   LinuxKeyringSecret() = delete;
   LinuxKeyringSecret(const LinuxKeyringSecret&) = delete;
@@ -38,10 +87,11 @@ class LinuxKeyringSecret {
   LinuxKeyringSecret(LinuxKeyringSecret&& other) noexcept
       : _len(std::exchange(other._len, 0)),
         _serial(std::exchange(other._serial, -1)) {};
+
   LinuxKeyringSecret& operator=(LinuxKeyringSecret&& other) noexcept {
     if (this != &other) {
       if (this->initialized()) {
-	this->reset();
+        this->reset();
       }
       _len = std::exchange(other._len, 0);
       _serial = std::exchange(other._serial, -1);
@@ -49,21 +99,17 @@ class LinuxKeyringSecret {
     return *this;
   };
 
-  ~LinuxKeyringSecret() noexcept;
+  ~LinuxKeyringSecret() noexcept override;
 
-  // Add a secret to the kernel process keyring. Beware: calling this
-  // with with an existing key _updates_ the value and causes multiple
-  // instances to refer to the same key.
-  static tl::expected<LinuxKeyringSecret, std::error_code> add(
-      const std::string& key, const std::string& secret) noexcept;
-  static bool supported(std::error_code* ec = nullptr) noexcept;
   // Initialize the process keyring. Do this before starting any
   // threads that want to share possession of keys in the process
   // keyring
   static void initialize_process_keyring() noexcept;
-  [[nodiscard]] std::error_code read(std::string& out) const;
-  [[nodiscard]] std::error_code remove() const;
-  [[nodiscard]] bool initialized() const;
+
+  [[nodiscard]] std::error_code read(std::string& out) const override;
+  [[nodiscard]] std::error_code remove() const override;
+  [[nodiscard]] bool initialized() const override;
+
  private:
   std::error_code reset();
 
@@ -75,10 +121,28 @@ class LinuxKeyringSecret {
     return os << "LinuxKeyringSecret{" << secret._serial << "}";
   }
   friend class LinuxKeyringTest_LifecycleMoveAssignResetsDestination_Test;
+  friend class LinuxKeyring;
+};
+
+class LinuxKeyring : public Keyring {
+ public:
+  // Add a secret to the kernel process keyring. Beware: calling this
+  // with with an existing key _updates_ the value and causes multiple
+  // instances to refer to the same key.
+  tl::expected<std::unique_ptr<KeyringSecret>, std::error_code> add(
+      const std::string& key, const std::string& secret) noexcept override;
+  bool supported(std::error_code* ec) noexcept override;
+  [[nodiscard]] std::string_view name() const noexcept override {
+    return "Linux Kernel Key Retention Service";
+  };
+
+  ~LinuxKeyring() override = default;
 };
 }  // namespace ceph
 
 #if FMT_VERSION >= 90000
-template <>
-struct fmt::formatter<ceph::LinuxKeyringSecret> : fmt::ostream_formatter {};
+template <typename T>
+struct fmt::formatter<
+    T, std::enable_if_t<std::is_base_of_v<ceph::KeyringSecret, T>, char>>
+    : fmt::ostream_formatter {};
 #endif
diff --git a/src/test/common/test_keyring.cc b/src/test/common/test_keyring.cc
index ced857e97772..1c591c514673 100644
--- a/src/test/common/test_keyring.cc
+++ b/src/test/common/test_keyring.cc
@@ -9,9 +9,13 @@ namespace ceph {
 
 class LinuxKeyringTest : public ::testing::Test {
  protected:
+  std::unique_ptr<Keyring> keyring;
+
+  LinuxKeyringTest() : keyring(new LinuxKeyring()) {}
+
   void SetUp() override {
     std::error_code ec;
-    if (!LinuxKeyringSecret::supported(&ec)) {
+    if (!keyring->supported(&ec)) {
       GTEST_SKIP() << "Linux Keyring is unsupported. " << ec
                    << ". Skipping test";
     }
@@ -20,37 +24,39 @@ class LinuxKeyringTest : public ::testing::Test {
 
 TEST_F(LinuxKeyringTest, Basics) {
   std::string secret("secret");
-  auto maybe_keyring_secret = LinuxKeyringSecret::add("testkey", secret);
+  auto maybe_keyring_secret = keyring->add("testkey", secret);
 
   ASSERT_TRUE(maybe_keyring_secret.has_value()) << maybe_keyring_secret.error();
   auto keyring_secret = std::move(maybe_keyring_secret.value());
 
   std::string out;
-  ASSERT_FALSE(keyring_secret.read(out));
+  ASSERT_FALSE(keyring_secret->read(out));
   ASSERT_EQ(secret, out);
 
-  ASSERT_FALSE(keyring_secret.remove());
-  ASSERT_TRUE(keyring_secret.read(out));
+  ASSERT_FALSE(keyring_secret->remove());
+  ASSERT_TRUE(keyring_secret->read(out));
 }
 
 TEST_F(LinuxKeyringTest, Lifecycle) {
   std::string secret("secret");
-  auto uut = LinuxKeyringSecret::add("testkey", secret);
-  ASSERT_TRUE(uut.has_value());
-  ASSERT_TRUE(uut->initialized());
-  auto next = std::move(uut);
-  ASSERT_FALSE(uut->initialized());
-  ASSERT_TRUE(next->initialized());
+  auto maybe = keyring->add("testkey", secret);
+  ASSERT_TRUE(maybe.has_value());
+  auto* ptr = dynamic_cast<LinuxKeyringSecret*>(maybe.value().get());
+  ASSERT_TRUE(ptr->initialized());
+  auto next = std::move(*ptr);
+  ASSERT_TRUE(next.initialized());
+  ASSERT_FALSE(ptr->initialized());
 }
 
 TEST_F(LinuxKeyringTest, LifecycleMoveAssignResetsDestination) {
   std::string secret("secret");
-  auto dest = LinuxKeyringSecret::add("testkey", secret);
-  auto source = LinuxKeyringSecret::add("testkey2", secret);
+  auto dest = keyring->add("testkey", secret);
+  auto source = keyring->add("testkey2", secret);
   ASSERT_TRUE(dest.has_value());
   ASSERT_TRUE(source.has_value());
 
-  auto dest_serial = dest->_serial;
+  const auto* lks = dynamic_cast<LinuxKeyringSecret*>(dest.value().get());
+  auto dest_serial = lks->_serial;
   ASSERT_NE(-1, dest_serial);
 
   dest = std::move(source);

From f2e42e8649594bc782c8caa7ceef56204376a0b8 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Thu, 19 Dec 2024 15:41:30 +0100
Subject: [PATCH 12/28] rgw: SSE-KMS Secrets Cache

Add SSE Key Management System secrets cache to RGW.

It is common to have secrets shared by many if not all objects in a
bucket. Without RGW-side caching every PUT/GET will cause a request to
an external KSM. This not only adds load to the KSM, but also slows
down read and writes.

Combine WebCache, ceph::async::call_once and LinuxKeyringSecret into
KMSCache. WebCache stores async::once_result to wrap results of a KMS
secret fetch to mitigate cache stampedes (concurrent cache requests to
the same key coalesce into one). The retrieved secrets are stored in
the Linux kernel key retention service (LinuxKeyringSecret) for safe
keeping and retrial by subsequent requests. KMSCache adds a TTL reaper
and life cycle.

Cache values and error handling: The cache stores positive
fetch results, permanent errors (e.g key does not exists) and
transient errors (e.g fetch timeout). Each with a different TTL.

Unit tests to cover cached / uncached KMS retrieve and runtime cache
disable via config.

Add perf counter `kms_fetch_lat` to track KMS fetch request latency
and error counters to track permanent, transient and key store
errors.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Fixes: https://tracker.ceph.com/issues/68524
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit b6186fc596dfab6442fb55f179ca72e8bc0fde60)
---
 src/common/options/rgw.yaml.in     |  68 ++++++
 src/rgw/CMakeLists.txt             |   1 +
 src/rgw/rgw_appmain.cc             |  19 ++
 src/rgw/rgw_crypt.cc               |   5 +-
 src/rgw/rgw_kms.cc                 |  82 +++++--
 src/rgw/rgw_kms.h                  |   6 +
 src/rgw/rgw_kms_cache.cc           | 256 ++++++++++++++++++++
 src/rgw/rgw_kms_cache.h            | 111 +++++++++
 src/rgw/rgw_main.cc                |   1 +
 src/rgw/rgw_main.h                 |   1 +
 src/rgw/rgw_perf_counters.cc       |   5 +
 src/rgw/rgw_perf_counters.h        |   5 +
 src/rgw/rgw_process_env.h          |   3 +
 src/test/rgw/CMakeLists.txt        |   7 +
 src/test/rgw/test_rgw_kms_cache.cc | 368 +++++++++++++++++++++++++++++
 15 files changed, 911 insertions(+), 27 deletions(-)
 create mode 100644 src/rgw/rgw_kms_cache.cc
 create mode 100644 src/rgw/rgw_kms_cache.h
 create mode 100644 src/test/rgw/test_rgw_kms_cache.cc

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index d2f5a59e2b98..48750020ea28 100644
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -2818,12 +2818,80 @@ options:
   with_legacy: true
 # extra keys that may be used for aws:kms
 # defined as map "key1=YmluCmJvb3N0CmJvb3N0LQ== key2=b3V0CnNyYwpUZXN0aW5nCg=="
+- name: rgw_crypt_s3_kms_cache_enabled
+  type: bool
+  level: advanced
+  desc: Enable caching of encryption keys for SSE-KMS.
+  default: true
+  with_legacy: true
+  services:
+  - rgw
+  see_also:
+  - rgw_crypt_s3_kms_cache_max_size
+  - rgw_crypt_s3_kms_cache_positive_ttl
+  - rgw_crypt_s3_kms_cache_transient_error_ttl
+  - rgw_crypt_s3_kms_cache_negative_ttl
+- name: rgw_crypt_s3_kms_cache_max_size
+  type: uint
+  level: advanced
+  desc: Maximum number of SSE-KMS secrets cached.
+    Each key consumes 32 byte (AES-256) + overhead in memory
+  default: 128
+  with_legacy: true
+  services:
+  - rgw
+  see_also:
+  - rgw_crypt_s3_kms_cache_enabled
+- name: rgw_crypt_s3_kms_cache_positive_ttl
+  type: uint
+  level: advanced
+  desc: Time in seconds after which the KMS cache evicts successful lookups.
+  services:
+  - rgw
+  with_legacy: true
+  default: 60
+  min: 10
+  max: 3600
+  see_also:
+  - rgw_crypt_s3_kms_cache_enabled
 - name: rgw_crypt_s3_kms_encryption_keys
   type: str
   level: dev
   services:
   - rgw
   with_legacy: true
+- name: rgw_crypt_s3_kms_cache_transient_error_ttl
+  type: uint
+  level: advanced
+  desc: Time in seconds after which the KMS cache evicts entries representing transient errors (timeouts, temporary outages, misconfiguration, etc).
+  services:
+  - rgw
+  with_legacy: true
+  default: 10
+  min: 0
+  max: 3600
+  see_also:
+  - rgw_crypt_s3_kms_cache_enabled
+- name: rgw_crypt_s3_kms_cache_negative_ttl
+  type: uint
+  level: advanced
+  desc: Time in seconds after which the KMS cache evicts permanent look-up errors (e.g key does not exist).
+  services:
+  - rgw
+  with_legacy: true
+  default: 120
+  min: 0
+  max: 3600
+  see_also:
+  - rgw_crypt_s3_kms_cache_enabled
+- name: rgw_crypt_s3_kms_testing_delay
+  type: uint
+  level: dev
+  desc: Add delay in milliseconds to the 'testing' KMS key retrieval.
+  services:
+  - rgw
+  with_legacy: true
+  default: 0
 - name: rgw_crypt_vault_auth
   type: str
   level: advanced
diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index 5e35b3c57fa6..216e6e6286c6 100644
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -141,6 +141,7 @@ set(librgw_common_srcs
   rgw_rest_iam.cc
   rgw_object_lock.cc
   rgw_kms.cc
+  rgw_kms_cache.cc
   rgw_kmip_client.cc
   rgw_url.cc
   rgw_oidc_provider.cc
diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc
index 73d645e45abc..e0b34c44023f 100644
--- a/src/rgw/rgw_appmain.cc
+++ b/src/rgw/rgw_appmain.cc
@@ -25,6 +25,7 @@
 #include "include/compat.h"
 #include "include/str_list.h"
 #include "include/stringify.h"
+#include "rgw_kms_cache.h"
 #include "rgw_main.h"
 #include "rgw_common.h"
 #include "rgw_sal.h"
@@ -472,6 +473,13 @@ int rgw::AppMain::init_frontends2(RGWLib* rgwlib)
     else if (framework == "beast") {
       need_context_pool();
       fe = new RGWAsioFrontend(env, config, *sched_ctx, *context_pool);
+      if (context_pool.has_value() &&
+          g_conf()->rgw_crypt_s3_kms_cache_enabled) {
+        env.kms_cache->initialize_ttl_reaper(
+            g_conf()->rgw_beast_enable_async
+                ? std::optional(context_pool.value().get_executor())
+                : nullopt);
+      }
     }
     else if (framework == "rgw-nfs") {
       fe = new RGWLibFrontend(env, config);
@@ -578,6 +586,15 @@ void rgw::AppMain::init_lua()
   }
 } /* init_lua */
 
+void rgw::AppMain::init_kms_cache()
+{
+  if (!g_conf().get_val<bool>("rgw_crypt_s3_kms_cache_enabled")) {
+    return;
+  }
+  env.kms_cache = std::make_unique<rgw::kms::KMSCache>(
+      dpp->get_cct(), Keyring::get_best());
+}
+
 void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
 {
   if (env.driver->get_name() == "rados") {
@@ -585,6 +602,8 @@ void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
     static_cast<rgw::sal::RadosLuaManager*>(env.lua.manager.get())->unwatch_reload(dpp);
   }
 
+  env.kms_cache.reset();
+
   for (auto& fe : fes) {
     fe->stop();
   }
diff --git a/src/rgw/rgw_crypt.cc b/src/rgw/rgw_crypt.cc
index 58874bf22b9d..5ad2d6987c55 100644
--- a/src/rgw/rgw_crypt.cc
+++ b/src/rgw/rgw_crypt.cc
@@ -17,6 +17,7 @@
 #include "crypto/crypto_accel.h"
 #include "crypto/crypto_plugin.h"
 #include "rgw/rgw_kms.h"
+#include "rgw/rgw_process_env.h"
 #include "rapidjson/document.h"
 #include "rapidjson/writer.h"
 #include "rapidjson/error/error.h"
@@ -1176,7 +1177,7 @@ int rgw_s3_prepare_encrypt(req_state* s, optional_yield y,
         set_attr(attrs, RGW_ATTR_CRYPT_KEYSEL, key_selector);
         set_attr(attrs, RGW_ATTR_CRYPT_CONTEXT, cooked_context);
         std::string actual_key;
-        res = make_actual_key_from_kms(s, attrs, y, actual_key);
+        res = make_actual_key_from_kms(s, attrs, s->penv.kms_cache.get(), y, actual_key);
         if (res != 0) {
           ldpp_dout(s, 5) << "ERROR: failed to retrieve actual key from key_id: " << key_id << dendl;
           s->err.message = "Failed to retrieve the actual key, kms-keyid: " + std::string(key_id);
@@ -1407,7 +1408,7 @@ int rgw_s3_prepare_decrypt(req_state* s, optional_yield y,
     /* try to retrieve actual key */
     std::string key_id = get_str_attribute(attrs, RGW_ATTR_CRYPT_KEYID);
     std::string actual_key;
-    res = reconstitute_actual_key_from_kms(s, attrs, y, actual_key);
+    res = reconstitute_actual_key_from_kms(s, attrs, s->penv.kms_cache.get(), y, actual_key);
     if (res != 0) {
       ldpp_dout(s, 10) << "ERROR: failed to retrieve actual key from key_id: " << key_id << dendl;
       s->err.message = "Failed to retrieve the actual key, kms-keyid: " + key_id;
diff --git a/src/rgw/rgw_kms.cc b/src/rgw/rgw_kms.cc
index ea30ff868fd9..c83a0ab85713 100644
--- a/src/rgw/rgw_kms.cc
+++ b/src/rgw/rgw_kms.cc
@@ -13,6 +13,9 @@
 #include "rgw/rgw_b64.h"
 #include "rgw/rgw_kms.h"
 #include "rgw/rgw_kmip_client.h"
+#include "rgw/rgw_perf_counters.h"
+#include "rgw_kms_cache.h"
+#include "rgw_string.h"
 #include <rapidjson/allocators.h>
 #include <rapidjson/document.h>
 #include <rapidjson/writer.h>
@@ -1166,41 +1169,70 @@ class SseS3Context : public SSEContext {
   };
 };
 
-int reconstitute_actual_key_from_kms(const DoutPrefixProvider *dpp,
-                                     map<string, bufferlist>& attrs,
-                                     optional_yield y,
-                                     std::string& actual_key)
-{
+static int maybe_cache_kms_fetch(
+    const DoutPrefixProvider* dpp, const std::string& cache_prefix,
+    const std::string& key_id, rgw::kms::KMSCache* kms_cache,
+    const kms::KMSCache::FetchFn& fetch, std::string& actual_key,
+    optional_yield y) {
+  if (kms_cache == nullptr ||
+      !dpp->get_cct()->_conf->rgw_crypt_s3_kms_cache_enabled) {
+    const auto ret = fetch(actual_key);
+    if (ret == -ENOENT) {
+      perfcounter->inc(l_rgw_kms_error_permanent);
+    } else if (ret < 0) {
+      perfcounter->inc(l_rgw_kms_error_transient);
+    }
+    return ret;
+  }
+  return kms_cache->do_cache(dpp, cache_prefix, key_id, fetch, actual_key, y);
+}
+
+int reconstitute_actual_key_from_kms(
+    const DoutPrefixProvider* dpp, map<string, bufferlist>& attrs,
+    rgw::kms::KMSCache* kms_cache, optional_yield y, std::string& actual_key) {
   std::string key_id = get_str_attribute(attrs, RGW_ATTR_CRYPT_KEYID);
-  KMSContext kctx { dpp->get_cct() };
-  const std::string &kms_backend { kctx.backend() };
+  KMSContext kctx{dpp->get_cct()};
+  const std::string& kms_backend{kctx.backend()};
 
-  ldpp_dout(dpp, 20) << "Getting KMS encryption key for key " << key_id << dendl;
+  ldpp_dout(dpp, 20) << "Getting KMS encryption key for key " << key_id
+                     << dendl;
   ldpp_dout(dpp, 20) << "SSE-KMS backend is " << kms_backend << dendl;
 
-  if (RGW_SSE_KMS_BACKEND_BARBICAN == kms_backend) {
-    return get_actual_key_from_barbican(dpp, key_id, y, actual_key);
-  }
-
-  if (RGW_SSE_KMS_BACKEND_VAULT == kms_backend) {
-    return reconstitute_actual_key_from_vault(dpp, kctx, attrs, y, actual_key);
-  }
+  const auto fetch = [&](std::string& out_secret) -> int {
+    PerfGuard perf(perfcounter, l_rgw_kms_fetch_lat);
+    if (RGW_SSE_KMS_BACKEND_BARBICAN == kms_backend) {
+      return get_actual_key_from_barbican(dpp, key_id, y, out_secret);
+    }
 
-  if (RGW_SSE_KMS_BACKEND_KMIP == kms_backend) {
-    return get_actual_key_from_kmip(dpp, key_id, y, actual_key);
-  }
+    if (RGW_SSE_KMS_BACKEND_VAULT == kms_backend) {
+      return reconstitute_actual_key_from_vault(
+          dpp, kctx, attrs, y, out_secret);
+    }
 
-  if (RGW_SSE_KMS_BACKEND_TESTING == kms_backend) {
-    std::string key_selector = get_str_attribute(attrs, RGW_ATTR_CRYPT_KEYSEL);
-    return get_actual_key_from_conf(dpp, key_id, key_selector, actual_key);
-  }
+    if (RGW_SSE_KMS_BACKEND_KMIP == kms_backend) {
+      return get_actual_key_from_kmip(dpp, key_id, y, out_secret);
+    }
 
-  ldpp_dout(dpp, 0) << "ERROR: Invalid rgw_crypt_s3_kms_backend: " << kms_backend << dendl;
-  return -EINVAL;
+    if (RGW_SSE_KMS_BACKEND_TESTING == kms_backend) {
+      std::string key_selector =
+          get_str_attribute(attrs, RGW_ATTR_CRYPT_KEYSEL);
+      std::this_thread::sleep_for(
+          std::chrono::milliseconds(
+              dpp->get_cct()->_conf->rgw_crypt_s3_kms_testing_delay));
+      return get_actual_key_from_conf(dpp, key_id, key_selector, out_secret);
+    }
+    ldpp_dout(dpp, 0) << "ERROR: Invalid rgw_crypt_s3_kms_backend: "
+                      << kms_backend << dendl;
+    return -EINVAL;
+  };
+  const std::string cache_prefix = string_cat_reserve("kms_", kms_backend);
+  return maybe_cache_kms_fetch(
+      dpp, cache_prefix, key_id, kms_cache, fetch, actual_key, y);
 }
 
 int make_actual_key_from_kms(const DoutPrefixProvider *dpp,
                              map<string, bufferlist>& attrs,
+                             rgw::kms::KMSCache* kms_cache,
                              optional_yield y,
                              std::string& actual_key)
 {
@@ -1208,7 +1240,7 @@ int make_actual_key_from_kms(const DoutPrefixProvider *dpp,
   const std::string &kms_backend { kctx.backend() };
   if (RGW_SSE_KMS_BACKEND_VAULT == kms_backend)
     return make_actual_key_from_vault(dpp, kctx, attrs, y, actual_key);
-  return reconstitute_actual_key_from_kms(dpp, attrs, y, actual_key);
+  return reconstitute_actual_key_from_kms(dpp, attrs, kms_cache, y, actual_key);
 }
 
 int reconstitute_actual_key_from_sse_s3(const DoutPrefixProvider *dpp,
diff --git a/src/rgw/rgw_kms.h b/src/rgw/rgw_kms.h
index ceece721dca0..4a2d905ce6a5 100644
--- a/src/rgw/rgw_kms.h
+++ b/src/rgw/rgw_kms.h
@@ -22,6 +22,10 @@ static const std::string RGW_SSE_KMS_VAULT_SE_KV = "kv";
 
 static const std::string RGW_SSE_KMS_KMIP_SE_KV = "kv";
 
+namespace rgw::kms {
+  class KMSCache;
+}
+
 /**
  * Retrieves the actual server-side encryption key from a KMS system given a
  * key ID. Currently supported KMS systems are OpenStack Barbican and HashiCorp
@@ -34,10 +38,12 @@ static const std::string RGW_SSE_KMS_KMIP_SE_KV = "kv";
  */
 int make_actual_key_from_kms(const DoutPrefixProvider *dpp,
                              std::map<std::string, bufferlist>& attrs,
+                             rgw::kms::KMSCache* kms_cache,
                              optional_yield y,
                              std::string& actual_key);
 int reconstitute_actual_key_from_kms(const DoutPrefixProvider *dpp,
                                      std::map<std::string, bufferlist>& attrs,
+                                     rgw::kms::KMSCache* kms_cache,
                                      optional_yield y,
                                      std::string& actual_key);
 int make_actual_key_from_sse_s3(const DoutPrefixProvider *dpp,
diff --git a/src/rgw/rgw_kms_cache.cc b/src/rgw/rgw_kms_cache.cc
new file mode 100644
index 000000000000..76d5aef89b84
--- /dev/null
+++ b/src/rgw/rgw_kms_cache.cc
@@ -0,0 +1,256 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2025 Clyso GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "rgw_kms_cache.h"
+
+#include <boost/asio/associated_executor.hpp>
+#include <boost/asio/bind_cancellation_slot.hpp>
+#include <boost/asio/bind_executor.hpp>
+#include <boost/asio/cancellation_signal.hpp>
+#include <boost/asio/cancellation_type.hpp>
+#include <boost/asio/detached.hpp>
+#include <boost/asio/use_future.hpp>
+#include <boost/asio/error.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/steady_timer.hpp>
+#include <boost/asio/strand.hpp>
+#include <utility>
+#include <variant>
+
+#include "common/async/yield_context.h"
+#include "common/dout.h"
+#include "common/dout_fmt.h"
+#include "common/keyring.h"
+#include "include/ceph_assert.h"
+#include "include/function2.hpp"
+#include "rgw_perf_counters.h"
+#include "rgw_string.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys  ceph_subsys_rgw
+
+namespace rgw::kms {
+
+std::jthread KMSCache::make_ttl_reaper_thread(
+    CephContext* cct, KMSSecretCache& cache, std::chrono::seconds ttl) {
+  return std::jthread([cct, &cache, ttl](std::stop_token stop) {
+    const std::string thread_name = fmt::format("{}-ttl-reaper", cache.name());
+    ceph_pthread_setname(thread_name.c_str());
+    std::mutex mutex;
+    std::condition_variable cond;
+    std::stop_callback on_stop(stop, [&cond]() { cond.notify_all(); });
+    ldout(cct, 10) << "KMS Cache: Starting TTL reaper thread " << thread_name
+                   << "/" << std::hex << std::this_thread::get_id() << std::dec
+                   << ", running every " << ttl << dendl;
+    while (!stop.stop_requested()) {
+      std::unique_lock<std::mutex> lock(mutex);
+      if (cond.wait_for(lock, ttl, [&stop] { return stop.stop_requested(); })) {
+        break;
+      }
+      const auto expired_count = cache.expire_erase();
+      ldout(cct, 20) << "KMS Cache: TTL reaper thread expired " << expired_count
+                     << " entries" << dendl;
+    }
+    ldout(cct, 10) << "KMS Cache: Stopping TTL reaper thread " << thread_name
+                   << "/" << std::hex << std::this_thread::get_id() << std::dec
+                   << dendl;
+  });
+}
+
+std::future<void> KMSCache::make_ttl_reaper_async(
+    CephContext* cct, KMSSecretCache& cache, std::chrono::seconds ttl,
+    const boost::asio::strand<boost::asio::io_context::executor_type>& strand,
+    boost::asio::cancellation_signal& cancel_signal) {
+  return boost::asio::spawn(
+      strand,
+      [cct, &cache, ttl](const boost::asio::yield_context& yield) {
+        ldout(cct, 10) << "KMS Cache: Starting async TTL reaper, running every "
+                       << ttl << dendl;
+        boost::asio::steady_timer timer(yield.get_executor());
+        while (true) {
+          timer.expires_after(ttl);
+          boost::system::error_code ec;
+          timer.async_wait(yield[ec]);
+          if (ec == boost::asio::error::operation_aborted) {
+            ldout(cct, 10) << "KMS Cache: Stopping async TTL reaper" << dendl;
+            break;
+          }
+          const auto expired_count = cache.expire_erase();
+          ldout(cct, 20) << "KMS Cache: Async TTL reaper expired "
+                         << expired_count << " entries" << dendl;
+        }
+      },
+      boost::asio::bind_cancellation_slot(
+          cancel_signal.slot(),
+          boost::asio::bind_executor(strand, boost::asio::use_future)));
+}
+
+KMSCache::KMSCache(CephContext* _cct, std::unique_ptr<Keyring> _keyring)
+    : cct(_cct), keyring(std::move(_keyring)) {
+  ldout(cct, 10) << fmt::format(
+                        "KMS Cache: Initializing size:{} TTL pos:{} "
+                        "neg:{} err:{}",
+                        cct->_conf->rgw_crypt_s3_kms_cache_max_size,
+                        cct->_conf->rgw_crypt_s3_kms_cache_positive_ttl,
+                        cct->_conf->rgw_crypt_s3_kms_cache_negative_ttl,
+                        cct->_conf->rgw_crypt_s3_kms_cache_transient_error_ttl)
+                 << dendl;
+  cache = std::make_unique<KMSSecretCache>(
+      cct, "kms-cache", cct->_conf->rgw_crypt_s3_kms_cache_max_size,
+      std::chrono::seconds(cct->_conf->rgw_crypt_s3_kms_cache_positive_ttl));
+
+  std::error_code ec;
+  if (!keyring->supported(&ec)) {
+    ldout(cct, 1) << "KMS Cache: " << keyring->name() << " unsupported (error "
+                  << ec << "). Disabling Cache." << dendl;
+    cct->_conf->rgw_crypt_s3_kms_cache_enabled = false;
+  }
+}
+
+KMSCache::~KMSCache() {
+  stop_ttl_reaper();
+}
+
+void KMSCache::initialize_ttl_reaper(
+    std::optional<boost::asio::io_context::executor_type> executor) {
+  ceph_assert(cache);
+  if (reaper_initialized()) {
+    return;
+  }
+  const auto min_ttl_secs = std::min(
+      {cct->_conf->rgw_crypt_s3_kms_cache_positive_ttl,
+       cct->_conf->rgw_crypt_s3_kms_cache_negative_ttl,
+       cct->_conf->rgw_crypt_s3_kms_cache_transient_error_ttl});
+  if (executor.has_value()) {
+    auto& state = reaper_state.emplace<AsyncState>(executor.value());
+    state.done = make_ttl_reaper_async(
+        cct, *cache, std::chrono::seconds(min_ttl_secs),
+        state.strand, state.cancel_signal);
+  } else {
+    reaper_state.emplace<std::jthread>(make_ttl_reaper_thread(
+        cct, *cache, std::chrono::seconds(min_ttl_secs)));
+  }
+}
+
+void KMSCache::stop_ttl_reaper() {
+  ceph_assert(cache);
+  std::visit(
+      fu2::overload(
+          [](const std::monostate& mono) {},
+          [](AsyncState& async_state) {
+            boost::asio::dispatch(
+                async_state.strand,
+                [&signal = async_state.cancel_signal]() {
+                  signal.emit(boost::asio::cancellation_type::terminal);
+            });
+            try {
+              async_state.done.wait();
+            } catch (const std::future_error& e) {
+              if (e.code() != std::future_errc::no_state) throw;
+            }
+          },
+          [&](const std::jthread&) { reaper_state.emplace<std::monostate>(); }),
+      reaper_state);
+}
+
+void KMSCache::clear_cache() const {
+  cache->clear();
+}
+
+int KMSCache::do_cache(
+    const DoutPrefixProvider* dpp, const std::string& key_prefix_kms,
+    const std::string& key_id, const FetchFn& fetch, std::string& actual_key,
+    optional_yield y) {
+  static std::string_view key_prefix("rgw_sse_");
+  const std::string cache_key =
+      string_cat_reserve(key_prefix, key_prefix_kms, "_", key_id);
+  std::shared_ptr<KMSCache::CacheValue> value =
+      cache->lookup_or(key_id, std::make_shared<KMSCache::CacheValue>());
+  auto result = call_once(
+      *value, y,
+      [&dpp, &fetch, &cache_key, &value, &key_prefix_kms, &key_id,
+       this]() -> KMSCache::CacheResult {
+        std::string secret;
+        const int ret = fetch(secret);
+        ldpp_dout(dpp, 20) << "KMS Cache: " << cache_key
+                           << " call_once fetched with ret " << ret << dendl;
+
+        if (ret == -ENOENT) {  // key does not exists, treat as permanent error
+          ldpp_dout(dpp, 15)
+              << "KMS Cache: " << cache_key
+              << " key does not exists. treating as permanent error " << dendl;
+          cache->update_ttl_if(
+              cache_key, value,
+              std::chrono::seconds(
+                  dpp->get_cct()->_conf->rgw_crypt_s3_kms_cache_negative_ttl));
+          perfcounter->inc(l_rgw_kms_error_permanent);
+          return tl::unexpected(ret);
+        } else if (ret < 0) {  // treat other errors as transient
+          ldpp_dout(dpp, 15)
+              << "KMS Cache: " << cache_key << " fetch error (" << ret << ") "
+              << std::strerror(ret) << " treating as transient error " << dendl;
+          cache->update_ttl_if(
+              cache_key, value,
+              std::chrono::seconds(
+                  dpp->get_cct()
+                      ->_conf->rgw_crypt_s3_kms_cache_transient_error_ttl));
+          perfcounter->inc(l_rgw_kms_error_transient);
+          return tl::unexpected(ret);
+        }
+
+        // This function might be in flight for the same key_id more than
+        // once. The keyring key must, however, be unique to not refer
+        // (and remove) the same key twice.
+        uuid_d uuid;
+        uuid.generate_random();
+        const std::string keyring_key = string_cat_reserve(
+            key_prefix, key_prefix_kms, "_", key_id, "_v", uuid.to_string());
+        auto keyring_secret = keyring->add(keyring_key, secret);
+        ceph::crypto::zeroize_for_security(secret.data(), secret.length());
+        if (!keyring_secret) {
+          ldpp_dout(dpp, 5)
+              << "KMS Cache: " << cache_key << " keyring add error ("
+              << keyring_secret.error()
+              << "). removing from cache. disabling cache." << dendl;
+          cache->remove_if(cache_key, value);
+          disable_cache();
+          perfcounter->inc(l_rgw_kms_error_secret_store);
+          return tl::unexpected(-ERR_INTERNAL_ERROR);
+        }
+        return std::move(keyring_secret.value());
+      });
+
+  ldpp_dout_fmt(
+      dpp, 20, "KMS Cache: {} -> {}/{}", cache_key,
+      result && result.has_value() ? fmt::format("{}", *result.value()) : "-",
+      !result ? result.error() : 0);
+
+  if (result) {
+    if (auto ret = result.value()->read(actual_key); ret.value() != 0) {
+      ldpp_dout(dpp, 5) << "KMS Cache: " << cache_key << " keyring "
+                        << *result.value() << " read error (" << ret
+                        << "). removing from cache. disabling cache." << dendl;
+      cache->remove_if(cache_key, value);
+      disable_cache();
+      perfcounter->inc(l_rgw_kms_error_secret_store);
+      return -ERR_INTERNAL_ERROR;
+    }
+    return 0;
+  } else {
+    return result.error();
+  }
+}
+
+}  // namespace rgw::kms
diff --git a/src/rgw/rgw_kms_cache.h b/src/rgw/rgw_kms_cache.h
new file mode 100644
index 000000000000..9fce8c80a413
--- /dev/null
+++ b/src/rgw/rgw_kms_cache.h
@@ -0,0 +1,111 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2025 Clyso GmbH
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <sys/stat.h>
+
+#include <boost/asio/executor.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/strand.hpp>
+#include <chrono>
+#include <future>
+#include <thread>
+#include <variant>
+
+#include "common/async/call_once.h"
+#include "common/async/yield_context.h"
+#include "common/dout.h"
+#include "common/keyring.h"
+#include "common/web_cache.h"
+#include "include/expected.hpp"
+
+namespace rgw::kms {
+
+// Cache RGW KMS Secrets
+//
+// Manages a WebCache instance plus TTL reaper. Adds cache stampede
+// mitigation when fetching new values using (async) call once.
+class KMSCache {
+  CephContext* cct;
+
+ public:
+  using SharedSecret = std::shared_ptr<KeyringSecret>;
+  using CacheResult = tl::expected<SharedSecret, int>;
+  using CacheValue = ceph::async::once_result<CacheResult>;
+  using KMSSecretCache = webcache::WebCache<std::string, CacheValue>;
+  using FetchFn = std::function<int(std::string&)>;
+
+ protected:
+  std::unique_ptr<KMSSecretCache> cache;
+  std::unique_ptr<Keyring> keyring;
+
+  // The TTL Reaper is either a service thread we own or async running
+  // on an executor elsewhere (where we keep a strand and cancellation
+  // signal).
+  struct AsyncState {
+    boost::asio::strand<boost::asio::io_context::executor_type> strand;
+    boost::asio::cancellation_signal cancel_signal;
+    std::future<void> done;
+    explicit AsyncState(boost::asio::io_context::executor_type ex)
+        : strand(boost::asio::make_strand(ex)) {}
+  };
+  std::variant<std::monostate, std::jthread, AsyncState> reaper_state;
+
+ public:
+  KMSCache() = delete;
+  KMSCache(const KMSCache&) = delete;
+  KMSCache(KMSCache&&) = delete;
+  KMSCache& operator=(const KMSCache&) = delete;
+  KMSCache& operator=(KMSCache&&) = delete;
+  KMSCache(CephContext* _cct, std::unique_ptr<Keyring> _keyring);
+  virtual ~KMSCache();
+
+  // Initialize a TTL reaper. Either threaded or async depending on
+  // the executor parameter
+  void initialize_ttl_reaper(
+      std::optional<boost::asio::io_context::executor_type> executor);
+
+  // Stop any type of TTL reaper running
+  void stop_ttl_reaper();
+
+  [[nodiscard]] bool reaper_initialized() const noexcept {
+    return !std::holds_alternative<std::monostate>(reaper_state);
+  }
+
+  static std::jthread make_ttl_reaper_thread(
+      CephContext* cct, KMSSecretCache& cache, std::chrono::seconds ttl);
+
+  static std::future<void> make_ttl_reaper_async(CephContext* cct,
+      KMSSecretCache& cache, std::chrono::seconds ttl,
+      const boost::asio::strand<boost::asio::io_context::executor_type>& strand,
+      boost::asio::cancellation_signal& cancel_signal);
+
+  void clear_cache() const;
+
+  // Retrieve a cached KMS secret or fetch, cache, and return a KMS
+  // secret.
+  //
+  // The secret is returned via actual_key and it is up to the caller
+  // to properly clear the secret from memory. The cache may be shared
+  // between SSE-KMS and SSE-S3, where key_kms_prefix is available to
+  // partition the key namespace.
+  int do_cache(
+      const DoutPrefixProvider* dpp, const std::string& key_kms_prefix,
+      const std::string& key_id, const FetchFn& fetch, std::string& actual_key,
+      optional_yield y);
+
+  void disable_cache() { cct->_conf->rgw_crypt_s3_kms_cache_enabled = false; }
+};
+}  // namespace rgw::kms
diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index 560b00dbf649..3e46d6075002 100644
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -165,6 +165,7 @@ int main(int argc, char *argv[])
   main.init_opslog();
   main.init_tracepoints();
   main.init_lua();
+  main.init_kms_cache();
   r = main.init_frontends2(nullptr /* RGWLib */);
   if (r != 0) {
     derr << "ERROR:  initialize frontend fail, r = " << r << dendl;
diff --git a/src/rgw/rgw_main.h b/src/rgw/rgw_main.h
index 3a729aa63090..3a6f1990e2a1 100644
--- a/src/rgw/rgw_main.h
+++ b/src/rgw/rgw_main.h
@@ -114,6 +114,7 @@ class AppMain {
   int init_frontends2(RGWLib* rgwlib = nullptr);
   void init_tracepoints();
   void init_lua();
+  void init_kms_cache();
 
   bool have_http() {
     return have_http_frontend;
diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index d316476a27b7..da36aa3f13f4 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -59,6 +59,11 @@ void add_rgw_frontend_counters(PerfCountersBuilder *pcb) {
   pcb->add_u64_counter(l_rgw_lua_script_ok, "lua_script_ok", "Successful executions of Lua scripts");
   pcb->add_u64_counter(l_rgw_lua_script_fail, "lua_script_fail", "Failed executions of Lua scripts");
   pcb->add_u64(l_rgw_lua_current_vms, "lua_current_vms", "Number of Lua VMs currently being executed");
+
+  pcb->add_time_avg(l_rgw_kms_fetch_lat, "kms_fetch_lat", "Uncached KMS secret fetch latency");
+  pcb->add_u64_counter(l_rgw_kms_error_permanent, "kms_error_permanent", "Permanent (e.g key not found) errors returned from KMS");
+  pcb->add_u64_counter(l_rgw_kms_error_transient, "kms_error_transient", "Trainsient (e.g timeout, overloaded) errors returned from KMS");
+  pcb->add_u64_counter(l_rgw_kms_error_secret_store, "kms_error_secret_store", "Secrt store errors (e.g kernel keyring quota)");
 }
 
 void add_rgw_op_counters(PerfCountersBuilder *lpcb) {
diff --git a/src/rgw/rgw_perf_counters.h b/src/rgw/rgw_perf_counters.h
index 7eb11c926c17..0a954d9ccf5f 100644
--- a/src/rgw/rgw_perf_counters.h
+++ b/src/rgw/rgw_perf_counters.h
@@ -49,6 +49,11 @@ enum {
   l_rgw_lua_script_ok,
   l_rgw_lua_script_fail,
 
+  l_rgw_kms_fetch_lat,
+  l_rgw_kms_error_transient,
+  l_rgw_kms_error_permanent,
+  l_rgw_kms_error_secret_store,
+
   l_rgw_last,
 };
 
diff --git a/src/rgw/rgw_process_env.h b/src/rgw/rgw_process_env.h
index d298c15069aa..9e3b04609bb8 100644
--- a/src/rgw/rgw_process_env.h
+++ b/src/rgw/rgw_process_env.h
@@ -5,6 +5,8 @@
 
 #include <memory>
 
+#include "rgw_kms_cache.h"
+
 class ActiveRateLimiter;
 class OpsLogSink;
 class RGWREST;
@@ -45,6 +47,7 @@ struct RGWProcessEnv {
   std::unique_ptr<OpsLogSink> olog;
   std::unique_ptr<rgw::auth::StrategyRegistry> auth_registry;
   ActiveRateLimiter* ratelimiting = nullptr;
+  std::unique_ptr<rgw::kms::KMSCache> kms_cache;
 
 #ifdef WITH_ARROW_FLIGHT
   // managed by rgw:flight::FlightFrontend in rgw_flight_frontend.cc
diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt
index 8e4235a1af88..0909b7464c16 100644
--- a/src/test/rgw/CMakeLists.txt
+++ b/src/test/rgw/CMakeLists.txt
@@ -257,6 +257,13 @@ target_include_directories(unittest_rgw_kms
   SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
 target_link_libraries(unittest_rgw_kms ${rgw_libs})
 
+# unittest_rgw_kms_cache
+add_executable(unittest_rgw_kms_cache test_rgw_kms_cache.cc)
+add_ceph_unittest(unittest_rgw_kms_cache)
+target_include_directories(unittest_rgw_kms_cache
+  SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
+target_link_libraries(unittest_rgw_kms_cache ${rgw_libs})
+
 # unittest_rgw_url
 add_executable(unittest_rgw_url test_rgw_url.cc)
 add_ceph_unittest(unittest_rgw_url)
diff --git a/src/test/rgw/test_rgw_kms_cache.cc b/src/test/rgw/test_rgw_kms_cache.cc
new file mode 100644
index 000000000000..15004a02e5f0
--- /dev/null
+++ b/src/test/rgw/test_rgw_kms_cache.cc
@@ -0,0 +1,368 @@
+#include <common/async/yield_context.h>
+#include <common/ceph_argparse.h>
+#include <common/common_init.h>
+#include <common/keyring.h>
+#include <common/perf_counters.h>
+#include <fmt/format.h>
+#include <fmt/ostream.h>
+#include <global/global_context.h>
+#include <global/global_init.h>
+#include <gtest/gtest.h>
+
+#include <boost/asio/cancellation_signal.hpp>
+#include <boost/asio/co_spawn.hpp>
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/use_future.hpp>
+#include <cerrno>
+#include <chrono>
+#include <variant>
+#include <thread>
+
+#include "common/web_cache.h"
+#include "rgw_common.h"
+#include "rgw_kms.h"
+#include "rgw_kms_cache.h"
+#include "rgw_perf_counters.h"
+
+class FakeKeyringSecret : public KeyringSecret {
+ public:
+  std::string key;
+  std::string val;
+
+  FakeKeyringSecret(std::string _key, std::string _val)
+      : key(std::move(_key)), val(std::move(_val)) {};
+
+  std::error_code read(std::string& out) const override {
+    out = val;
+    return {};
+  }
+  std::error_code remove() const override { return {}; }
+  bool initialized() const override { return true; }
+};
+
+class FakeKeyring : public Keyring {
+ public:
+  tl::expected<std::unique_ptr<KeyringSecret>, std::error_code> add(
+      const std::string& key, const std::string& val) noexcept override {
+    return std::make_unique<FakeKeyringSecret>(key, val);
+  }
+  bool supported(std::error_code* ec) noexcept override { return true; }
+  std::string_view name() const noexcept override { return "fake"; };
+};
+
+class TestKMSCacheReaperLifecycle : public ::testing::Test,
+                                    public rgw::kms::KMSCache {
+ public:
+  TestKMSCacheReaperLifecycle()
+      : rgw::kms::KMSCache(g_ceph_context, std::make_unique<FakeKeyring>()) {};
+};
+
+static void rethrow(const std::exception_ptr& eptr) {
+  if (eptr) {
+    std::rethrow_exception(eptr);
+  }
+}
+
+TEST_F(TestKMSCacheReaperLifecycle, Threaded) {
+  initialize_ttl_reaper(std::nullopt);
+  EXPECT_TRUE(reaper_initialized());
+  EXPECT_TRUE(std::holds_alternative<std::jthread>(reaper_state));
+
+  stop_ttl_reaper();
+  EXPECT_FALSE(reaper_initialized());
+  EXPECT_FALSE(std::holds_alternative<std::jthread>(reaper_state));
+}
+
+TEST_F(TestKMSCacheReaperLifecycle, NoInit) {
+  EXPECT_FALSE(reaper_initialized());
+  EXPECT_TRUE(std::holds_alternative<std::monostate>(reaper_state));
+  this->stop_ttl_reaper();
+  EXPECT_FALSE(reaper_initialized());
+  EXPECT_TRUE(std::holds_alternative<std::monostate>(reaper_state));
+}
+
+TEST_F(TestKMSCacheReaperLifecycle, Async) {
+  boost::asio::io_context io;
+  auto work = boost::asio::make_work_guard(io);
+  std::jthread io_thread([&]() {
+    io.run();
+  });
+  auto drain = [&] {
+    boost::asio::post(io, boost::asio::use_future).get();
+  };
+  this->initialize_ttl_reaper(io.get_executor());
+  drain();
+  EXPECT_TRUE(reaper_initialized());
+  EXPECT_TRUE(
+      std::holds_alternative<AsyncState>(reaper_state));
+  stop_ttl_reaper();
+  drain();
+  work.reset();
+  io.restart();
+  EXPECT_EQ(io.run(), 0);
+}
+
+class TestKMSCache : public ::testing::Test {
+ protected:
+  CephContext* cct = g_ceph_context;
+  const NoDoutPrefix no_dpp{cct, ceph_subsys_rgw};
+
+ public:
+  std::unique_ptr<rgw::kms::KMSCache> uut =
+      std::make_unique<rgw::kms::KMSCache>(
+          cct, std::make_unique<FakeKeyring>());
+
+  void TearDown() override { perfcounter->reset(); }
+};
+
+TEST_F(TestKMSCache, TransientFetchError) {
+  std::string tmp;
+  ASSERT_EQ(
+      uut->do_cache(
+          &no_dpp, "testing", "foo", [](std::string&) { return -EINVAL; }, tmp,
+          null_yield),
+      -EINVAL);
+  ASSERT_EQ(perfcounter->get(l_rgw_kms_error_transient), 1);
+  ASSERT_EQ(perfcounter->get(l_rgw_kms_error_permanent), 0);
+  ASSERT_EQ(perfcounter->get(l_rgw_kms_error_secret_store), 0);
+  ASSERT_TRUE(cct->_conf->rgw_crypt_s3_kms_cache_enabled);
+}
+
+TEST_F(TestKMSCache, PermanentFetchError) {
+  std::string tmp;
+  ASSERT_EQ(
+      uut->do_cache(
+          &no_dpp, "testing", "foo", [](std::string&) { return -ENOENT; }, tmp,
+          null_yield),
+      -ENOENT);
+  ASSERT_EQ(perfcounter->get(l_rgw_kms_error_transient), 0);
+  ASSERT_EQ(perfcounter->get(l_rgw_kms_error_permanent), 1);
+  ASSERT_EQ(perfcounter->get(l_rgw_kms_error_secret_store), 0);
+  ASSERT_TRUE(cct->_conf->rgw_crypt_s3_kms_cache_enabled);
+}
+
+TEST_F(TestKMSCache, Cache) {
+  const std::string test_key = "37u481923789123u72189ou3jsdf978of";
+  std::string_view test_val = "dlksafj3029jfmsjf8322ty7nghb67435";
+  std::string cache_return;
+  ASSERT_EQ(
+      uut->do_cache(
+          &no_dpp, "testing", test_key,
+          [&](std::string& val) {
+            val = test_val;
+            return 0;
+          },
+          cache_return, null_yield),
+      0);
+  EXPECT_EQ(perfcounter->get(l_rgw_kms_error_transient), 0);
+  EXPECT_EQ(perfcounter->get(l_rgw_kms_error_permanent), 0);
+  EXPECT_EQ(perfcounter->get(l_rgw_kms_error_secret_store), 0);
+  EXPECT_TRUE(cct->_conf->rgw_crypt_s3_kms_cache_enabled);
+  ASSERT_EQ(cache_return, test_val);
+
+  cache_return.clear();
+  ASSERT_EQ(
+      uut->do_cache(
+          &no_dpp, "testing", test_key,
+          [&](std::string&) -> int {
+            EXPECT_TRUE(false) << "fetch must not be called";
+            return -2342;
+          },
+          cache_return, null_yield),
+      0);
+  EXPECT_EQ(perfcounter->get(l_rgw_kms_error_transient), 0);
+  EXPECT_EQ(perfcounter->get(l_rgw_kms_error_permanent), 0);
+  EXPECT_EQ(perfcounter->get(l_rgw_kms_error_secret_store), 0);
+  EXPECT_TRUE(cct->_conf->rgw_crypt_s3_kms_cache_enabled);
+  ASSERT_EQ(cache_return, test_val);
+}
+
+class TestSSEKMSWithTestingKMS : public ::testing::Test {
+ protected:
+  CephContext* cct = g_ceph_context;
+  const NoDoutPrefix no_dpp{cct, ceph_subsys_rgw};
+  std::map<std::string, bufferlist> attrs = {
+      {RGW_ATTR_CRYPT_KEYID,
+       []() {
+         bufferlist bl;
+         bl.append("foo");
+         return bl;
+       }()},
+      {RGW_ATTR_CRYPT_KEYSEL, []() {
+         // AES_ECB(32*"#").decrypt(32*"*")
+         bufferlist bl;
+         bl.append(
+             "\xc6\xb1/\x12\xdc\xf7"
+             "e"
+             "\xe3;\xea\x14\xa4x\x1f"
+             "bX"
+             "\xc6\xb1/\x12\xdc\xf7"
+             "e"
+             "\xe3;\xea\x14\xa4x\x1f"
+             "bX");
+         return bl;
+       }()}};
+  rgw::kms::KMSCache* uut;
+  PerfCounters* cache_perf = nullptr;
+
+  void SetUp() override {
+    // Simulate RGW app KMSCache livecyle. A single instance started
+    // during app init and may be disabled via config.
+    uut = &cct->lookup_or_create_singleton_object<rgw::kms::KMSCache>(
+        "TestSSEKMSWithTestingKMS::kms-cache", false, cct,
+        std::make_unique<FakeKeyring>());
+    cct->get_perfcounters_collection()->with_counters(
+        [&](const PerfCountersCollectionImpl::CounterMap& by_path) {
+          for (const auto& i : by_path) {
+            const auto& perf_counters = i.second.perf_counters;
+            if (perf_counters->get_name() == "kms-cache") {
+              cache_perf = perf_counters;
+              return;
+            }
+          }
+        });
+
+    ASSERT_NE(perfcounter, nullptr);
+    ASSERT_NE(cache_perf, nullptr);
+  }
+
+  void TearDown() override {
+    JSONFormatter f(true);
+    cache_perf->dump_formatted(&f, false, select_labeled_t::labeled);
+    f.flush(std::cout);
+    cct->get_perfcounters_collection()->with_counters(
+        [&](const PerfCountersCollectionImpl::CounterMap& by_path) {
+          for (const auto& i : by_path) {
+            const auto& perf_counters = i.second.perf_counters;
+            if (perf_counters->get_name() == "rgw") {
+              auto [sum, count] =
+                  perf_counters->get_tavg_ns(l_rgw_kms_fetch_lat);
+              fmt::println(
+                  std::cout,
+                  "RGW KMS perf counters: err_trans={} err_perm={} err_sec={} "
+                  "avg_fetch_lat={} fetch_cnt={}",
+                  perf_counters->get(l_rgw_kms_error_transient),
+                  perf_counters->get(l_rgw_kms_error_permanent),
+                  perf_counters->get(l_rgw_kms_error_secret_store),
+                  std::chrono::nanoseconds(sum / count), count);
+              return;
+            }
+          }
+        });
+    uut->clear_cache();
+  }
+
+  void test_do_reconstitue() {
+    std::string actual_key;
+    const int ret = reconstitute_actual_key_from_kms(
+        &no_dpp, attrs, uut, null_yield, actual_key);
+    ASSERT_EQ(ret, 0);
+    ASSERT_EQ(actual_key, "********************************");
+    ASSERT_EQ(perfcounter->get(l_rgw_kms_error_secret_store), 0);
+    ASSERT_EQ(perfcounter->get(l_rgw_kms_error_permanent), 0);
+    ASSERT_EQ(perfcounter->get(l_rgw_kms_error_transient), 0);
+  }
+};
+
+TEST_F(
+    TestSSEKMSWithTestingKMS, TestReconstituteActualKeyFromKMSBasicsDefault) {
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 0);
+  ASSERT_TRUE(cct->_conf->rgw_crypt_s3_kms_cache_enabled);
+  test_do_reconstitue();
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::clear)), 0);
+  EXPECT_EQ(
+      cache_perf->get(static_cast<int>(webcache::Metric::capacity)),
+      cct->_conf->rgw_crypt_s3_kms_cache_max_size);
+}
+
+TEST_F(
+    TestSSEKMSWithTestingKMS,
+    TestReconstituteActualKeyFromKMSBasicsWithoutCache) {
+  cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "false");
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 1);
+  test_do_reconstitue();
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 0);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 0);
+  ASSERT_FALSE(cct->_conf->rgw_crypt_s3_kms_cache_enabled);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 2);
+}
+
+TEST_F(
+    TestSSEKMSWithTestingKMS, TestReconstituteActualKeyFromKMSBasicsWithCache) {
+  ASSERT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 2);
+
+  cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "true");
+  test_do_reconstitue();
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 3);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
+
+  test_do_reconstitue();
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 3);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
+
+  test_do_reconstitue();
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 3);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 2);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
+}
+
+TEST_F(TestSSEKMSWithTestingKMS, TestRuntimeEnableDisable) {
+  ASSERT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 3);
+
+  cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "true");
+  test_do_reconstitue();
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 4);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
+
+  cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "false");
+  test_do_reconstitue();
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 5);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
+
+  cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "true");
+  test_do_reconstitue();
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 5);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
+
+  cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "false");
+  uut->clear_cache();
+  test_do_reconstitue();
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 6);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 0);
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 0);
+}
+
+int main(int argc, char** argv) {
+  auto args = argv_to_vec(argc, argv);
+  std::map<std::string, std::string> defaults{
+      {"rgw_crypt_s3_kms_backend", RGW_SSE_KMS_BACKEND_TESTING},
+      {"rgw_crypt_s3_kms_encryption_keys",
+       "foo=IyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyM="},
+      {"debug_rgw", "20"}};
+  auto cct = global_init(
+      &defaults, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY,
+      CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  rgw_perf_start(g_ceph_context);
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

From c8ecdd22aa0860951641371f6f3561a89aef75bb Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Fri, 19 Dec 2025 10:20:13 +0100
Subject: [PATCH 13/28] PendingReleaseNotes: Add SSE-KMS Cache

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit aa4ead34a16e8582fea1c5a6740a48924f33cc92)
---
 PendingReleaseNotes | 110 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)

diff --git a/PendingReleaseNotes b/PendingReleaseNotes
index c1289d2fd10f..0a8ae908214a 100644
--- a/PendingReleaseNotes
+++ b/PendingReleaseNotes
@@ -1,3 +1,113 @@
+* librados/neorados: The C++ APIs for executing Ceph Class (CLS) methods
+  have undergone a breaking change to enforce compile-time type safety, replacing
+  legacy string-based parameters with strongly-typed ClsMethod structs. C++
+  developers must update their method calls to use these new structs, which ensure
+  read/write semantics are correctly applied.
+
+* RGW: S3 ListObjects and ListObjectVersions now support the
+  ``x-amz-optional-object-attributes: RestoreStatus`` request header to include
+  restore status in listing responses. Restore status is stored in the bucket
+  index, so only objects written or restored after this upgrade will populate
+  the field. Existing objects are unaffected.
+* RGW: New S3 Control APIs to apply PublicAccessBlock configuration to User Accounts.
+* ceph-volume: Raw BlueStore OSD preparation now pre-formats NVMe devices and
+  skips the slower BlueStore discard phase,reducing mkfs time on
+  very large namespaces.
+* RGW: Bucket Logging suppports creating log buckets in EC pools.
+  Implicit logging object commits are now performed asynchronously.
+* RGW: radosgw-admin bucket list now supports pagination for versioned buckets by using
+  both --marker and --object-version options together (e.g., ``--marker=obj1 --object-version=abc123``).
+  This enables proper pagination through versioned bucket listings without duplicates or
+  infinite loops. For non-versioned buckets, use only --marker as before (backward compatible).
+* RGW: OpenSSL engine support is deprecated in favor of provider support.
+  - Removed the `openssl_engine_opts` configuration option. OpenSSL engine configuration in string format is no longer supported.
+  - Added the `openssl_conf` configuration option for loading specified providers as default providers.
+    Configuration file syntax follows the OpenSSL standard (see https://github.com/openssl/openssl/blob/master/doc/man5/config.pod).
+    If the default provider is required when also using custom providers,
+    it must be explicitly loaded in the configuration file or code (see https://github.com/openssl/openssl/blob/master/README-PROVIDERS.md).
+* RGW: Fixed bucket notification events so the 'x_amz_request_id' in NotificationEvent now matches the 'x_amz_request_id' returned by the corresponding S3 operation.
+
+* DASHBOARD: Introduces a new landing page - "Overview". This revamps UX and adds more information in the landing page - overall cluster health, health checks, resilency panel (showing PG status, active/clean percent), total and used raw capacity, alerts, capacity breakdown by object, file and block and performance charts - throughput, latency and IOPS. This renames the landing page from "Dashboard" to "Overview"
+* DASHBOARD: Removed the older landing page which was deprecated in Quincy.
+  Admins can no longer enable the older, deprecated landing page layout by
+  adjusting FEATURE_TOGGLE_DASHBOARD.
+* DASHBOARD: RGW Service form updated to take input regarding QAT compression.
+  - QAT compression is an optional field which can be set to 'Hardware' or 'Software' by selecting options from provided dropdwon. If 'None' is selected, compression is removed altogether.
+* Monitoring: Added new Prometheus alerts for certificate management:
+  - CephCertificateError: Fires when a Ceph certificate has expired (critical severity).
+  - CephCertificateWarning: Fires when a Ceph certificate is about to expire (warning severity).
+* CephFS: The `peer_add` command is deprecated in favor of the `peer_bootstrap` command.
+* RGW: For `radosgw-admin account rm`, the `--purge-data` option previously applied only
+  to buckets and objects, but now deletes account users, roles, groups, and oidc-providers
+  too instead of failing with ENOTEMPTY.
+* RADOS: When objects are read during deep scrubs, the data is read in strides,
+  and the scrubbing process is delayed between each read in order to avoid monopolizing
+  the I/O capacity of the OSD.
+  The default stride size (``osd_deep_scrub_stride``) was 512 KBytes, and is now 4 MBytes.
+* RADOS: When an OSD is overloaded with queued snap-trim operations, no
+  regular (non-urgent) scrubs will be scheduled on that OSD. This is
+  determined by comparing the total snap-trim queue lengths for all PGs
+  for which the OSD is a primary against ``osd_scrub_queued_snaptrims_limit``,
+  which defaults to 500.
+  This restriction does not apply to operator-initiated scrubs, nor to repair scrubs.
+  It can be disabled by setting ``osd_scrub_queued_snaptrims_limit`` to 0.
+* RGW: Add SSE-KMS secrets cache
+
+* RADOS: Stretch mode can now be entered even if the two dividing buckets differ
+  in weight by a small fraction (default 0.1). This is tunable via
+  `mon_stretch_max_bucket_weight_delta`.
+
+* CephFS: The offline CephFS tools (cephfs-data-scan, cephfs-journal-tool,
+  and cephfs-table-tool) now include progress tracking with ETA (Estimated Time of
+  Arrival) for long-running operations. Progress updates are displayed automatically
+  at regular intervals, showing completion percentage, processed items, and time
+  estimates. This feature is enabled by default for relevant commands including
+  scan_extents, scan_inodes, and other state-changing operations.
+  Related Tracker: https://tracker.ceph.com/issues/63191
+* RBD: Fixed incorrect behavior of the "start-time" argument for mirror
+  snapshot and trash purge schedules, where it previously offset the schedule
+  anchor instead of defining it. The argument now requires an ISO 8601
+  date-time. The `schedule ls` output displays the start time in UTC, including
+  the date and time in the format "%Y-%m-%d %H:%M:00". The `schedule status`
+  output now displays the next schedule time in UTC.
+* CephFS Mirroring:  Now utilizes a multi-threaded architecture to improve synchronization
+  performance. The workload is split into two distinct thread pools: a crawler thread pool, which
+  manages snapshot crawl and a data synchronization thread pool, which handles concurrent file
+  transfers. Users can fine-tune these operations using configuration parameters:
+  cephfs_mirror_max_concurrent_directory_syncs (controlling the number of concurrent snapshots being crawled)
+  and cephfs_mirror_max_datasync_threads (controlling the total threads available for data sync).
+  For more information, see https://tracker.ceph.com/issues/73452
+* CephFS Mirroring: Improved incremental synchronization behavior in CephFS mirroring. Previously,
+  block-level delta synchronization was used for all files regardless of size. With this change,
+  blockdiff is applied only to files larger than a configurable threshold, while smaller files are
+  synchronized using full copy, as blockdiff is not efficient for small files. The threshold is
+  controlled by the new configuration option cephfs_mirror_blockdiff_min_file_size (default: 16_M).
+  For more information, see https://tracker.ceph.com/issues/73452
+* CephFS Mirroring: Improved mirror daemon status reporting. The command
+  ``ceph fs snapshot mirror daemon status`` now shows the remote cluster's
+  monitor addresses and cluster ID for each configured peer, making it easier
+  to verify peer connectivity and troubleshoot mirroring issues.
+* RBD: Mirror snapshot creation and trash purge schedules are now automatically
+  staggered when no explicit "start-time" is specified. This reduces scheduling
+  spikes and distributes work more evenly over time.
+* RBD: Introduced a new ``RBD_LOCK_MODE_EXCLUSIVE_TRANSIENT`` policy for
+  ``rbd_lock_acquire()``. This is a low-level interface intended to allow
+  a peer to grab exclusive lock manually for short periods of time with other
+  peers pausing their activity and waiting for the lock to be released rather
+  than instantly aborting I/O and returning an error. It's possible to switch
+  from ``RBD_LOCK_MODE_EXCLUSIVE`` to ``RBD_LOCK_MODE_EXCLUSIVE_TRANSIENT``
+  policy and vice versa even if the lock is already held.
+* OSD: A health warning is reported when BlueFS usage exceeds the
+  configured ratio of the main OSD data device size. This warning is
+  informational and can be muted with:
+  ``ceph health mute BLUESTORE_BLUEFS_OVERSIZED``
+* MGR: The Manager now automatically increases ``mgr_stats_period`` when its
+  message queue is congested, reducing daemon reporting frequency to prevent
+  overload. The period recovers automatically once the queue clears. This
+  behavior is controlled by the new ``mgr_stats_period_autotune`` (default:
+  ``true``) and ``mgr_stats_period_autotune_queue_threshold`` (default: ``100``)
+  config options.
+
 >=20.0.0
 
 * RGW: Lua scripts will not run against health checks.

From 52bc0234b964a26dedf054bb45d2435c77d3b19b Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 3 Mar 2026 21:24:48 +0100
Subject: [PATCH 14/28] rgw: Fix typos in perf counter descriptions

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit fb5bbc7d81a0cf6745f2b47b523909014b842ac6)
---
 src/rgw/rgw_perf_counters.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index da36aa3f13f4..4395f8d100f2 100644
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -62,8 +62,8 @@ void add_rgw_frontend_counters(PerfCountersBuilder *pcb) {
 
   pcb->add_time_avg(l_rgw_kms_fetch_lat, "kms_fetch_lat", "Uncached KMS secret fetch latency");
   pcb->add_u64_counter(l_rgw_kms_error_permanent, "kms_error_permanent", "Permanent (e.g key not found) errors returned from KMS");
-  pcb->add_u64_counter(l_rgw_kms_error_transient, "kms_error_transient", "Trainsient (e.g timeout, overloaded) errors returned from KMS");
-  pcb->add_u64_counter(l_rgw_kms_error_secret_store, "kms_error_secret_store", "Secrt store errors (e.g kernel keyring quota)");
+  pcb->add_u64_counter(l_rgw_kms_error_transient, "kms_error_transient", "Transient (e.g timeout, overloaded) errors returned from KMS");
+  pcb->add_u64_counter(l_rgw_kms_error_secret_store, "kms_error_secret_store", "Secret store errors (e.g kernel keyring quota)");
 }
 
 void add_rgw_op_counters(PerfCountersBuilder *lpcb) {

From ce8d261a78d348acffba2e7ad067f6cd3365598b Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 3 Mar 2026 21:24:10 +0100
Subject: [PATCH 15/28] rgw: SSE-KMS: Handle Vault Transit Key Per Object

KMS backends Barbican, Vault KV, and KMIP have a static key per
key_id. However, with Vault Transit, each object has a unique DEK
wrapped by the transit key.

Keying th cache with key_id in Transit mode results in only the first
DEK to be cached for all subsequent objects.

Fix this by appending a hash of the wrapped DEK to the cache key.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 4d7c556461a13a93988b58abb9690aa8017e064c)
---
 src/rgw/rgw_kms.cc | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/rgw/rgw_kms.cc b/src/rgw/rgw_kms.cc
index c83a0ab85713..0b4817455294 100644
--- a/src/rgw/rgw_kms.cc
+++ b/src/rgw/rgw_kms.cc
@@ -1198,6 +1198,19 @@ int reconstitute_actual_key_from_kms(
                      << dendl;
   ldpp_dout(dpp, 20) << "SSE-KMS backend is " << kms_backend << dendl;
 
+  std::string cache_key_id = key_id;
+  if (RGW_SSE_KMS_BACKEND_VAULT == kms_backend &&
+      kctx.secret_engine() == RGW_SSE_KMS_VAULT_SE_TRANSIT) {
+    const std::string wrapped_key =
+        get_str_attribute(attrs, RGW_ATTR_CRYPT_DATAKEY);
+    // Vault Transit in "old" compat mode behaves like the other K/V
+    // style KMS in that it does not use DATAKEY.
+    if (!wrapped_key.empty()) {
+      cache_key_id = string_cat_reserve(
+          key_id, "T", calc_hash_sha256(wrapped_key).to_str());
+    }
+  }
+
   const auto fetch = [&](std::string& out_secret) -> int {
     PerfGuard perf(perfcounter, l_rgw_kms_fetch_lat);
     if (RGW_SSE_KMS_BACKEND_BARBICAN == kms_backend) {
@@ -1227,7 +1240,7 @@ int reconstitute_actual_key_from_kms(
   };
   const std::string cache_prefix = string_cat_reserve("kms_", kms_backend);
   return maybe_cache_kms_fetch(
-      dpp, cache_prefix, key_id, kms_cache, fetch, actual_key, y);
+      dpp, cache_prefix, cache_key_id, kms_cache, fetch, actual_key, y);
 }
 
 int make_actual_key_from_kms(const DoutPrefixProvider *dpp,

From 46fe994e91cf0518c630453018bbcc91c465f502 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 3 Mar 2026 21:24:34 +0100
Subject: [PATCH 16/28] rgw: SSE-KMS: Fix wrong cache key in in lookup_or()
 call

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 78f2ec3429f8ba52502c6ba03f605a0e0fbf30cc)
---
 src/rgw/rgw_kms_cache.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/rgw/rgw_kms_cache.cc b/src/rgw/rgw_kms_cache.cc
index 76d5aef89b84..c221d8bb25e8 100644
--- a/src/rgw/rgw_kms_cache.cc
+++ b/src/rgw/rgw_kms_cache.cc
@@ -177,7 +177,7 @@ int KMSCache::do_cache(
   const std::string cache_key =
       string_cat_reserve(key_prefix, key_prefix_kms, "_", key_id);
   std::shared_ptr<KMSCache::CacheValue> value =
-      cache->lookup_or(key_id, std::make_shared<KMSCache::CacheValue>());
+      cache->lookup_or(cache_key, std::make_shared<KMSCache::CacheValue>());
   auto result = call_once(
       *value, y,
       [&dpp, &fetch, &cache_key, &value, &key_prefix_kms, &key_id,

From d6facbe172af8d15807e11dd44108e41bd544c9d Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 3 Mar 2026 21:24:26 +0100
Subject: [PATCH 17/28] common/web_cache: delete perfcounters on destruction

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 898cd5b86837c67e68345c35a6b098a3cf152f7a)
---
 src/common/web_cache.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/common/web_cache.h b/src/common/web_cache.h
index 97b362d003b1..5f56aba03e25 100644
--- a/src/common/web_cache.h
+++ b/src/common/web_cache.h
@@ -325,6 +325,7 @@ template <typename Key, typename Value>
 WebCache<Key, Value>::~WebCache() {
   if (_cct != nullptr && _perf != nullptr) {
     _cct->get_perfcounters_collection()->remove(_perf);
+    delete _perf;
   }
 }
 

From 509ac112d9afbd82c8027665cb7cb3d8cb4e6ca9 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 3 Mar 2026 21:24:57 +0100
Subject: [PATCH 18/28] common/web_cache: Fix _sieve_hand dangling pointer

sieve_expire_erase_unmutexed did not update the sieve hand passed as
advertised. Make it return the updated hand and use that to
update the global _sieve_hand in expire_erase

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 3518aaf27bb5469e795c87d65442fd1dab2d5cb2)
---
 src/common/web_cache.h | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/common/web_cache.h b/src/common/web_cache.h
index 5f56aba03e25..c0667e149307 100644
--- a/src/common/web_cache.h
+++ b/src/common/web_cache.h
@@ -138,8 +138,8 @@ class WebCache {
 
   // sieve_expire_erase_unmutexed removes all expired nodes from the
   // sieve_queue in place. It writes the expired nodes to out_expired.
-  // Updates the sieve hand
-  static void sieve_expire_erase_unmutexed(
+  // Returns the updated sieve hand.
+  static Node* sieve_expire_erase_unmutexed(
       SieveQueue& sieve_queue, Node* sieve_hand,
       ceph::real_time eviction_cutoff, SieveQueue& out_expired);
 
@@ -511,7 +511,7 @@ WebCache<Key, Value>::lookup(const Key& key) {
 }
 
 template <typename Key, typename Value>
-void WebCache<Key, Value>::sieve_expire_erase_unmutexed(
+typename WebCache<Key, Value>::Node* WebCache<Key, Value>::sieve_expire_erase_unmutexed(
     SieveQueue& sieve_queue, Node* sieve_hand, ceph::real_time eviction_cutoff,
     SieveQueue& out_expired) {
   // The sieve queue is ordered by ascending insertion time which
@@ -532,6 +532,7 @@ void WebCache<Key, Value>::sieve_expire_erase_unmutexed(
       ++it;
     }
   }
+  return sieve_hand;
 }
 
 template <typename Key, typename Value>
@@ -540,7 +541,7 @@ size_t WebCache<Key, Value>::expire_erase() {
   const auto expiration_cutoff = ceph::real_clock::now();
   const auto lookup_size_before = _lookup.size();
   SieveQueue expired;
-  sieve_expire_erase_unmutexed(
+  _sieve_hand = sieve_expire_erase_unmutexed(
       _sieve_queue, _sieve_hand, expiration_cutoff, expired);
   const auto expired_size = expired.size();
   expired.clear_and_dispose(

From 1bd84b8da4057402304c4c0501a462afd7aa092d Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 3 Mar 2026 21:25:05 +0100
Subject: [PATCH 19/28] common/keyring: Fix reset error checking

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit f96f7a6665aad696a77d8112882a08ad672e8120)
---
 src/common/keyring.cc           |  2 +-
 src/common/keyring.h            |  1 +
 src/test/common/test_keyring.cc | 12 ++++++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/common/keyring.cc b/src/common/keyring.cc
index b6823753efb8..a14c49534ba8 100644
--- a/src/common/keyring.cc
+++ b/src/common/keyring.cc
@@ -134,7 +134,7 @@ void LinuxKeyringSecret::initialize_process_keyring() noexcept {
   if (_serial == -1) {
     return {-EINVAL, std::generic_category()};
   }
-  if (const auto ret = remove(); !ret) {
+  if (const auto ret = remove(); ret) {
     return ret;
   }
   _serial = -1;
diff --git a/src/common/keyring.h b/src/common/keyring.h
index 48e1ee59a99c..6d7478adbaea 100644
--- a/src/common/keyring.h
+++ b/src/common/keyring.h
@@ -121,6 +121,7 @@ class LinuxKeyringSecret : public KeyringSecret {
     return os << "LinuxKeyringSecret{" << secret._serial << "}";
   }
   friend class LinuxKeyringTest_LifecycleMoveAssignResetsDestination_Test;
+  friend class LinuxKeyringTest_ResetClearsState_Test;
   friend class LinuxKeyring;
 };
 
diff --git a/src/test/common/test_keyring.cc b/src/test/common/test_keyring.cc
index 1c591c514673..0eeb69be8999 100644
--- a/src/test/common/test_keyring.cc
+++ b/src/test/common/test_keyring.cc
@@ -68,4 +68,16 @@ TEST_F(LinuxKeyringTest, LifecycleMoveAssignResetsDestination) {
   ASSERT_EQ(ENOKEY, errno);
 }
 
+TEST_F(LinuxKeyringTest, ResetClearsState) {
+  auto maybe = keyring->add("testkey", "secret");
+  auto* ptr = dynamic_cast<LinuxKeyringSecret*>(maybe.value().get());
+  auto err = ptr->reset();
+  ASSERT_FALSE(err) << err;
+  ASSERT_FALSE(ptr->initialized());
+  ASSERT_EQ(-1, ptr->_serial);
+
+  err = ptr->reset();
+  ASSERT_TRUE(err);
+}
+
 }  // namespace ceph

From 1dc6521bb719dbf4ed0946f75f930e400e7e8b48 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Fri, 6 Mar 2026 10:44:08 +0100
Subject: [PATCH 20/28] rgw: KMS Cache: Reset Reaper State in Async+Threaded

Reset reaper state to monostate in the async and threaded case.
Fixes a possible use after free in the async reaper case.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit e1043721018590b02e2ffd9e4001cf42d024fbe9)
---
 src/rgw/rgw_kms_cache.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_kms_cache.cc b/src/rgw/rgw_kms_cache.cc
index c221d8bb25e8..334487c33ebe 100644
--- a/src/rgw/rgw_kms_cache.cc
+++ b/src/rgw/rgw_kms_cache.cc
@@ -148,7 +148,7 @@ void KMSCache::stop_ttl_reaper() {
   ceph_assert(cache);
   std::visit(
       fu2::overload(
-          [](const std::monostate& mono) {},
+          [](const std::monostate&) {},
           [](AsyncState& async_state) {
             boost::asio::dispatch(
                 async_state.strand,
@@ -161,8 +161,9 @@ void KMSCache::stop_ttl_reaper() {
               if (e.code() != std::future_errc::no_state) throw;
             }
           },
-          [&](const std::jthread&) { reaper_state.emplace<std::monostate>(); }),
+          [](const std::jthread&) {}),
       reaper_state);
+  reaper_state.emplace<std::monostate>();
 }
 
 void KMSCache::clear_cache() const {

From 5fc8f93e869509ffa7c041ad02ea8dee67d6e2fb Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 10 Mar 2026 11:51:30 +0100
Subject: [PATCH 21/28] rgw: KMS Cache Shutdown: Reaper first

1. Don't delete the KMS cache before draining/joining the frontend
coroutine threads. They may still depend on the KMS cache.
2. Stop the TTL reaper early to get it off the coroutine pool.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 62ba03f3ff1a174afbd64bf7d3727baa8c5586de)
---
 src/rgw/rgw_appmain.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc
index e0b34c44023f..7898f1e1ad4e 100644
--- a/src/rgw/rgw_appmain.cc
+++ b/src/rgw/rgw_appmain.cc
@@ -602,12 +602,13 @@ void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
     static_cast<rgw::sal::RadosLuaManager*>(env.lua.manager.get())->unwatch_reload(dpp);
   }
 
-  env.kms_cache.reset();
-
   for (auto& fe : fes) {
     fe->stop();
   }
 
+  if (env.kms_cache) {
+    env.kms_cache->stop_ttl_reaper();
+  }
   ldh.reset(nullptr); // deletes ldap helper if it was created
   rgw_log_usage_finalize();
 

From fb31517c08c48f39f7367aa8e622f2ba1ff29adc Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Tue, 5 May 2026 14:21:03 +0200
Subject: [PATCH 22/28] rgw: SSE-KMS: Handle Testing Key Per Object

The testing backend uses a 'keysel' attribute to derive a per object
key from the KEK in the config. A single key_id with distinct keysel
has different keys and need to be cached as such.

Add the keysel to the cache key id to handle these collisions.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
(cherry picked from commit 12de72b3f34f53357e5a918b55f4a5d20c200761)
---
 src/rgw/rgw_kms.cc                 |  9 +++++++++
 src/test/rgw/test_rgw_kms.cc       | 13 ++++++++++++
 src/test/rgw/test_rgw_kms_cache.cc | 32 ++++++++++++++++++++++++++++++
 3 files changed, 54 insertions(+)

diff --git a/src/rgw/rgw_kms.cc b/src/rgw/rgw_kms.cc
index 0b4817455294..6dc48badb8c2 100644
--- a/src/rgw/rgw_kms.cc
+++ b/src/rgw/rgw_kms.cc
@@ -1209,6 +1209,15 @@ int reconstitute_actual_key_from_kms(
       cache_key_id = string_cat_reserve(
           key_id, "T", calc_hash_sha256(wrapped_key).to_str());
     }
+  } else if (RGW_SSE_KMS_BACKEND_TESTING == kms_backend) {
+    // The testing backend uses keysel to derive per-object keys with
+    // the same keyid.
+    const std::string key_selector =
+        get_str_attribute(attrs, RGW_ATTR_CRYPT_KEYSEL);
+    if (!key_selector.empty()) {
+      cache_key_id = string_cat_reserve(
+          key_id, "S", calc_hash_sha256(key_selector).to_str());
+    }
   }
 
   const auto fetch = [&](std::string& out_secret) -> int {
diff --git a/src/test/rgw/test_rgw_kms.cc b/src/test/rgw/test_rgw_kms.cc
index 9fc6145d2fe7..b0323b2112e6 100644
--- a/src/test/rgw/test_rgw_kms.cc
+++ b/src/test/rgw/test_rgw_kms.cc
@@ -1,6 +1,8 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
+#include "gtest/gtest.h"
+#include <common/random_string.h>
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
 #include "common/ceph_context.h"
@@ -293,3 +295,14 @@ TEST_F(TestSSEKMS, test_transit_backend_empty_response)
   ASSERT_EQ(res, -EINVAL);
   ASSERT_EQ(actual_key, from_base64(""));
 }
+
+TEST_F(TestSSEKMS, TestingBackendDifferentKeyselSameKeyIdDoNotCollide) {
+  map<string, bufferlist> attrs;
+  const NoDoutPrefix no_dpp(cct, 1);
+  cct->_conf->rgw_crypt_s3_kms_encryption_keys =
+      "foo=IyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyM=";
+  std::string out_a, out_b;
+  ASSERT_EQ(0, get_actual_key_from_conf(&no_dpp, "foo", gen_rand_alphanumeric(cct, AES_256_KEYSIZE), out_a));
+  ASSERT_EQ(0, get_actual_key_from_conf(&no_dpp, "foo", gen_rand_alphanumeric(cct, AES_256_KEYSIZE), out_b));
+  ASSERT_NE(out_a, out_b);
+}
diff --git a/src/test/rgw/test_rgw_kms_cache.cc b/src/test/rgw/test_rgw_kms_cache.cc
index 15004a02e5f0..7fc3f18fc341 100644
--- a/src/test/rgw/test_rgw_kms_cache.cc
+++ b/src/test/rgw/test_rgw_kms_cache.cc
@@ -8,6 +8,7 @@
 #include <global/global_context.h>
 #include <global/global_init.h>
 #include <gtest/gtest.h>
+#include <rgw_crypt.h>
 
 #include <boost/asio/cancellation_signal.hpp>
 #include <boost/asio/co_spawn.hpp>
@@ -351,6 +352,37 @@ TEST_F(TestSSEKMSWithTestingKMS, TestRuntimeEnableDisable) {
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 0);
 }
 
+TEST_F(TestSSEKMSWithTestingKMS, KeyselCollisionAfterClearCache) {
+  cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "true");
+  auto attrs_A = attrs;
+  auto attrs_B = attrs;
+  set_attr(attrs_B, RGW_ATTR_CRYPT_KEYSEL,
+      "\x3a\xa8\x16\xc2\x48\x0e\xb4\x3e\x9b\xff\xab\x7b\xfc\xc2\xc7\xfd"
+      "\x3a\xa8\x16\xc2\x48\x0e\xb4\x3e\x9b\xff\xab\x7b\xfc\xc2\xc7\xfd");
+
+  std::string out_A1;
+  ASSERT_EQ(reconstitute_actual_key_from_kms(
+                &no_dpp, attrs_A, uut, null_yield, out_A1),
+      0);
+  EXPECT_EQ(out_A1, std::string(32, '*'));
+
+  uut->clear_cache();
+  EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 0);
+
+  std::string out_B1;
+  ASSERT_EQ(reconstitute_actual_key_from_kms(
+                &no_dpp, attrs_B, uut, null_yield, out_B1),
+      0);
+  EXPECT_EQ(out_B1, std::string(32, '+'));
+
+  std::string out_A2;
+  ASSERT_EQ(reconstitute_actual_key_from_kms(
+                &no_dpp, attrs_A, uut, null_yield, out_A2),
+      0);
+  EXPECT_EQ(out_A2, out_A1);
+  EXPECT_NE(out_A2, out_B1);
+}
+
 int main(int argc, char** argv) {
   auto args = argv_to_vec(argc, argv);
   std::map<std::string, std::string> defaults{

From ec7d1687295efd61f15f40e2bb488dec3c830843 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Wed, 11 Jan 2023 13:41:50 -0500
Subject: [PATCH 23/28] common/async: add service template for
 execution_context shutdown

Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit be4ddd75012781ed621c2cf75f4e315a95b7ffd0)
---
 src/common/async/service.h | 75 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 src/common/async/service.h

diff --git a/src/common/async/service.h b/src/common/async/service.h
new file mode 100644
index 000000000000..f611e7c81298
--- /dev/null
+++ b/src/common/async/service.h
@@ -0,0 +1,75 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <mutex>
+#include <boost/asio/execution_context.hpp>
+#include <boost/intrusive/list.hpp>
+
+namespace ceph::async {
+
+struct service_tag {};
+using service_list_base_hook = boost::intrusive::list_base_hook<
+    boost::intrusive::tag<service_tag>>;
+
+/// Service for two-phase execution_context shutdown, which breaks ownership
+/// cycles between completion handlers and their io objects. Tracks objects
+/// which may have outstanding completion handlers, and calls their member
+/// function service_shutdown() when the execution_context is shutting down.
+/// This member function should destroy any memory associated with its
+/// outstanding completion handlers.
+///
+/// Requirements for IoObject:
+/// * Inherits publicly from service_list_base_hook
+/// * Has public member function service_shutdown()
+/// * Calls add(*this) on construction and remove(*this) on destruction.
+template <typename IoObject>
+class service : public boost::asio::execution_context::service {
+  using base_hook = boost::intrusive::base_hook<service_list_base_hook>;
+  boost::intrusive::list<IoObject, base_hook> entries;
+  std::mutex mutex;
+
+  /// Called by the execution_context on shutdown
+  void shutdown() override {
+    while (!entries.empty()) {
+      auto& entry = entries.front();
+      entries.pop_front();
+      entry.service_shutdown();
+    }
+  }
+ public:
+  using key_type = service;
+  static inline boost::asio::execution_context::id id;
+
+  explicit service(boost::asio::execution_context& ctx)
+      : boost::asio::execution_context::service(ctx) {}
+
+  /// Register an io object for notification of service_shutdown()
+  void add(IoObject& entry) {
+    auto lock = std::scoped_lock{mutex};
+    entries.push_back(entry);
+  }
+  /// Unregister an object
+  void remove(IoObject& entry) {
+    auto lock = std::scoped_lock{mutex};
+    if (entries.empty()) {
+      // already shut down
+    } else {
+      entries.erase(entries.iterator_to(entry));
+    }
+  }
+};
+
+} // namespace ceph::async

From f748a3f4636adedc83a9d69cfb20d93ca0fdc60b Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Thu, 24 Jul 2025 18:32:16 +0200
Subject: [PATCH 24/28] backport: common/web_cache: perf counters don't have
 select_labeled_t

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
---
 src/test/common/test_web_cache.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/common/test_web_cache.cc b/src/test/common/test_web_cache.cc
index 6f1f133f0195..034c3801faba 100644
--- a/src/test/common/test_web_cache.cc
+++ b/src/test/common/test_web_cache.cc
@@ -451,7 +451,7 @@ class WebCacheConcurrencyTest : public WebCacheTest {
   void TearDown() override {
     if (_uut->perf() != nullptr) {
       JSONFormatter f(true);
-      _uut->perf()->dump_formatted(&f, false, select_labeled_t::labeled);
+      _uut->perf()->dump_formatted(&f, false, true);
       f.flush(std::cout);
       _uut->perf()->reset();
     }
@@ -568,7 +568,7 @@ class WebCacheRandomizedTest : public WebCacheTest {
   void TearDown() override {
     if (_uut->perf() != nullptr) {
       JSONFormatter f(true);
-      _uut->perf()->dump_formatted(&f, false, select_labeled_t::labeled);
+      _uut->perf()->dump_formatted(&f, false, true);
       f.flush(std::cout);
       _uut->perf()->reset();
     }

From 92d8f18718bc97ab2baeaa0535959af2df53768d Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Thu, 24 Jul 2025 18:33:24 +0200
Subject: [PATCH 25/28] backport: rgw/kms_cache: perf counters don't have
 select_labeled_t

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
---
 src/test/rgw/test_rgw_kms_cache.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/rgw/test_rgw_kms_cache.cc b/src/test/rgw/test_rgw_kms_cache.cc
index 7fc3f18fc341..78aebb9aebdf 100644
--- a/src/test/rgw/test_rgw_kms_cache.cc
+++ b/src/test/rgw/test_rgw_kms_cache.cc
@@ -231,7 +231,7 @@ class TestSSEKMSWithTestingKMS : public ::testing::Test {
 
   void TearDown() override {
     JSONFormatter f(true);
-    cache_perf->dump_formatted(&f, false, select_labeled_t::labeled);
+    cache_perf->dump_formatted(&f, false, true);
     f.flush(std::cout);
     cct->get_perfcounters_collection()->with_counters(
         [&](const PerfCountersCollectionImpl::CounterMap& by_path) {

From 296971c7e1b4283bc214ec4abd4066f78fee3731 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Fri, 20 Feb 2026 12:55:06 +0100
Subject: [PATCH 26/28] backport: no fmt::println

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
---
 src/test/rgw/test_rgw_kms_cache.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/rgw/test_rgw_kms_cache.cc b/src/test/rgw/test_rgw_kms_cache.cc
index 78aebb9aebdf..386b2fda9a1a 100644
--- a/src/test/rgw/test_rgw_kms_cache.cc
+++ b/src/test/rgw/test_rgw_kms_cache.cc
@@ -240,10 +240,10 @@ class TestSSEKMSWithTestingKMS : public ::testing::Test {
             if (perf_counters->get_name() == "rgw") {
               auto [sum, count] =
                   perf_counters->get_tavg_ns(l_rgw_kms_fetch_lat);
-              fmt::println(
+              fmt::print(
                   std::cout,
                   "RGW KMS perf counters: err_trans={} err_perm={} err_sec={} "
-                  "avg_fetch_lat={} fetch_cnt={}",
+                  "avg_fetch_lat={} fetch_cnt={}\n",
                   perf_counters->get(l_rgw_kms_error_transient),
                   perf_counters->get(l_rgw_kms_error_permanent),
                   perf_counters->get(l_rgw_kms_error_secret_store),

From 132f1314dbf55d4ff679fb77f08bffd6ab03a4d3 Mon Sep 17 00:00:00 2001
From: Marcel Lauhoff <marcel.lauhoff@clyso.com>
Date: Fri, 20 Feb 2026 13:35:48 +0100
Subject: [PATCH 27/28] backport: Workaround get_tavg_ns return parameter
 change

c826e155538 common/perf_counters: track maximum increment value for LONGRUNAVG perf
intruduced a return parameter change regression.
Since this only affects kms cache unit tests, apply the reversal there.

Signed-off-by: Marcel Lauhoff <marcel.lauhoff@clyso.com>
On-behalf-of: SAP marcel.lauhoff@sap.com
---
 src/test/rgw/test_rgw_kms_cache.cc | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/test/rgw/test_rgw_kms_cache.cc b/src/test/rgw/test_rgw_kms_cache.cc
index 386b2fda9a1a..fdd775efe223 100644
--- a/src/test/rgw/test_rgw_kms_cache.cc
+++ b/src/test/rgw/test_rgw_kms_cache.cc
@@ -238,7 +238,7 @@ class TestSSEKMSWithTestingKMS : public ::testing::Test {
           for (const auto& i : by_path) {
             const auto& perf_counters = i.second.perf_counters;
             if (perf_counters->get_name() == "rgw") {
-              auto [sum, count] =
+              auto [count, sum] =
                   perf_counters->get_tavg_ns(l_rgw_kms_fetch_lat);
               fmt::print(
                   std::cout,
@@ -269,10 +269,10 @@ class TestSSEKMSWithTestingKMS : public ::testing::Test {
 
 TEST_F(
     TestSSEKMSWithTestingKMS, TestReconstituteActualKeyFromKMSBasicsDefault) {
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 0);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 0);
   ASSERT_TRUE(cct->_conf->rgw_crypt_s3_kms_cache_enabled);
   test_do_reconstitue();
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 1);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
@@ -286,59 +286,59 @@ TEST_F(
     TestSSEKMSWithTestingKMS,
     TestReconstituteActualKeyFromKMSBasicsWithoutCache) {
   cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "false");
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 1);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 1);
   test_do_reconstitue();
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 0);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 0);
   ASSERT_FALSE(cct->_conf->rgw_crypt_s3_kms_cache_enabled);
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 2);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 2);
 }
 
 TEST_F(
     TestSSEKMSWithTestingKMS, TestReconstituteActualKeyFromKMSBasicsWithCache) {
-  ASSERT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 2);
+  ASSERT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 2);
 
   cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "true");
   test_do_reconstitue();
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 3);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 3);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
 
   test_do_reconstitue();
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 3);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 3);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
 
   test_do_reconstitue();
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 3);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 3);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 2);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
 }
 
 TEST_F(TestSSEKMSWithTestingKMS, TestRuntimeEnableDisable) {
-  ASSERT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 3);
+  ASSERT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 3);
 
   cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "true");
   test_do_reconstitue();
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 4);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 4);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
 
   cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "false");
   test_do_reconstitue();
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 5);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 5);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
 
   cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "true");
   test_do_reconstitue();
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 5);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 5);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 1);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 1);
@@ -346,7 +346,7 @@ TEST_F(TestSSEKMSWithTestingKMS, TestRuntimeEnableDisable) {
   cct->_conf.set_val("rgw_crypt_s3_kms_cache_enabled", "false");
   uut->clear_cache();
   test_do_reconstitue();
-  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).second, 6);
+  EXPECT_EQ(perfcounter->get_tavg_ns(l_rgw_kms_fetch_lat).first, 6);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::hit)), 0);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::miss)), 0);
   EXPECT_EQ(cache_perf->get(static_cast<int>(webcache::Metric::size)), 0);

From 55b27b4eb99cdbb6c48f00327bb2682fd1eb7ef8 Mon Sep 17 00:00:00 2001
From: Casey Bodley <cbodley@redhat.com>
Date: Mon, 24 Jun 2024 11:23:36 -0400
Subject: [PATCH 28/28] dout: add macros for libfmt-style logging

new dout macros use fmt::print() to write output directly to the
underlying ostream. this enables libfmt's printf-style formatting syntax
for ceph log output, without the string allocation overhead of:

    dout(10) << fmt::format("str={} count={}", str, count) << dendl;

which becomes:

    dout_fmt(10, "str={} count={}", str, count);

Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit a58d07aaa39b7852fc6c79563c2ae8c92d853d4d)
---
 src/common/dout_fmt.h            | 56 +++++++++++++++++++++++++++++++
 src/test/common/CMakeLists.txt   |  4 +++
 src/test/common/test_dout_fmt.cc | 57 ++++++++++++++++++++++++++++++++
 3 files changed, 117 insertions(+)
 create mode 100644 src/common/dout_fmt.h
 create mode 100644 src/test/common/test_dout_fmt.cc

diff --git a/src/common/dout_fmt.h b/src/common/dout_fmt.h
new file mode 100644
index 000000000000..c22fdf30cfe4
--- /dev/null
+++ b/src/common/dout_fmt.h
@@ -0,0 +1,56 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#pragma once
+
+#include <iosfwd>
+#include <iterator>
+#include <fmt/ostream.h>
+#include "dout.h"
+
+/// \file dout_fmt.h
+///
+/// \brief dout macros to format log statements with libfmt
+///
+/// A set of dout macros taking a format string and its corresponding argument
+/// list. Log output is written directly to the underlying std::ostream by
+/// fmt::print() rather than exposing the stream for ostream operator
+/// chaining.
+
+// work around "warning: value computed is not used" with default dout_prefix
+inline void dout_fmt_use_prefix(std::ostream&) {}
+
+#define lsubdout_fmt(cct, sub, v, ...) \
+  dout_impl(cct, ceph_subsys_##sub, v) \
+  dout_fmt_use_prefix(dout_prefix); \
+  fmt::print(*_dout, __VA_ARGS__); \
+  *_dout << dendl
+
+#define ldout_fmt(cct, v, ...) \
+  dout_impl(cct, dout_subsys, v) \
+  dout_fmt_use_prefix(dout_prefix); \
+  fmt::print(*_dout, __VA_ARGS__); \
+  *_dout << dendl
+
+#define dout_fmt(v, ...) \
+  ldout_fmt((dout_context), v, __VA_ARGS__)
+
+#define ldpp_dout_fmt(dpp, v, ...) \
+  if (decltype(auto) pdpp = (dpp); pdpp) { /* workaround -Wnonnull-compare for 'this' */ \
+    dout_impl(pdpp->get_cct(), ceph::dout::need_dynamic(pdpp->get_subsys()), v) \
+    pdpp->gen_prefix(*_dout); \
+    fmt::print(*_dout, __VA_ARGS__); \
+    *_dout << dendl; \
+  }
diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt
index 0296690f2dee..5504c68f7e5e 100644
--- a/src/test/common/CMakeLists.txt
+++ b/src/test/common/CMakeLists.txt
@@ -334,6 +334,10 @@ target_link_libraries(unittest_dns_resolve global)
 add_ceph_unittest(unittest_dns_resolve)
 endif()
 
+add_executable(unittest_dout_fmt test_dout_fmt.cc $<TARGET_OBJECTS:unit-main>)
+target_link_libraries(unittest_dout_fmt global)
+add_ceph_unittest(unittest_dout_fmt)
+
 # We're getting an ICE when trying to compile this test using mingw-gcc and
 # recent Boost versions. Note that mingw-llvm works fine.
 if (NOT WIN32 OR (NOT(CMAKE_CXX_COMPILER_ID STREQUAL GNU)))
diff --git a/src/test/common/test_dout_fmt.cc b/src/test/common/test_dout_fmt.cc
new file mode 100644
index 000000000000..7d6b519fe35d
--- /dev/null
+++ b/src/test/common/test_dout_fmt.cc
@@ -0,0 +1,57 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright contributors to the Ceph project
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "common/dout_fmt.h"
+#include <gtest/gtest.h>
+
+TEST(DoutFmt, SubDout)
+{
+  // expect level 0 to always be gathered
+  lsubdout_fmt(g_ceph_context, test, 0, "{}: {}", "value", 42);
+  // expect level 99 to be compiled out
+  lsubdout_fmt(g_ceph_context, test, 99, "{}: {}", "value", 42);
+}
+
+#define dout_subsys ceph_subsys_test
+
+TEST(DoutFmt, Dout)
+{
+  ldout_fmt(g_ceph_context, 0, "{}: {}", "value", 42);
+  ldout_fmt(g_ceph_context, 99, "{}: {}", "value", 42);
+}
+
+#define dout_context g_ceph_context
+
+TEST(DoutFmt, DoutContext)
+{
+  dout_fmt(0, "{}: {}", "value", 42);
+  dout_fmt(99, "{}: {}", "value", 42);
+}
+
+#undef dout_prefix
+#define dout_prefix *_dout << "prefix: "
+
+TEST(DoutFmt, DoutPrefix)
+{
+  ldout_fmt(g_ceph_context, 0, "{}: {}", "value", 42);
+  ldout_fmt(g_ceph_context, 99, "{}: {}", "value", 42);
+}
+
+TEST(DoutFmt, DppDout)
+{
+  const DoutPrefix dpp{g_ceph_context, dout_subsys, "prefix: "};
+  ldpp_dout_fmt(&dpp, 0, "{}: {}", "value", 42);
+  ldpp_dout_fmt(&dpp, 99, "{}: {}", "value", 42);
+}