From f4648df874595146b189e810fda735ee99f58a6e Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Fri, 16 Oct 2026 12:10:32 +0800 Subject: [PATCH 1/2] expose data-corrupting api for scrubber testing --- conanfile.py | 2 +- src/lib/homestore_backend/CMakeLists.txt | 16 +- src/lib/homestore_backend/gc_manager.cpp | 4 +- src/lib/homestore_backend/gc_manager.hpp | 44 ++-- src/lib/homestore_backend/hs_blob_manager.cpp | 6 +- src/lib/homestore_backend/hs_debug_inject.cpp | 160 ++++++++++++ src/lib/homestore_backend/hs_debug_inject.hpp | 35 +++ src/lib/homestore_backend/hs_http_manager.cpp | 76 ++++++ src/lib/homestore_backend/hs_http_manager.hpp | 1 + src/lib/homestore_backend/hs_pg_manager.cpp | 26 +- .../replication_state_machine.hpp | 2 +- .../snapshot_receive_handler.cpp | 18 +- .../tests/homeobj_misc_tests.cpp | 20 +- .../homestore_backend/tests/hs_pg_tests.cpp | 4 +- .../tests/hs_scrubber_tests.cpp | 227 ++++++------------ 15 files changed, 418 insertions(+), 223 deletions(-) create mode 100644 src/lib/homestore_backend/hs_debug_inject.cpp create mode 100644 src/lib/homestore_backend/hs_debug_inject.hpp diff --git a/conanfile.py b/conanfile.py index 86a2cbcf7..3aa12a9aa 100644 --- a/conanfile.py +++ b/conanfile.py @@ -10,7 +10,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "4.3.6" + version = "4.3.7" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeStore" diff --git a/src/lib/homestore_backend/CMakeLists.txt b/src/lib/homestore_backend/CMakeLists.txt index 39f4c3f23..487a0007d 100644 --- a/src/lib/homestore_backend/CMakeLists.txt +++ b/src/lib/homestore_backend/CMakeLists.txt @@ -31,6 +31,7 @@ target_sources("${PROJECT_NAME}_homestore" PRIVATE hs_http_manager.cpp gc_manager.cpp scrub_manager.cpp + hs_debug_inject.cpp MPMCPriorityQueue.hpp $ ) @@ -186,7 +187,19 @@ target_sources(homestore_test_scrubber PRIVATE $ kRatioBucketLabels = { - "00-10", "10-20", "20-30", "30-40", "40-50", - "50-60", "60-70", "70-80", "80-90", "90-100"}; + "00-10", "10-20", "20-30", "30-40", "40-50", "50-60", "60-70", "70-80", "80-90", "90-100"}; for (size_t i = 0; i < kRatioBucketLabels.size(); ++i) { const auto lo = i * 10; const auto hi = (i + 1) * 10; - const auto desc = fmt::format( - "Snapshot count of pending chunks with garbage ratio in ({}, {}]% " - "(bucket={})", - lo, hi, kRatioBucketLabels[i]); - ratio_bucket_indices_[i] = m_impl_ptr->register_gauge( - "pending_gc_chunks_ratio", desc, "" /* report_name */, - sisl::metric_label{"bucket", kRatioBucketLabels[i]}); + const auto desc = fmt::format("Snapshot count of pending chunks with garbage ratio in ({}, {}]% " + "(bucket={})", + lo, hi, kRatioBucketLabels[i]); + ratio_bucket_indices_[i] = + m_impl_ptr->register_gauge("pending_gc_chunks_ratio", desc, "" /* report_name */, + sisl::metric_label{"bucket", kRatioBucketLabels[i]}); } register_me_to_farm(); @@ -230,9 +224,8 @@ class GCManager { // Bypass GAUGE_UPDATE for the same reason as bucket registration: we need to // address 10 distinct gauge indices that share one metric name. for (size_t i = 0; i < ratio_bucket_indices_.size(); ++i) { - m_impl_ptr->gauge_update( - ratio_bucket_indices_[i], - static_cast< int64_t >(gc_actor_.get_pending_ratio_bucket(i))); + m_impl_ptr->gauge_update(ratio_bucket_indices_[i], + static_cast< int64_t >(gc_actor_.get_pending_ratio_bucket(i))); } } @@ -315,12 +308,8 @@ class GCManager { // Snapshot readers used by pdev_gc_metrics::on_gather. Return the last value published // by GCManager::scan_chunks_for_gc for this pdev; 0 before the first scan completes. - uint64_t get_pending_gc_bytes() const { - return m_pending_gc_bytes.load(std::memory_order_relaxed); - } - uint64_t get_eligible_gc_bytes() const { - return m_eligible_gc_bytes.load(std::memory_order_relaxed); - } + uint64_t get_pending_gc_bytes() const { return m_pending_gc_bytes.load(std::memory_order_relaxed); } + uint64_t get_eligible_gc_bytes() const { return m_eligible_gc_bytes.load(std::memory_order_relaxed); } uint32_t get_eligible_gc_chunk_count() const { return m_eligible_gc_chunk_count.load(std::memory_order_relaxed); } @@ -332,8 +321,7 @@ class GCManager { // the totals locally over all chunks on this pdev, then hand them in via one call so the // metrics stay internally consistent within a scan cycle. Between-gauge drift is bounded // by one scan interval; individual scalars are aligned and therefore torn-read safe. - void publish_scan_snapshot(uint64_t pending_bytes, uint64_t eligible_bytes, - uint32_t eligible_chunks, + void publish_scan_snapshot(uint64_t pending_bytes, uint64_t eligible_bytes, uint32_t eligible_chunks, const std::array< uint32_t, 10 >& ratio_buckets) { m_pending_gc_bytes.store(pending_bytes, std::memory_order_relaxed); m_eligible_gc_bytes.store(eligible_bytes, std::memory_order_relaxed); diff --git a/src/lib/homestore_backend/hs_blob_manager.cpp b/src/lib/homestore_backend/hs_blob_manager.cpp index 6dc82be30..c229592f8 100644 --- a/src/lib/homestore_backend/hs_blob_manager.cpp +++ b/src/lib/homestore_backend/hs_blob_manager.cpp @@ -14,8 +14,8 @@ SISL_LOGGING_DECL(blobmgr) #define BLOG(level, trace_id, shard_id, blob_id, msg, ...) \ LOG##level##MOD(blobmgr, "[traceID={},shardID=0x{:x},pg={},shard=0x{:x},blob={}] " msg, trace_id, shard_id, \ - (shard_id >> homeobject::shard_width), (shard_id & homeobject::shard_mask), blob_id, \ - ##__VA_ARGS__) + (shard_id >> homeobject::shard_width), (shard_id & homeobject::shard_mask), \ + blob_id, ##__VA_ARGS__) #define BLOGT(trace_id, shard_id, blob_id, msg, ...) BLOG(TRACE, trace_id, shard_id, blob_id, msg, ##__VA_ARGS__) #define BLOGD(trace_id, shard_id, blob_id, msg, ...) BLOG(DEBUG, trace_id, shard_id, blob_id, msg, ##__VA_ARGS__) @@ -650,6 +650,8 @@ void HSHomeObject::on_blob_del_commit(int64_t lsn, sisl::blob const& header, sis BLOGE(tid, msg_header->shard_id, *r_cast< blob_id_t const* >(key.cbytes()), "replication message header is corrupted with crc error, lsn={} header={}", lsn, msg_header->to_string()); if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError(BlobErrorCode::CHECKSUM_MISMATCH))); } + RELEASE_ASSERT(false, "replication message header is corrupted with crc error, lsn={}, header={}", lsn, + msg_header->to_string()); return; } diff --git a/src/lib/homestore_backend/hs_debug_inject.cpp b/src/lib/homestore_backend/hs_debug_inject.cpp new file mode 100644 index 000000000..39d6485ff --- /dev/null +++ b/src/lib/homestore_backend/hs_debug_inject.cpp @@ -0,0 +1,160 @@ +#include "hs_debug_inject.hpp" + +#include +#include + +#include +#include +#include +#include + +#include "hs_homeobject.hpp" + +namespace homeobject { +namespace hs_debug_inject { + +InjectResult delete_blob_from_index(HSHomeObject& ho, pg_id_t pg_id, shard_id_t shard_id, blob_id_t blob_id) { + auto hs_pg = ho.get_hs_pg(pg_id); + if (!hs_pg) { return folly::makeUnexpected(fmt::format("pg {} not found", pg_id)); } + + BlobRouteKey blob_key{BlobRoute{shard_id, blob_id}}; + BlobRouteValue out_value; + homestore::BtreeSingleRemoveRequest remove_req{&blob_key, &out_value}; + auto status = hs_pg->index_table_->remove(remove_req); + if (status != homestore::btree_status_t::success) { + return folly::makeUnexpected( + fmt::format("failed to remove blob {} from index, status={}", blob_id, static_cast< int >(status))); + } + return folly::Unit(); +} + +InjectResult delete_shard_from_index(HSHomeObject& ho, pg_id_t pg_id, shard_id_t shard_id) { + auto hs_pg = ho.get_hs_pg(pg_id); + if (!hs_pg) { return folly::makeUnexpected(fmt::format("pg {} not found", pg_id)); } + + auto start_key = BlobRouteKey{BlobRoute{shard_id, 0}}; + auto end_key = BlobRouteKey{BlobRoute{shard_id, std::numeric_limits< uint64_t >::max()}}; + homestore::BtreeRangeRemoveRequest< BlobRouteKey > range_remove_req{ + homestore::BtreeKeyRange< BlobRouteKey >{std::move(start_key), true /* inclusive */, std::move(end_key), + true /* inclusive */}, + nullptr, std::numeric_limits< uint32_t >::max(), + [](homestore::BtreeKey const&, homestore::BtreeValue const&) -> bool { return true; }}; + + auto status = hs_pg->index_table_->remove(range_remove_req); + if (status != homestore::btree_status_t::success && status != homestore::btree_status_t::not_found) { + return folly::makeUnexpected( + fmt::format("failed to remove shard {} from index, status={}", shard_id, static_cast< int >(status))); + } + return folly::Unit(); +} + +InjectResult corrupt_blob_data(HSHomeObject& ho, pg_id_t pg_id, shard_id_t shard_id, blob_id_t blob_id) { + auto hs_pg = ho.get_hs_pg(pg_id); + if (!hs_pg) { return folly::makeUnexpected(fmt::format("pg {} not found", pg_id)); } + + auto& data_service = homestore::data_service(); + const auto blk_size = data_service.get_blk_size(); + + BlobRouteKey blob_key{BlobRoute{shard_id, blob_id}}; + BlobRouteValue out_value; + homestore::BtreeSingleGetRequest blob_get_req{&blob_key, &out_value}; + auto status = hs_pg->index_table_->get(blob_get_req); + if (status != homestore::btree_status_t::success) { + return folly::makeUnexpected( + fmt::format("blob {} not found in index, status={}", blob_id, static_cast< int >(status))); + } + + auto pbas = out_value.pbas(); + auto total_size = pbas.blk_count() * blk_size; + sisl::sg_list data_sgs; + data_sgs.size = total_size; + data_sgs.iovs.emplace_back(iovec{.iov_base = iomanager.iobuf_alloc(blk_size, total_size), .iov_len = total_size}); + + std::string err_msg; + try { + data_service.async_read(pbas, data_sgs, total_size) + .thenValue([&](auto&& err) { + if (err) { throw std::runtime_error(fmt::format("read failed: {}", err.message())); } + + auto* data_ptr = reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base); + for (size_t i = 0; i < data_sgs.iovs[0].iov_len / 2; i++) { + data_ptr[i] ^= 0xFF; + } + + return data_service.async_write(data_sgs, pbas).thenValue([](auto&& werr) { + if (werr) { throw std::runtime_error(fmt::format("write failed: {}", werr.message())); } + }); + }) + .get(); + } catch (const std::exception& e) { err_msg = e.what(); } + + iomanager.iobuf_free(reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base)); + if (!err_msg.empty()) { return folly::makeUnexpected(err_msg); } + return folly::Unit(); +} + +InjectResult make_blob_inconsistent(HSHomeObject& ho, pg_id_t pg_id, shard_id_t shard_id, blob_id_t blob_id) { + auto hs_pg = ho.get_hs_pg(pg_id); + if (!hs_pg) { return folly::makeUnexpected(fmt::format("pg {} not found", pg_id)); } + + auto& data_service = homestore::data_service(); + const auto blk_size = data_service.get_blk_size(); + + BlobRouteKey blob_key{BlobRoute{shard_id, blob_id}}; + BlobRouteValue out_value; + homestore::BtreeSingleGetRequest blob_get_req{&blob_key, &out_value}; + auto status = hs_pg->index_table_->get(blob_get_req); + if (status != homestore::btree_status_t::success) { + return folly::makeUnexpected( + fmt::format("blob {} not found in index, status={}", blob_id, static_cast< int >(status))); + } + + auto pbas = out_value.pbas(); + auto total_size = pbas.blk_count() * blk_size; + sisl::sg_list data_sgs; + data_sgs.size = total_size; + data_sgs.iovs.emplace_back(iovec{.iov_base = iomanager.iobuf_alloc(blk_size, total_size), .iov_len = total_size}); + + std::string err_msg; + try { + data_service.async_read(pbas, data_sgs, total_size) + .thenValue([&](auto&& err) { + if (err) { throw std::runtime_error(fmt::format("read failed: {}", err.message())); } + + uint8_t* read_buf = reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base); + auto* header = reinterpret_cast< HSHomeObject::BlobHeader* >(read_buf); + uint8_t* blob_bytes = read_buf + header->data_offset; + + std::mt19937 rng{std::random_device{}()}; + std::uniform_int_distribution< int > dist(0, 255); + for (size_t i = 0; i < header->blob_size / 2; i++) { + blob_bytes[i] ^= static_cast< uint8_t >(dist(rng)); + } + + uint8_t computed_hash[HSHomeObject::BlobHeader::blob_max_hash_len]{}; + ho.compute_blob_payload_hash(header->hash_algorithm, blob_bytes, header->blob_size, computed_hash, + HSHomeObject::BlobHeader::blob_max_hash_len); + std::memcpy(header->hash, computed_hash, HSHomeObject::BlobHeader::blob_max_hash_len); + std::memset(header->header_hash, 0, HSHomeObject::BlobHeader::blob_max_hash_len); + uint32_t computed_header_hash = crc32_ieee(0, (uint8_t*)header, sizeof(HSHomeObject::BlobHeader)); + std::memcpy(header->header_hash, &computed_header_hash, sizeof(uint32_t)); + + if (!ho.verify_blob(data_sgs.iovs[0].iov_base, header->shard_id, header->blob_id)) { + throw std::runtime_error( + fmt::format("blob verification failed after modification, blob_id={}", blob_id)); + } + + return data_service.async_write(data_sgs, pbas).thenValue([](auto&& werr) { + if (werr) { throw std::runtime_error(fmt::format("write failed: {}", werr.message())); } + }); + }) + .get(); + } catch (const std::exception& e) { err_msg = e.what(); } + + iomanager.iobuf_free(reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base)); + if (!err_msg.empty()) { return folly::makeUnexpected(err_msg); } + return folly::Unit(); +} + +} // namespace hs_debug_inject +} // namespace homeobject diff --git a/src/lib/homestore_backend/hs_debug_inject.hpp b/src/lib/homestore_backend/hs_debug_inject.hpp new file mode 100644 index 000000000..7c1e3a34c --- /dev/null +++ b/src/lib/homestore_backend/hs_debug_inject.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include "homeobject/common.hpp" + +namespace homeobject { + +class HSHomeObject; + +namespace hs_debug_inject { + +using InjectResult = folly::Expected< folly::Unit, std::string >; + +// Deletes a single blob's entry from its PG's local B-tree index, simulating a +// missing blob on this replica without going through the replication layer. +InjectResult delete_blob_from_index(HSHomeObject& ho, pg_id_t pg_id, shard_id_t shard_id, blob_id_t blob_id); + +// Deletes all blob entries for a shard from its PG's local B-tree index, +// simulating a missing shard on this replica. +InjectResult delete_shard_from_index(HSHomeObject& ho, pg_id_t pg_id, shard_id_t shard_id); + +// Flips bits in the first half of a blob's on-disk data, simulating silent +// bit-rot. The blob's stored hash is left untouched, so scrub should report MISMATCH. +InjectResult corrupt_blob_data(HSHomeObject& ho, pg_id_t pg_id, shard_id_t shard_id, blob_id_t blob_id); + +// Flips bits in a blob's on-disk data and recomputes a self-consistent hash, +// simulating a replica whose copy differs from its peers but passes its own +// integrity check. +InjectResult make_blob_inconsistent(HSHomeObject& ho, pg_id_t pg_id, shard_id_t shard_id, blob_id_t blob_id); + +} // namespace hs_debug_inject +} // namespace homeobject diff --git a/src/lib/homestore_backend/hs_http_manager.cpp b/src/lib/homestore_backend/hs_http_manager.cpp index 8cb52df00..cc84d19d1 100644 --- a/src/lib/homestore_backend/hs_http_manager.cpp +++ b/src/lib/homestore_backend/hs_http_manager.cpp @@ -24,6 +24,7 @@ #include "hs_http_manager.hpp" #include "hs_homeobject.hpp" +#include "hs_debug_inject.hpp" namespace homeobject { @@ -73,6 +74,8 @@ HttpManager::HttpManager(HSHomeObject& ho) : ho_(ho) { #ifdef _PRERELEASE {Pistache::Http::Method::Post, "/api/v1/crashSystem", Pistache::Rest::Routes::bind(&HttpManager::crash_system, this)}, + {Pistache::Http::Method::Post, "/api/v1/debug/inject_corruption", + Pistache::Rest::Routes::bind(&HttpManager::inject_corruption, this)}, #endif {Pistache::Http::Method::Get, "/api/v1/pg", Pistache::Rest::Routes::bind(&HttpManager::get_pg, this)}, {Pistache::Http::Method::Get, "/api/v1/chunks", @@ -1266,6 +1269,79 @@ void HttpManager::crash_system(const Pistache::Rest::Request& request, Pistache: } response.send(Pistache::Http::Code::Ok, resp); } + +void HttpManager::inject_corruption(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response) { + LOGINFO("inject_corruption handler invoked, raw query resource={}", request.resource()); + const auto pg_id_param = request.query().get("pg_id"); + const auto shard_id_param = request.query().get("shard_id"); + const auto blob_id_param = request.query().get("blob_id"); + const auto mode_param = request.query().get("mode"); + + if (!pg_id_param || !shard_id_param || !blob_id_param || !mode_param) { + LOGWARN("inject_corruption missing param(s): pg_id={} shard_id={} blob_id={} mode={}", pg_id_param.has_value(), + shard_id_param.has_value(), blob_id_param.has_value(), mode_param.has_value()); + nlohmann::json error; + error["error"] = "Missing required parameter(s)"; + error["usage"] = "POST /api/v1/debug/inject_corruption?pg_id=&shard_id=&blob_id=" + "&mode="; + response.send(Pistache::Http::Code::Bad_Request, error.dump()); + return; + } + + uint16_t pg_id; + shard_id_t shard_id; + blob_id_t blob_id; + try { + auto pg_val = std::stoul(pg_id_param.value()); + if (pg_val > std::numeric_limits< uint16_t >::max()) { throw std::out_of_range("pg_id"); } + pg_id = static_cast< uint16_t >(pg_val); + shard_id = std::stoull(shard_id_param.value(), nullptr, 0); + blob_id = std::stoull(blob_id_param.value(), nullptr, 0); + } catch (const std::exception& e) { + LOGWARN("inject_corruption invalid numeric parameter: {}", e.what()); + nlohmann::json error; + error["error"] = fmt::format("Invalid numeric parameter: {}", e.what()); + response.send(Pistache::Http::Code::Bad_Request, error.dump()); + return; + } + + const auto& mode = mode_param.value(); + LOGINFO("inject_corruption pg_id={} shard_id={} blob_id={} mode={}", pg_id, shard_id, blob_id, mode); + hs_debug_inject::InjectResult result; + if (mode == "corrupt_data") { + result = hs_debug_inject::corrupt_blob_data(ho_, pg_id, shard_id, blob_id); + } else if (mode == "inconsistent") { + result = hs_debug_inject::make_blob_inconsistent(ho_, pg_id, shard_id, blob_id); + } else if (mode == "delete_blob") { + result = hs_debug_inject::delete_blob_from_index(ho_, pg_id, shard_id, blob_id); + } else if (mode == "delete_shard") { + result = hs_debug_inject::delete_shard_from_index(ho_, pg_id, shard_id); + } else { + LOGWARN("inject_corruption unknown mode: {}", mode); + nlohmann::json error; + error["error"] = "Unknown mode: " + mode; + error["usage"] = "mode must be one of corrupt_data|inconsistent|delete_blob|delete_shard"; + response.send(Pistache::Http::Code::Bad_Request, error.dump()); + return; + } + + if (!result) { + LOGWARN("inject_corruption failed: {}", result.error()); + nlohmann::json error; + error["error"] = result.error(); + response.send(Pistache::Http::Code::Not_Found, error.dump()); + return; + } + + LOGINFO("inject_corruption succeeded for pg_id={} shard_id={} blob_id={} mode={}", pg_id, shard_id, blob_id, mode); + nlohmann::json ok; + ok["pg_id"] = pg_id; + ok["shard_id"] = shard_id; + ok["blob_id"] = blob_id; + ok["mode"] = mode; + ok["message"] = "Corruption injected"; + response.send(Pistache::Http::Code::Ok, ok.dump()); +} #endif } // namespace homeobject diff --git a/src/lib/homestore_backend/hs_http_manager.hpp b/src/lib/homestore_backend/hs_http_manager.hpp index 02a24dcc4..7439cb154 100644 --- a/src/lib/homestore_backend/hs_http_manager.hpp +++ b/src/lib/homestore_backend/hs_http_manager.hpp @@ -56,6 +56,7 @@ class HttpManager { #ifdef _PRERELEASE void crash_system(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); + void inject_corruption(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response); #endif private: diff --git a/src/lib/homestore_backend/hs_pg_manager.cpp b/src/lib/homestore_backend/hs_pg_manager.cpp index 2a0acdc0f..bb60b0277 100644 --- a/src/lib/homestore_backend/hs_pg_manager.cpp +++ b/src/lib/homestore_backend/hs_pg_manager.cpp @@ -174,19 +174,19 @@ PGManager::NullAsyncResult HSHomeObject::_create_pg(PGInfo&& pg_info, std::set< // if don't have repl dev, it will return ReplServiceError::SERVER_NOT_FOUND return hs_repl_service() .remove_repl_dev(repl_dev_group_id) - .deferValue([r, repl_dev_group_id, tid, this, - finish_creating_pg](auto&& e) -> PGManager::NullAsyncResult { - if (e != ReplServiceError::OK) { - LOGW("Failed to remove repl device which group_id={}, error={}, trace_id={}", - repl_dev_group_id, e, tid); - } - // Drop the inflight mark only after chunk/repldev cleanup, so a retry cannot - // select the same chunks while they are being returned. - finish_creating_pg(); - decr_pending_request_num(); - // still return the original error - return folly::makeUnexpected(r.error()); - }); + .deferValue( + [r, repl_dev_group_id, tid, this, finish_creating_pg](auto&& e) -> PGManager::NullAsyncResult { + if (e != ReplServiceError::OK) { + LOGW("Failed to remove repl device which group_id={}, error={}, trace_id={}", + repl_dev_group_id, e, tid); + } + // Drop the inflight mark only after chunk/repldev cleanup, so a retry cannot + // select the same chunks while they are being returned. + finish_creating_pg(); + decr_pending_request_num(); + // still return the original error + return folly::makeUnexpected(r.error()); + }); } finish_creating_pg(); decr_pending_request_num(); diff --git a/src/lib/homestore_backend/replication_state_machine.hpp b/src/lib/homestore_backend/replication_state_machine.hpp index 75d2d4187..6d01ea590 100644 --- a/src/lib/homestore_backend/replication_state_machine.hpp +++ b/src/lib/homestore_backend/replication_state_machine.hpp @@ -277,7 +277,7 @@ class ReplicationStateMachine : public homestore::ReplDevListener { std::pair< homestore::repl_lsn_t, homestore::chunk_num_t > get_no_space_left_error_info() const; - void handle_no_space_left(homestore ::repl_lsn_t lsn, homestore ::chunk_num_t chunk_id); + void handle_no_space_left(homestore::repl_lsn_t lsn, homestore::chunk_num_t chunk_id); }; } // namespace homeobject diff --git a/src/lib/homestore_backend/snapshot_receive_handler.cpp b/src/lib/homestore_backend/snapshot_receive_handler.cpp index 4ea5cf0da..624a60fe4 100644 --- a/src/lib/homestore_backend/snapshot_receive_handler.cpp +++ b/src/lib/homestore_backend/snapshot_receive_handler.cpp @@ -35,8 +35,8 @@ int HSHomeObject::SnapshotReceiveHandler::process_pg_snapshot_data(ResyncPGMetaD pg_member.priority = member->priority(); pg_info.members.insert(pg_member); } - LOGD("Resync PG membership: pg={}, expected_members={}, members={}", pg_meta.pg_id(), - pg_info.expected_member_num, pg_meta.members()->size()) + LOGD("Resync PG membership: pg={}, expected_members={}, members={}", pg_meta.pg_id(), pg_info.expected_member_num, + pg_meta.members()->size()) #ifdef _PRERELEASE if (iomgr_flip::instance()->test_flip("snapshot_receiver_pg_error")) { @@ -105,8 +105,8 @@ int HSHomeObject::SnapshotReceiveHandler::process_shard_snapshot_data(ResyncShar } ctx_->shard_cursor = shard_meta.shard_id(); ctx_->cur_batch_num = 0; - LOGI("Processed resync shard metadata: pg={}, shard_id=0x{:x}, state={}", shard_meta.pg_id(), - shard_meta.shard_id(), shard_meta.state()); + LOGI("Processed resync shard metadata: pg={}, shard_id=0x{:x}, state={}", shard_meta.pg_id(), shard_meta.shard_id(), + shard_meta.state()); return 0; } @@ -267,8 +267,8 @@ int HSHomeObject::SnapshotReceiveHandler::process_blobs_snapshot_data(ResyncBlob } #endif auto blob_id = blob->blob_id(); - LOGT("Writing resync blob: pg={}, shard_id=0x{:x}, blob={}, blkid={}", ctx_->pg_id, ctx_->shard_cursor, - blob_id, blk_id.to_string()); + LOGT("Writing resync blob: pg={}, shard_id=0x{:x}, blob={}, blkid={}", ctx_->pg_id, ctx_->shard_cursor, blob_id, + blk_id.to_string()); // ToDo: limit the max concurrent? futs.emplace_back( @@ -315,9 +315,9 @@ int HSHomeObject::SnapshotReceiveHandler::process_blobs_snapshot_data(ResyncBlob if (!all_io_submitted || ec != std::error_code{}) { if (!all_io_submitted) { - LOGE( - "Failed to submit complete resync shard batch: pg={}, shard_id=0x{:x}, batch={}, expected_blobs={}, submitted_blobs={}", - ctx_->pg_id, ctx_->shard_cursor, batch_num, data_blobs.blob_list()->size(), futs.size()); + LOGE("Failed to submit complete resync shard batch: pg={}, shard_id=0x{:x}, batch={}, expected_blobs={}, " + "submitted_blobs={}", + ctx_->pg_id, ctx_->shard_cursor, batch_num, data_blobs.blob_list()->size(), futs.size()); } else { LOGE("Failed to write resync shard batch: pg={}, shard_id=0x{:x}, batch={}, error_code={}, error={}", ctx_->pg_id, ctx_->shard_cursor, batch_num, ec.value(), ec.message()); diff --git a/src/lib/homestore_backend/tests/homeobj_misc_tests.cpp b/src/lib/homestore_backend/tests/homeobj_misc_tests.cpp index 6848063dc..2c0de9c21 100644 --- a/src/lib/homestore_backend/tests/homeobj_misc_tests.cpp +++ b/src/lib/homestore_backend/tests/homeobj_misc_tests.cpp @@ -508,10 +508,9 @@ TEST_F(HomeObjectFixture, SnapshotReceiveHandler) { for (uint64_t i = 1; i <= num_shards_per_pg; i++) { shard_ids.push_back(i); } - auto pg_entry = - CreateResyncPGMetaDataDirect(builder, pg_id, &uuid, pg->pg_info_.size, pg->pg_info_.expected_member_num, - pg->pg_info_.chunk_size, blob_seq_num, num_shards_per_pg, &members, &shard_ids, - blob_seq_num /* total_blobs_to_transfer */); + auto pg_entry = CreateResyncPGMetaDataDirect( + builder, pg_id, &uuid, pg->pg_info_.size, pg->pg_info_.expected_member_num, pg->pg_info_.chunk_size, + blob_seq_num, num_shards_per_pg, &members, &shard_ids, blob_seq_num /* total_blobs_to_transfer */); builder.Finish(pg_entry); auto pg_meta = GetResyncPGMetaData(builder.GetBufferPointer()); auto ret = handler->process_pg_snapshot_data(*pg_meta); @@ -672,7 +671,8 @@ TEST_F(HomeObjectFixture, SnapshotReceiveHandler) { builder, retry_blob_id, static_cast< uint8_t >(ResyncBlobState::NORMAL), &retry_data)); blob_map[retry_blob_id] = std::make_tuple< Blob, bool >(std::move(retry_blob), false); } - builder.Finish(CreateResyncBlobDataBatchDirect(builder, &retry_blob_entries, j == num_batches_per_shard)); + builder.Finish( + CreateResyncBlobDataBatchDirect(builder, &retry_blob_entries, j == num_batches_per_shard)); auto retry_blob_batch = GetResyncBlobDataBatch(builder.GetBufferPointer()); ASSERT_EQ(handler->process_blobs_snapshot_data(*retry_blob_batch, j, j == num_batches_per_shard), 0); } else { @@ -759,10 +759,9 @@ TEST_F(HomeObjectFixture, SnapshotReceiveHandlerAllocatorResyncAfterCrash) { members.push_back(CreateMemberDirect(builder, &id, member.name.c_str(), priority)); } std::vector< uint64_t > shard_ids = {1}; - auto pg_entry = CreateResyncPGMetaDataDirect(builder, pg_id, &uuid, pg->pg_info_.size, - pg->pg_info_.expected_member_num, pg->pg_info_.chunk_size, - num_blobs_total /*blob_seq_num*/, 1 /*shard_seq_num*/, &members, - &shard_ids); + auto pg_entry = CreateResyncPGMetaDataDirect( + builder, pg_id, &uuid, pg->pg_info_.size, pg->pg_info_.expected_member_num, pg->pg_info_.chunk_size, + num_blobs_total /*blob_seq_num*/, 1 /*shard_seq_num*/, &members, &shard_ids); builder.Finish(pg_entry); ASSERT_EQ(handler->process_pg_snapshot_data(*GetResyncPGMetaData(builder.GetBufferPointer())), 0); builder.Reset(); @@ -816,7 +815,8 @@ TEST_F(HomeObjectFixture, SnapshotReceiveHandlerAllocatorResyncAfterCrash) { uint64_t total_bytes{0}; for (blob_id_t blob_id = start_blob_id; blob_id < end_blob_id; blob_id++) { auto blob = build_blob(blob_id); - total_bytes += sisl::round_up(sizeof(HSHomeObject::BlobHeader), _obj_inst->_data_block_size) + blob.body.size(); + total_bytes += + sisl::round_up(sizeof(HSHomeObject::BlobHeader), _obj_inst->_data_block_size) + blob.body.size(); } return total_bytes; }; diff --git a/src/lib/homestore_backend/tests/hs_pg_tests.cpp b/src/lib/homestore_backend/tests/hs_pg_tests.cpp index a544e81b5..15936d3f9 100644 --- a/src/lib/homestore_backend/tests/hs_pg_tests.cpp +++ b/src/lib/homestore_backend/tests/hs_pg_tests.cpp @@ -382,8 +382,8 @@ TEST_F(HomeObjectFixture, CreatePGRetryWhileInflight) { auto info = homeobject::PGInfo(pg_id); info.size = pg_size; for (const auto& member : members) { - info.members.insert(homeobject::PGMember{member.first, name + std::to_string(member.second), - member.second == 0 ? 1 : 0}); + info.members.insert( + homeobject::PGMember{member.first, name + std::to_string(member.second), member.second == 0 ? 1 : 0}); } return info; }; diff --git a/src/lib/homestore_backend/tests/hs_scrubber_tests.cpp b/src/lib/homestore_backend/tests/hs_scrubber_tests.cpp index eb06b1d44..901e56359 100644 --- a/src/lib/homestore_backend/tests/hs_scrubber_tests.cpp +++ b/src/lib/homestore_backend/tests/hs_scrubber_tests.cpp @@ -1,144 +1,9 @@ #include "homeobj_fixture.hpp" -#include -#include -#include -#include +#include "homeobj_fixture_http.hpp" #include "lib/homestore_backend/hs_homeobject.hpp" +#include "lib/homestore_backend/hs_debug_inject.hpp" using namespace homeobject; -using BlobHeader = HSHomeObject::BlobHeader; - -// Helper function to delete a blob from index table -static void delete_blob_from_index(shared< homestore::IndexTable< BlobRouteKey, BlobRouteValue > > pg_index_table, - shard_id_t shard_id, blob_id_t blob_id) { - LOGINFO("Deleting blob from index, shard_id={}, blob_id={}", shard_id, blob_id); - BlobRouteKey blob_key{BlobRoute{shard_id, blob_id}}; - BlobRouteValue out_value; - homestore::BtreeSingleRemoveRequest remove_req{&blob_key, &out_value}; - auto status = pg_index_table->remove(remove_req); - ASSERT_TRUE(status == homestore::btree_status_t::success) - << "Failed to remove blob key from index table, status=" << status; -} - -static void delete_shard_from_index(shared< homestore::IndexTable< BlobRouteKey, BlobRouteValue > > pg_index_table, - shard_id_t missing_shard_id) { - LOGINFO("Deleting shard from index, shard_id={}", missing_shard_id); - auto start_key = BlobRouteKey{BlobRoute{missing_shard_id, 0}}; - auto end_key = BlobRouteKey{BlobRoute{missing_shard_id, std::numeric_limits< uint64_t >::max()}}; - homestore::BtreeRangeRemoveRequest< BlobRouteKey > range_remove_req{ - homestore::BtreeKeyRange< BlobRouteKey >{ - std::move(start_key), true /* inclusive */, std::move(end_key), true /* inclusive */ - }, - nullptr, std::numeric_limits< uint32_t >::max(), - [](homestore::BtreeKey const& key, homestore::BtreeValue const& value) -> bool { return true; }}; - - auto status = pg_index_table->remove(range_remove_req); - ASSERT_TRUE(status == homestore::btree_status_t::success || status == homestore::btree_status_t::not_found) - << "Failed to remove shard keys from index table, status=" << status; -} - -// Helper function to corrupt a blob's data -static void corrupt_blob_data(shared< homestore::IndexTable< BlobRouteKey, BlobRouteValue > > pg_index_table, - shard_id_t shard_id, blob_id_t blob_id) { - auto& data_service = homestore::data_service(); - const auto blk_size = data_service.get_blk_size(); - - BlobRouteKey blob_key{BlobRoute{shard_id, blob_id}}; - BlobRouteValue out_value; - homestore::BtreeSingleGetRequest blob_get_req{&blob_key, &out_value}; - - auto status = pg_index_table->get(blob_get_req); - ASSERT_TRUE(status == homestore::btree_status_t::success) - << "Failed to get blob key from index table, status=" << status; - - auto pbas = out_value.pbas(); - auto total_size = pbas.blk_count() * blk_size; - sisl::sg_list data_sgs; - data_sgs.size = total_size; - data_sgs.iovs.emplace_back(iovec{.iov_base = iomanager.iobuf_alloc(blk_size, total_size), .iov_len = total_size}); - - data_service.async_read(pbas, data_sgs, total_size) - .thenValue([&](auto&& err) { - if (err) { - LOGE("Failed to read blob data, blob_id={}, err={}", blob_id, err.message()); - iomanager.iobuf_free(reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base)); - throw std::runtime_error(fmt::format("Failed to read blob data: {}", err.message())); - } - - auto* data_ptr = reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base); - for (size_t i = 0; i < data_sgs.iovs[0].iov_len / 2; i++) { - data_ptr[i] ^= 0xFF; // Flip first half of data - } - - return data_service.async_write(data_sgs, pbas).thenValue([data_sgs = std::move(data_sgs)](auto&& err) { - iomanager.iobuf_free(reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base)); - ASSERT_FALSE(err) << "Failed to write corrupted blob data"; - }); - }) - .get(); -} - -// Helper function to make a blob inconsistent (valid but different hash) -static void make_blob_inconsistent(shared< homestore::IndexTable< BlobRouteKey, BlobRouteValue > > pg_index_table, - shard_id_t shard_id, blob_id_t blob_id, HSHomeObject* obj_inst) { - auto& data_service = homestore::data_service(); - const auto blk_size = data_service.get_blk_size(); - - BlobRouteKey blob_key{BlobRoute{shard_id, blob_id}}; - BlobRouteValue out_value; - homestore::BtreeSingleGetRequest blob_get_req{&blob_key, &out_value}; - - auto status = pg_index_table->get(blob_get_req); - ASSERT_TRUE(status == homestore::btree_status_t::success) << "Failed to get blob key from index table"; - - auto pbas = out_value.pbas(); - auto total_size = pbas.blk_count() * blk_size; - sisl::sg_list data_sgs; - data_sgs.size = total_size; - data_sgs.iovs.emplace_back(iovec{.iov_base = iomanager.iobuf_alloc(blk_size, total_size), .iov_len = total_size}); - - data_service.async_read(pbas, data_sgs, total_size) - .thenValue([&](auto&& err) { - if (err) { - LOGE("Failed to read blob data, blob_id={}, err={}", blob_id, err.message()); - iomanager.iobuf_free(reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base)); - throw std::runtime_error(fmt::format("Failed to read blob data: {}", err.message())); - } - - // Modify blob data and recompute valid hash - uint8_t* read_buf = r_cast< uint8_t* >(data_sgs.iovs[0].iov_base); - auto header = r_cast< BlobHeader* >(read_buf); - uint8_t* blob_bytes = read_buf + header->data_offset; - - std::mt19937 rng{std::random_device{}()}; - std::uniform_int_distribution< int > dist(0, 255); - - for (size_t i = 0; i < header->blob_size / 2; i++) { - blob_bytes[i] ^= static_cast< uint8_t >(dist(rng)); - } - - uint8_t computed_hash[BlobHeader::blob_max_hash_len]{}; - obj_inst->compute_blob_payload_hash(header->hash_algorithm, blob_bytes, header->blob_size, computed_hash, - BlobHeader::blob_max_hash_len); - - std::memcpy(header->hash, computed_hash, BlobHeader::blob_max_hash_len); - std::memset(header->header_hash, 0, BlobHeader::blob_max_hash_len); - uint32_t computed_header_hash = crc32_ieee(0, (uint8_t*)header, sizeof(BlobHeader)); - std::memcpy(header->header_hash, &computed_header_hash, sizeof(uint32_t)); - - if (!obj_inst->verify_blob(data_sgs.iovs[0].iov_base, header->shard_id, header->blob_id)) { - LOGE("Blob verification failed after modification, blob_id={}", blob_id); - iomanager.iobuf_free(reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base)); - throw std::runtime_error(fmt::format("Blob verification failed for blob_id={}", blob_id)); - } - - return data_service.async_write(data_sgs, pbas).thenValue([data_sgs = std::move(data_sgs)](auto&& err) { - iomanager.iobuf_free(reinterpret_cast< uint8_t* >(data_sgs.iovs[0].iov_base)); - ASSERT_FALSE(err) << "Failed to write inconsistent blob data"; - }); - }) - .get(); -} // Helper function to verify missing blobs in scrub report // missing_blobs[blob_route] = set of peers that HAVE the blob; peer_id is missing it iff not in that set. @@ -331,19 +196,17 @@ TEST_F(HomeObjectFixture, BasicScrubTest) { // Corrupt data on followers run_on_pg_follower(pg_id, [&]() { - auto& pg_index_table = hs_pg->index_table_; - // 1. Remove missing_shard_id to simulate missing shard - delete_shard_from_index(pg_index_table, missing_shard_id); + ASSERT_TRUE(hs_debug_inject::delete_shard_from_index(*_obj_inst, pg_id, missing_shard_id)); // 2. Delete missing_blob_id from blob_op_shard_id (different shard, still exists on follower) - delete_blob_from_index(pg_index_table, blob_op_shard_id, missing_blob_id); + ASSERT_TRUE(hs_debug_inject::delete_blob_from_index(*_obj_inst, pg_id, blob_op_shard_id, missing_blob_id)); // 3. Make corrupted_blob_id corrupted - corrupt_blob_data(pg_index_table, blob_op_shard_id, corrupted_blob_id); + ASSERT_TRUE(hs_debug_inject::corrupt_blob_data(*_obj_inst, pg_id, blob_op_shard_id, corrupted_blob_id)); // 4. Make inconsistent_blob_id inconsistent (valid but different hash) - make_blob_inconsistent(pg_index_table, blob_op_shard_id, inconsistent_blob_id, _obj_inst.get()); + ASSERT_TRUE(hs_debug_inject::make_blob_inconsistent(*_obj_inst, pg_id, blob_op_shard_id, inconsistent_blob_id)); }); g_helper->sync(); @@ -478,18 +341,18 @@ TEST_F(HomeObjectFixture, BasicScrubTest) { // Corrupt data on leader run_on_pg_leader(pg_id, [&]() { - auto& pg_index_table = hs_pg->index_table_; - // 1. Delete leader_missing_blob_id from pg_index table on leader - delete_blob_from_index(pg_index_table, blob_op_shard_id, leader_missing_blob_id); + ASSERT_TRUE( + hs_debug_inject::delete_blob_from_index(*_obj_inst, pg_id, blob_op_shard_id, leader_missing_blob_id)); LOGINFO("Deleted blob {} from leader index table", leader_missing_blob_id); // 2. Make leader_corrupted_blob_id corrupted on leader - corrupt_blob_data(pg_index_table, blob_op_shard_id, leader_corrupted_blob_id); + ASSERT_TRUE(hs_debug_inject::corrupt_blob_data(*_obj_inst, pg_id, blob_op_shard_id, leader_corrupted_blob_id)); LOGINFO("Corrupted blob {} on leader", leader_corrupted_blob_id); // 3. Make leader_inconsistent_blob_id inconsistent on leader - make_blob_inconsistent(pg_index_table, blob_op_shard_id, leader_inconsistent_blob_id, _obj_inst.get()); + ASSERT_TRUE( + hs_debug_inject::make_blob_inconsistent(*_obj_inst, pg_id, blob_op_shard_id, leader_inconsistent_blob_id)); LOGINFO("Made blob {} inconsistent on leader", leader_inconsistent_blob_id); }); @@ -631,7 +494,7 @@ TEST_F(HomeObjectFixture, LeaderMissingShardTest) { // Simulate leader losing the shard by removing it from the local B-tree index only. // Followers still have the shard, so scrub should detect the leader as missing it. run_on_pg_leader(pg_id, [&]() { - delete_shard_from_index(hs_pg->index_table_, leader_missing_shard_id); + ASSERT_TRUE(hs_debug_inject::delete_shard_from_index(*_obj_inst, pg_id, leader_missing_shard_id)); LOGINFO("Deleted shard {} from leader index to simulate leader missing shard", leader_missing_shard_id); }); @@ -895,9 +758,8 @@ TEST_F(HomeObjectFixture, ReconcileScrubReportTest) { // Delete both blobs from index table to simulate missing blobs on followers run_on_pg_follower(pg_id, [&]() { - auto& pg_index_table = hs_pg->index_table_; - delete_blob_from_index(pg_index_table, shard_id, missing_blob_to_delete); - delete_blob_from_index(pg_index_table, shard_id, missing_blob_not_deleted); + ASSERT_TRUE(hs_debug_inject::delete_blob_from_index(*_obj_inst, pg_id, shard_id, missing_blob_to_delete)); + ASSERT_TRUE(hs_debug_inject::delete_blob_from_index(*_obj_inst, pg_id, shard_id, missing_blob_not_deleted)); LOGINFO("Deleted blobs {} and {} from follower index table", missing_blob_to_delete, missing_blob_not_deleted); }); @@ -1051,8 +913,7 @@ TEST_F(HomeObjectFixture, LocalScrubMethodsTest) { // corrupt one blob's data in the index table to simulate a blob-level corruption that should be detected by local // blob scrub. - auto& pg_index_table = hs_pg->index_table_; - corrupt_blob_data(pg_index_table, shard_id, corrupted_blob_id); + ASSERT_TRUE(hs_debug_inject::corrupt_blob_data(*_obj_inst, pg_id, shard_id, corrupted_blob_id)); auto my_uuid = _obj_inst->our_uuid(); const int64_t scrub_lsn = hs_pg->repl_dev_->get_last_commit_lsn(); @@ -1248,3 +1109,61 @@ TEST_F(HomeObjectFixture, ScrubResultSerializationTest) { EXPECT_TRUE(loaded->entries.empty()) << "Empty result should round-trip with no entries"; } } + +// Registered as its own ctest (HomestoreTestScrubberHttpDebug, see CMakeLists.txt) with +// --enable_http true and --gtest_filter limiting the binary to just this test. The shared +// process-wide iomgr http_server's route table does not support being re-registered across +// multiple HSHomeObject instances within one process (the first test to construct one "wins" +// the route bindings, which then stay bound to its now-destroyed HSHomeObject for the rest of +// the run), so this test must be the only one ever constructing an HSHomeObject in its process. +TEST_F(HomeObjectFixture, DebugInjectCorruptionHttpTest) { +#ifndef _PRERELEASE + GTEST_SKIP() << "/api/v1/debug/inject_corruption only exists in _PRERELEASE builds; this test is a no-op here " + "(registered as its own ctest, HomestoreTestScrubberHttpDebug, which Release CI also runs)"; +#else + const pg_id_t pg_id = 11; + create_pg(pg_id); + + const uint64_t shard_size = 64 * Mi; + auto shard_info = create_shard(pg_id, shard_size, "shard meta"); + + std::map< pg_id_t, std::vector< shard_id_t > > pg_shard_id_vec; + std::map< pg_id_t, blob_id_t > pg_blob_id; + pg_shard_id_vec[pg_id].push_back(shard_info.id); + pg_blob_id[pg_id] = 0; + + const uint64_t num_blobs = 3; + auto shard_blob_ids_map = put_blobs(pg_shard_id_vec, num_blobs, pg_blob_id); + const auto shard_id = shard_info.id; + auto& shard_blobs = shard_blob_ids_map[shard_id]; + const auto corrupted_blob_id = shard_blobs.begin()->first; + + HttpHelper http_helper("127.0.0.1", 5000 + g_helper->replica_num()); + + // Diagnostic: crashSystem is an existing _PRERELEASE-only route. An unrecognized `type` value + // returns 200 without crashing, so this is a safe way to confirm _PRERELEASE routes are + // registered at all in this build before blaming the new inject_corruption route specifically. + auto probe = http_helper.post("/api/v1/crashSystem?type=diagnostic_probe", ""); + ASSERT_EQ(probe.code(), Pistache::Http::Code::Ok) + << "_PRERELEASE route probe (crashSystem) failed, so _PRERELEASE routes are not registered in this build " + << "at all -- actual_code=" << static_cast< int >(probe.code()) << " body=[" << probe.body() << "]"; + + auto resource = fmt::format("/api/v1/debug/inject_corruption?pg_id={}&shard_id={}&blob_id={}&mode=corrupt_data", + pg_id, shard_id, corrupted_blob_id); + auto r = http_helper.post(resource, ""); + ASSERT_EQ(r.code(), Pistache::Http::Code::Ok) + << "inject_corruption resource=" << resource << " actual_code=" << static_cast< int >(r.code()) + << " body_len=" << r.body().size() << " body=[" << r.body() << "]"; + + auto scrub_report = submit_scrub_with_retry(pg_id, true /* is_deep */); + ASSERT_NE(scrub_report, nullptr) << "Deep scrub report should not be null"; + auto deep_scrub_report = std::dynamic_pointer_cast< ScrubManager::DeepScrubReport >(scrub_report); + ASSERT_NE(deep_scrub_report, nullptr) << "Should be DeepScrubReport"; + + const auto& corrupted_blobs = deep_scrub_report->get_corrupted_blobs(); + auto it = corrupted_blobs.find(_obj_inst->our_uuid()); + ASSERT_TRUE(it != corrupted_blobs.end()) << "Corrupted blob should be reported for this peer"; + EXPECT_TRUE(it->second.count(BlobRoute{shard_id, corrupted_blob_id}) == 1) + << "Blob corrupted via HTTP endpoint should be in the scrub report"; +#endif // _PRERELEASE +} From a0dc6fbaef8d186c6fe5f8fdbe6343c6ced9b1e1 Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Mon, 19 Oct 2026 12:13:30 +0800 Subject: [PATCH 2/2] update --- src/lib/homestore_backend/hs_blob_manager.cpp | 7 +++++-- src/lib/homestore_backend/scrub_manager.cpp | 16 ++++++++++++++++ src/lib/homestore_backend/scrub_manager.hpp | 1 + 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/lib/homestore_backend/hs_blob_manager.cpp b/src/lib/homestore_backend/hs_blob_manager.cpp index c229592f8..f00b8504f 100644 --- a/src/lib/homestore_backend/hs_blob_manager.cpp +++ b/src/lib/homestore_backend/hs_blob_manager.cpp @@ -650,8 +650,11 @@ void HSHomeObject::on_blob_del_commit(int64_t lsn, sisl::blob const& header, sis BLOGE(tid, msg_header->shard_id, *r_cast< blob_id_t const* >(key.cbytes()), "replication message header is corrupted with crc error, lsn={} header={}", lsn, msg_header->to_string()); if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError(BlobErrorCode::CHECKSUM_MISMATCH))); } - RELEASE_ASSERT(false, "replication message header is corrupted with crc error, lsn={}, header={}", lsn, - msg_header->to_string()); + + // TODO:: add RELEASE_ASSERT or proper error handling for corrupted replication message header + // RELEASE_ASSERT(false, "replication message header is corrupted with crc error, lsn={}, header={}", + // lsn,msg_header->to_string()); + return; } diff --git a/src/lib/homestore_backend/scrub_manager.cpp b/src/lib/homestore_backend/scrub_manager.cpp index 58086b70f..c29d48905 100644 --- a/src/lib/homestore_backend/scrub_manager.cpp +++ b/src/lib/homestore_backend/scrub_manager.cpp @@ -230,7 +230,23 @@ bool ScrubManager::is_eligible_for_shallow_scrub(const pg_id_t& pg_id) { return false; } +void ScrubManager::add_missing_pg_scrub_superblks() { + // pgs created before scrub manager tracked scrub superblocks (or that otherwise missed getting one, e.g. a crash + // between create_pg and add_pg) won't have an entry in m_pg_scrub_sb_map. Backfill those here. + std::vector< pg_id_t > pg_ids; + m_hs_home_object->get_pg_ids(pg_ids); + for (auto const& pg_id : pg_ids) { + if (m_pg_scrub_sb_map.find(pg_id) == m_pg_scrub_sb_map.end()) { + LOGINFOMOD(scrubmgr, "pg={} has no scrub superblock, backfilling one", pg_id); + add_pg(pg_id); + } + } +} + void ScrubManager::start() { + // backfill scrub superblocks for any pre-existing pgs that don't have one yet. + add_missing_pg_scrub_superblks(); + // 1 set scrub task handling threads. // TODO :: make thread count configurable, thread number is the most concurrent scrub tasks that can be handled // concurrently. Too many concurrent scrub tasks may bring too much pressure to the node diff --git a/src/lib/homestore_backend/scrub_manager.hpp b/src/lib/homestore_backend/scrub_manager.hpp index 8c4fb591c..3f53cfdbd 100644 --- a/src/lib/homestore_backend/scrub_manager.hpp +++ b/src/lib/homestore_backend/scrub_manager.hpp @@ -351,6 +351,7 @@ class ScrubManager { } }; + void add_missing_pg_scrub_superblks(); void scan_pg_for_scrub(); void handle_pg_scrub_task(scrub_task task); bool is_eligible_for_deep_scrub(const pg_id_t& pg_id);