From 009615b58630a124e8b1cc4e7388c6f118d29991 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 15 Jul 2026 18:02:46 +0800 Subject: [PATCH 01/16] [feature](be) Add asynchronous file cache writes ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: On a file-cache miss, CachedRemoteFileReader currently reads the requested data from remote storage and then appends and finalizes the corresponding local cache blocks on the query thread. The remote read is required to satisfy the query, but the subsequent local writes are best-effort cache population. Coupling the two makes local filesystem latency and backpressure part of foreground scan latency. Moving only the append call to a background thread is not sufficient. Concurrent readers may fetch the same missing range, FileBlock downloader ownership has thread-affine cleanup semantics, and cache clear/remove can invalidate queued work. Warm-up, prefetch, dry-run, and other explicit cache-population callers also require synchronous completion semantics. This change introduces an opt-in asynchronous write path with the following architecture: 1. BlockFileCache provides a read-only probe API that reports downloaded, downloading, empty, and missing ranges without creating cache cells or taking downloader ownership. 2. Each cache instance owns an InflightWriteBufferIndex. It publishes remote-read buffers with insert-if-absent semantics so later readers can reuse bytes that have already been fetched and are awaiting persistence. 3. Each cache disk owns an AsyncCacheWriteService with a bounded MPMC queue, tracked-buffer accounting, dynamically resizable workers, task-age protection, and an explicit shutdown protocol. 4. The ordinary read path combines inflight buffers and downloaded cache blocks, reads the remaining middle range from remote storage once, copies all required bytes into the caller buffer, and submits background tasks only for true cache misses. 5. Workers revalidate the cache write epoch and current block state before writing. Conditional index removal and epoch changes prevent stale callbacks or queued tasks from deleting newer entries or recreating data after cache invalidation. Queue rejection, tracked-buffer pressure, or asynchronous persistence failure does not fail a remote read that has already produced the requested data. The inflight entry is rolled back and the operation falls back to best-effort cache behavior. Explicit cache-population paths continue to use the synchronous implementation. The feature is disabled by default and can be switched online. Worker counts, pending-task limits, batch size, and task-age thresholds are validated through configuration. New bvars and runtime-profile counters expose submissions, inflight reuse, probe results, rejections, failures, queue depth, buffer memory, and write latency. The asynchronous reader implementation is isolated in cached_remote_file_reader_async_write.cpp. The top-level read function only orchestrates planning, covered-range materialization, a single remote middle read, and task submission; the detailed steps are kept in cohesive helper functions. ### Release note Add an opt-in asynchronous file-cache write path controlled by `enable_async_file_cache_write`. It is disabled by default. ### Check List (For Author) - Test: - [x] Regression test - Docker cloud suite `test_async_file_cache_write`: 1 suite passed - [x] Unit Test - BE ASAN targeted tests: 26 cases from 4 suites passed - [x] Build - `./build.sh --be --fe --cloud -j100` - `./build.sh --be -j100` - [x] Code style - `build-support/check-format.sh` - clang-tidy was intentionally not run for this change - Behavior changed: - [x] Yes. When explicitly enabled, ordinary file-cache misses return after the caller buffer is complete and persist missing cache blocks in background workers. The default and explicit cache-population behavior are unchanged. - Does this need documentation: - [x] No. The feature is experimental and disabled by default. --- be/src/common/config.cpp | 19 + be/src/common/config.h | 8 + be/src/exec/scan/olap_scanner.cpp | 9 +- be/src/io/cache/async_cache_write_service.cpp | 465 ++++++++++++++ be/src/io/cache/async_cache_write_service.h | 219 +++++++ be/src/io/cache/block_file_cache.cpp | 124 ++++ be/src/io/cache/block_file_cache.h | 40 ++ .../io/cache/block_file_cache_downloader.cpp | 4 +- be/src/io/cache/block_file_cache_factory.cpp | 27 + be/src/io/cache/block_file_cache_factory.h | 3 + be/src/io/cache/block_file_cache_profile.cpp | 45 ++ be/src/io/cache/block_file_cache_profile.h | 11 + be/src/io/cache/cached_remote_file_reader.cpp | 28 +- be/src/io/cache/cached_remote_file_reader.h | 122 ++++ .../cached_remote_file_reader_async_write.cpp | 568 +++++++++++++++++ be/src/io/cache/file_cache_common.h | 11 + .../io/cache/inflight_write_buffer_index.cpp | 183 ++++++ be/src/io/cache/inflight_write_buffer_index.h | 144 +++++ be/src/io/fs/file_reader.h | 14 + be/src/io/io_common.h | 28 + .../segment_index_file_cache_loader.cpp | 1 + .../cache/async_cache_write_service_test.cpp | 583 ++++++++++++++++++ .../io/cache/block_file_cache_probe_test.cpp | 183 ++++++ ...block_file_cache_profile_reporter_test.cpp | 33 + .../cache/cached_remote_file_reader_test.cpp | 525 ++++++++++++++++ .../inflight_write_buffer_index_test.cpp | 160 +++++ .../cache/test_async_file_cache_write.out | 9 + .../cache/test_async_file_cache_write.groovy | 140 +++++ 28 files changed, 3702 insertions(+), 4 deletions(-) create mode 100644 be/src/io/cache/async_cache_write_service.cpp create mode 100644 be/src/io/cache/async_cache_write_service.h create mode 100644 be/src/io/cache/cached_remote_file_reader_async_write.cpp create mode 100644 be/src/io/cache/inflight_write_buffer_index.cpp create mode 100644 be/src/io/cache/inflight_write_buffer_index.h create mode 100644 be/test/io/cache/async_cache_write_service_test.cpp create mode 100644 be/test/io/cache/block_file_cache_probe_test.cpp create mode 100644 be/test/io/cache/inflight_write_buffer_index_test.cpp create mode 100644 regression-test/data/cloud_p0/cache/test_async_file_cache_write.out create mode 100644 regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index ca20cb63990447..20d3fd4202dee8 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1261,6 +1261,25 @@ DEFINE_mInt64(file_cache_background_block_lru_update_interval_ms, "5000"); DEFINE_mInt64(file_cache_background_block_lru_update_qps_limit, "1000"); DEFINE_mInt64(file_cache_background_block_lru_update_queue_max_size, "500000"); DEFINE_mBool(enable_file_cache_async_touch_on_get_or_set, "false"); +DEFINE_mBool(enable_async_file_cache_write, "false"); +DEFINE_mInt32(async_file_cache_write_workers_per_disk, "2"); +DEFINE_mInt64(async_file_cache_write_max_pending_tasks_per_disk, "256"); +DEFINE_mInt32(async_file_cache_write_batch_size, "16"); +DEFINE_mInt64(async_file_cache_write_watchdog_warn_secs, "30"); +DEFINE_mInt64(async_file_cache_write_watchdog_drop_secs, "120"); +DEFINE_mBool(enable_inflight_write_buffer_index, "true"); +DEFINE_Int32(inflight_write_buffer_index_shard_count, "64"); +DEFINE_Validator(async_file_cache_write_workers_per_disk, + [](int32_t value) { return value > 0 && value <= 128; }); +DEFINE_Validator(async_file_cache_write_max_pending_tasks_per_disk, + [](int64_t value) { return value > 0; }); +DEFINE_Validator(async_file_cache_write_batch_size, [](int32_t value) { return value > 0; }); +DEFINE_Validator(async_file_cache_write_watchdog_warn_secs, [](int64_t value) { + return value >= 0 && value < async_file_cache_write_watchdog_drop_secs; +}); +DEFINE_Validator(async_file_cache_write_watchdog_drop_secs, + [](int64_t value) { return value > async_file_cache_write_watchdog_warn_secs; }); +DEFINE_Validator(inflight_write_buffer_index_shard_count, [](int32_t value) { return value > 0; }); DEFINE_mBool(enable_reader_dryrun_when_download_file_cache, "true"); DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000"); DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "180000"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 2a6b21ccf8c0e3..987aad42fad568 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1301,6 +1301,14 @@ DECLARE_mInt64(file_cache_background_block_lru_update_interval_ms); DECLARE_mInt64(file_cache_background_block_lru_update_qps_limit); DECLARE_mInt64(file_cache_background_block_lru_update_queue_max_size); DECLARE_mBool(enable_file_cache_async_touch_on_get_or_set); +DECLARE_mBool(enable_async_file_cache_write); +DECLARE_mInt32(async_file_cache_write_workers_per_disk); +DECLARE_mInt64(async_file_cache_write_max_pending_tasks_per_disk); +DECLARE_mInt32(async_file_cache_write_batch_size); +DECLARE_mInt64(async_file_cache_write_watchdog_warn_secs); +DECLARE_mInt64(async_file_cache_write_watchdog_drop_secs); +DECLARE_mBool(enable_inflight_write_buffer_index); +DECLARE_Int32(inflight_write_buffer_index_shard_count); DECLARE_mBool(enable_reader_dryrun_when_download_file_cache); DECLARE_mInt64(file_cache_background_monitor_interval_ms); DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms); diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 834ee012737ecd..8dc619619711bf 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -146,7 +146,14 @@ static bool has_file_cache_statistics(const io::FileCacheStatistics& stats) { stats.write_cache_io_timer != 0 || stats.bytes_write_into_cache != 0 || stats.num_skip_cache_io_total != 0 || stats.read_cache_file_directly_timer != 0 || stats.cache_get_or_set_timer != 0 || stats.lock_wait_timer != 0 || - stats.get_timer != 0 || stats.set_timer != 0 || + stats.get_timer != 0 || stats.set_timer != 0 || stats.async_cache_write_submitted != 0 || + stats.async_cache_write_rejected != 0 || + stats.async_cache_write_buffer_alloc_fail != 0 || + stats.async_cache_write_drop_stale_epoch != 0 || + stats.inflight_write_buffer_index_hit != 0 || + stats.inflight_write_buffer_index_miss != 0 || stats.probe_downloaded_hit != 0 || + stats.probe_downloading_hit != 0 || stats.probe_miss != 0 || + stats.block_wait_success != 0 || stats.block_wait_timeout != 0 || stats.inverted_index_num_local_io_total != 0 || stats.inverted_index_num_remote_io_total != 0 || stats.inverted_index_num_peer_io_total != 0 || diff --git a/be/src/io/cache/async_cache_write_service.cpp b/be/src/io/cache/async_cache_write_service.cpp new file mode 100644 index 00000000000000..4afaac844798d2 --- /dev/null +++ b/be/src/io/cache/async_cache_write_service.cpp @@ -0,0 +1,465 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/async_cache_write_service.h" + +#include +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "core/allocator.h" +#include "cpp/sync_point.h" +#include "io/cache/block_file_cache.h" +#include "runtime/thread_context.h" +#include "util/defer_op.h" +#include "util/time.h" + +namespace doris::io { + +using AsyncCacheWriteAllocator = Allocator; + +AsyncCacheWriteBuffer::AsyncCacheWriteBuffer(size_t size, + std::shared_ptr tracker) + : _size(size), _tracker(std::move(tracker)) { + AsyncCacheWriteAllocator allocator; + _data = reinterpret_cast(allocator.alloc(_size)); +} + +AsyncCacheWriteBuffer::~AsyncCacheWriteBuffer() { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_tracker); + AsyncCacheWriteAllocator allocator; + allocator.free(_data, _size); +} + +AsyncCacheWriteServiceOptions AsyncCacheWriteServiceOptions::from_config() { + return AsyncCacheWriteServiceOptions { + .worker_count = static_cast(config::async_file_cache_write_workers_per_disk), + .max_pending_tasks = + static_cast(config::async_file_cache_write_max_pending_tasks_per_disk), + .batch_size = static_cast(config::async_file_cache_write_batch_size), + .watchdog_warn_secs = config::async_file_cache_write_watchdog_warn_secs, + .watchdog_drop_secs = config::async_file_cache_write_watchdog_drop_secs, + .follow_global_config = true, + }; +} + +AsyncCacheWriteService::AsyncCacheWriteService(BlockFileCache* cache, + AsyncCacheWriteServiceOptions options) + : _cache(cache), _options(std::move(options)) { + DORIS_CHECK(_cache != nullptr); + DORIS_CHECK(_options.worker_count > 0); + DORIS_CHECK(_options.max_pending_tasks > 0); + DORIS_CHECK(_options.batch_size > 0); + DORIS_CHECK(_options.watchdog_drop_secs > _options.watchdog_warn_secs); + + const char* prefix = _cache->get_base_path().c_str(); + _mem_tracker = MemTrackerLimiter::create_shared( + MemTrackerLimiter::Type::CACHE, + fmt::format("AsyncFileCacheWrite:{}", _cache->get_base_path())); + _pending_count_metric = std::make_shared>( + prefix, "async_cache_write_pending_count", + [](void* service) { + return static_cast(service)->pending_count(); + }, + this); + _buffer_memory_metric = std::make_shared>( + prefix, "async_cache_write_buffer_memory_bytes", + [](void* service) { + return static_cast(service)->buffer_memory_bytes(); + }, + this); + _submitted_metric = + std::make_shared>(prefix, "async_cache_write_submitted_total"); + _rejected_metric = + std::make_shared>(prefix, "async_cache_write_rejected_total"); + _buffer_alloc_fail_metric = std::make_shared>( + prefix, "async_cache_write_buffer_alloc_fail_total"); + _latency_metric = + std::make_shared(prefix, "async_cache_write_latency_us"); + _skip_downloaded_metric = std::make_shared>( + prefix, "async_cache_write_skip_downloaded_total"); + _skip_downloading_metric = std::make_shared>( + prefix, "async_cache_write_skip_downloading_total"); + _skip_partial_overlap_metric = std::make_shared>( + prefix, "async_cache_write_skip_partial_overlap_total"); + _drop_stale_epoch_metric = std::make_shared>( + prefix, "async_cache_write_drop_stale_epoch_total"); + _skip_deleting_metric = std::make_shared>( + prefix, "async_cache_write_skip_deleting_total"); + _append_fail_metric = + std::make_shared>(prefix, "async_cache_write_append_fail_total"); + _watchdog_timeout_metric = std::make_shared>( + prefix, "async_cache_write_watchdog_timeout_total"); +} + +AsyncCacheWriteService::~AsyncCacheWriteService() { + shutdown(); +} + +Status AsyncCacheWriteService::start() { + bool expected = false; + if (!_started.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + return Status::OK(); + } + + const size_t worker_count = _options.worker_count; + RETURN_IF_ERROR( + ThreadPoolBuilder(fmt::format("AsyncFileCacheWrite-{}", + std::hash {}(_cache->get_base_path()))) + .set_min_threads(0) + .set_max_threads(static_cast(worker_count)) + .set_max_queue_size(128) + .build(&_worker_pool)); + _desired_worker_count.store(worker_count, std::memory_order_release); + _worker_scheduled.resize(worker_count, false); + for (size_t worker_id = 0; worker_id < worker_count; ++worker_id) { + RETURN_IF_ERROR(_schedule_worker(worker_id)); + } + return Status::OK(); +} + +bool AsyncCacheWriteService::try_submit(AsyncCacheWriteTask task) { + _active_submitters.fetch_add(1, std::memory_order_acq_rel); + Defer submitter_done {[&]() { _active_submitters.fetch_sub(1, std::memory_order_acq_rel); }}; + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::try_submit:after_register", &task); + if (!_accepting.load(std::memory_order_acquire)) { + *_rejected_metric << 1; + return false; + } + + const size_t max_pending = _max_pending_tasks(); + size_t current = _pending_count.load(std::memory_order_relaxed); + while (current < max_pending) { + if (_pending_count.compare_exchange_weak(current, current + 1, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + if (_queue.enqueue(std::move(task))) { + *_submitted_metric << 1; + _cv.notify_one(); + return true; + } + _pending_count.fetch_sub(1, std::memory_order_acq_rel); + *_rejected_metric << 1; + return false; + } + } + *_rejected_metric << 1; + return false; +} + +Status AsyncCacheWriteService::allocate_tracked_buffer(size_t size, + AsyncCacheWriteBufferPtr* buffer) { + DORIS_CHECK(buffer != nullptr); + DORIS_CHECK(size > 0); + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); + Status status = Status::OK(); + try { + *buffer = AsyncCacheWriteBufferPtr(new AsyncCacheWriteBuffer(size, _mem_tracker)); + } catch (const std::exception& e) { + status = Status::MemoryAllocFailed("allocate async file cache write buffer failed: {}", + e.what()); + } + if (!status.ok()) { + *_buffer_alloc_fail_metric << 1; + } + return status; +} + +Status AsyncCacheWriteService::_schedule_worker(size_t worker_id) { + { + std::lock_guard lock(_worker_state_mutex); + if (_worker_scheduled.size() <= worker_id) { + _worker_scheduled.resize(worker_id + 1, false); + } + DORIS_CHECK(!_worker_scheduled[worker_id]); + _worker_scheduled[worker_id] = true; + } + Status status = _worker_pool->submit_func([this, worker_id]() { _worker_loop(worker_id); }); + if (!status.ok()) { + std::lock_guard lock(_worker_state_mutex); + _worker_scheduled[worker_id] = false; + _worker_state_cv.notify_all(); + } + return status; +} + +void AsyncCacheWriteService::_worker_loop(size_t worker_id) { + Defer mark_stopped {[&]() { + std::lock_guard lock(_worker_state_mutex); + _worker_scheduled[worker_id] = false; + _worker_state_cv.notify_all(); + }}; + + while (true) { + if (!_shutdown_requested.load(std::memory_order_acquire) && + worker_id >= _desired_worker_count.load(std::memory_order_acquire)) { + return; + } + + size_t processed = 0; + const size_t batch_size = _batch_size(); + AsyncCacheWriteTask task; + while (processed < batch_size && _queue.try_dequeue(task)) { + ++processed; + Defer finish {[&]() { + _finish_task(task); + task = AsyncCacheWriteTask {}; + }}; + + const int64_t age_us = MonotonicMicros() - task.submit_ts_us; + const int64_t warn_us = _watchdog_warn_secs() * 1000 * 1000; + const int64_t drop_us = _watchdog_drop_secs() * 1000 * 1000; + if (age_us > warn_us) { + *_watchdog_timeout_metric << 1; + LOG(WARNING) << "Async file cache write task waited " << age_us + << " us, cache=" << _cache->get_base_path() + << ", hash=" << task.cache_hash.to_string() + << ", offset=" << task.file_offset; + } + if (age_us > drop_us) { + continue; + } + if (!is_current_write_epoch(task.write_epoch)) { + *_drop_stale_epoch_metric << 1; + continue; + } + + const int64_t start_us = MonotonicMicros(); + Status status = _write_one(task); + *_latency_metric << (MonotonicMicros() - start_us); + if (!status.ok()) { + LOG(WARNING) << "Async file cache write failed, cache=" << _cache->get_base_path() + << ", hash=" << task.cache_hash.to_string() + << ", offset=" << task.file_offset << ", size=" << task.file_size + << ", status=" << status; + } + } + + if (_shutdown_requested.load(std::memory_order_acquire) && + _pending_count.load(std::memory_order_acquire) == 0) { + return; + } + if (processed == 0) { + std::unique_lock lock(_cv_mutex); + _cv.wait_for(lock, std::chrono::milliseconds(1), [&]() { + return _queue.size_approx() != 0 || + _shutdown_requested.load(std::memory_order_acquire) || + worker_id >= _desired_worker_count.load(std::memory_order_acquire); + }); + } + } +} + +Status AsyncCacheWriteService::_write_one(const AsyncCacheWriteTask& task) { + DORIS_CHECK(task.buffer != nullptr); + DORIS_CHECK(task.file_size == task.buffer->size()); + if (!is_current_write_epoch(task.write_epoch)) { + *_drop_stale_epoch_metric << 1; + return Status::OK(); + } + + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:before_get_or_set", &task); + ReadStatistics dummy_stats; + CacheContext context; + context.query_id = task.admission_ctx.query_id; + context.cache_type = task.admission_ctx.cache_type; + context.expiration_time = task.admission_ctx.expiration_time; + context.tablet_id = task.admission_ctx.tablet_id; + context.is_warmup = task.admission_ctx.is_warmup; + context.stats = &dummy_stats; + auto holder = _cache->get_or_set(task.cache_hash, task.file_offset, task.file_size, context); + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:after_get_or_set", &task); + + if (!is_current_write_epoch(task.write_epoch)) { + *_drop_stale_epoch_metric << 1; + return Status::OK(); + } + + const size_t task_end = task.file_offset + task.file_size; + for (const auto& block : holder.file_blocks) { + if (block->range().left < task.file_offset || block->range().right >= task_end) { + *_skip_partial_overlap_metric << 1; + continue; + } + if (!is_current_write_epoch(task.write_epoch)) { + *_drop_stale_epoch_metric << 1; + return Status::OK(); + } + if (_cache->is_block_deleting(block)) { + *_skip_deleting_metric << 1; + continue; + } + + switch (block->state()) { + case FileBlock::State::DOWNLOADED: + *_skip_downloaded_metric << 1; + continue; + case FileBlock::State::DOWNLOADING: + *_skip_downloading_metric << 1; + continue; + case FileBlock::State::SKIP_CACHE: + continue; + case FileBlock::State::EMPTY: + break; + } + + if (block->get_or_set_downloader() != FileBlock::get_caller_id()) { + *_skip_downloading_metric << 1; + continue; + } + const size_t buffer_offset = block->range().left - task.file_offset; + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:before_append", &task); + Status status = + block->append(Slice(task.buffer->data() + buffer_offset, block->range().size())); + if (!status.ok()) { + *_append_fail_metric << 1; + LOG(WARNING) << "Append async file cache block failed, cache=" + << _cache->get_base_path() << ", hash=" << task.cache_hash.to_string() + << ", offset=" << block->offset() << ", size=" << block->range().size() + << ", status=" << status; + continue; + } + status = block->finalize(); + if (!status.ok()) { + *_append_fail_metric << 1; + LOG(WARNING) << "Finalize async file cache block failed, cache=" + << _cache->get_base_path() << ", hash=" << task.cache_hash.to_string() + << ", offset=" << block->offset() << ", size=" << block->range().size() + << ", status=" << status; + } + } + return Status::OK(); +} + +void AsyncCacheWriteService::_finish_task(const AsyncCacheWriteTask& task) { + const size_t old_pending = _pending_count.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(old_pending > 0); + if (task.on_finalized) { + task.on_finalized(task); + } +} + +Status AsyncCacheWriteService::resize_workers(size_t worker_count) { + if (worker_count == 0) { + return Status::InvalidArgument("async file cache write worker count must be positive"); + } + std::lock_guard resize_lock(_resize_mutex); + if (!_started.load(std::memory_order_acquire)) { + _options.worker_count = worker_count; + return Status::OK(); + } + if (_shutdown_requested.load(std::memory_order_acquire)) { + return Status::InternalError("async file cache write service is shutting down"); + } + + const size_t old_count = + _desired_worker_count.exchange(worker_count, std::memory_order_acq_rel); + if (old_count == worker_count) { + return Status::OK(); + } + _cv.notify_all(); + + if (worker_count < old_count) { + std::unique_lock state_lock(_worker_state_mutex); + _worker_state_cv.wait(state_lock, [&]() { + for (size_t id = worker_count; id < old_count; ++id) { + if (id < _worker_scheduled.size() && _worker_scheduled[id]) { + return false; + } + } + return true; + }); + state_lock.unlock(); + RETURN_IF_ERROR(_worker_pool->set_max_threads(static_cast(worker_count))); + return Status::OK(); + } + + RETURN_IF_ERROR(_worker_pool->set_max_threads(static_cast(worker_count))); + for (size_t worker_id = old_count; worker_id < worker_count; ++worker_id) { + RETURN_IF_ERROR(_schedule_worker(worker_id)); + } + return Status::OK(); +} + +void AsyncCacheWriteService::shutdown() { + std::lock_guard resize_lock(_resize_mutex); + if (!_accepting.exchange(false, std::memory_order_acq_rel)) { + return; + } + TEST_SYNC_POINT("AsyncCacheWriteService::shutdown:after_stop_accepting"); + while (_active_submitters.load(std::memory_order_acquire) != 0) { + std::this_thread::yield(); + } + _shutdown_requested.store(true, std::memory_order_release); + _cv.notify_all(); + + if (_worker_pool) { + { + std::unique_lock lock(_worker_state_mutex); + _worker_state_cv.wait(lock, [&]() { + return std::none_of(_worker_scheduled.begin(), _worker_scheduled.end(), + [](bool scheduled) { return scheduled; }); + }); + } + DORIS_CHECK(_pending_count.load(std::memory_order_acquire) == 0); + _worker_pool->shutdown(); + } +} + +AsyncCacheWriteServiceStats AsyncCacheWriteService::stats() const { + return AsyncCacheWriteServiceStats { + .pending_count = pending_count(), + .submitted = _submitted_metric->get_value(), + .rejected = _rejected_metric->get_value(), + .buffer_alloc_fail = _buffer_alloc_fail_metric->get_value(), + .skip_downloaded = _skip_downloaded_metric->get_value(), + .skip_downloading = _skip_downloading_metric->get_value(), + .skip_partial_overlap = _skip_partial_overlap_metric->get_value(), + .drop_stale_epoch = _drop_stale_epoch_metric->get_value(), + .skip_deleting = _skip_deleting_metric->get_value(), + .append_fail = _append_fail_metric->get_value(), + .watchdog_timeout = _watchdog_timeout_metric->get_value(), + }; +} + +size_t AsyncCacheWriteService::_max_pending_tasks() const { + if (_options.follow_global_config) { + return static_cast(config::async_file_cache_write_max_pending_tasks_per_disk); + } + return _options.max_pending_tasks; +} + +size_t AsyncCacheWriteService::_batch_size() const { + if (_options.follow_global_config) { + return static_cast(config::async_file_cache_write_batch_size); + } + return _options.batch_size; +} + +int64_t AsyncCacheWriteService::_watchdog_warn_secs() const { + return _options.follow_global_config ? config::async_file_cache_write_watchdog_warn_secs + : _options.watchdog_warn_secs; +} + +int64_t AsyncCacheWriteService::_watchdog_drop_secs() const { + return _options.follow_global_config ? config::async_file_cache_write_watchdog_drop_secs + : _options.watchdog_drop_secs; +} + +} // namespace doris::io diff --git a/be/src/io/cache/async_cache_write_service.h b/be/src/io/cache/async_cache_write_service.h new file mode 100644 index 00000000000000..9fac7927d005f3 --- /dev/null +++ b/be/src/io/cache/async_cache_write_service.h @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/cache/file_cache_common.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "util/threadpool.h" + +namespace doris::io { + +class BlockFileCache; + +/// Cache admission attributes captured on the query thread and replayed by a write worker. +struct CacheAdmissionContext { + /// Query identity used by per-query cache admission and accounting. + TUniqueId query_id; + FileCacheType cache_type {FileCacheType::NORMAL}; + int64_t expiration_time {0}; + int64_t tablet_id {0}; + bool is_warmup {false}; +}; + +/// Reference-counted payload whose allocation is charged to the async-write memory tracker. +class AsyncCacheWriteBuffer { +public: + ~AsyncCacheWriteBuffer(); + + char* data() { return _data; } + const char* data() const { return _data; } + size_t size() const { return _size; } + +private: + friend class AsyncCacheWriteService; + + AsyncCacheWriteBuffer(size_t size, std::shared_ptr tracker); + + char* _data = nullptr; + size_t _size = 0; + std::shared_ptr _tracker; +}; + +using AsyncCacheWriteBufferPtr = std::shared_ptr; + +/// One block-aligned cache write. `buffer` contains exactly `file_size` bytes starting at +/// `file_offset`; `write_epoch` prevents a worker from resurrecting data after cache invalidation. +struct AsyncCacheWriteTask { + UInt128Wrapper cache_hash; + size_t file_offset {0}; + size_t file_size {0}; + AsyncCacheWriteBufferPtr buffer; + CacheAdmissionContext admission_ctx; + int64_t submit_ts_us {0}; + uint64_t write_epoch {0}; + std::function on_finalized; +}; + +/// Per-cache-disk worker and queue limits. Tests may supply fixed values; production instances set +/// `follow_global_config` so mutable limits are read when each task is handled. +struct AsyncCacheWriteServiceOptions { + size_t worker_count {1}; + size_t max_pending_tasks {1}; + size_t batch_size {1}; + int64_t watchdog_warn_secs {30}; + int64_t watchdog_drop_secs {120}; + bool follow_global_config {false}; + + /// Build production options from the current BE configuration. + static AsyncCacheWriteServiceOptions from_config(); +}; + +/// Snapshot of service counters used by tests and cache-stat reporting. +struct AsyncCacheWriteServiceStats { + size_t pending_count {0}; + uint64_t submitted {0}; + uint64_t rejected {0}; + uint64_t buffer_alloc_fail {0}; + uint64_t skip_downloaded {0}; + uint64_t skip_downloading {0}; + uint64_t skip_partial_overlap {0}; + uint64_t drop_stale_epoch {0}; + uint64_t skip_deleting {0}; + uint64_t append_fail {0}; + uint64_t watchdog_timeout {0}; +}; + +/// Owns the bounded async-write queue and workers for one BlockFileCache (one cache disk). +/// +/// The referenced cache must outlive this service. Shutdown stops new producers, waits registered +/// producers, and drains all accepted tasks before worker resources are released. +class AsyncCacheWriteService { +public: + /// @param cache Non-owning target cache; it must outlive this service. + /// @param options Initial worker, queue, batch, and watchdog limits. + AsyncCacheWriteService(BlockFileCache* cache, AsyncCacheWriteServiceOptions options); + ~AsyncCacheWriteService(); + + /// Create the worker pool and schedule the configured long-running workers. Idempotent. + Status start(); + + /// Reserve one pending slot and enqueue `task` without blocking the query thread. + /// @return true if ownership was transferred to the queue; false on shutdown, backpressure, or + /// queue rejection. A rejected task's finalization callback is not invoked. + bool try_submit(AsyncCacheWriteTask task); + + /// Allocate `size` payload bytes charged to the service tracker and return them in `buffer`. + Status allocate_tracked_buffer(size_t size, AsyncCacheWriteBufferPtr* buffer); + + /// Return the epoch accepted by workers and inflight-buffer readers. + uint64_t current_write_epoch() const { return _write_epoch.load(std::memory_order_acquire); } + + /// Test whether `epoch` still belongs to the current cache contents. + bool is_current_write_epoch(uint64_t epoch) const { return epoch == current_write_epoch(); } + + /// Advance the epoch so queued/inflight writes captured before invalidation become stale. + /// @return The newly active epoch. + uint64_t invalidate_pending_writes() { + return _write_epoch.fetch_add(1, std::memory_order_acq_rel) + 1; + } + + /// Resize the number of active workers. A shrink waits only for retiring worker loops. + /// @param worker_count Positive target worker count for this cache disk. + Status resize_workers(size_t worker_count); + + /// Stop submissions, drain all accepted tasks, and join worker loops. Idempotent. + void shutdown(); + + /// Return a point-in-time counter snapshot. + AsyncCacheWriteServiceStats stats() const; + + /// Return accepted tasks that have not yet completed finalization. + size_t pending_count() const { return _pending_count.load(std::memory_order_relaxed); } + + /// Return bytes currently held by tracked task buffers. + int64_t buffer_memory_bytes() const { return _mem_tracker->consumption(); } + +private: + /// Mark `worker_id` active and submit its persistent loop to the thread pool. + Status _schedule_worker(size_t worker_id); + + /// Drain batches until shutdown or until this worker id is retired by a resize. + void _worker_loop(size_t worker_id); + + /// Revalidate epoch/cache state and persist the task's still-empty complete blocks. + Status _write_one(const AsyncCacheWriteTask& task); + + /// Release one pending slot and invoke the inflight-index cleanup callback. + void _finish_task(const AsyncCacheWriteTask& task); + + /// Read queue/batch/watchdog limits from mutable config or fixed test options. + size_t _max_pending_tasks() const; + size_t _batch_size() const; + int64_t _watchdog_warn_secs() const; + int64_t _watchdog_drop_secs() const; + + BlockFileCache* _cache; + AsyncCacheWriteServiceOptions _options; + moodycamel::ConcurrentQueue _queue; + std::atomic _pending_count {0}; + std::condition_variable _cv; + std::mutex _cv_mutex; + std::atomic _accepting {true}; + std::atomic _active_submitters {0}; + std::atomic _shutdown_requested {false}; + std::atomic _started {false}; + std::atomic _write_epoch {1}; + + std::shared_ptr _mem_tracker; + std::unique_ptr _worker_pool; + std::atomic _desired_worker_count {0}; + std::mutex _resize_mutex; + std::mutex _worker_state_mutex; + std::condition_variable _worker_state_cv; + std::vector _worker_scheduled; + + std::shared_ptr> _pending_count_metric; + std::shared_ptr> _buffer_memory_metric; + std::shared_ptr> _submitted_metric; + std::shared_ptr> _rejected_metric; + std::shared_ptr> _buffer_alloc_fail_metric; + std::shared_ptr _latency_metric; + std::shared_ptr> _skip_downloaded_metric; + std::shared_ptr> _skip_downloading_metric; + std::shared_ptr> _skip_partial_overlap_metric; + std::shared_ptr> _drop_stale_epoch_metric; + std::shared_ptr> _skip_deleting_metric; + std::shared_ptr> _append_fail_metric; + std::shared_ptr> _watchdog_timeout_metric; +}; + +} // namespace doris::io diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index b44679a94fa150..c1dc89a4b4ff8c 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -548,6 +548,12 @@ Status BlockFileCache::initialize_unlocked(std::lock_guard& cache_lo } RETURN_IF_ERROR(_storage->init(this)); + _inflight_write_buffer_index = std::make_unique( + static_cast(config::inflight_write_buffer_index_shard_count), _cache_base_path); + _async_write_service = std::make_unique( + this, AsyncCacheWriteServiceOptions::from_config()); + RETURN_IF_ERROR(_async_write_service->start()); + if (auto* fs_storage = dynamic_cast(_storage.get())) { if (auto* meta_store = fs_storage->get_meta_store()) { _ttl_mgr = std::make_unique(this, meta_store); @@ -822,6 +828,118 @@ Status BlockFileCache::get_downloaded_blocks_if_fully_covered(const UInt128Wrapp return Status::OK(); } +FileBlocksProbeResult BlockFileCache::probe(const UInt128Wrapper& hash, size_t offset, size_t size, + const CacheContext& context) { + DORIS_CHECK(size > 0); + const FileBlock::Range range(offset, offset + size - 1); + FileBlocks blocks; + std::vector gaps; + std::lock_guard cache_lock(_mutex); + + auto file_iterator = _files.find(hash); + if (file_iterator == _files.end() && !_async_open_done) { + FileCacheKey key; + key.hash = hash; + key.meta.type = context.cache_type; + key.meta.expiration_time = context.expiration_time; + key.meta.tablet_id = context.tablet_id; + _storage->load_blocks_directly_unlocked(this, key, cache_lock); + file_iterator = _files.find(hash); + } + if (file_iterator == _files.end()) { + gaps.emplace_back(range); + return FileBlocksProbeResult(std::move(blocks), std::move(gaps)); + } + + auto& file_blocks = file_iterator->second; + DORIS_CHECK(!file_blocks.empty()); + auto block_iterator = file_blocks.lower_bound(range.left); + if (block_iterator != file_blocks.begin()) { + auto previous = std::prev(block_iterator); + if (previous->second.file_block->range().right >= range.left) { + block_iterator = previous; + } + } + + size_t current = range.left; + for (; block_iterator != file_blocks.end(); ++block_iterator) { + const auto& block = block_iterator->second.file_block; + const auto& block_range = block->range(); + if (block_range.left > range.right) { + break; + } + if (block_range.right < range.left) { + continue; + } + const size_t clipped_left = std::max(block_range.left, range.left); + const size_t clipped_right = std::min(block_range.right, range.right); + if (current < clipped_left) { + gaps.emplace_back(current, clipped_left - 1); + } + blocks.emplace_back(block); + current = clipped_right + 1; + if (current > range.right) { + break; + } + } + if (current <= range.right) { + gaps.emplace_back(current, range.right); + } + return FileBlocksProbeResult(std::move(blocks), std::move(gaps)); +} + +void BlockFileCache::touch_probe_block_if_cached(const FileBlockSPtr& block, + const CacheContext& context) { + DORIS_CHECK(block != nullptr); + FileBlockSPtr async_touch; + { + std::lock_guard cache_lock(_mutex); + auto file_iterator = _files.find(block->get_hash_value()); + if (file_iterator == _files.end()) { + return; + } + auto cell_iterator = file_iterator->second.find(block->offset()); + if (cell_iterator == file_iterator->second.end() || + cell_iterator->second.file_block != block) { + return; + } + { + std::lock_guard block_lock(block->_mutex); + if (block->_is_deleting) { + return; + } + } + auto& cell = cell_iterator->second; + const bool move_iter = need_to_move(block->cache_type(), context.cache_type); + if (config::enable_file_cache_async_touch_on_get_or_set) { + cell.update_atime(); + if (move_iter) { + async_touch = block; + } + } else { + use_cell(cell, nullptr, move_iter, cache_lock); + } + } + if (async_touch) { + add_need_update_lru_block(std::move(async_touch)); + } +} + +bool BlockFileCache::is_block_deleting(const FileBlockSPtr& block) const { + DORIS_CHECK(block != nullptr); + std::lock_guard cache_lock(_mutex); + auto file_iterator = _files.find(block->get_hash_value()); + if (file_iterator == _files.end()) { + return true; + } + auto cell_iterator = file_iterator->second.find(block->offset()); + if (cell_iterator == file_iterator->second.end() || cell_iterator->second.file_block != block) { + return true; + } + std::lock_guard block_lock(block->_mutex); + return block->_is_deleting; +} + std::string BlockFileCache::clear_file_cache_async() { return clear_file_cache_impl(false); } @@ -831,6 +949,8 @@ std::string BlockFileCache::clear_file_cache_sync() { } std::string BlockFileCache::clear_file_cache_impl(bool sync_remove) { + DORIS_CHECK(_async_write_service != nullptr); + _async_write_service->invalidate_pending_writes(); const char* action = sync_remove ? "clear_file_cache_sync" : "clear_file_cache_async"; LOG(INFO) << "start " << action << ", path=" << _cache_base_path; _lru_dumper->remove_lru_dump_files(); @@ -1370,6 +1490,8 @@ void BlockFileCache::try_evict_in_advance(size_t size, std::lock_guardinvalidate_pending_writes(); std::string reason = "remove_if_cached"; SCOPED_CACHE_LOCK(_mutex, this); auto iter = _files.find(file_key); @@ -1390,6 +1512,8 @@ void BlockFileCache::remove_if_cached(const UInt128Wrapper& file_key) { // cache meta is deleted synchronously if not in use, and the block file is deleted asynchronously // if in use, cache meta will be deleted after use and the block file is then deleted asynchronously void BlockFileCache::remove_if_cached_async(const UInt128Wrapper& file_key) { + DORIS_CHECK(_async_write_service != nullptr); + _async_write_service->invalidate_pending_writes(); std::string reason = "remove_if_cached_async"; SCOPED_CACHE_LOCK(_mutex, this); diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 9a01d07d136a62..c52129242d0dac 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -33,11 +33,13 @@ #include #include +#include "io/cache/async_cache_write_service.h" #include "io/cache/block_file_cache_ttl_mgr.h" #include "io/cache/cache_lru_dumper.h" #include "io/cache/file_block.h" #include "io/cache/file_cache_common.h" #include "io/cache/file_cache_storage.h" +#include "io/cache/inflight_write_buffer_index.h" #include "io/cache/lru_queue_recorder.h" #include "runtime/runtime_profile.h" #include "util/threadpool.h" @@ -80,6 +82,18 @@ class LockScopedTimer { class FSFileCacheStorage; +struct FileBlocksProbeResult { + FileBlocksProbeResult(FileBlocks blocks, std::vector gaps_) + : holder(std::move(blocks)), gaps(std::move(gaps_)) {} + FileBlocksProbeResult(FileBlocksProbeResult&&) noexcept = default; + FileBlocksProbeResult& operator=(FileBlocksProbeResult&&) noexcept = delete; + FileBlocksProbeResult(const FileBlocksProbeResult&) = delete; + FileBlocksProbeResult& operator=(const FileBlocksProbeResult&) = delete; + + FileBlocksHolder holder; + std::vector gaps; +}; + // NeedUpdateLRUBlocks keeps FileBlockSPtr entries that require LRU updates in a // deduplicated, sharded container. Entries are keyed by the raw FileBlock // pointer so that multiple shared_ptr copies of the same block are treated as a @@ -180,6 +194,9 @@ class BlockFileCache { BlockFileCache(const std::string& cache_base_path, const FileCacheSettings& cache_settings); virtual ~BlockFileCache() { + if (_async_write_service) { + _async_write_service->shutdown(); + } { std::lock_guard lock(_close_mtx); _close = true; @@ -237,6 +254,26 @@ class BlockFileCache { FileBlocksHolder get_or_set(const UInt128Wrapper& hash, size_t offset, size_t size, CacheContext& context); + /// Return existing blocks and uncovered gaps for `[offset, offset + size)` without creating + /// cache cells or touching LRU state. `context` supplies cache-type visibility rules. + FileBlocksProbeResult probe(const UInt128Wrapper& hash, size_t offset, size_t size, + const CacheContext& context); + + /// Touch `block` after a successful local read, provided it is still the cached cell. The + /// supplied `context` controls the target LRU queue and query-level accounting. + void touch_probe_block_if_cached(const FileBlockSPtr& block, const CacheContext& context); + + /// Check whether `block` is being deleted while taking cache/block locks in canonical order. + bool is_block_deleting(const FileBlockSPtr& block) const; + + /// Return this cache disk's async-write service. + AsyncCacheWriteService* async_write_service() const { return _async_write_service.get(); } + + /// Return this cache disk's inflight payload index. + InflightWriteBufferIndex* inflight_write_buffer_index() const { + return _inflight_write_buffer_index.get(); + } + /** * Return existing downloaded blocks only if they fully cover [offset, offset + size). * This lookup is read-only: it does not reserve cache space or create EMPTY blocks. @@ -570,6 +607,9 @@ class BlockFileCache { std::unique_ptr _lru_dumper; std::unique_ptr _ttl_mgr; + std::unique_ptr _inflight_write_buffer_index; + std::unique_ptr _async_write_service; + // metrics std::shared_ptr> _cache_capacity_metrics; std::shared_ptr> _cur_cache_size_metrics; diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index e5d1005d24aa53..be8373ef4d99b4 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -328,6 +328,8 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met size_t task_num = (download_size + one_single_task_size - 1) / one_single_task_size; std::unique_ptr buffer(new char[one_single_task_size]); + IOContext download_ctx = meta.ctx; + download_ctx.cache_write_mode_override = CacheWriteMode::SYNC_WRITE; DBUG_EXECUTE_IF("FileCacheBlockDownloader::download_segment_file_sleep", { if (meta.ctx.is_warmup) { @@ -352,7 +354,7 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met // 1. Directly append buffer data to file cache // 2. Provide `FileReader::async_read()` interface DCHECK(meta.ctx.is_dryrun == config::enable_reader_dryrun_when_download_file_cache); - st = file_reader->read_at(offset, {buffer.get(), size}, &bytes_read, &meta.ctx); + st = file_reader->read_at(offset, {buffer.get(), size}, &bytes_read, &download_ctx); if (!st.ok()) { LOG(WARNING) << "failed to download file path=" << meta.path << ", st=" << st; if (meta.download_done) { diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index 0ffc9cea365d6f..060f023da60ed2 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -70,6 +70,14 @@ size_t FileCacheFactory::try_release(const std::string& base_path) { return 0; } +Status FileCacheFactory::resize_async_write_workers(size_t worker_count) { + std::lock_guard lock(_mtx); + for (const auto& cache : _caches) { + RETURN_IF_ERROR(cache->async_write_service()->resize_workers(worker_count)); + } + return Status::OK(); +} + Status FileCacheFactory::create_file_cache(const std::string& cache_base_path, FileCacheSettings file_cache_settings) { if (file_cache_settings.storage == "memory") { @@ -394,3 +402,22 @@ void FileCacheFactory::get_cache_stats_block(Block* block) { } // namespace io } // namespace doris + +namespace doris::config { + +DEFINE_ON_UPDATE(async_file_cache_write_workers_per_disk, [](int32_t old_value, int32_t new_value) { + if (old_value == new_value) { + return; + } + auto* factory = ExecEnv::GetInstance()->file_cache_factory(); + if (factory == nullptr) { + return; + } + Status status = factory->resize_async_write_workers(static_cast(new_value)); + if (!status.ok()) { + LOG(WARNING) << "Failed to resize async file cache write workers from " << old_value + << " to " << new_value << ": " << status.to_string(); + } +}); + +} // namespace doris::config diff --git a/be/src/io/cache/block_file_cache_factory.h b/be/src/io/cache/block_file_cache_factory.h index f5be334de8b75f..8a986a94cba793 100644 --- a/be/src/io/cache/block_file_cache_factory.h +++ b/be/src/io/cache/block_file_cache_factory.h @@ -95,6 +95,9 @@ class FileCacheFactory { std::vector get_base_paths(); + /// Apply a positive per-disk async-write worker count to every initialized cache instance. + Status resize_async_write_workers(size_t worker_count); + /** * Clears data of all file cache instances * diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 751d422f044c1d..e541890a575e52 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -99,6 +99,17 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(lock_wait_timer); SUBTRACT_FIELD(get_timer); SUBTRACT_FIELD(set_timer); + SUBTRACT_FIELD(async_cache_write_submitted); + SUBTRACT_FIELD(async_cache_write_rejected); + SUBTRACT_FIELD(async_cache_write_buffer_alloc_fail); + SUBTRACT_FIELD(async_cache_write_drop_stale_epoch); + SUBTRACT_FIELD(inflight_write_buffer_index_hit); + SUBTRACT_FIELD(inflight_write_buffer_index_miss); + SUBTRACT_FIELD(probe_downloaded_hit); + SUBTRACT_FIELD(probe_downloading_hit); + SUBTRACT_FIELD(probe_miss); + SUBTRACT_FIELD(block_wait_success); + SUBTRACT_FIELD(block_wait_timeout); SUBTRACT_FIELD(inverted_index_num_local_io_total); SUBTRACT_FIELD(inverted_index_num_remote_io_total); @@ -163,6 +174,27 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _p lock_wait_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LockWaitTimer", cache_profile, 1); get_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "GetTimer", cache_profile, 1); set_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "SetTimer", cache_profile, 1); + async_cache_write_submitted = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "AsyncCacheWriteSubmitted", + TUnit::UNIT, cache_profile, 1); + async_cache_write_rejected = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "AsyncCacheWriteRejected", + TUnit::UNIT, cache_profile, 1); + async_cache_write_buffer_alloc_fail = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AsyncCacheWriteBufferAllocFail", TUnit::UNIT, cache_profile, 1); + async_cache_write_drop_stale_epoch = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AsyncCacheWriteDropStaleEpoch", TUnit::UNIT, cache_profile, 1); + inflight_write_buffer_index_hit = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InflightWriteBufferIndexHit", TUnit::UNIT, cache_profile, 1); + inflight_write_buffer_index_miss = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InflightWriteBufferIndexMiss", TUnit::UNIT, cache_profile, 1); + probe_downloaded_hit = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ProbeDownloadedHit", TUnit::UNIT, + cache_profile, 1); + probe_downloading_hit = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ProbeDownloadingHit", + TUnit::UNIT, cache_profile, 1); + probe_miss = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ProbeMiss", TUnit::UNIT, cache_profile, 1); + block_wait_success = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "BlockWaitSuccess", TUnit::UNIT, + cache_profile, 1); + block_wait_timeout = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "BlockWaitTimeout", TUnit::UNIT, + cache_profile, 1); remote_only_on_miss_triggered = profile->AddHighWaterMarkCounter("RemoteOnlyOnMissTriggered", TUnit::UNIT, cache_profile, 1); remote_only_on_miss_threshold_bytes = profile->AddHighWaterMarkCounter( @@ -257,6 +289,19 @@ void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) con COUNTER_UPDATE(lock_wait_timer, statistics->lock_wait_timer); COUNTER_UPDATE(get_timer, statistics->get_timer); COUNTER_UPDATE(set_timer, statistics->set_timer); + COUNTER_UPDATE(async_cache_write_submitted, statistics->async_cache_write_submitted); + COUNTER_UPDATE(async_cache_write_rejected, statistics->async_cache_write_rejected); + COUNTER_UPDATE(async_cache_write_buffer_alloc_fail, + statistics->async_cache_write_buffer_alloc_fail); + COUNTER_UPDATE(async_cache_write_drop_stale_epoch, + statistics->async_cache_write_drop_stale_epoch); + COUNTER_UPDATE(inflight_write_buffer_index_hit, statistics->inflight_write_buffer_index_hit); + COUNTER_UPDATE(inflight_write_buffer_index_miss, statistics->inflight_write_buffer_index_miss); + COUNTER_UPDATE(probe_downloaded_hit, statistics->probe_downloaded_hit); + COUNTER_UPDATE(probe_downloading_hit, statistics->probe_downloading_hit); + COUNTER_UPDATE(probe_miss, statistics->probe_miss); + COUNTER_UPDATE(block_wait_success, statistics->block_wait_success); + COUNTER_UPDATE(block_wait_timeout, statistics->block_wait_timeout); remote_only_on_miss_triggered->set(statistics->remote_only_on_miss_triggered); remote_only_on_miss_threshold_bytes->set(statistics->remote_only_on_miss_threshold_bytes); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index 2a6e7b33980bb2..c2db1a704c0706 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -89,6 +89,17 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* lock_wait_timer = nullptr; RuntimeProfile::Counter* get_timer = nullptr; RuntimeProfile::Counter* set_timer = nullptr; + RuntimeProfile::Counter* async_cache_write_submitted = nullptr; + RuntimeProfile::Counter* async_cache_write_rejected = nullptr; + RuntimeProfile::Counter* async_cache_write_buffer_alloc_fail = nullptr; + RuntimeProfile::Counter* async_cache_write_drop_stale_epoch = nullptr; + RuntimeProfile::Counter* inflight_write_buffer_index_hit = nullptr; + RuntimeProfile::Counter* inflight_write_buffer_index_miss = nullptr; + RuntimeProfile::Counter* probe_downloaded_hit = nullptr; + RuntimeProfile::Counter* probe_downloading_hit = nullptr; + RuntimeProfile::Counter* probe_miss = nullptr; + RuntimeProfile::Counter* block_wait_success = nullptr; + RuntimeProfile::Counter* block_wait_timeout = nullptr; RuntimeProfile::HighWaterMarkCounter* remote_only_on_miss_triggered = nullptr; RuntimeProfile::HighWaterMarkCounter* remote_only_on_miss_threshold_bytes = nullptr; diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index 47fe02ee2c3e92..91edf328475f8c 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -65,6 +65,7 @@ #include "util/concurrency_stats.h" #include "util/debug_points.h" #include "util/defer_op.h" +#include "util/time.h" namespace doris::io { @@ -118,6 +119,8 @@ static bool use_remote_only_on_cache_miss(const IOContext* io_ctx) { CachedRemoteFileReader::CachedRemoteFileReader(FileReaderSPtr remote_file_reader, const FileReaderOptions& opts) : _is_doris_table(opts.is_doris_table), + _cache_align_mode(opts.align_mode), + _cache_write_mode(opts.cache_write_mode), _tablet_id(opts.tablet_id), _storage_resource_id(opts.storage_resource_id), _remote_file_reader(std::move(remote_file_reader)) { @@ -1258,8 +1261,17 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* return Status::OK(); } - read_st = _read_from_indirect_cache(offset, result, bytes_req, already_read, is_dryrun, - bytes_read, stats, source_read_breakdown, io_ctx); + const CacheWriteMode cache_write_mode = _resolve_cache_write_mode(io_ctx); + DORIS_CHECK(_cache_align_mode == CacheAlignMode::ALIGN_TO_BLOCK); + DORIS_CHECK(cache_write_mode == CacheWriteMode::SYNC_WRITE || + cache_write_mode == CacheWriteMode::ASYNC_WRITE); + if (cache_write_mode == CacheWriteMode::ASYNC_WRITE) { + read_st = _read_async_write_path(offset, result, bytes_req, already_read, bytes_read, stats, + source_read_breakdown, io_ctx); + } else { + read_st = _read_from_indirect_cache(offset, result, bytes_req, already_read, is_dryrun, + bytes_read, stats, source_read_breakdown, io_ctx); + } return read_st; } @@ -1280,6 +1292,7 @@ void CachedRemoteFileReader::prefetch_range(size_t offset, size_t size, const IO dryrun_ctx = *io_ctx; } dryrun_ctx.is_dryrun = true; + dryrun_ctx.cache_write_mode_override = CacheWriteMode::SYNC_WRITE; dryrun_ctx.query_id = nullptr; dryrun_ctx.file_cache_stats = nullptr; dryrun_ctx.file_reader_stats = nullptr; @@ -1360,6 +1373,17 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->lock_wait_timer += read_stats.lock_wait_timer; statis->get_timer += read_stats.get_timer; statis->set_timer += read_stats.set_timer; + statis->async_cache_write_submitted += read_stats.async_cache_write_submitted; + statis->async_cache_write_rejected += read_stats.async_cache_write_rejected; + statis->async_cache_write_buffer_alloc_fail += read_stats.async_cache_write_buffer_alloc_fail; + statis->async_cache_write_drop_stale_epoch += read_stats.async_cache_write_drop_stale_epoch; + statis->inflight_write_buffer_index_hit += read_stats.inflight_write_buffer_index_hit; + statis->inflight_write_buffer_index_miss += read_stats.inflight_write_buffer_index_miss; + statis->probe_downloaded_hit += read_stats.probe_downloaded_hit; + statis->probe_downloading_hit += read_stats.probe_downloading_hit; + statis->probe_miss += read_stats.probe_miss; + statis->block_wait_success += read_stats.block_wait_success; + statis->block_wait_timeout += read_stats.block_wait_timeout; auto update_index_stats = [&](int64_t& num_local_io_total, int64_t& num_remote_io_total, int64_t& num_peer_io_total, int64_t& bytes_read_from_local, diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h index 5c562c82513f7b..ce6c4cc0749ea3 100644 --- a/be/src/io/cache/cached_remote_file_reader.h +++ b/be/src/io/cache/cached_remote_file_reader.h @@ -118,6 +118,10 @@ class CachedRemoteFileReader final : public FileReader, const IOContext* io_ctx) override; private: + struct AsyncBlockCoverage; + struct AsyncReadPlan; + struct AsyncRemoteRange; + enum class FileCacheReadType { DATA, INVERTED_INDEX, @@ -142,6 +146,122 @@ class CachedRemoteFileReader final : public FileReader, /// @return true if peer read is enabled for this request; otherwise false. bool _should_read_from_peer(const IOContext* io_ctx) const; + /// Resolve the write policy for the current read instead of freezing a global setting in the + /// reader constructor. Explicit cache-population reads always remain synchronous. + /// @param[in] io_ctx Per-read flags and an optional write-mode override. + /// @return The effective synchronous or asynchronous cache-write mode for this read. + CacheWriteMode _resolve_cache_write_mode(const IOContext* io_ctx) const; + + /// Serve a normal read while moving cache-miss writes off the query thread. This method only + /// coordinates the four stages: build coverage, materialize covered sides, read one remote + /// middle span, and submit tasks for blocks still owned by this reader. + /// @param[in] offset Original user read offset. + /// @param[out] result Destination buffer for the complete user request. + /// @param[in] bytes_req Requested user bytes. + /// @param[in] already_read Prefix bytes already filled by the direct-cache path. + /// @param[out] bytes_read Total completed user bytes. + /// @param[in,out] stats Per-read cache statistics. + /// @param[in,out] source_read_breakdown Local/remote byte attribution for query profiles. + /// @param[in] io_ctx Context passed to cache lookup and remote IO. + /// @return OK on success; otherwise the remote-read error. + Status _read_async_write_path(size_t offset, Slice result, size_t bytes_req, + size_t already_read, size_t* bytes_read, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + const IOContext* io_ctx); + + /// Build block-aligned source coverage for the unread suffix. Inflight buffers are resolved + /// first; only contiguous runs still uncovered are passed to the read-only cache probe. + /// @param[in] remaining_offset First user byte not filled by the direct-cache path. + /// @param[in] remaining_size Number of unread user bytes. + /// @param[in] write_epoch Epoch captured before any lookup or remote IO. + /// @param[in] io_ctx Context used to build the cache admission/probe context. + /// @param[in,out] stats Lookup and probe counters updated during planning. + /// @param[out] plan Coverage plan that owns probe holders until materialization finishes. + /// @return None. + void _build_async_read_plan(size_t remaining_offset, size_t remaining_size, + uint64_t write_epoch, const IOContext* io_ctx, + ReadStatistics& stats, AsyncReadPlan* plan); + + /// Merge one sorted probe result into the corresponding consecutive coverage entries. The + /// merge uses monotonic block/gap cursors and never rescans earlier probe results. + /// @param[in,out] plan Plan whose entries in [begin_index, end_index) are classified. + /// @param[in] begin_index First coverage entry represented by probe_result. + /// @param[in] end_index One-past-last coverage entry represented by probe_result. + /// @param[in] probe_result Existing cache blocks and uncovered gaps for the same range. + /// @param[in,out] stats Per-block probe classifications added here. + /// @return None. + void _classify_async_probe_result(AsyncReadPlan* plan, size_t begin_index, size_t end_index, + const FileBlocksProbeResult& probe_result, + ReadStatistics& stats); + + /// Materialize one boundary coverage entry from an inflight buffer or cache file. A + /// DOWNLOADING entry is waited only because the caller invokes this method for a boundary. + /// @param[in,out] coverage Source entry; failures convert it to MISS. + /// @param[in] user_offset Original user request offset used to locate the destination slice. + /// @param[out] result Destination buffer for the complete user request. + /// @param[in] user_left First user byte handled by the async path. + /// @param[in] user_right Last user byte of the request, inclusive. + /// @param[in] cache_context Context used only when a successful local read touches LRU. + /// @param[in,out] stats Wait and local-read timing counters. + /// @param[in,out] source_read_breakdown Local bytes committed by a fully successful entry. + /// @param[in,out] indirect_read_bytes User bytes copied by the indirect path. + /// @param[in,out] need_self_heal Set when a cache file disappears during a local read. + /// @return true when the whole aligned entry is available; false when it becomes a MISS. + bool _materialize_async_coverage(AsyncBlockCoverage* coverage, size_t user_offset, Slice result, + size_t user_left, size_t user_right, + const CacheContext& cache_context, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, bool* need_self_heal); + + /// Consume the maximum consecutive available coverage from both ends and leave one middle + /// range bounded by the first and last MISS. + /// @param[in,out] plan Coverage entries may transition from DOWNLOADING to DOWNLOADED or MISS. + /// @param[in] user_offset Original user request offset. + /// @param[out] result Destination buffer for side data. + /// @param[in] cache_context Context used for successful local-cache touches. + /// @param[in,out] stats Wait/local and middle-span statistics. + /// @param[in,out] source_read_breakdown Local bytes copied from the covered sides. + /// @param[in,out] indirect_read_bytes User bytes copied from the covered sides. + /// @param[in,out] need_self_heal Whether a missing local cache file requires async cleanup. + /// @param[out] remote_range The single middle span when one remains. + /// @return true when remote IO is required; false when both sides cover the request. + bool _materialize_async_covered_sides(AsyncReadPlan* plan, size_t user_offset, Slice result, + const CacheContext& cache_context, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, bool* need_self_heal, + AsyncRemoteRange* remote_range); + + /// Read the planned middle span once and copy only its overlap with the unread user range. + /// @param[in] plan Source plan containing user boundaries. + /// @param[in] remote_range Middle span bounded by the first and last MISS. + /// @param[in] user_offset Original user request offset. + /// @param[out] result Destination buffer for the complete user request. + /// @param[in] need_self_heal Whether stale cache metadata should be removed before remote IO. + /// @param[in] io_ctx Context passed to remote storage. + /// @param[in,out] stats Remote-read timing and source flags. + /// @param[in,out] source_read_breakdown Remote user bytes copied from this span. + /// @param[in,out] indirect_read_bytes User bytes copied by the indirect path. + /// @param[out] remote_buffer Full aligned middle-span payload retained for async tasks. + /// @return OK on success; otherwise the remote-read error. + Status _read_async_remote_range(const AsyncReadPlan& plan, const AsyncRemoteRange& remote_range, + size_t user_offset, Slice result, bool need_self_heal, + const IOContext* io_ctx, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, + std::unique_ptr* remote_buffer); + + /// Copy each MISS block from the remote middle buffer into tracked memory and enqueue a + /// per-block write task. A final insert-if-absent prevents duplicate ownership after IO. + /// @param[in] plan Classified coverage and the epoch captured before remote IO. + /// @param[in] remote_range Span represented by remote_buffer. + /// @param[in] remote_buffer Full remote payload for remote_range. + /// @param[in] io_ctx Context converted to the worker's admission context. + /// @param[in,out] stats Submission, rejection, allocation, and dedup counters. + /// @return None. + void _submit_async_write_tasks(const AsyncReadPlan& plan, const AsyncRemoteRange& remote_range, + const std::unique_ptr& remote_buffer, + const IOContext* io_ctx, ReadStatistics& stats); + /// Register a downloaded block in the direct-read map owned by this reader. /// @param[in] file_block Downloaded cache block to insert. /// @return None. @@ -320,6 +440,8 @@ class CachedRemoteFileReader final : public FileReader, FileCacheReadType read_type) const; bool _is_doris_table = false; + CacheAlignMode _cache_align_mode {CacheAlignMode::ALIGN_TO_BLOCK}; + CacheWriteMode _cache_write_mode {CacheWriteMode::DEFAULT}; int64_t _tablet_id = -1; std::string _storage_resource_id; FileReaderSPtr _remote_file_reader; diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp new file mode 100644 index 00000000000000..5d6cddf4b98896 --- /dev/null +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -0,0 +1,568 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "io/cache/async_cache_write_service.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/cached_remote_file_reader.h" +#include "io/cache/inflight_write_buffer_index.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "util/time.h" + +namespace doris::io { + +// These counters are shared by the synchronous and asynchronous indirect-read paths, so their +// definitions remain in cached_remote_file_reader.cpp. +extern bvar::Adder g_read_cache_indirect_num; +extern bvar::Adder g_read_cache_indirect_bytes; +extern bvar::Adder g_read_cache_indirect_total_bytes; +extern bvar::Adder g_read_cache_self_heal_on_not_found; + +namespace { + +bvar::Adder g_cached_remote_reader_probe_total("cached_remote_file_reader_probe_total"); +bvar::Adder g_cached_remote_reader_probe_downloaded( + "cached_remote_file_reader_probe_hit_downloaded_total"); +bvar::Adder g_cached_remote_reader_probe_downloading( + "cached_remote_file_reader_probe_hit_downloading_total"); +bvar::Adder g_cached_remote_reader_probe_miss( + "cached_remote_file_reader_probe_miss_total"); +bvar::Adder g_cached_remote_reader_inflight_hit( + "cached_remote_file_reader_inflight_write_buffer_hit_total"); +bvar::Adder g_cached_remote_reader_async_skip_existing( + "cached_remote_file_reader_async_write_skip_inflight_existing_total"); +bvar::Adder g_cached_remote_reader_block_wait( + "cached_remote_file_reader_block_wait_total"); +bvar::Adder g_cached_remote_reader_block_wait_timeout( + "cached_remote_file_reader_block_wait_timeout_total"); +bvar::Adder g_cached_remote_reader_remote_after_dedup_miss( + "cached_remote_file_reader_remote_read_after_all_dedup_miss_total"); +bvar::Adder g_cached_remote_reader_middle_span_read_bytes( + "cached_remote_file_reader_middle_span_read_bytes"); +bvar::Adder g_cached_remote_reader_middle_span_miss_bytes( + "cached_remote_file_reader_middle_span_miss_bytes"); + +} // namespace + +// One block-aligned ownership decision. Only MISS entries can create a new async write task. +struct CachedRemoteFileReader::AsyncBlockCoverage { + enum class Source { + INFLIGHT, + DOWNLOADED, + DOWNLOADING, + MISS, + }; + + explicit AsyncBlockCoverage(FileBlock::Range range_) : range(range_) {} + + FileBlock::Range range; + Source source {Source::MISS}; + std::shared_ptr inflight_entry; + std::vector probe_blocks; +}; + +// Probe holders must outlive copied block references. Declaration order makes coverage destruct +// first, before FileBlocksHolder performs EMPTY/deleting cleanup. +struct CachedRemoteFileReader::AsyncReadPlan { + uint64_t write_epoch {0}; + size_t user_left {0}; + size_t user_right {0}; + std::vector probe_results; + std::vector coverage; +}; + +// The one aligned remote GET left after consuming the maximum covered prefix and suffix. +struct CachedRemoteFileReader::AsyncRemoteRange { + size_t left {0}; + size_t size {0}; +}; + +// Resolve mode on every read so online configuration changes affect existing readers. +CacheWriteMode CachedRemoteFileReader::_resolve_cache_write_mode(const IOContext* io_ctx) const { + if (io_ctx->is_dryrun || io_ctx->is_warmup || _should_read_from_peer(io_ctx)) { + return CacheWriteMode::SYNC_WRITE; + } + if (io_ctx->cache_write_mode_override.has_value()) { + return *io_ctx->cache_write_mode_override; + } + if (_cache_write_mode != CacheWriteMode::DEFAULT) { + return _cache_write_mode; + } + return config::enable_async_file_cache_write ? CacheWriteMode::ASYNC_WRITE + : CacheWriteMode::SYNC_WRITE; +} + +// Build per-block coverage with the required priority: inflight memory first, then read-only +// probing only for consecutive runs that remain uncovered. +void CachedRemoteFileReader::_build_async_read_plan(size_t remaining_offset, size_t remaining_size, + uint64_t write_epoch, const IOContext* io_ctx, + ReadStatistics& stats, AsyncReadPlan* plan) { + DORIS_CHECK(plan != nullptr); + DORIS_CHECK(plan->probe_results.empty()); + DORIS_CHECK(plan->coverage.empty()); + + plan->write_epoch = write_epoch; + plan->user_left = remaining_offset; + plan->user_right = remaining_offset + remaining_size - 1; + + const auto [align_left, align_size] = s_align_size(remaining_offset, remaining_size, size()); + const size_t cache_block_size = static_cast(config::file_cache_each_block_size); + DORIS_CHECK(cache_block_size > 0); + + std::vector block_offsets; + const size_t align_end = align_left + align_size; + for (size_t block_offset = align_left; block_offset < align_end; + block_offset += cache_block_size) { + const size_t block_size = std::min(cache_block_size, size() - block_offset); + DORIS_CHECK(block_size > 0); + plan->coverage.emplace_back(FileBlock::Range(block_offset, block_offset + block_size - 1)); + block_offsets.emplace_back(block_offset); + } + + auto* inflight_index = _cache->inflight_write_buffer_index(); + DORIS_CHECK(inflight_index != nullptr); + if (config::enable_inflight_write_buffer_index) { + auto lookup_results = inflight_index->lookup_all(_cache_hash, block_offsets, write_epoch); + DORIS_CHECK(lookup_results.size() == plan->coverage.size()); + for (size_t index = 0; index < lookup_results.size(); ++index) { + auto& lookup_result = lookup_results[index]; + if (!lookup_result.entry) { + ++stats.inflight_write_buffer_index_miss; + continue; + } + + auto& coverage = plan->coverage[index]; + DORIS_CHECK(lookup_result.entry->buffer != nullptr); + DORIS_CHECK(lookup_result.entry->buffer_offset <= coverage.range.left); + DORIS_CHECK(lookup_result.entry->buffer_offset + lookup_result.entry->buffer_size > + coverage.range.right); + coverage.source = AsyncBlockCoverage::Source::INFLIGHT; + coverage.inflight_entry = std::move(lookup_result.entry); + ++stats.inflight_write_buffer_index_hit; + g_cached_remote_reader_inflight_hit << 1; + } + } + + CacheContext cache_context(io_ctx); + cache_context.stats = &stats; + cache_context.tablet_id = _tablet_id; + size_t index = 0; + while (index < plan->coverage.size()) { + while (index < plan->coverage.size() && + plan->coverage[index].source == AsyncBlockCoverage::Source::INFLIGHT) { + ++index; + } + if (index == plan->coverage.size()) { + break; + } + + const size_t begin_index = index; + while (index < plan->coverage.size() && + plan->coverage[index].source != AsyncBlockCoverage::Source::INFLIGHT) { + ++index; + } + const size_t end_index = index; + const size_t probe_left = plan->coverage[begin_index].range.left; + const size_t probe_right = plan->coverage[end_index - 1].range.right; + plan->probe_results.emplace_back(_cache->probe( + _cache_hash, probe_left, probe_right - probe_left + 1, cache_context)); + g_cached_remote_reader_probe_total << 1; + _classify_async_probe_result(plan, begin_index, end_index, plan->probe_results.back(), + stats); + } +} + +// Classify a consecutive probe run with two monotonic cursors. Each cache block/gap is considered +// only while it can overlap the current aligned coverage entry. +void CachedRemoteFileReader::_classify_async_probe_result(AsyncReadPlan* plan, size_t begin_index, + size_t end_index, + const FileBlocksProbeResult& probe_result, + ReadStatistics& stats) { + DORIS_CHECK(plan != nullptr); + DORIS_CHECK(begin_index < end_index); + DORIS_CHECK(end_index <= plan->coverage.size()); + + const auto& cache_blocks = probe_result.holder.file_blocks; + auto cache_block = cache_blocks.begin(); + size_t gap_index = 0; + for (size_t index = begin_index; index < end_index; ++index) { + auto& coverage = plan->coverage[index]; + DORIS_CHECK(coverage.source == AsyncBlockCoverage::Source::MISS); + + while (cache_block != cache_blocks.end() && + (*cache_block)->range().right < coverage.range.left) { + ++cache_block; + } + while (gap_index < probe_result.gaps.size() && + probe_result.gaps[gap_index].right < coverage.range.left) { + ++gap_index; + } + + bool has_miss = gap_index < probe_result.gaps.size() && + probe_result.gaps[gap_index].left <= coverage.range.right; + bool has_downloading = false; + auto overlapping_block = cache_block; + while (overlapping_block != cache_blocks.end() && + (*overlapping_block)->range().left <= coverage.range.right) { + const auto& block = *overlapping_block; + coverage.probe_blocks.emplace_back(block); + if (_cache->is_block_deleting(block)) { + has_miss = true; + } else { + const FileBlock::State state = block->state(); + has_miss = has_miss || state == FileBlock::State::EMPTY || + state == FileBlock::State::SKIP_CACHE; + has_downloading = has_downloading || state == FileBlock::State::DOWNLOADING; + } + ++overlapping_block; + } + + if (has_miss) { + ++stats.probe_miss; + g_cached_remote_reader_probe_miss << 1; + continue; + } + + DORIS_CHECK(!coverage.probe_blocks.empty()); + if (has_downloading) { + coverage.source = AsyncBlockCoverage::Source::DOWNLOADING; + ++stats.probe_downloading_hit; + g_cached_remote_reader_probe_downloading << 1; + } else { + coverage.source = AsyncBlockCoverage::Source::DOWNLOADED; + ++stats.probe_downloaded_hit; + g_cached_remote_reader_probe_downloaded << 1; + } + } +} + +// Materialize a single side entry. The caller deliberately never invokes this for entries already +// enclosed by the middle span, which keeps waits and local reads out of that span. +bool CachedRemoteFileReader::_materialize_async_coverage( + AsyncBlockCoverage* coverage, size_t user_offset, Slice result, size_t user_left, + size_t user_right, const CacheContext& cache_context, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, size_t* indirect_read_bytes, + bool* need_self_heal) { + DORIS_CHECK(coverage != nullptr); + DORIS_CHECK(indirect_read_bytes != nullptr); + DORIS_CHECK(need_self_heal != nullptr); + if (coverage->source == AsyncBlockCoverage::Source::MISS) { + return false; + } + + const size_t copy_left = std::max(coverage->range.left, user_left); + const size_t copy_right = std::min(coverage->range.right, user_right); + const bool has_user_bytes = copy_left <= copy_right; + if (coverage->source == AsyncBlockCoverage::Source::INFLIGHT) { + if (has_user_bytes) { + const auto& entry = coverage->inflight_entry; + DORIS_CHECK(entry != nullptr); + const size_t entry_offset = copy_left - entry->buffer_offset; + const size_t copy_size = copy_right - copy_left + 1; + DORIS_CHECK(entry_offset + copy_size <= entry->buffer_size); + memcpy(result.data + (copy_left - user_offset), entry->buffer->data() + entry_offset, + copy_size); + *indirect_read_bytes += copy_size; + source_read_breakdown.local_bytes += copy_size; + } + return true; + } + + DORIS_CHECK(!coverage->probe_blocks.empty()); + size_t local_read_bytes = 0; + std::vector consumed_blocks; + for (const auto& block : coverage->probe_blocks) { + if (_cache->is_block_deleting(block)) { + coverage->source = AsyncBlockCoverage::Source::MISS; + return false; + } + + FileBlock::State state = block->state(); + if (state == FileBlock::State::DOWNLOADING) { + DORIS_CHECK(coverage->source == AsyncBlockCoverage::Source::DOWNLOADING); + { + SCOPED_RAW_TIMER(&stats.remote_wait_timer); + state = block->wait(); + } + if (state != FileBlock::State::DOWNLOADED) { + ++stats.block_wait_timeout; + g_cached_remote_reader_block_wait_timeout << 1; + coverage->source = AsyncBlockCoverage::Source::MISS; + return false; + } + ++stats.block_wait_success; + g_cached_remote_reader_block_wait << 1; + } + if (state != FileBlock::State::DOWNLOADED) { + coverage->source = AsyncBlockCoverage::Source::MISS; + return false; + } + if (!has_user_bytes || block->range().right < copy_left || + block->range().left > copy_right) { + continue; + } + + const size_t read_left = std::max(block->range().left, copy_left); + const size_t read_right = std::min(block->range().right, copy_right); + const size_t read_size = read_right - read_left + 1; + Status status; + { + SCOPED_RAW_TIMER(&stats.local_read_timer); + status = block->read(Slice(result.data + (read_left - user_offset), read_size), + read_left - block->range().left); + } + if (!status.ok()) { + if (status.is()) { + *need_self_heal = true; + g_read_cache_self_heal_on_not_found << 1; + } + LOG_EVERY_N(WARNING, 100) + << "Read probed file cache block failed, falling back to remote. path=" + << path().native() << ", hash=" << _cache_hash.to_string() + << ", offset=" << block->offset() << ", status=" << status; + coverage->source = AsyncBlockCoverage::Source::MISS; + return false; + } + local_read_bytes += read_size; + consumed_blocks.emplace_back(block); + } + + for (const auto& block : consumed_blocks) { + _cache->touch_probe_block_if_cached(block, cache_context); + } + coverage->source = AsyncBlockCoverage::Source::DOWNLOADED; + *indirect_read_bytes += local_read_bytes; + source_read_breakdown.local_bytes += local_read_bytes; + return true; +} + +// Consume only boundary entries. Once either side reaches a MISS, everything between the two +// boundaries is intentionally covered by one remote GET without further waits or local reads. +bool CachedRemoteFileReader::_materialize_async_covered_sides( + AsyncReadPlan* plan, size_t user_offset, Slice result, const CacheContext& cache_context, + ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, bool* need_self_heal, AsyncRemoteRange* remote_range) { + DORIS_CHECK(plan != nullptr); + DORIS_CHECK(remote_range != nullptr); + + size_t middle_begin = 0; + while (middle_begin < plan->coverage.size() && + _materialize_async_coverage(&plan->coverage[middle_begin], user_offset, result, + plan->user_left, plan->user_right, cache_context, stats, + source_read_breakdown, indirect_read_bytes, + need_self_heal)) { + ++middle_begin; + } + + size_t middle_end = plan->coverage.size(); + while (middle_end > middle_begin && + _materialize_async_coverage(&plan->coverage[middle_end - 1], user_offset, result, + plan->user_left, plan->user_right, cache_context, stats, + source_read_breakdown, indirect_read_bytes, + need_self_heal)) { + --middle_end; + } + if (middle_begin == middle_end) { + return false; + } + + remote_range->left = plan->coverage[middle_begin].range.left; + remote_range->size = plan->coverage[middle_end - 1].range.right - remote_range->left + 1; + size_t miss_bytes = 0; + for (size_t index = middle_begin; index < middle_end; ++index) { + if (plan->coverage[index].source == AsyncBlockCoverage::Source::MISS) { + miss_bytes += plan->coverage[index].range.size(); + } + } + DORIS_CHECK(miss_bytes > 0); + g_cached_remote_reader_remote_after_dedup_miss << 1; + g_cached_remote_reader_middle_span_read_bytes << remote_range->size; + g_cached_remote_reader_middle_span_miss_bytes << miss_bytes; + return true; +} + +// Execute one middle-span remote read and expose only the requested user overlap. The full aligned +// payload stays alive for per-block task construction in the next stage. +Status CachedRemoteFileReader::_read_async_remote_range( + const AsyncReadPlan& plan, const AsyncRemoteRange& remote_range, size_t user_offset, + Slice result, bool need_self_heal, const IOContext* io_ctx, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, size_t* indirect_read_bytes, + std::unique_ptr* remote_buffer) { + DORIS_CHECK(indirect_read_bytes != nullptr); + DORIS_CHECK(remote_buffer != nullptr); + DORIS_CHECK(remote_range.size > 0); + + stats.hit_cache = false; + stats.from_peer_cache = false; + if (need_self_heal) { + _cache->remove_if_cached_async(_cache_hash); + } + + const std::vector no_peer_blocks; + RETURN_IF_ERROR(_execute_remote_read(no_peer_blocks, remote_range.left, remote_range.size, + *remote_buffer, nullptr, stats, io_ctx)); + DORIS_CHECK(*remote_buffer != nullptr); + + const size_t remote_right = remote_range.left + remote_range.size - 1; + const size_t copy_left = std::max(remote_range.left, plan.user_left); + const size_t copy_right = std::min(remote_right, plan.user_right); + if (copy_left <= copy_right) { + const size_t copy_size = copy_right - copy_left + 1; + memcpy(result.data + (copy_left - user_offset), + remote_buffer->get() + (copy_left - remote_range.left), copy_size); + *indirect_read_bytes += copy_size; + source_read_breakdown.remote_bytes += copy_size; + } + return Status::OK(); +} + +// Submit exactly the MISS entries contained in the remote span. Inflight insertion happens after +// remote IO and immediately before enqueueing, so a concurrent owner wins without duplicate work. +void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan, + const AsyncRemoteRange& remote_range, + const std::unique_ptr& remote_buffer, + const IOContext* io_ctx, + ReadStatistics& stats) { + auto* service = _cache->async_write_service(); + auto* inflight_index = _cache->inflight_write_buffer_index(); + DORIS_CHECK(service != nullptr); + DORIS_CHECK(inflight_index != nullptr); + DORIS_CHECK(remote_buffer != nullptr); + + CacheContext cache_context(io_ctx); + CacheAdmissionContext admission_context { + .query_id = cache_context.query_id, + .cache_type = cache_context.cache_type, + .expiration_time = cache_context.expiration_time, + .tablet_id = _tablet_id, + .is_warmup = cache_context.is_warmup, + }; + const size_t remote_right = remote_range.left + remote_range.size - 1; + for (const auto& coverage : plan.coverage) { + if (coverage.source != AsyncBlockCoverage::Source::MISS || + coverage.range.left < remote_range.left || coverage.range.right > remote_right) { + continue; + } + if (!service->is_current_write_epoch(plan.write_epoch)) { + ++stats.async_cache_write_drop_stale_epoch; + continue; + } + + AsyncCacheWriteBufferPtr tracked_buffer; + Status status = service->allocate_tracked_buffer(coverage.range.size(), &tracked_buffer); + if (!status.ok()) { + ++stats.async_cache_write_buffer_alloc_fail; + continue; + } + memcpy(tracked_buffer->data(), + remote_buffer.get() + (coverage.range.left - remote_range.left), + coverage.range.size()); + + AsyncCacheWriteTask task { + .cache_hash = _cache_hash, + .file_offset = coverage.range.left, + .file_size = coverage.range.size(), + .buffer = tracked_buffer, + .admission_ctx = admission_context, + .submit_ts_us = MonotonicMicros(), + .write_epoch = plan.write_epoch, + .on_finalized = nullptr, + }; + std::shared_ptr entry; + if (config::enable_inflight_write_buffer_index) { + entry = std::make_shared(tracked_buffer, coverage.range.left, + coverage.range.size(), + task.submit_ts_us, plan.write_epoch); + auto existing = + inflight_index->insert_if_absent(_cache_hash, coverage.range.left, entry); + if (existing) { + g_cached_remote_reader_async_skip_existing << 1; + continue; + } + task.on_finalized = [cache_hash = _cache_hash, offset = coverage.range.left, + inflight_index, entry](const AsyncCacheWriteTask&) { + inflight_index->remove_if(cache_hash, offset, entry); + }; + } + + if (!service->try_submit(std::move(task))) { + if (entry) { + inflight_index->remove_if(_cache_hash, coverage.range.left, entry); + inflight_index->record_backpressure_rollback(); + } + ++stats.async_cache_write_rejected; + } else { + ++stats.async_cache_write_submitted; + } + } +} + +// Orchestrate the async-write path while keeping ownership decisions inside the stage helpers. +Status CachedRemoteFileReader::_read_async_write_path(size_t offset, Slice result, size_t bytes_req, + size_t already_read, size_t* bytes_read, + ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + const IOContext* io_ctx) { + DORIS_CHECK(_cache_align_mode == CacheAlignMode::ALIGN_TO_BLOCK); + DORIS_CHECK(!io_ctx->is_dryrun); + DORIS_CHECK(result.data != nullptr); + g_read_cache_indirect_num << 1; + if (already_read == bytes_req) { + *bytes_read = bytes_req; + return Status::OK(); + } + + auto* service = _cache->async_write_service(); + DORIS_CHECK(service != nullptr); + AsyncReadPlan plan; + _build_async_read_plan(offset + already_read, bytes_req - already_read, + service->current_write_epoch(), io_ctx, stats, &plan); + + CacheContext cache_context(io_ctx); + cache_context.stats = &stats; + cache_context.tablet_id = _tablet_id; + size_t indirect_read_bytes = 0; + bool need_self_heal = false; + AsyncRemoteRange remote_range; + if (_materialize_async_covered_sides(&plan, offset, result, cache_context, stats, + source_read_breakdown, &indirect_read_bytes, + &need_self_heal, &remote_range)) { + std::unique_ptr remote_buffer; + RETURN_IF_ERROR(_read_async_remote_range(plan, remote_range, offset, result, need_self_heal, + io_ctx, stats, source_read_breakdown, + &indirect_read_bytes, &remote_buffer)); + _submit_async_write_tasks(plan, remote_range, remote_buffer, io_ctx, stats); + } + + *bytes_read = bytes_req; + g_read_cache_indirect_bytes << indirect_read_bytes; + g_read_cache_indirect_total_bytes << bytes_req; + return Status::OK(); +} + +} // namespace doris::io diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index 8b52af8d161b9a..868aceee06b9de 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -86,6 +86,17 @@ struct ReadStatistics { int64_t lock_wait_timer = 0; int64_t get_timer = 0; int64_t set_timer = 0; + int64_t async_cache_write_submitted = 0; + int64_t async_cache_write_rejected = 0; + int64_t async_cache_write_buffer_alloc_fail = 0; + int64_t async_cache_write_drop_stale_epoch = 0; + int64_t inflight_write_buffer_index_hit = 0; + int64_t inflight_write_buffer_index_miss = 0; + int64_t probe_downloaded_hit = 0; + int64_t probe_downloading_hit = 0; + int64_t probe_miss = 0; + int64_t block_wait_success = 0; + int64_t block_wait_timeout = 0; }; class BlockFileCache; diff --git a/be/src/io/cache/inflight_write_buffer_index.cpp b/be/src/io/cache/inflight_write_buffer_index.cpp new file mode 100644 index 00000000000000..b80a5f684c523a --- /dev/null +++ b/be/src/io/cache/inflight_write_buffer_index.cpp @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/inflight_write_buffer_index.h" + +#include + +#include "common/logging.h" + +namespace doris::io { + +InflightWriteBufferIndex::InflightWriteBufferIndex(size_t shard_count, std::string metric_prefix) { + DORIS_CHECK(shard_count > 0); + _shards.reserve(shard_count); + for (size_t index = 0; index < shard_count; ++index) { + _shards.emplace_back(std::make_unique()); + } + + const char* prefix = metric_prefix.c_str(); + _size_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_size", + [](void* index) { return static_cast(index)->size(); }, + this); + _memory_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_memory_bytes", + [](void* index) { + return static_cast(index)->memory_usage(); + }, + this); + _lookup_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_lookup_total"); + _hit_metric = std::make_shared>(prefix, + "inflight_write_buffer_index_hit_total"); + _miss_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_miss_total"); + _insert_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_insert_total"); + _insert_existing_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_insert_existing_total"); + _remove_success_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_remove_if_success_total"); + _remove_failed_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_remove_if_failed_total"); + _stale_epoch_miss_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_stale_epoch_miss_total"); + _stale_epoch_replace_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_stale_epoch_replace_total"); + _rollback_on_backpressure_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_rollback_on_backpressure_total"); +} + +std::shared_ptr InflightWriteBufferIndex::insert_if_absent( + const UInt128Wrapper& cache_hash, size_t block_offset, + std::shared_ptr entry) { + DORIS_CHECK(entry != nullptr); + Key key {.cache_hash = cache_hash, .block_offset = block_offset}; + auto& shard = *_shards[_shard_index(key)]; + bool inserted = false; + { + std::lock_guard lock(shard.mutex); + auto iterator = shard.entries.find(key); + if (iterator == shard.entries.end()) { + shard.entries.emplace(key, std::move(entry)); + _size.fetch_add(1, std::memory_order_relaxed); + inserted = true; + } else if (iterator->second->write_epoch < entry->write_epoch) { + iterator->second = std::move(entry); + *_stale_epoch_replace_metric << 1; + *_insert_metric << 1; + return nullptr; + } else { + *_insert_existing_metric << 1; + return iterator->second; + } + } + + DORIS_CHECK(inserted); + *_insert_metric << 1; + return nullptr; +} + +std::shared_ptr InflightWriteBufferIndex::lookup( + const UInt128Wrapper& cache_hash, size_t block_offset, uint64_t expected_epoch) { + *_lookup_metric << 1; + Key key {.cache_hash = cache_hash, .block_offset = block_offset}; + auto& shard = *_shards[_shard_index(key)]; + { + std::lock_guard lock(shard.mutex); + auto iterator = shard.entries.find(key); + if (iterator == shard.entries.end()) { + *_miss_metric << 1; + return nullptr; + } + if (iterator->second->write_epoch == expected_epoch) { + *_hit_metric << 1; + return iterator->second; + } + *_stale_epoch_miss_metric << 1; + if (iterator->second->write_epoch < expected_epoch) { + shard.entries.erase(iterator); + const size_t old_size = _size.fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_size > 0); + } + } + *_miss_metric << 1; + return nullptr; +} + +std::vector InflightWriteBufferIndex::lookup_all( + const UInt128Wrapper& cache_hash, const std::vector& block_offsets, + uint64_t expected_epoch) { + std::vector results; + results.reserve(block_offsets.size()); + for (size_t block_offset : block_offsets) { + results.emplace_back(LookupResult { + .block_offset = block_offset, + .entry = lookup(cache_hash, block_offset, expected_epoch), + }); + } + return results; +} + +bool InflightWriteBufferIndex::remove_if( + const UInt128Wrapper& cache_hash, size_t block_offset, + const std::shared_ptr& expected) { + DORIS_CHECK(expected != nullptr); + Key key {.cache_hash = cache_hash, .block_offset = block_offset}; + auto& shard = *_shards[_shard_index(key)]; + bool removed = false; + { + std::lock_guard lock(shard.mutex); + auto iterator = shard.entries.find(key); + if (iterator != shard.entries.end() && iterator->second == expected) { + shard.entries.erase(iterator); + const size_t old_size = _size.fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_size > 0); + removed = true; + } + } + if (removed) { + *_remove_success_metric << 1; + return true; + } + *_remove_failed_metric << 1; + return false; +} + +void InflightWriteBufferIndex::for_each_entry( + const std::function& visitor) const { + std::vector> entries; + entries.reserve(size()); + for (const auto& shard : _shards) { + std::lock_guard lock(shard->mutex); + for (const auto& [_, entry] : shard->entries) { + entries.emplace_back(entry); + } + } + for (const auto& entry : entries) { + visitor(*entry); + } +} + +size_t InflightWriteBufferIndex::memory_usage() const { + constexpr size_t entry_overhead = sizeof(Key) + sizeof(InflightWriteBufferEntry) + + sizeof(std::shared_ptr) + 32; + return size() * entry_overhead; +} + +} // namespace doris::io diff --git a/be/src/io/cache/inflight_write_buffer_index.h b/be/src/io/cache/inflight_write_buffer_index.h new file mode 100644 index 00000000000000..1c6e13662791d7 --- /dev/null +++ b/be/src/io/cache/inflight_write_buffer_index.h @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io/cache/async_cache_write_service.h" +#include "io/cache/file_cache_common.h" + +namespace doris::io { + +/// Metadata for one block whose remote payload is waiting for asynchronous persistence. +/// `buffer_offset` and `buffer_size` describe the file interval represented by `buffer`. +struct InflightWriteBufferEntry { + AsyncCacheWriteBufferPtr buffer; + size_t buffer_offset {0}; + size_t buffer_size {0}; + int64_t submit_ts_us {0}; + uint64_t write_epoch {0}; + + InflightWriteBufferEntry(AsyncCacheWriteBufferPtr buffer_, size_t offset_, size_t size_, + int64_t submit_ts_us_, uint64_t write_epoch_) + : buffer(std::move(buffer_)), + buffer_offset(offset_), + buffer_size(size_), + submit_ts_us(submit_ts_us_), + write_epoch(write_epoch_) {} +}; + +/// Sharded index from (cache key, aligned block offset) to an accepted async-write payload. +/// Readers use it before probing disk cache so concurrent misses can reuse remote bytes in memory. +class InflightWriteBufferIndex { +public: + /// A batch-lookup result retains the requested offset even when `entry` is null. + struct LookupResult { + size_t block_offset {0}; + std::shared_ptr entry; + }; + + /// @param shard_count Positive number of independently locked hash shards. + /// @param metric_prefix Prefix that makes per-cache-disk bvar names unique. + explicit InflightWriteBufferIndex(size_t shard_count, std::string metric_prefix = {}); + + /// Claim `(cache_hash, block_offset)` for `entry` if no entry from the same/newer epoch exists. + /// An older entry is atomically replaced so invalidation cannot block the current epoch. + /// @return null when insertion/replacement succeeds; otherwise the existing owner. + std::shared_ptr insert_if_absent( + const UInt128Wrapper& cache_hash, size_t block_offset, + std::shared_ptr entry); + + /// Find the payload only when its epoch equals `expected_epoch`. An older stale entry is erased; + /// a newer entry is preserved for readers already operating in that epoch. + std::shared_ptr lookup(const UInt128Wrapper& cache_hash, + size_t block_offset, uint64_t expected_epoch); + + /// Look up aligned `block_offsets` in input order using the same epoch rule as `lookup`. + std::vector lookup_all(const UInt128Wrapper& cache_hash, + const std::vector& block_offsets, + uint64_t expected_epoch); + + /// Remove the key only if it still points to `expected`, preventing an old task callback from + /// deleting a replacement entry. + /// @return true if this exact owner was removed. + bool remove_if(const UInt128Wrapper& cache_hash, size_t block_offset, + const std::shared_ptr& expected); + + /// Visit a retained snapshot of entries without holding shard locks during callbacks. + void for_each_entry(const std::function& visitor) const; + + /// Return the number of indexed block payloads. + size_t size() const { return _size.load(std::memory_order_relaxed); } + + /// Estimate index metadata usage; shared payload bytes are tracked by the write service. + size_t memory_usage() const; + + /// Record removal of an inserted entry because queue submission hit backpressure. + void record_backpressure_rollback() { *_rollback_on_backpressure_metric << 1; } + +private: + struct Key { + UInt128Wrapper cache_hash; + size_t block_offset {0}; + + bool operator==(const Key& other) const { + return cache_hash == other.cache_hash && block_offset == other.block_offset; + } + }; + + struct KeyHash { + size_t operator()(const Key& key) const { + return doris::io::KeyHash()(key.cache_hash) ^ std::hash()(key.block_offset); + } + }; + + struct Shard { + mutable std::mutex mutex; + std::unordered_map, KeyHash> entries; + }; + + size_t _shard_index(const Key& key) const { return KeyHash()(key) % _shards.size(); } + + std::vector> _shards; + std::atomic _size {0}; + + std::shared_ptr> _size_metric; + std::shared_ptr> _memory_metric; + std::shared_ptr> _lookup_metric; + std::shared_ptr> _hit_metric; + std::shared_ptr> _miss_metric; + std::shared_ptr> _insert_metric; + std::shared_ptr> _insert_existing_metric; + std::shared_ptr> _remove_success_metric; + std::shared_ptr> _remove_failed_metric; + std::shared_ptr> _stale_epoch_miss_metric; + std::shared_ptr> _stale_epoch_replace_metric; + std::shared_ptr> _rollback_on_backpressure_metric; +}; + +} // namespace doris::io diff --git a/be/src/io/fs/file_reader.h b/be/src/io/fs/file_reader.h index ab00a9823520e8..6633841dc45c66 100644 --- a/be/src/io/fs/file_reader.h +++ b/be/src/io/fs/file_reader.h @@ -44,6 +44,18 @@ enum class FileCachePolicy : uint8_t { FILE_BLOCK_CACHE, }; +enum class CacheAlignMode : uint8_t { + ALIGN_TO_BLOCK, + UNALIGNED, +}; + +enum class CacheWriteMode : uint8_t { + DEFAULT, + NO_WRITE, + SYNC_WRITE, + ASYNC_WRITE, +}; + inline FileCachePolicy cache_type_from_string(std::string_view type) { if (type == "file_block_cache") { return FileCachePolicy::FILE_BLOCK_CACHE; @@ -55,6 +67,8 @@ inline FileCachePolicy cache_type_from_string(std::string_view type) { // Only affects remote file readers struct FileReaderOptions { FileCachePolicy cache_type {FileCachePolicy::NO_CACHE}; + CacheAlignMode align_mode {CacheAlignMode::ALIGN_TO_BLOCK}; + CacheWriteMode cache_write_mode {CacheWriteMode::DEFAULT}; bool is_doris_table = false; std::string cache_base_path; // Length of the file in bytes, -1 means unset. diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index f42e486eb40aea..a7c7005708a189 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -19,6 +19,8 @@ #include +#include +#include #include #include @@ -41,6 +43,7 @@ enum class ReaderType : uint8_t { namespace io { class RemoteScanCacheWriteLimiter; +enum class CacheWriteMode : uint8_t; enum class FileCacheMissPolicy : uint8_t { READ_THROUGH_AND_WRITE_BACK = 0, @@ -73,6 +76,17 @@ struct FileCacheStatistics { int64_t lock_wait_timer = 0; int64_t get_timer = 0; int64_t set_timer = 0; + int64_t async_cache_write_submitted = 0; + int64_t async_cache_write_rejected = 0; + int64_t async_cache_write_buffer_alloc_fail = 0; + int64_t async_cache_write_drop_stale_epoch = 0; + int64_t inflight_write_buffer_index_hit = 0; + int64_t inflight_write_buffer_index_miss = 0; + int64_t probe_downloaded_hit = 0; + int64_t probe_downloading_hit = 0; + int64_t probe_miss = 0; + int64_t block_wait_success = 0; + int64_t block_wait_timeout = 0; int64_t inverted_index_num_local_io_total = 0; int64_t inverted_index_num_remote_io_total = 0; @@ -134,6 +148,17 @@ struct FileCacheStatistics { lock_wait_timer += other.lock_wait_timer; get_timer += other.get_timer; set_timer += other.set_timer; + async_cache_write_submitted += other.async_cache_write_submitted; + async_cache_write_rejected += other.async_cache_write_rejected; + async_cache_write_buffer_alloc_fail += other.async_cache_write_buffer_alloc_fail; + async_cache_write_drop_stale_epoch += other.async_cache_write_drop_stale_epoch; + inflight_write_buffer_index_hit += other.inflight_write_buffer_index_hit; + inflight_write_buffer_index_miss += other.inflight_write_buffer_index_miss; + probe_downloaded_hit += other.probe_downloaded_hit; + probe_downloading_hit += other.probe_downloading_hit; + probe_miss += other.probe_miss; + block_wait_success += other.block_wait_success; + block_wait_timeout += other.block_wait_timeout; inverted_index_num_local_io_total += other.inverted_index_num_local_io_total; inverted_index_num_remote_io_total += other.inverted_index_num_remote_io_total; @@ -212,6 +237,9 @@ struct IOContext { int64_t predicate_filtered_rows = 0; // if true, bypass peer read / peer-vs-S3 race and read directly from remote storage bool bypass_peer_read {false}; + // Per-call override for cache write completion semantics. An unset value follows the reader + // option and the global async file-cache write switch. + std::optional cache_write_mode_override = std::nullopt; FileCacheMissPolicy file_cache_miss_policy = FileCacheMissPolicy::READ_THROUGH_AND_WRITE_BACK; RemoteScanCacheWriteLimiter* remote_scan_cache_write_limiter = nullptr; // Ref }; diff --git a/be/src/storage/segment/segment_index_file_cache_loader.cpp b/be/src/storage/segment/segment_index_file_cache_loader.cpp index 13a93460cbc2a3..46a1ffa9525be3 100644 --- a/be/src/storage/segment/segment_index_file_cache_loader.cpp +++ b/be/src/storage/segment/segment_index_file_cache_loader.cpp @@ -150,6 +150,7 @@ Status SegmentIndexFileCacheLoader::load_segment_index_to_file_cache( .is_index_data = true, .is_dryrun = true, .is_warmup = false, + .cache_write_mode_override = io::CacheWriteMode::SYNC_WRITE, }; TEST_SYNC_POINT_RETURN_WITH_VALUE( "SegmentIndexFileCacheLoader::load_segment_index_to_file_cache", Status::OK(), &ctx, diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp new file mode 100644 index 00000000000000..114abe2be25199 --- /dev/null +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -0,0 +1,583 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/async_cache_write_service.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "cpp/sync_point.h" +#include "io/cache/block_file_cache_test_common.h" +#include "util/defer_op.h" +#include "util/time.h" + +namespace doris::io { +namespace { + +FileCacheSettings async_write_cache_settings() { + FileCacheSettings settings; + settings.query_queue_size = 4_mb; + settings.query_queue_elements = 1024; + settings.index_queue_size = 1_mb; + settings.index_queue_elements = 256; + settings.disposable_queue_size = 1_mb; + settings.disposable_queue_elements = 256; + settings.capacity = 8_mb; + settings.max_file_block_size = 4096; + settings.max_query_cache_size = 0; + return settings; +} + +class AsyncCacheWriteServiceTest : public BlockFileCacheTest { +protected: + std::unique_ptr create_cache(const std::string& name) { + auto path = caches_dir / name; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + _paths.emplace_back(path); + auto cache = std::make_unique(path.string(), async_write_cache_settings()); + EXPECT_TRUE(cache->initialize().ok()); + wait_until_cache_ready(*cache); + return cache; + } + + void TearDown() override { + for (const auto& path : _paths) { + std::error_code error; + fs::remove_all(path, error); + } + } + +private: + std::vector _paths; +}; + +TEST_F(AsyncCacheWriteServiceTest, TaskWritesDownloadedBlockAndCleansInflightEntry) { + auto cache = create_cache("async_write_service_single_task"); + auto* service = cache->async_write_service(); + auto* index = cache->inflight_write_buffer_index(); + ASSERT_NE(service, nullptr); + ASSERT_NE(index, nullptr); + + constexpr size_t block_size = 4096; + const auto hash = BlockFileCache::hash("async_single_task"); + const std::string payload(block_size, 'a'); + const int64_t baseline_memory = service->buffer_memory_bytes(); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(block_size, &buffer).ok()); + memcpy(buffer->data(), payload.data(), payload.size()); + EXPECT_GE(service->buffer_memory_bytes(), baseline_memory + static_cast(block_size)); + + const uint64_t epoch = service->current_write_epoch(); + auto entry = std::make_shared(buffer, 0, block_size, + MonotonicMicros(), epoch); + ASSERT_EQ(index->insert_if_absent(hash, 0, entry), nullptr); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = block_size, + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = epoch, + .on_finalized = + [index, hash, entry, &finished](const AsyncCacheWriteTask&) { + index->remove_if(hash, 0, entry); + finished.set_value(); + }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(index->lookup(hash, 0, epoch), nullptr); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + FileBlocks blocks; + bool fully_covered = false; + ASSERT_TRUE(cache->get_downloaded_blocks_if_fully_covered(hash, 0, block_size, context, &blocks, + &fully_covered) + .ok()); + ASSERT_TRUE(fully_covered); + ASSERT_EQ(blocks.size(), 1); + std::string actual(block_size, '\0'); + ASSERT_TRUE(blocks.front()->read(Slice(actual.data(), actual.size()), 0).ok()); + EXPECT_EQ(actual, payload); + + blocks.clear(); + entry.reset(); + buffer.reset(); + for (int attempt = 0; attempt < 100 && service->buffer_memory_bytes() != baseline_memory; + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(service->buffer_memory_bytes(), baseline_memory); +} + +TEST_F(AsyncCacheWriteServiceTest, PendingLimitRejectsWhileWorkerIsActive) { + const int64_t old_max_pending = config::async_file_cache_write_max_pending_tasks_per_disk; + config::async_file_cache_write_max_pending_tasks_per_disk = 1; + Defer restore_config { + [&]() { config::async_file_cache_write_max_pending_tasks_per_disk = old_max_pending; }}; + auto cache = create_cache("async_write_service_backpressure"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + std::mutex mutex; + std::condition_variable cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + AsyncCacheWriteBufferPtr first_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &first_buffer).ok()); + memset(first_buffer->data(), 'b', first_buffer->size()); + std::promise first_finished; + auto first_future = first_finished.get_future(); + AsyncCacheWriteTask first_task { + .cache_hash = BlockFileCache::hash("backpressure_first"), + .file_offset = 0, + .file_size = first_buffer->size(), + .buffer = first_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&first_finished](const AsyncCacheWriteTask&) { first_finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(first_task))); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); + } + + AsyncCacheWriteBufferPtr rejected_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &rejected_buffer).ok()); + AsyncCacheWriteTask rejected_task { + .cache_hash = BlockFileCache::hash("backpressure_rejected"), + .file_offset = 0, + .file_size = rejected_buffer->size(), + .buffer = rejected_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = nullptr, + }; + EXPECT_FALSE(service->try_submit(std::move(rejected_task))); + EXPECT_EQ(service->pending_count(), 1); + EXPECT_GE(service->stats().rejected, 1); + + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + ASSERT_EQ(first_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); +} + +TEST_F(AsyncCacheWriteServiceTest, WatchdogDropsExpiredTaskAndCleansInflightEntry) { + const int64_t old_warn_secs = config::async_file_cache_write_watchdog_warn_secs; + const int64_t old_drop_secs = config::async_file_cache_write_watchdog_drop_secs; + config::async_file_cache_write_watchdog_warn_secs = 0; + config::async_file_cache_write_watchdog_drop_secs = 1; + Defer restore_config {[&]() { + config::async_file_cache_write_watchdog_warn_secs = old_warn_secs; + config::async_file_cache_write_watchdog_drop_secs = old_drop_secs; + }}; + auto cache = create_cache("async_write_service_watchdog"); + auto* service = cache->async_write_service(); + auto* index = cache->inflight_write_buffer_index(); + ASSERT_NE(service, nullptr); + ASSERT_NE(index, nullptr); + + const auto hash = BlockFileCache::hash("watchdog_drop"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), 'w', buffer->size()); + const uint64_t epoch = service->current_write_epoch(); + auto entry = std::make_shared(buffer, 0, buffer->size(), + MonotonicMicros(), epoch); + ASSERT_EQ(index->insert_if_absent(hash, 0, entry), nullptr); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros() - 2 * 1000 * 1000, + .write_epoch = epoch, + .on_finalized = + [index, hash, entry, &finished](const AsyncCacheWriteTask&) { + index->remove_if(hash, 0, entry); + finished.set_value(); + }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(index->lookup(hash, 0, epoch), nullptr); + EXPECT_GE(service->stats().watchdog_timeout, 1); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + auto probe_result = cache->probe(hash, 0, 4096, context); + EXPECT_TRUE(probe_result.holder.file_blocks.empty()); + ASSERT_EQ(probe_result.gaps.size(), 1); + EXPECT_EQ(probe_result.gaps.front().left, 0); + EXPECT_EQ(probe_result.gaps.front().right, 4095); +} + +TEST_F(AsyncCacheWriteServiceTest, ShutdownDrainsAcceptedTask) { + auto cache = create_cache("async_write_service_shutdown_drain"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + const auto hash = BlockFileCache::hash("shutdown_drain"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), 's', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + service->shutdown(); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(0)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + FileBlocks blocks; + bool fully_covered = false; + ASSERT_TRUE(cache->get_downloaded_blocks_if_fully_covered(hash, 0, 4096, context, &blocks, + &fully_covered) + .ok()); + EXPECT_TRUE(fully_covered); + ASSERT_EQ(blocks.size(), 1); + std::string actual(4096, '\0'); + ASSERT_TRUE(blocks.front()->read(Slice(actual.data(), actual.size()), 0).ok()); + EXPECT_EQ(actual, std::string(4096, 's')); +} + +TEST_F(AsyncCacheWriteServiceTest, OneTaskWritesMultipleContainedCells) { + auto cache = create_cache("async_write_service_multiple_cells"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t cell_size = 4096; + constexpr size_t task_size = cell_size * 2; + const auto hash = BlockFileCache::hash("multiple_cells"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(task_size, &buffer).ok()); + memset(buffer->data(), 'm', cell_size); + memset(buffer->data() + cell_size, 'n', cell_size); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = task_size, + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + FileBlocks blocks; + bool fully_covered = false; + ASSERT_TRUE(cache->get_downloaded_blocks_if_fully_covered(hash, 0, task_size, context, &blocks, + &fully_covered) + .ok()); + EXPECT_TRUE(fully_covered); + ASSERT_EQ(blocks.size(), 2); + auto iterator = blocks.begin(); + std::string first(cell_size, '\0'); + ASSERT_TRUE((*iterator)->read(Slice(first.data(), first.size()), 0).ok()); + EXPECT_EQ(first, std::string(cell_size, 'm')); + ++iterator; + std::string second(cell_size, '\0'); + ASSERT_TRUE((*iterator)->read(Slice(second.data(), second.size()), 0).ok()); + EXPECT_EQ(second, std::string(cell_size, 'n')); +} + +TEST_F(AsyncCacheWriteServiceTest, PartialOverlapIsSkippedWithoutOutOfBoundsWrite) { + auto cache = create_cache("async_write_service_partial_overlap"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t existing_size = 4096; + const auto hash = BlockFileCache::hash("partial_overlap"); + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + { + auto holder = cache->get_or_set(hash, 0, existing_size, context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + ASSERT_EQ(block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(existing_size, 'x'); + ASSERT_TRUE(block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(block->finalize().ok()); + } + + constexpr size_t task_offset = 1024; + constexpr size_t task_size = 4096; + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(task_size, &buffer).ok()); + memset(buffer->data(), 'y', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = task_offset, + .file_size = task_size, + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_GE(service->stats().skip_partial_overlap, 1); + + FileBlocks blocks; + bool fully_covered = false; + ASSERT_TRUE(cache->get_downloaded_blocks_if_fully_covered(hash, 0, task_offset + task_size, + context, &blocks, &fully_covered) + .ok()); + EXPECT_TRUE(fully_covered); + ASSERT_EQ(blocks.size(), 2); + auto iterator = blocks.begin(); + std::string existing(existing_size, '\0'); + ASSERT_TRUE((*iterator)->read(Slice(existing.data(), existing.size()), 0).ok()); + EXPECT_EQ(existing, std::string(existing_size, 'x')); + ++iterator; + std::string tail(task_offset, '\0'); + ASSERT_TRUE((*iterator)->read(Slice(tail.data(), tail.size()), 0).ok()); + EXPECT_EQ(tail, std::string(task_offset, 'y')); +} + +TEST_F(AsyncCacheWriteServiceTest, EpochChangeAfterFirstCheckDropsTaskAndCleansEmptyCell) { + auto cache = create_cache("async_write_service_epoch"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + std::mutex mutex; + std::condition_variable cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + const auto hash = BlockFileCache::hash("epoch_drop"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), 'c', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); + } + service->invalidate_pending_writes(); + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_GE(service->stats().drop_stale_epoch, 1); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + auto probe_result = cache->probe(hash, 0, 4096, context); + EXPECT_TRUE(probe_result.holder.file_blocks.empty()); + ASSERT_EQ(probe_result.gaps.size(), 1); + EXPECT_EQ(probe_result.gaps.front().left, 0); + EXPECT_EQ(probe_result.gaps.front().right, 4095); +} + +TEST_F(AsyncCacheWriteServiceTest, ResizeWorkersAtRuntime) { + auto cache = create_cache("async_write_service_resize"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + ASSERT_TRUE(service->resize_workers(3).ok()); + EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 3); + ASSERT_TRUE(service->resize_workers(1).ok()); + EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 1); +} + +TEST_F(AsyncCacheWriteServiceTest, ShutdownWaitsForRegisteredSubmitterAndRejectsItsTask) { + auto cache = create_cache("async_write_service_shutdown"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + std::mutex mutex; + std::condition_variable cv; + bool submitter_registered = false; + bool release_submitter = false; + std::promise shutdown_stopped_accepting; + auto shutdown_stopped_accepting_future = shutdown_stopped_accepting.get_future(); + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard submit_guard; + SyncPoint::CallbackGuard shutdown_guard; + sync_point->set_call_back( + "AsyncCacheWriteService::try_submit:after_register", + [&](auto&&) { + std::unique_lock lock(mutex); + submitter_registered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_submitter; }); + }, + &submit_guard); + sync_point->set_call_back( + "AsyncCacheWriteService::shutdown:after_stop_accepting", + [&](auto&&) { shutdown_stopped_accepting.set_value(); }, &shutdown_guard); + sync_point->enable_processing(); + std::future submit_future; + std::future shutdown_future; + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_submitter = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + AsyncCacheWriteTask task { + .cache_hash = BlockFileCache::hash("shutdown_submitter"), + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = nullptr, + }; + submit_future = std::async(std::launch::async, [service, task = std::move(task)]() mutable { + return service->try_submit(std::move(task)); + }); + { + std::unique_lock lock(mutex); + ASSERT_TRUE( + cv.wait_for(lock, std::chrono::seconds(5), [&]() { return submitter_registered; })); + } + + shutdown_future = std::async(std::launch::async, [service]() { service->shutdown(); }); + ASSERT_EQ(shutdown_stopped_accepting_future.wait_for(std::chrono::seconds(5)), + std::future_status::ready); + EXPECT_EQ(shutdown_future.wait_for(std::chrono::milliseconds(0)), std::future_status::timeout); + { + std::lock_guard lock(mutex); + release_submitter = true; + } + cv.notify_all(); + + EXPECT_FALSE(submit_future.get()); + ASSERT_EQ(shutdown_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); +} + +} // namespace +} // namespace doris::io diff --git a/be/test/io/cache/block_file_cache_probe_test.cpp b/be/test/io/cache/block_file_cache_probe_test.cpp new file mode 100644 index 00000000000000..ba1982bcd72bf4 --- /dev/null +++ b/be/test/io/cache/block_file_cache_probe_test.cpp @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "io/cache/block_file_cache_test_common.h" +#include "util/defer_op.h" + +namespace doris::io { +namespace { + +FileCacheSettings probe_cache_settings() { + FileCacheSettings settings; + settings.query_queue_size = 4_mb; + settings.query_queue_elements = 1024; + settings.index_queue_size = 1_mb; + settings.index_queue_elements = 256; + settings.disposable_queue_size = 1_mb; + settings.disposable_queue_elements = 256; + settings.capacity = 8_mb; + settings.max_file_block_size = 4096; + settings.max_query_cache_size = 0; + return settings; +} + +TEST_F(BlockFileCacheTest, ProbeMissDoesNotCreateCacheCell) { + const auto path = caches_dir / "block_file_cache_probe_miss"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_miss"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + const size_t cache_size_before = cache._cur_cache_size; + + auto result = cache.probe(hash, 1024, 4096, context); + EXPECT_TRUE(result.holder.file_blocks.empty()); + ASSERT_EQ(result.gaps.size(), 1); + EXPECT_EQ(result.gaps.front().left, 1024); + EXPECT_EQ(result.gaps.front().right, 5119); + EXPECT_EQ(cache._cur_cache_size, cache_size_before); + std::lock_guard cache_lock(cache._mutex); + EXPECT_EQ(cache._files.find(hash), cache._files.end()); + } + fs::remove_all(path, error); +} + +TEST_F(BlockFileCacheTest, ProbeReturnsDownloadedBlockAndIndependentGap) { + const auto path = caches_dir / "block_file_cache_probe_mixed"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_mixed"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + { + auto holder = cache.get_or_set(hash, 0, 4096, context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + ASSERT_EQ(block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(4096, 'p'); + ASSERT_TRUE(block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(block->finalize().ok()); + } + const size_t cache_size_before = cache._cur_cache_size; + { + std::lock_guard cache_lock(cache._mutex); + cache._files.at(hash).begin()->second.atime = 123; + } + + auto result = cache.probe(hash, 0, 8192, context); + ASSERT_EQ(result.holder.file_blocks.size(), 1); + EXPECT_EQ(result.holder.file_blocks.front()->state(), FileBlock::State::DOWNLOADED); + EXPECT_EQ(result.holder.file_blocks.front()->range().left, 0); + EXPECT_EQ(result.holder.file_blocks.front()->range().right, 4095); + ASSERT_EQ(result.gaps.size(), 1); + EXPECT_EQ(result.gaps.front().left, 4096); + EXPECT_EQ(result.gaps.front().right, 8191); + EXPECT_EQ(cache._cur_cache_size, cache_size_before); + { + std::lock_guard cache_lock(cache._mutex); + EXPECT_EQ(cache._files.at(hash).begin()->second.atime, 123); + } + } + fs::remove_all(path, error); +} + +TEST_F(BlockFileCacheTest, ProbeObservesDownloadingAndEmptyBlocksWithoutTakingDownloader) { + const auto path = caches_dir / "block_file_cache_probe_states"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_states"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + std::mutex mutex; + std::condition_variable cv; + bool downloader_ready = false; + bool release_downloader = false; + + std::thread downloader([&]() { + ReadStatistics downloader_stats; + CacheContext downloader_context; + downloader_context.stats = &downloader_stats; + auto holder = cache.get_or_set(hash, 0, 8192, downloader_context); + DORIS_CHECK(holder.file_blocks.size() == 2); + DORIS_CHECK(holder.file_blocks.front()->get_or_set_downloader() == + FileBlock::get_caller_id()); + { + std::lock_guard lock(mutex); + downloader_ready = true; + } + cv.notify_all(); + std::unique_lock lock(mutex); + cv.wait(lock, [&]() { return release_downloader; }); + }); + Defer stop_downloader {[&]() { + { + std::lock_guard lock(mutex); + release_downloader = true; + } + cv.notify_all(); + if (downloader.joinable()) { + downloader.join(); + } + }}; + bool ready = false; + { + std::unique_lock lock(mutex); + ready = cv.wait_for(lock, std::chrono::seconds(5), [&]() { return downloader_ready; }); + } + if (!ready) { + FAIL() << "downloader thread did not become ready"; + } + + { + auto result = cache.probe(hash, 0, 8192, context); + ASSERT_EQ(result.holder.file_blocks.size(), 2); + auto iterator = result.holder.file_blocks.begin(); + EXPECT_EQ((*iterator)->state(), FileBlock::State::DOWNLOADING); + ++iterator; + EXPECT_EQ((*iterator)->state(), FileBlock::State::EMPTY); + EXPECT_TRUE(result.gaps.empty()); + } + } + fs::remove_all(path, error); +} + +} // namespace +} // namespace doris::io diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index c2aea1cd825253..bbbea1fe2f6600 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -54,6 +54,17 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.inverted_index_io_timer = multiplier * 28; stats.remote_only_on_miss_triggered = multiplier * 29; stats.remote_only_on_miss_threshold_bytes = multiplier * 30; + stats.async_cache_write_submitted = multiplier * 31; + stats.async_cache_write_rejected = multiplier * 32; + stats.async_cache_write_buffer_alloc_fail = multiplier * 33; + stats.async_cache_write_drop_stale_epoch = multiplier * 34; + stats.inflight_write_buffer_index_hit = multiplier * 35; + stats.inflight_write_buffer_index_miss = multiplier * 36; + stats.probe_downloaded_hit = multiplier * 37; + stats.probe_downloading_hit = multiplier * 38; + stats.probe_miss = multiplier * 39; + stats.block_wait_success = multiplier * 40; + stats.block_wait_timeout = multiplier * 41; return stats; } @@ -94,6 +105,19 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.remote_only_on_miss_triggered, expected.remote_only_on_miss_triggered); EXPECT_EQ(actual.remote_only_on_miss_threshold_bytes, expected.remote_only_on_miss_threshold_bytes); + EXPECT_EQ(actual.async_cache_write_submitted, expected.async_cache_write_submitted); + EXPECT_EQ(actual.async_cache_write_rejected, expected.async_cache_write_rejected); + EXPECT_EQ(actual.async_cache_write_buffer_alloc_fail, + expected.async_cache_write_buffer_alloc_fail); + EXPECT_EQ(actual.async_cache_write_drop_stale_epoch, + expected.async_cache_write_drop_stale_epoch); + EXPECT_EQ(actual.inflight_write_buffer_index_hit, expected.inflight_write_buffer_index_hit); + EXPECT_EQ(actual.inflight_write_buffer_index_miss, expected.inflight_write_buffer_index_miss); + EXPECT_EQ(actual.probe_downloaded_hit, expected.probe_downloaded_hit); + EXPECT_EQ(actual.probe_downloading_hit, expected.probe_downloading_hit); + EXPECT_EQ(actual.probe_miss, expected.probe_miss); + EXPECT_EQ(actual.block_wait_success, expected.block_wait_success); + EXPECT_EQ(actual.block_wait_timeout, expected.block_wait_timeout); } } // namespace @@ -139,6 +163,15 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot EXPECT_EQ(profile->get_counter("CacheGetOrSetTimer")->value(), after_second_report.cache_get_or_set_timer); EXPECT_EQ(profile->get_counter("LockWaitTimer")->value(), after_second_report.lock_wait_timer); + EXPECT_EQ(profile->get_counter("AsyncCacheWriteSubmitted")->value(), + after_second_report.async_cache_write_submitted); + EXPECT_EQ(profile->get_counter("AsyncCacheWriteRejected")->value(), + after_second_report.async_cache_write_rejected); + EXPECT_EQ(profile->get_counter("InflightWriteBufferIndexHit")->value(), + after_second_report.inflight_write_buffer_index_hit); + EXPECT_EQ(profile->get_counter("ProbeMiss")->value(), after_second_report.probe_miss); + EXPECT_EQ(profile->get_counter("BlockWaitTimeout")->value(), + after_second_report.block_wait_timeout); } } // namespace doris diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index 84cce46c9980aa..89b7a99be5d1e0 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -16,8 +16,62 @@ // under the License. #include "block_file_cache_test_common.h" +#include "cloud/config.h" namespace doris::io { +namespace { + +FileCacheSettings async_reader_cache_settings() { + FileCacheSettings settings; + settings.query_queue_size = 8_mb; + settings.query_queue_elements = 16; + settings.index_queue_size = 2_mb; + settings.index_queue_elements = 4; + settings.disposable_queue_size = 2_mb; + settings.disposable_queue_elements = 4; + settings.capacity = 16_mb; + settings.max_file_block_size = 1_mb; + settings.max_query_cache_size = 0; + return settings; +} + +void reset_async_reader_cache_factory() { + FileCacheFactory::instance()->_caches.clear(); + FileCacheFactory::instance()->_path_to_cache.clear(); + FileCacheFactory::instance()->_capacity = 0; +} + +class CountingFileReader final : public FileReader { +public: + explicit CountingFileReader(FileReaderSPtr delegate) : _delegate(std::move(delegate)) {} + + Status close() override { return _delegate->close(); } + const Path& path() const override { return _delegate->path(); } + size_t size() const override { return _delegate->size(); } + bool closed() const override { return _delegate->closed(); } + int64_t mtime() const override { return _delegate->mtime(); } + + size_t read_count() const { return _read_count; } + size_t last_offset() const { return _last_offset; } + size_t last_size() const { return _last_size; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override { + ++_read_count; + _last_offset = offset; + _last_size = result.size; + return _delegate->read_at(offset, result, bytes_read, io_ctx); + } + +private: + FileReaderSPtr _delegate; + size_t _read_count {0}; + size_t _last_offset {0}; + size_t _last_size {0}; +}; + +} // namespace TEST_F(BlockFileCacheTest, direct_partial_hit_with_downloaded_remainder_should_not_read_remote_again) { @@ -122,4 +176,475 @@ TEST_F(BlockFileCacheTest, config::enable_read_cache_file_directly = false; } +TEST_F(BlockFileCacheTest, async_write_reuses_inflight_buffer_then_reads_downloaded_block) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + config::enable_async_file_cache_write = true; + config::enable_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_async_write"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + std::mutex mutex; + std::condition_variable cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(local_reader, opts); + + std::string first_page(4096, '\0'); + FileCacheStatistics first_stats; + IOContext first_ctx; + first_ctx.file_cache_stats = &first_stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(first_page.data(), first_page.size()), &bytes_read, &first_ctx) + .ok()); + EXPECT_EQ(bytes_read, first_page.size()); + EXPECT_EQ(first_page, std::string(first_page.size(), '0')); + EXPECT_EQ(first_stats.async_cache_write_submitted, 1); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); + } + + std::string second_page(4096, '\0'); + FileCacheStatistics second_stats; + IOContext second_ctx; + second_ctx.file_cache_stats = &second_stats; + bytes_read = 0; + ASSERT_TRUE(reader->read_at(4096, Slice(second_page.data(), second_page.size()), &bytes_read, + &second_ctx) + .ok()); + EXPECT_EQ(bytes_read, second_page.size()); + EXPECT_EQ(second_page, std::string(second_page.size(), '0')); + EXPECT_EQ(second_stats.inflight_write_buffer_index_hit, 1); + EXPECT_EQ(second_stats.bytes_read_from_remote, 0); + EXPECT_EQ(second_stats.async_cache_write_submitted, 0); + + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + for (int attempt = 0; attempt < 500 && cache->async_write_service()->pending_count() != 0; + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_EQ(cache->async_write_service()->pending_count(), 0); + for (int attempt = 0; attempt < 500 && cache->inflight_write_buffer_index()->size() != 0; + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_EQ(cache->inflight_write_buffer_index()->size(), 0); + + std::string third_page(4096, '\0'); + FileCacheStatistics third_stats; + IOContext third_ctx; + third_ctx.file_cache_stats = &third_stats; + bytes_read = 0; + ASSERT_TRUE(reader->read_at(8192, Slice(third_page.data(), third_page.size()), &bytes_read, + &third_ctx) + .ok()); + EXPECT_EQ(bytes_read, third_page.size()); + EXPECT_EQ(third_page, std::string(third_page.size(), '0')); + EXPECT_EQ(third_stats.probe_downloaded_hit, 1); + EXPECT_EQ(third_stats.bytes_read_from_remote, 0); +} + +TEST_F(BlockFileCacheTest, async_write_reads_only_middle_span_between_cached_sides) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const int64_t old_block_size = config::file_cache_each_block_size; + config::enable_async_file_cache_write = true; + config::enable_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + config::file_cache_each_block_size = old_block_size; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_middle_span"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(local_reader, opts); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto populate_block = [&](size_t offset, char value) { + auto holder = cache->get_or_set(reader->_cache_hash, offset, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + ASSERT_EQ(block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(1_mb, value); + ASSERT_TRUE(block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(block->finalize().ok()); + }; + populate_block(0, '0'); + populate_block(2_mb, '2'); + + std::string result(3_mb, '\0'); + FileCacheStatistics read_stats; + IOContext context; + context.file_cache_stats = &read_stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result.substr(0, 1_mb), std::string(1_mb, '0')); + EXPECT_EQ(result.substr(1_mb, 1_mb), std::string(1_mb, '1')); + EXPECT_EQ(result.substr(2_mb, 1_mb), std::string(1_mb, '2')); + EXPECT_EQ(read_stats.bytes_read_from_local, 2_mb); + EXPECT_EQ(read_stats.bytes_read_from_remote, 1_mb); + EXPECT_EQ(read_stats.probe_downloaded_hit, 2); + EXPECT_EQ(read_stats.probe_miss, 1); + EXPECT_EQ(read_stats.async_cache_write_submitted, 1); + + for (int attempt = 0; attempt < 5000 && (cache->async_write_service()->pending_count() != 0 || + cache->inflight_write_buffer_index()->size() != 0); + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(cache->async_write_service()->pending_count(), 0); + EXPECT_EQ(cache->inflight_write_buffer_index()->size(), 0); +} + +TEST_F(BlockFileCacheTest, async_write_middle_hit_uses_one_remote_span_and_submits_only_misses) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const int64_t old_block_size = config::file_cache_each_block_size; + config::enable_async_file_cache_write = true; + config::enable_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + config::file_cache_each_block_size = old_block_size; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_middle_hit"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + auto counting_reader = std::make_shared(local_reader); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(counting_reader, opts); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache->get_or_set(reader->_cache_hash, 1_mb, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& middle_block = holder.file_blocks.front(); + ASSERT_EQ(middle_block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(1_mb, '1'); + ASSERT_TRUE(middle_block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(middle_block->finalize().ok()); + + std::string result(3_mb, '\0'); + FileCacheStatistics read_stats; + IOContext context; + context.file_cache_stats = &read_stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result.substr(0, 1_mb), std::string(1_mb, '0')); + EXPECT_EQ(result.substr(1_mb, 1_mb), std::string(1_mb, '1')); + EXPECT_EQ(result.substr(2_mb, 1_mb), std::string(1_mb, '2')); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 0); + EXPECT_EQ(counting_reader->last_size(), 3_mb); + EXPECT_EQ(read_stats.bytes_read_from_local, 0); + EXPECT_EQ(read_stats.bytes_read_from_remote, 3_mb); + EXPECT_EQ(read_stats.probe_downloaded_hit, 1); + EXPECT_EQ(read_stats.probe_miss, 2); + EXPECT_EQ(read_stats.async_cache_write_submitted, 2); + + for (int attempt = 0; attempt < 5000 && (cache->async_write_service()->pending_count() != 0 || + cache->inflight_write_buffer_index()->size() != 0); + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(cache->async_write_service()->pending_count(), 0); + EXPECT_EQ(cache->inflight_write_buffer_index()->size(), 0); +} + +TEST_F(BlockFileCacheTest, async_write_backpressure_rolls_back_inflight_entry) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const int64_t old_block_size = config::file_cache_each_block_size; + const int64_t old_max_pending = config::async_file_cache_write_max_pending_tasks_per_disk; + config::enable_async_file_cache_write = true; + config::enable_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + config::async_file_cache_write_max_pending_tasks_per_disk = 1; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + config::file_cache_each_block_size = old_block_size; + config::async_file_cache_write_max_pending_tasks_per_disk = old_max_pending; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_backpressure"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + std::mutex mutex; + std::condition_variable cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(local_reader, opts); + + std::string first_page(4096, '\0'); + FileCacheStatistics first_stats; + IOContext first_context; + first_context.file_cache_stats = &first_stats; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, Slice(first_page.data(), first_page.size()), &bytes_read, + &first_context) + .ok()); + EXPECT_EQ(first_page, std::string(first_page.size(), '0')); + EXPECT_EQ(first_stats.async_cache_write_submitted, 1); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); + } + + std::string second_page(4096, '\0'); + FileCacheStatistics second_stats; + IOContext second_context; + second_context.file_cache_stats = &second_stats; + bytes_read = 0; + ASSERT_TRUE(reader->read_at(1_mb, Slice(second_page.data(), second_page.size()), &bytes_read, + &second_context) + .ok()); + EXPECT_EQ(second_page, std::string(second_page.size(), '1')); + EXPECT_EQ(second_stats.async_cache_write_rejected, 1); + EXPECT_EQ(second_stats.bytes_read_from_remote, 4096); + EXPECT_EQ(cache->async_write_service()->pending_count(), 1); + EXPECT_EQ( + cache->inflight_write_buffer_index()->lookup( + reader->_cache_hash, 1_mb, cache->async_write_service()->current_write_epoch()), + nullptr); + EXPECT_EQ(cache->inflight_write_buffer_index()->size(), 1); + + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + for (int attempt = 0; attempt < 5000 && (cache->async_write_service()->pending_count() != 0 || + cache->inflight_write_buffer_index()->size() != 0); + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(cache->async_write_service()->pending_count(), 0); + EXPECT_EQ(cache->inflight_write_buffer_index()->size(), 0); +} + +TEST_F(BlockFileCacheTest, cache_write_mode_is_resolved_for_each_read_context) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_peer = config::enable_cache_read_from_peer; + config::enable_cache_read_from_peer = false; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_cache_read_from_peer = old_enable_peer; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_mode_resolution"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(local_reader, opts); + IOContext context; + + config::enable_async_file_cache_write = false; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + config::enable_async_file_cache_write = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::ASYNC_WRITE); + + context.cache_write_mode_override = CacheWriteMode::SYNC_WRITE; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + context.cache_write_mode_override.reset(); + context.is_dryrun = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + context.is_dryrun = false; + context.is_warmup = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + + context.is_warmup = false; + opts.cache_write_mode = CacheWriteMode::SYNC_WRITE; + auto sync_reader = std::make_shared(local_reader, opts); + EXPECT_EQ(sync_reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + context.cache_write_mode_override = CacheWriteMode::ASYNC_WRITE; + EXPECT_EQ(sync_reader->_resolve_cache_write_mode(&context), CacheWriteMode::ASYNC_WRITE); +} + } // namespace doris::io diff --git a/be/test/io/cache/inflight_write_buffer_index_test.cpp b/be/test/io/cache/inflight_write_buffer_index_test.cpp new file mode 100644 index 00000000000000..4445a690632e95 --- /dev/null +++ b/be/test/io/cache/inflight_write_buffer_index_test.cpp @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/inflight_write_buffer_index.h" + +#include + +#include +#include +#include +#include + +#include "io/cache/block_file_cache.h" +#include "util/time.h" + +namespace doris::io { +namespace { + +std::shared_ptr make_entry(uint64_t epoch, size_t offset = 0, + size_t size = 4096) { + return std::make_shared(nullptr, offset, size, MonotonicMicros(), + epoch); +} + +TEST(InflightWriteBufferIndexTest, InsertLookupAndConditionalRemove) { + InflightWriteBufferIndex index(8, "inflight_index_basic_test"); + const auto hash = BlockFileCache::hash("basic"); + auto entry = make_entry(7); + + EXPECT_EQ(index.insert_if_absent(hash, 0, entry), nullptr); + EXPECT_EQ(index.size(), 1); + EXPECT_EQ(index.lookup(hash, 0, 7), entry); + + auto unexpected = make_entry(7); + EXPECT_FALSE(index.remove_if(hash, 0, unexpected)); + EXPECT_EQ(index.lookup(hash, 0, 7), entry); + EXPECT_TRUE(index.remove_if(hash, 0, entry)); + EXPECT_EQ(index.size(), 0); + EXPECT_EQ(index.lookup(hash, 0, 7), nullptr); +} + +TEST(InflightWriteBufferIndexTest, LookupAllSupportsPartialHit) { + InflightWriteBufferIndex index(8, "inflight_index_lookup_all_test"); + const auto hash = BlockFileCache::hash("lookup_all"); + auto first = make_entry(3, 0); + auto third = make_entry(3, 8192); + ASSERT_EQ(index.insert_if_absent(hash, 0, first), nullptr); + ASSERT_EQ(index.insert_if_absent(hash, 8192, third), nullptr); + + auto results = index.lookup_all(hash, {0, 4096, 8192}, 3); + ASSERT_EQ(results.size(), 3); + EXPECT_EQ(results[0].block_offset, 0); + EXPECT_EQ(results[0].entry, first); + EXPECT_EQ(results[1].block_offset, 4096); + EXPECT_EQ(results[1].entry, nullptr); + EXPECT_EQ(results[2].block_offset, 8192); + EXPECT_EQ(results[2].entry, third); +} + +TEST(InflightWriteBufferIndexTest, NewEpochReplacesOldWithoutOldCallbackDeletingNew) { + InflightWriteBufferIndex index(8, "inflight_index_epoch_test"); + const auto hash = BlockFileCache::hash("epoch"); + auto old_entry = make_entry(10); + auto new_entry = make_entry(11); + ASSERT_EQ(index.insert_if_absent(hash, 0, old_entry), nullptr); + + EXPECT_EQ(index.lookup(hash, 0, 11), nullptr); + EXPECT_EQ(index.size(), 0); + ASSERT_EQ(index.insert_if_absent(hash, 0, old_entry), nullptr); + EXPECT_EQ(index.insert_if_absent(hash, 0, new_entry), nullptr); + EXPECT_EQ(index.size(), 1); + EXPECT_FALSE(index.remove_if(hash, 0, old_entry)); + EXPECT_EQ(index.lookup(hash, 0, 11), new_entry); + + auto stale_candidate = make_entry(10); + EXPECT_EQ(index.insert_if_absent(hash, 0, stale_candidate), new_entry); + EXPECT_EQ(index.lookup(hash, 0, 11), new_entry); +} + +TEST(InflightWriteBufferIndexTest, ConcurrentInsertPublishesExactlyOneEntry) { + InflightWriteBufferIndex index(64, "inflight_index_concurrent_test"); + const auto hash = BlockFileCache::hash("concurrent"); + constexpr size_t thread_count = 32; + std::vector> candidates; + candidates.reserve(thread_count); + for (size_t i = 0; i < thread_count; ++i) { + candidates.emplace_back(make_entry(20)); + } + + std::atomic inserted_count {0}; + std::vector threads; + threads.reserve(thread_count); + for (size_t i = 0; i < thread_count; ++i) { + threads.emplace_back([&, i]() { + if (index.insert_if_absent(hash, 0, candidates[i]) == nullptr) { + inserted_count.fetch_add(1, std::memory_order_relaxed); + } + }); + } + for (auto& thread : threads) { + thread.join(); + } + + EXPECT_EQ(inserted_count.load(std::memory_order_relaxed), 1); + EXPECT_EQ(index.size(), 1); + auto published = index.lookup(hash, 0, 20); + ASSERT_NE(published, nullptr); + EXPECT_TRUE(std::find(candidates.begin(), candidates.end(), published) != candidates.end()); +} + +TEST(InflightWriteBufferIndexTest, ConcurrentNewEpochReplacementSurvivesOldRemoval) { + InflightWriteBufferIndex index(16, "inflight_index_concurrent_replace_test"); + const auto hash = BlockFileCache::hash("concurrent_replace"); + auto old_entry = make_entry(30); + auto new_entry = make_entry(31); + ASSERT_EQ(index.insert_if_absent(hash, 0, old_entry), nullptr); + + std::atomic start {false}; + std::vector threads; + threads.emplace_back([&]() { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + EXPECT_EQ(index.insert_if_absent(hash, 0, new_entry), nullptr); + }); + for (size_t thread_index = 0; thread_index < 16; ++thread_index) { + threads.emplace_back([&]() { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + index.remove_if(hash, 0, old_entry); + }); + } + start.store(true, std::memory_order_release); + for (auto& thread : threads) { + thread.join(); + } + + EXPECT_EQ(index.size(), 1); + EXPECT_EQ(index.lookup(hash, 0, 31), new_entry); + EXPECT_FALSE(index.remove_if(hash, 0, old_entry)); + EXPECT_EQ(index.lookup(hash, 0, 31), new_entry); +} + +} // namespace +} // namespace doris::io diff --git a/regression-test/data/cloud_p0/cache/test_async_file_cache_write.out b/regression-test/data/cloud_p0/cache/test_async_file_cache_write.out new file mode 100644 index 00000000000000..d53f5b895d051e --- /dev/null +++ b/regression-test/data/cloud_p0/cache/test_async_file_cache_write.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !async_file_cache_first -- +536854528 16777216 + +-- !async_file_cache_second -- +536854528 16777216 + +-- !async_file_cache_sync_fallback -- +536854528 16777216 diff --git a/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy b/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy new file mode 100644 index 00000000000000..44a215f39b90a2 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_async_file_cache_write", "docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + options.msNum = 1 + options.beConfigs += [ + "enable_file_cache=true", + "enable_async_file_cache_write=true", + "enable_inflight_write_buffer_index=true", + "enable_read_cache_file_directly=false", + "enable_cache_read_from_peer=false", + "disable_storage_page_cache=true", + "disable_segment_cache=true", + "enable_evict_file_cache_in_advance=false", + "file_cache_each_block_size=262144", + "file_cache_enter_disk_resource_limit_mode_percent=99", + "file_cache_path=[{\"path\":\"/opt/apache-doris/be/storage/file_cache\",\"total_size\":134217728,\"query_limit\":134217728}]" + ] + + docker(options) { + def clusters = sql "SHOW CLUSTERS" + assertFalse(clusters.isEmpty()) + sql "use @${clusters[0][0]}" + + def backends = sql "SHOW BACKENDS" + assertEquals(backends.size(), 1) + def beHost = backends[0][1] + def beHttpPort = backends[0][4] + def beBrpcPort = backends[0][5] + + def clearFileCache = { + def (code, out, err) = curl( + "GET", "http://${beHost}:${beHttpPort}/api/file_cache?op=clear&sync=true") + assertTrue(code == 0 && out.contains("OK"), + "clear file cache failed, out=${out}, err=${err}") + } + + def readMetric = { String metricSuffix -> + def metrics = new URL("http://${beHost}:${beBrpcPort}/brpc_metrics").text + long total = 0 + metrics.eachLine { String line -> + if (!line.startsWith("#") && line.contains(metricSuffix)) { + def matcher = line =~ ~".*${metricSuffix}\\s+([0-9]+)\\s*" + if (matcher.matches()) { + total += matcher[0][1] as long + } + } + } + return total + } + + def setBeConfig = { String name, String value -> + def (code, out, err) = curl( + "POST", "http://${beHost}:${beHttpPort}/api/update_config?${name}=${value}") + assertTrue(code == 0 && out.contains("OK"), + "update_config ${name}=${value} failed, out=${out}, err=${err}") + } + + def waitForPendingWrites = { + long deadline = System.currentTimeMillis() + 30000L + while (System.currentTimeMillis() < deadline && + readMetric("async_cache_write_pending_count") != 0L) { + sleep(100) + } + assertEquals(readMetric("async_cache_write_pending_count"), 0L) + } + + sql "DROP TABLE IF EXISTS test_async_file_cache_write_phase1" + sql """ + CREATE TABLE test_async_file_cache_write_phase1 ( + k BIGINT NOT NULL, + v STRING NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ) + """ + sql """ + INSERT INTO test_async_file_cache_write_phase1 + SELECT number, repeat(md5(cast(number AS STRING)), 16) + FROM numbers("number" = "32768") + """ + sql "SYNC" + + clearFileCache() + long submittedBefore = readMetric("async_cache_write_submitted_total") + order_qt_async_file_cache_first """ + SELECT sum(k), sum(length(v)) + FROM test_async_file_cache_write_phase1 + """ + long submittedAfter = readMetric("async_cache_write_submitted_total") + assertTrue(submittedAfter > submittedBefore, + "cold query should submit async cache writes: before=${submittedBefore}, " + + "after=${submittedAfter}") + + waitForPendingWrites() + long probeHitBefore = readMetric("cached_remote_file_reader_probe_hit_downloaded_total") + order_qt_async_file_cache_second """ + SELECT sum(k), sum(length(v)) + FROM test_async_file_cache_write_phase1 + """ + long probeHitAfter = readMetric("cached_remote_file_reader_probe_hit_downloaded_total") + assertTrue(probeHitAfter > probeHitBefore, + "second query should read downloaded cache blocks: before=${probeHitBefore}, " + + "after=${probeHitAfter}") + + setBeConfig("enable_async_file_cache_write", "false") + clearFileCache() + submittedBefore = readMetric("async_cache_write_submitted_total") + order_qt_async_file_cache_sync_fallback """ + SELECT sum(k), sum(length(v)) + FROM test_async_file_cache_write_phase1 + """ + submittedAfter = readMetric("async_cache_write_submitted_total") + assertEquals(submittedAfter, submittedBefore) + } +} From 7ff6668ca9e6cdf8851055c7e0d810370e92b4a4 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 16 Jul 2026 13:54:58 +0800 Subject: [PATCH 02/16] [refactor](be) Simplify asynchronous cache read planning ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The phase-one asynchronous file-cache reader built detailed coverage runs, maintained multiple cursors, and materialized individual holes inside a read even though most read_at requests span only one or two cache blocks. This made the query-side orchestration difficult to review and maintain without providing meaningful value for the common case. Replace that logic with one aligned inflight lookup, one read-only cache probe, and a simple per-block source plan. The reader still gives inflight buffers priority and still distinguishes downloaded, downloading, and missing cache blocks. Downloading blocks outside the remote span retain their wait behavior. When real misses exist, the reader takes the first through last miss as one remote range, intentionally rereads any cache or inflight blocks inside that range, and submits background writes only for the blocks that were actual misses. A cache-side race falls back to one full aligned remote read. This preserves caller-buffer completeness, inflight deduplication, existing-block reads, cache wait semantics, non-blocking write submission, and backpressure rollback while substantially reducing the amount of control flow in CachedRemoteFileReader::_read_async_write_path and its helpers. ### Release note None ### Check List (For Author) - Test: - Unit Test: six targeted BlockFileCacheTest cases passed under ASAN, covering inflight reuse, DOWNLOADING wait, cached sides, one remote middle span, real-miss-only submission, backpressure rollback, and per-read mode selection - Build: ./build.sh --be -j100 passed - Style check: build-support/check-format.sh and git diff --check passed - Behavior changed: No. This refactor preserves the phase-one asynchronous cache-write behavior while simplifying how the read range is assembled. - Does this need documentation: No --- be/src/io/cache/cached_remote_file_reader.h | 103 ++-- .../cached_remote_file_reader_async_write.cpp | 442 ++++++++---------- .../cache/cached_remote_file_reader_test.cpp | 81 ++++ 3 files changed, 329 insertions(+), 297 deletions(-) diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h index ce6c4cc0749ea3..dcf0d91c0ef68e 100644 --- a/be/src/io/cache/cached_remote_file_reader.h +++ b/be/src/io/cache/cached_remote_file_reader.h @@ -118,9 +118,8 @@ class CachedRemoteFileReader final : public FileReader, const IOContext* io_ctx) override; private: - struct AsyncBlockCoverage; + struct AsyncReadBlock; struct AsyncReadPlan; - struct AsyncRemoteRange; enum class FileCacheReadType { DATA, @@ -152,9 +151,9 @@ class CachedRemoteFileReader final : public FileReader, /// @return The effective synchronous or asynchronous cache-write mode for this read. CacheWriteMode _resolve_cache_write_mode(const IOContext* io_ctx) const; - /// Serve a normal read while moving cache-miss writes off the query thread. This method only - /// coordinates the four stages: build coverage, materialize covered sides, read one remote - /// middle span, and submit tasks for blocks still owned by this reader. + /// Serve a normal read while moving cache-miss writes off the query thread. The method uses + /// the first and last remotely covered blocks as one range, matching the synchronous path's + /// preference for a single remote operation over fine-grained hole processing. /// @param[in] offset Original user read offset. /// @param[out] result Destination buffer for the complete user request. /// @param[in] bytes_req Requested user bytes. @@ -169,71 +168,56 @@ class CachedRemoteFileReader final : public FileReader, SourceReadBreakdown& source_read_breakdown, const IOContext* io_ctx); - /// Build block-aligned source coverage for the unread suffix. Inflight buffers are resolved - /// first; only contiguous runs still uncovered are passed to the read-only cache probe. + /// Build block-aligned source coverage for the unread suffix with one inflight batch lookup + /// and one read-only cache probe. Per-block scans intentionally favor simple code because a + /// typical read_at request intersects only one or two cache blocks. /// @param[in] remaining_offset First user byte not filled by the direct-cache path. /// @param[in] remaining_size Number of unread user bytes. /// @param[in] write_epoch Epoch captured before any lookup or remote IO. /// @param[in] io_ctx Context used to build the cache admission/probe context. /// @param[in,out] stats Lookup and probe counters updated during planning. - /// @param[out] plan Coverage plan that owns probe holders until materialization finishes. - /// @return None. - void _build_async_read_plan(size_t remaining_offset, size_t remaining_size, - uint64_t write_epoch, const IOContext* io_ctx, - ReadStatistics& stats, AsyncReadPlan* plan); - - /// Merge one sorted probe result into the corresponding consecutive coverage entries. The - /// merge uses monotonic block/gap cursors and never rescans earlier probe results. - /// @param[in,out] plan Plan whose entries in [begin_index, end_index) are classified. - /// @param[in] begin_index First coverage entry represented by probe_result. - /// @param[in] end_index One-past-last coverage entry represented by probe_result. - /// @param[in] probe_result Existing cache blocks and uncovered gaps for the same range. - /// @param[in,out] stats Per-block probe classifications added here. - /// @return None. - void _classify_async_probe_result(AsyncReadPlan* plan, size_t begin_index, size_t end_index, - const FileBlocksProbeResult& probe_result, - ReadStatistics& stats); - - /// Materialize one boundary coverage entry from an inflight buffer or cache file. A - /// DOWNLOADING entry is waited only because the caller invokes this method for a boundary. - /// @param[in,out] coverage Source entry; failures convert it to MISS. + /// @return Plan that owns the probe holder and first-to-last remote range. + AsyncReadPlan _build_async_read_plan(size_t remaining_offset, size_t remaining_size, + uint64_t write_epoch, const IOContext* io_ctx, + ReadStatistics& stats); + + /// Copy one block already available from an inflight buffer or downloaded cache file. Cache + /// state is revalidated before IO; a race is reported to the caller as a simple full-range + /// remote fallback. + /// @param[in] plan Plan owning the probe holder and user boundaries. + /// @param[in] read_block Aligned block classified as INFLIGHT, CACHE, or DOWNLOADING. /// @param[in] user_offset Original user request offset used to locate the destination slice. /// @param[out] result Destination buffer for the complete user request. - /// @param[in] user_left First user byte handled by the async path. - /// @param[in] user_right Last user byte of the request, inclusive. /// @param[in] cache_context Context used only when a successful local read touches LRU. - /// @param[in,out] stats Wait and local-read timing counters. - /// @param[in,out] source_read_breakdown Local bytes committed by a fully successful entry. - /// @param[in,out] indirect_read_bytes User bytes copied by the indirect path. + /// @param[in,out] stats Local-read timing counters. + /// @param[in,out] materialized_bytes User bytes copied from cache or inflight memory. /// @param[in,out] need_self_heal Set when a cache file disappears during a local read. - /// @return true when the whole aligned entry is available; false when it becomes a MISS. - bool _materialize_async_coverage(AsyncBlockCoverage* coverage, size_t user_offset, Slice result, - size_t user_left, size_t user_right, - const CacheContext& cache_context, ReadStatistics& stats, - SourceReadBreakdown& source_read_breakdown, - size_t* indirect_read_bytes, bool* need_self_heal); - - /// Consume the maximum consecutive available coverage from both ends and leave one middle - /// range bounded by the first and last MISS. - /// @param[in,out] plan Coverage entries may transition from DOWNLOADING to DOWNLOADED or MISS. + /// @return true when the block was copied; false when the caller should use remote fallback. + bool _materialize_async_block(const AsyncReadPlan& plan, const AsyncReadBlock& read_block, + size_t user_offset, Slice result, + const CacheContext& cache_context, ReadStatistics& stats, + size_t* materialized_bytes, bool* need_self_heal); + + /// Copy only blocks before the first and after the last REMOTE block. When no REMOTE block + /// exists, copy the entire request. DOWNLOADING blocks outside the remote span keep the + /// existing wait behavior; blocks inside the span are covered by the same remote read. + /// @param[in] plan Classified block list and remote boundaries. /// @param[in] user_offset Original user request offset. /// @param[out] result Destination buffer for side data. /// @param[in] cache_context Context used for successful local-cache touches. - /// @param[in,out] stats Wait/local and middle-span statistics. + /// @param[in,out] stats Local-read statistics. /// @param[in,out] source_read_breakdown Local bytes copied from the covered sides. /// @param[in,out] indirect_read_bytes User bytes copied from the covered sides. /// @param[in,out] need_self_heal Whether a missing local cache file requires async cleanup. - /// @param[out] remote_range The single middle span when one remains. - /// @return true when remote IO is required; false when both sides cover the request. - bool _materialize_async_covered_sides(AsyncReadPlan* plan, size_t user_offset, Slice result, - const CacheContext& cache_context, ReadStatistics& stats, - SourceReadBreakdown& source_read_breakdown, - size_t* indirect_read_bytes, bool* need_self_heal, - AsyncRemoteRange* remote_range); + /// @return true when all selected cache/inflight blocks were copied; false on a race/read error. + bool _materialize_async_cached_sides(const AsyncReadPlan& plan, size_t user_offset, + Slice result, const CacheContext& cache_context, + ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, bool* need_self_heal); /// Read the planned middle span once and copy only its overlap with the unread user range. /// @param[in] plan Source plan containing user boundaries. - /// @param[in] remote_range Middle span bounded by the first and last MISS. /// @param[in] user_offset Original user request offset. /// @param[out] result Destination buffer for the complete user request. /// @param[in] need_self_heal Whether stale cache metadata should be removed before remote IO. @@ -243,22 +227,21 @@ class CachedRemoteFileReader final : public FileReader, /// @param[in,out] indirect_read_bytes User bytes copied by the indirect path. /// @param[out] remote_buffer Full aligned middle-span payload retained for async tasks. /// @return OK on success; otherwise the remote-read error. - Status _read_async_remote_range(const AsyncReadPlan& plan, const AsyncRemoteRange& remote_range, - size_t user_offset, Slice result, bool need_self_heal, - const IOContext* io_ctx, ReadStatistics& stats, + Status _read_async_remote_range(const AsyncReadPlan& plan, size_t user_offset, Slice result, + bool need_self_heal, const IOContext* io_ctx, + ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, size_t* indirect_read_bytes, std::unique_ptr* remote_buffer); - /// Copy each MISS block from the remote middle buffer into tracked memory and enqueue a + /// Copy each real cache-miss block from the remote span into tracked memory and enqueue a /// per-block write task. A final insert-if-absent prevents duplicate ownership after IO. - /// @param[in] plan Classified coverage and the epoch captured before remote IO. - /// @param[in] remote_range Span represented by remote_buffer. - /// @param[in] remote_buffer Full remote payload for remote_range. + /// @param[in] plan Classified blocks, remote boundaries, and the epoch captured before IO. + /// @param[in] remote_buffer Full payload for the plan's first-to-last remote span. /// @param[in] io_ctx Context converted to the worker's admission context. /// @param[in,out] stats Submission, rejection, allocation, and dedup counters. /// @return None. - void _submit_async_write_tasks(const AsyncReadPlan& plan, const AsyncRemoteRange& remote_range, + void _submit_async_write_tasks(const AsyncReadPlan& plan, const std::unique_ptr& remote_buffer, const IOContext* io_ctx, ReadStatistics& stats); diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp index 5d6cddf4b98896..ea771ed8024f4b 100644 --- a/be/src/io/cache/cached_remote_file_reader_async_write.cpp +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -69,37 +69,42 @@ bvar::Adder g_cached_remote_reader_middle_span_miss_bytes( } // namespace -// One block-aligned ownership decision. Only MISS entries can create a new async write task. -struct CachedRemoteFileReader::AsyncBlockCoverage { +// One aligned cache block in the simplified read plan. REMOTE blocks delimit the single remote +// range; submit_write distinguishes real cache misses from blocks already being downloaded. +struct CachedRemoteFileReader::AsyncReadBlock { enum class Source { INFLIGHT, - DOWNLOADED, + CACHE, DOWNLOADING, - MISS, + REMOTE, }; - explicit AsyncBlockCoverage(FileBlock::Range range_) : range(range_) {} + explicit AsyncReadBlock(FileBlock::Range range_) : range(range_) {} FileBlock::Range range; - Source source {Source::MISS}; + Source source {Source::REMOTE}; + bool submit_write {false}; std::shared_ptr inflight_entry; - std::vector probe_blocks; }; -// Probe holders must outlive copied block references. Declaration order makes coverage destruct -// first, before FileBlocksHolder performs EMPTY/deleting cleanup. +// A read uses one inflight lookup and one read-only cache probe. first_remote_block and +// remote_block_count identify the inclusive first-to-last uncovered span, including any cache +// hits between those boundaries. struct CachedRemoteFileReader::AsyncReadPlan { + AsyncReadPlan(uint64_t write_epoch_, size_t user_left_, size_t user_right_, + FileBlocksProbeResult probe_result_) + : write_epoch(write_epoch_), + user_left(user_left_), + user_right(user_right_), + probe_result(std::move(probe_result_)) {} + uint64_t write_epoch {0}; size_t user_left {0}; size_t user_right {0}; - std::vector probe_results; - std::vector coverage; -}; - -// The one aligned remote GET left after consuming the maximum covered prefix and suffix. -struct CachedRemoteFileReader::AsyncRemoteRange { - size_t left {0}; - size_t size {0}; + FileBlocksProbeResult probe_result; + std::vector blocks; + size_t first_remote_block {0}; + size_t remote_block_count {0}; }; // Resolve mode on every read so online configuration changes affect existing readers. @@ -117,120 +122,73 @@ CacheWriteMode CachedRemoteFileReader::_resolve_cache_write_mode(const IOContext : CacheWriteMode::SYNC_WRITE; } -// Build per-block coverage with the required priority: inflight memory first, then read-only -// probing only for consecutive runs that remain uncovered. -void CachedRemoteFileReader::_build_async_read_plan(size_t remaining_offset, size_t remaining_size, - uint64_t write_epoch, const IOContext* io_ctx, - ReadStatistics& stats, AsyncReadPlan* plan) { - DORIS_CHECK(plan != nullptr); - DORIS_CHECK(plan->probe_results.empty()); - DORIS_CHECK(plan->coverage.empty()); - - plan->write_epoch = write_epoch; - plan->user_left = remaining_offset; - plan->user_right = remaining_offset + remaining_size - 1; - +// Build a deliberately small plan: one aligned block list, one inflight lookup, and one cache +// probe. The per-block scans below favor readability because read_at normally spans very few +// cache blocks. +CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_plan( + size_t remaining_offset, size_t remaining_size, uint64_t write_epoch, + const IOContext* io_ctx, ReadStatistics& stats) { + DORIS_CHECK(remaining_size > 0); const auto [align_left, align_size] = s_align_size(remaining_offset, remaining_size, size()); const size_t cache_block_size = static_cast(config::file_cache_each_block_size); DORIS_CHECK(cache_block_size > 0); + CacheContext cache_context(io_ctx); + cache_context.stats = &stats; + cache_context.tablet_id = _tablet_id; + AsyncReadPlan plan(write_epoch, remaining_offset, remaining_offset + remaining_size - 1, + _cache->probe(_cache_hash, align_left, align_size, cache_context)); + g_cached_remote_reader_probe_total << 1; + std::vector block_offsets; const size_t align_end = align_left + align_size; for (size_t block_offset = align_left; block_offset < align_end; block_offset += cache_block_size) { const size_t block_size = std::min(cache_block_size, size() - block_offset); DORIS_CHECK(block_size > 0); - plan->coverage.emplace_back(FileBlock::Range(block_offset, block_offset + block_size - 1)); + plan.blocks.emplace_back(FileBlock::Range(block_offset, block_offset + block_size - 1)); block_offsets.emplace_back(block_offset); } + DORIS_CHECK(!plan.blocks.empty()); auto* inflight_index = _cache->inflight_write_buffer_index(); DORIS_CHECK(inflight_index != nullptr); + std::vector inflight_results; if (config::enable_inflight_write_buffer_index) { - auto lookup_results = inflight_index->lookup_all(_cache_hash, block_offsets, write_epoch); - DORIS_CHECK(lookup_results.size() == plan->coverage.size()); - for (size_t index = 0; index < lookup_results.size(); ++index) { - auto& lookup_result = lookup_results[index]; - if (!lookup_result.entry) { - ++stats.inflight_write_buffer_index_miss; - continue; - } - - auto& coverage = plan->coverage[index]; - DORIS_CHECK(lookup_result.entry->buffer != nullptr); - DORIS_CHECK(lookup_result.entry->buffer_offset <= coverage.range.left); - DORIS_CHECK(lookup_result.entry->buffer_offset + lookup_result.entry->buffer_size > - coverage.range.right); - coverage.source = AsyncBlockCoverage::Source::INFLIGHT; - coverage.inflight_entry = std::move(lookup_result.entry); - ++stats.inflight_write_buffer_index_hit; - g_cached_remote_reader_inflight_hit << 1; - } - } - - CacheContext cache_context(io_ctx); - cache_context.stats = &stats; - cache_context.tablet_id = _tablet_id; - size_t index = 0; - while (index < plan->coverage.size()) { - while (index < plan->coverage.size() && - plan->coverage[index].source == AsyncBlockCoverage::Source::INFLIGHT) { - ++index; - } - if (index == plan->coverage.size()) { - break; - } - - const size_t begin_index = index; - while (index < plan->coverage.size() && - plan->coverage[index].source != AsyncBlockCoverage::Source::INFLIGHT) { - ++index; - } - const size_t end_index = index; - const size_t probe_left = plan->coverage[begin_index].range.left; - const size_t probe_right = plan->coverage[end_index - 1].range.right; - plan->probe_results.emplace_back(_cache->probe( - _cache_hash, probe_left, probe_right - probe_left + 1, cache_context)); - g_cached_remote_reader_probe_total << 1; - _classify_async_probe_result(plan, begin_index, end_index, plan->probe_results.back(), - stats); + inflight_results = inflight_index->lookup_all(_cache_hash, block_offsets, write_epoch); + DORIS_CHECK(inflight_results.size() == plan.blocks.size()); } -} -// Classify a consecutive probe run with two monotonic cursors. Each cache block/gap is considered -// only while it can overlap the current aligned coverage entry. -void CachedRemoteFileReader::_classify_async_probe_result(AsyncReadPlan* plan, size_t begin_index, - size_t end_index, - const FileBlocksProbeResult& probe_result, - ReadStatistics& stats) { - DORIS_CHECK(plan != nullptr); - DORIS_CHECK(begin_index < end_index); - DORIS_CHECK(end_index <= plan->coverage.size()); - - const auto& cache_blocks = probe_result.holder.file_blocks; - auto cache_block = cache_blocks.begin(); - size_t gap_index = 0; - for (size_t index = begin_index; index < end_index; ++index) { - auto& coverage = plan->coverage[index]; - DORIS_CHECK(coverage.source == AsyncBlockCoverage::Source::MISS); - - while (cache_block != cache_blocks.end() && - (*cache_block)->range().right < coverage.range.left) { - ++cache_block; - } - while (gap_index < probe_result.gaps.size() && - probe_result.gaps[gap_index].right < coverage.range.left) { - ++gap_index; + const auto overlaps = [](const FileBlock::Range& lhs, const FileBlock::Range& rhs) { + return lhs.left <= rhs.right && rhs.left <= lhs.right; + }; + for (size_t index = 0; index < plan.blocks.size(); ++index) { + auto& read_block = plan.blocks[index]; + if (config::enable_inflight_write_buffer_index) { + auto& entry = inflight_results[index].entry; + if (entry) { + DORIS_CHECK(entry->buffer != nullptr); + DORIS_CHECK(entry->buffer_offset <= read_block.range.left); + DORIS_CHECK(entry->buffer_offset + entry->buffer_size > read_block.range.right); + read_block.source = AsyncReadBlock::Source::INFLIGHT; + read_block.inflight_entry = std::move(entry); + ++stats.inflight_write_buffer_index_hit; + g_cached_remote_reader_inflight_hit << 1; + continue; + } + ++stats.inflight_write_buffer_index_miss; } - bool has_miss = gap_index < probe_result.gaps.size() && - probe_result.gaps[gap_index].left <= coverage.range.right; + bool has_miss = std::any_of( + plan.probe_result.gaps.begin(), plan.probe_result.gaps.end(), + [&](const FileBlock::Range& gap) { return overlaps(gap, read_block.range); }); bool has_downloading = false; - auto overlapping_block = cache_block; - while (overlapping_block != cache_blocks.end() && - (*overlapping_block)->range().left <= coverage.range.right) { - const auto& block = *overlapping_block; - coverage.probe_blocks.emplace_back(block); + bool has_cache_block = false; + for (const auto& block : plan.probe_result.holder.file_blocks) { + if (!overlaps(block->range(), read_block.range)) { + continue; + } + has_cache_block = true; if (_cache->is_block_deleting(block)) { has_miss = true; } else { @@ -239,72 +197,75 @@ void CachedRemoteFileReader::_classify_async_probe_result(AsyncReadPlan* plan, s state == FileBlock::State::SKIP_CACHE; has_downloading = has_downloading || state == FileBlock::State::DOWNLOADING; } - ++overlapping_block; } + DORIS_CHECK(has_miss || has_cache_block); if (has_miss) { + read_block.submit_write = true; ++stats.probe_miss; g_cached_remote_reader_probe_miss << 1; - continue; - } - - DORIS_CHECK(!coverage.probe_blocks.empty()); - if (has_downloading) { - coverage.source = AsyncBlockCoverage::Source::DOWNLOADING; + } else if (has_downloading) { + read_block.source = AsyncReadBlock::Source::DOWNLOADING; ++stats.probe_downloading_hit; g_cached_remote_reader_probe_downloading << 1; + continue; } else { - coverage.source = AsyncBlockCoverage::Source::DOWNLOADED; + read_block.source = AsyncReadBlock::Source::CACHE; ++stats.probe_downloaded_hit; g_cached_remote_reader_probe_downloaded << 1; + continue; + } + + if (plan.remote_block_count == 0) { + plan.first_remote_block = index; } + plan.remote_block_count = index - plan.first_remote_block + 1; } + return plan; } -// Materialize a single side entry. The caller deliberately never invokes this for entries already -// enclosed by the middle span, which keeps waits and local reads out of that span. -bool CachedRemoteFileReader::_materialize_async_coverage( - AsyncBlockCoverage* coverage, size_t user_offset, Slice result, size_t user_left, - size_t user_right, const CacheContext& cache_context, ReadStatistics& stats, - SourceReadBreakdown& source_read_breakdown, size_t* indirect_read_bytes, - bool* need_self_heal) { - DORIS_CHECK(coverage != nullptr); - DORIS_CHECK(indirect_read_bytes != nullptr); +// Copy one block available from inflight memory or cache. DOWNLOADING blocks outside the remote +// span retain the existing wait semantics. Any race or read failure asks the caller to replace the +// whole planned request with one remote read instead of incrementally repairing the range. +bool CachedRemoteFileReader::_materialize_async_block( + const AsyncReadPlan& plan, const AsyncReadBlock& read_block, size_t user_offset, + Slice result, const CacheContext& cache_context, ReadStatistics& stats, + size_t* materialized_bytes, bool* need_self_heal) { + DORIS_CHECK(read_block.source != AsyncReadBlock::Source::REMOTE); + DORIS_CHECK(materialized_bytes != nullptr); DORIS_CHECK(need_self_heal != nullptr); - if (coverage->source == AsyncBlockCoverage::Source::MISS) { - return false; - } - const size_t copy_left = std::max(coverage->range.left, user_left); - const size_t copy_right = std::min(coverage->range.right, user_right); - const bool has_user_bytes = copy_left <= copy_right; - if (coverage->source == AsyncBlockCoverage::Source::INFLIGHT) { - if (has_user_bytes) { - const auto& entry = coverage->inflight_entry; - DORIS_CHECK(entry != nullptr); - const size_t entry_offset = copy_left - entry->buffer_offset; - const size_t copy_size = copy_right - copy_left + 1; - DORIS_CHECK(entry_offset + copy_size <= entry->buffer_size); - memcpy(result.data + (copy_left - user_offset), entry->buffer->data() + entry_offset, - copy_size); - *indirect_read_bytes += copy_size; - source_read_breakdown.local_bytes += copy_size; - } + const size_t copy_left = std::max(read_block.range.left, plan.user_left); + const size_t copy_right = std::min(read_block.range.right, plan.user_right); + if (copy_left > copy_right) { + return true; + } + const size_t copy_size = copy_right - copy_left + 1; + + if (read_block.source == AsyncReadBlock::Source::INFLIGHT) { + const auto& entry = read_block.inflight_entry; + DORIS_CHECK(entry != nullptr); + const size_t entry_offset = copy_left - entry->buffer_offset; + DORIS_CHECK(entry_offset + copy_size <= entry->buffer_size); + memcpy(result.data + (copy_left - user_offset), entry->buffer->data() + entry_offset, + copy_size); + *materialized_bytes += copy_size; return true; } - DORIS_CHECK(!coverage->probe_blocks.empty()); size_t local_read_bytes = 0; std::vector consumed_blocks; - for (const auto& block : coverage->probe_blocks) { + for (const auto& block : plan.probe_result.holder.file_blocks) { + if (block->range().right < copy_left || block->range().left > copy_right) { + continue; + } if (_cache->is_block_deleting(block)) { - coverage->source = AsyncBlockCoverage::Source::MISS; return false; } FileBlock::State state = block->state(); if (state == FileBlock::State::DOWNLOADING) { - DORIS_CHECK(coverage->source == AsyncBlockCoverage::Source::DOWNLOADING); + DORIS_CHECK(read_block.source == AsyncReadBlock::Source::DOWNLOADING); { SCOPED_RAW_TIMER(&stats.remote_wait_timer); state = block->wait(); @@ -312,20 +273,14 @@ bool CachedRemoteFileReader::_materialize_async_coverage( if (state != FileBlock::State::DOWNLOADED) { ++stats.block_wait_timeout; g_cached_remote_reader_block_wait_timeout << 1; - coverage->source = AsyncBlockCoverage::Source::MISS; return false; } ++stats.block_wait_success; g_cached_remote_reader_block_wait << 1; } if (state != FileBlock::State::DOWNLOADED) { - coverage->source = AsyncBlockCoverage::Source::MISS; return false; } - if (!has_user_bytes || block->range().right < copy_left || - block->range().left > copy_right) { - continue; - } const size_t read_left = std::max(block->range().left, copy_left); const size_t read_right = std::min(block->range().right, copy_right); @@ -345,77 +300,74 @@ bool CachedRemoteFileReader::_materialize_async_coverage( << "Read probed file cache block failed, falling back to remote. path=" << path().native() << ", hash=" << _cache_hash.to_string() << ", offset=" << block->offset() << ", status=" << status; - coverage->source = AsyncBlockCoverage::Source::MISS; return false; } local_read_bytes += read_size; consumed_blocks.emplace_back(block); } + DORIS_CHECK(local_read_bytes == copy_size); for (const auto& block : consumed_blocks) { _cache->touch_probe_block_if_cached(block, cache_context); } - coverage->source = AsyncBlockCoverage::Source::DOWNLOADED; - *indirect_read_bytes += local_read_bytes; - source_read_breakdown.local_bytes += local_read_bytes; + *materialized_bytes += local_read_bytes; return true; } -// Consume only boundary entries. Once either side reaches a MISS, everything between the two -// boundaries is intentionally covered by one remote GET without further waits or local reads. -bool CachedRemoteFileReader::_materialize_async_covered_sides( - AsyncReadPlan* plan, size_t user_offset, Slice result, const CacheContext& cache_context, - ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, - size_t* indirect_read_bytes, bool* need_self_heal, AsyncRemoteRange* remote_range) { - DORIS_CHECK(plan != nullptr); - DORIS_CHECK(remote_range != nullptr); - - size_t middle_begin = 0; - while (middle_begin < plan->coverage.size() && - _materialize_async_coverage(&plan->coverage[middle_begin], user_offset, result, - plan->user_left, plan->user_right, cache_context, stats, - source_read_breakdown, indirect_read_bytes, - need_self_heal)) { - ++middle_begin; - } +// Read cache/inflight blocks outside the first-to-last remote span. A side failure returns false; +// the caller then performs one remote read for the whole aligned request. +bool CachedRemoteFileReader::_materialize_async_cached_sides( + const AsyncReadPlan& plan, size_t user_offset, Slice result, + const CacheContext& cache_context, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, size_t* indirect_read_bytes, + bool* need_self_heal) { + DORIS_CHECK(indirect_read_bytes != nullptr); + DORIS_CHECK(need_self_heal != nullptr); - size_t middle_end = plan->coverage.size(); - while (middle_end > middle_begin && - _materialize_async_coverage(&plan->coverage[middle_end - 1], user_offset, result, - plan->user_left, plan->user_right, cache_context, stats, - source_read_breakdown, indirect_read_bytes, - need_self_heal)) { - --middle_end; - } - if (middle_begin == middle_end) { + size_t materialized_bytes = 0; + const auto materialize_range = [&](size_t begin, size_t end) { + for (size_t index = begin; index < end; ++index) { + if (!_materialize_async_block(plan, plan.blocks[index], user_offset, result, + cache_context, stats, &materialized_bytes, + need_self_heal)) { + return false; + } + } + return true; + }; + + const size_t prefix_end = + plan.remote_block_count == 0 ? plan.blocks.size() : plan.first_remote_block; + if (!materialize_range(0, prefix_end)) { return false; } - - remote_range->left = plan->coverage[middle_begin].range.left; - remote_range->size = plan->coverage[middle_end - 1].range.right - remote_range->left + 1; - size_t miss_bytes = 0; - for (size_t index = middle_begin; index < middle_end; ++index) { - if (plan->coverage[index].source == AsyncBlockCoverage::Source::MISS) { - miss_bytes += plan->coverage[index].range.size(); + if (plan.remote_block_count != 0) { + const size_t suffix_begin = plan.first_remote_block + plan.remote_block_count; + DORIS_CHECK(suffix_begin <= plan.blocks.size()); + if (!materialize_range(suffix_begin, plan.blocks.size())) { + return false; } } - DORIS_CHECK(miss_bytes > 0); - g_cached_remote_reader_remote_after_dedup_miss << 1; - g_cached_remote_reader_middle_span_read_bytes << remote_range->size; - g_cached_remote_reader_middle_span_miss_bytes << miss_bytes; + + *indirect_read_bytes += materialized_bytes; + source_read_breakdown.local_bytes += materialized_bytes; return true; } -// Execute one middle-span remote read and expose only the requested user overlap. The full aligned -// payload stays alive for per-block task construction in the next stage. +// Read the single aligned range from the first REMOTE block through the last REMOTE block. Cache +// hits inside that span are intentionally reread to keep one straightforward remote operation. Status CachedRemoteFileReader::_read_async_remote_range( - const AsyncReadPlan& plan, const AsyncRemoteRange& remote_range, size_t user_offset, - Slice result, bool need_self_heal, const IOContext* io_ctx, ReadStatistics& stats, - SourceReadBreakdown& source_read_breakdown, size_t* indirect_read_bytes, - std::unique_ptr* remote_buffer) { + const AsyncReadPlan& plan, size_t user_offset, Slice result, bool need_self_heal, + const IOContext* io_ctx, ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, std::unique_ptr* remote_buffer) { DORIS_CHECK(indirect_read_bytes != nullptr); DORIS_CHECK(remote_buffer != nullptr); - DORIS_CHECK(remote_range.size > 0); + DORIS_CHECK(plan.remote_block_count > 0); + const size_t remote_end = plan.first_remote_block + plan.remote_block_count; + DORIS_CHECK(remote_end <= plan.blocks.size()); + const size_t remote_left = plan.blocks[plan.first_remote_block].range.left; + const size_t remote_right = plan.blocks[remote_end - 1].range.right; + const size_t remote_size = remote_right - remote_left + 1; stats.hit_cache = false; stats.from_peer_cache = false; @@ -424,27 +376,37 @@ Status CachedRemoteFileReader::_read_async_remote_range( } const std::vector no_peer_blocks; - RETURN_IF_ERROR(_execute_remote_read(no_peer_blocks, remote_range.left, remote_range.size, - *remote_buffer, nullptr, stats, io_ctx)); + RETURN_IF_ERROR(_execute_remote_read(no_peer_blocks, remote_left, remote_size, *remote_buffer, + nullptr, stats, io_ctx)); DORIS_CHECK(*remote_buffer != nullptr); - const size_t remote_right = remote_range.left + remote_range.size - 1; - const size_t copy_left = std::max(remote_range.left, plan.user_left); + const size_t copy_left = std::max(remote_left, plan.user_left); const size_t copy_right = std::min(remote_right, plan.user_right); if (copy_left <= copy_right) { const size_t copy_size = copy_right - copy_left + 1; memcpy(result.data + (copy_left - user_offset), - remote_buffer->get() + (copy_left - remote_range.left), copy_size); + remote_buffer->get() + (copy_left - remote_left), copy_size); *indirect_read_bytes += copy_size; source_read_breakdown.remote_bytes += copy_size; } + + size_t miss_bytes = 0; + for (size_t index = plan.first_remote_block; index < remote_end; ++index) { + if (plan.blocks[index].submit_write) { + miss_bytes += plan.blocks[index].range.size(); + } + } + if (miss_bytes > 0) { + g_cached_remote_reader_remote_after_dedup_miss << 1; + } + g_cached_remote_reader_middle_span_read_bytes << remote_size; + g_cached_remote_reader_middle_span_miss_bytes << miss_bytes; return Status::OK(); } -// Submit exactly the MISS entries contained in the remote span. Inflight insertion happens after +// Submit exactly the real cache misses contained in the remote span. Inflight insertion happens after // remote IO and immediately before enqueueing, so a concurrent owner wins without duplicate work. void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan, - const AsyncRemoteRange& remote_range, const std::unique_ptr& remote_buffer, const IOContext* io_ctx, ReadStatistics& stats) { @@ -462,31 +424,34 @@ void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan .tablet_id = _tablet_id, .is_warmup = cache_context.is_warmup, }; - const size_t remote_right = remote_range.left + remote_range.size - 1; - for (const auto& coverage : plan.coverage) { - if (coverage.source != AsyncBlockCoverage::Source::MISS || - coverage.range.left < remote_range.left || coverage.range.right > remote_right) { + DORIS_CHECK(plan.remote_block_count > 0); + const size_t remote_end = plan.first_remote_block + plan.remote_block_count; + DORIS_CHECK(remote_end <= plan.blocks.size()); + const size_t remote_left = plan.blocks[plan.first_remote_block].range.left; + for (size_t index = plan.first_remote_block; index < remote_end; ++index) { + const auto& read_block = plan.blocks[index]; + if (!read_block.submit_write) { continue; } + DORIS_CHECK(read_block.source == AsyncReadBlock::Source::REMOTE); if (!service->is_current_write_epoch(plan.write_epoch)) { ++stats.async_cache_write_drop_stale_epoch; continue; } AsyncCacheWriteBufferPtr tracked_buffer; - Status status = service->allocate_tracked_buffer(coverage.range.size(), &tracked_buffer); + Status status = service->allocate_tracked_buffer(read_block.range.size(), &tracked_buffer); if (!status.ok()) { ++stats.async_cache_write_buffer_alloc_fail; continue; } - memcpy(tracked_buffer->data(), - remote_buffer.get() + (coverage.range.left - remote_range.left), - coverage.range.size()); + memcpy(tracked_buffer->data(), remote_buffer.get() + (read_block.range.left - remote_left), + read_block.range.size()); AsyncCacheWriteTask task { .cache_hash = _cache_hash, - .file_offset = coverage.range.left, - .file_size = coverage.range.size(), + .file_offset = read_block.range.left, + .file_size = read_block.range.size(), .buffer = tracked_buffer, .admission_ctx = admission_context, .submit_ts_us = MonotonicMicros(), @@ -495,16 +460,16 @@ void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan }; std::shared_ptr entry; if (config::enable_inflight_write_buffer_index) { - entry = std::make_shared(tracked_buffer, coverage.range.left, - coverage.range.size(), - task.submit_ts_us, plan.write_epoch); + entry = std::make_shared( + tracked_buffer, read_block.range.left, read_block.range.size(), + task.submit_ts_us, plan.write_epoch); auto existing = - inflight_index->insert_if_absent(_cache_hash, coverage.range.left, entry); + inflight_index->insert_if_absent(_cache_hash, read_block.range.left, entry); if (existing) { g_cached_remote_reader_async_skip_existing << 1; continue; } - task.on_finalized = [cache_hash = _cache_hash, offset = coverage.range.left, + task.on_finalized = [cache_hash = _cache_hash, offset = read_block.range.left, inflight_index, entry](const AsyncCacheWriteTask&) { inflight_index->remove_if(cache_hash, offset, entry); }; @@ -512,7 +477,7 @@ void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan if (!service->try_submit(std::move(task))) { if (entry) { - inflight_index->remove_if(_cache_hash, coverage.range.left, entry); + inflight_index->remove_if(_cache_hash, read_block.range.left, entry); inflight_index->record_backpressure_rollback(); } ++stats.async_cache_write_rejected; @@ -522,7 +487,8 @@ void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan } } -// Orchestrate the async-write path while keeping ownership decisions inside the stage helpers. +// Orchestrate one simple plan: read cache/inflight blocks outside the remote span, issue at most +// one remote read, then enqueue only blocks that were actual cache misses. Status CachedRemoteFileReader::_read_async_write_path(size_t offset, Slice result, size_t bytes_req, size_t already_read, size_t* bytes_read, ReadStatistics& stats, @@ -539,24 +505,26 @@ Status CachedRemoteFileReader::_read_async_write_path(size_t offset, Slice resul auto* service = _cache->async_write_service(); DORIS_CHECK(service != nullptr); - AsyncReadPlan plan; - _build_async_read_plan(offset + already_read, bytes_req - already_read, - service->current_write_epoch(), io_ctx, stats, &plan); + auto plan = _build_async_read_plan(offset + already_read, bytes_req - already_read, + service->current_write_epoch(), io_ctx, stats); CacheContext cache_context(io_ctx); cache_context.stats = &stats; cache_context.tablet_id = _tablet_id; size_t indirect_read_bytes = 0; bool need_self_heal = false; - AsyncRemoteRange remote_range; - if (_materialize_async_covered_sides(&plan, offset, result, cache_context, stats, + if (!_materialize_async_cached_sides(plan, offset, result, cache_context, stats, source_read_breakdown, &indirect_read_bytes, - &need_self_heal, &remote_range)) { + &need_self_heal)) { + plan.first_remote_block = 0; + plan.remote_block_count = plan.blocks.size(); + } + if (plan.remote_block_count > 0) { std::unique_ptr remote_buffer; - RETURN_IF_ERROR(_read_async_remote_range(plan, remote_range, offset, result, need_self_heal, - io_ctx, stats, source_read_breakdown, - &indirect_read_bytes, &remote_buffer)); - _submit_async_write_tasks(plan, remote_range, remote_buffer, io_ctx, stats); + RETURN_IF_ERROR(_read_async_remote_range(plan, offset, result, need_self_heal, io_ctx, + stats, source_read_breakdown, &indirect_read_bytes, + &remote_buffer)); + _submit_async_write_tasks(plan, remote_buffer, io_ctx, stats); } *bytes_read = bytes_req; diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index 89b7a99be5d1e0..03e30d762c5a32 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include + #include "block_file_cache_test_common.h" #include "cloud/config.h" @@ -303,6 +305,85 @@ TEST_F(BlockFileCacheTest, async_write_reuses_inflight_buffer_then_reads_downloa EXPECT_EQ(third_stats.bytes_read_from_remote, 0); } +TEST_F(BlockFileCacheTest, async_write_waits_for_downloading_block_outside_remote_span) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const int64_t old_block_size = config::file_cache_each_block_size; + config::enable_async_file_cache_write = true; + config::enable_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + config::file_cache_each_block_size = old_block_size; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_wait_downloading"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + auto counting_reader = std::make_shared(local_reader); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(counting_reader, opts); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache->get_or_set(reader->_cache_hash, 0, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& downloading_block = holder.file_blocks.front(); + ASSERT_EQ(downloading_block->get_or_set_downloader(), FileBlock::get_caller_id()); + + std::string result(4096, '\0'); + FileCacheStatistics read_stats; + IOContext context; + context.file_cache_stats = &read_stats; + size_t bytes_read = 0; + auto read_future = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context); + }); + EXPECT_EQ(read_future.wait_for(std::chrono::milliseconds(100)), std::future_status::timeout); + + const std::string payload(1_mb, '0'); + ASSERT_TRUE(downloading_block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(downloading_block->finalize().ok()); + ASSERT_TRUE(read_future.get().ok()); + + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result, std::string(result.size(), '0')); + EXPECT_EQ(counting_reader->read_count(), 0); + EXPECT_EQ(read_stats.bytes_read_from_remote, 0); + EXPECT_EQ(read_stats.probe_downloading_hit, 1); + EXPECT_EQ(read_stats.block_wait_success, 1); + EXPECT_EQ(read_stats.block_wait_timeout, 0); + EXPECT_EQ(read_stats.async_cache_write_submitted, 0); +} + TEST_F(BlockFileCacheTest, async_write_reads_only_middle_span_between_cached_sides) { const bool old_enable_async = config::enable_async_file_cache_write; const bool old_enable_inflight = config::enable_inflight_write_buffer_index; From fd8c503be09580cead8eabc28c7d2ef5abdd9b6e Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 16 Jul 2026 14:35:24 +0800 Subject: [PATCH 03/16] [improvement](be) Avoid cache probe on full inflight hit ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The asynchronous cache read planner called BlockFileCache::probe before consulting the inflight write-buffer index. BlockFileCache::probe acquires the cache mutex, so a request already covered entirely by inflight buffers still contended on BlockFileCache even though it needed no cache metadata. Build the aligned block list and perform the batch inflight lookup first. If every requested block is covered for the current write epoch, return the plan immediately and materialize the caller buffer directly from inflight memory. If any block is not covered, retain the existing mixed-source behavior by issuing one whole-range read-only cache probe and classifying only the non-inflight blocks as downloaded, downloading, or remote misses. Make the probe result optional in the read plan so ownership matches the conditional probe. Extend the inflight reuse unit test to hold the BlockFileCache mutex during the second read; the read must still complete, directly proving that the full-inflight fast path does not enter BlockFileCache::probe. ### Release note None ### Check List (For Author) - Test: - Unit Test: six targeted BlockFileCacheTest cases passed under ASAN, including full inflight coverage while the BlockFileCache mutex is held, partial cache coverage, downloading waits, middle-span reads, backpressure rollback, and per-read mode selection - Build: ./build.sh --be -j100 passed - Style check: build-support/check-format.sh and git diff --check passed - Behavior changed: Yes. Reads fully covered by current-epoch inflight buffers no longer call BlockFileCache::probe or acquire its cache mutex; partial inflight coverage still probes and combines existing cache blocks. - Does this need documentation: No --- be/src/io/cache/cached_remote_file_reader.h | 8 +- .../cached_remote_file_reader_async_write.cpp | 91 +++++++++++-------- .../cache/cached_remote_file_reader_test.cpp | 16 +++- 3 files changed, 68 insertions(+), 47 deletions(-) diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h index dcf0d91c0ef68e..364bad55ec9653 100644 --- a/be/src/io/cache/cached_remote_file_reader.h +++ b/be/src/io/cache/cached_remote_file_reader.h @@ -168,15 +168,15 @@ class CachedRemoteFileReader final : public FileReader, SourceReadBreakdown& source_read_breakdown, const IOContext* io_ctx); - /// Build block-aligned source coverage for the unread suffix with one inflight batch lookup - /// and one read-only cache probe. Per-block scans intentionally favor simple code because a - /// typical read_at request intersects only one or two cache blocks. + /// Build block-aligned source coverage for the unread suffix. It first performs one inflight + /// batch lookup; a fully covered request returns without taking the BlockFileCache lock, while + /// incomplete coverage triggers one read-only cache probe for the whole aligned range. /// @param[in] remaining_offset First user byte not filled by the direct-cache path. /// @param[in] remaining_size Number of unread user bytes. /// @param[in] write_epoch Epoch captured before any lookup or remote IO. /// @param[in] io_ctx Context used to build the cache admission/probe context. /// @param[in,out] stats Lookup and probe counters updated during planning. - /// @return Plan that owns the probe holder and first-to-last remote range. + /// @return Plan that owns any required probe holder and first-to-last remote range. AsyncReadPlan _build_async_read_plan(size_t remaining_offset, size_t remaining_size, uint64_t write_epoch, const IOContext* io_ctx, ReadStatistics& stats); diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp index ea771ed8024f4b..ef2a8e56c7b6bf 100644 --- a/be/src/io/cache/cached_remote_file_reader_async_write.cpp +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -87,21 +88,17 @@ struct CachedRemoteFileReader::AsyncReadBlock { std::shared_ptr inflight_entry; }; -// A read uses one inflight lookup and one read-only cache probe. first_remote_block and -// remote_block_count identify the inclusive first-to-last uncovered span, including any cache -// hits between those boundaries. +// A read first checks inflight buffers and owns a cache probe result only when that lookup leaves +// uncovered blocks. first_remote_block and remote_block_count identify the inclusive first-to-last +// uncovered span, including any cache hits between those boundaries. struct CachedRemoteFileReader::AsyncReadPlan { - AsyncReadPlan(uint64_t write_epoch_, size_t user_left_, size_t user_right_, - FileBlocksProbeResult probe_result_) - : write_epoch(write_epoch_), - user_left(user_left_), - user_right(user_right_), - probe_result(std::move(probe_result_)) {} + AsyncReadPlan(uint64_t write_epoch_, size_t user_left_, size_t user_right_) + : write_epoch(write_epoch_), user_left(user_left_), user_right(user_right_) {} uint64_t write_epoch {0}; size_t user_left {0}; size_t user_right {0}; - FileBlocksProbeResult probe_result; + std::optional probe_result; std::vector blocks; size_t first_remote_block {0}; size_t remote_block_count {0}; @@ -122,9 +119,10 @@ CacheWriteMode CachedRemoteFileReader::_resolve_cache_write_mode(const IOContext : CacheWriteMode::SYNC_WRITE; } -// Build a deliberately small plan: one aligned block list, one inflight lookup, and one cache -// probe. The per-block scans below favor readability because read_at normally spans very few -// cache blocks. +// Build a deliberately small plan. The inflight index is checked first so a fully covered read +// avoids BlockFileCache::probe and its cache mutex. Only an incomplete inflight lookup performs one +// whole-range cache probe; the per-block scans favor readability because read_at normally spans +// very few cache blocks. CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_plan( size_t remaining_offset, size_t remaining_size, uint64_t write_epoch, const IOContext* io_ctx, ReadStatistics& stats) { @@ -133,12 +131,7 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ const size_t cache_block_size = static_cast(config::file_cache_each_block_size); DORIS_CHECK(cache_block_size > 0); - CacheContext cache_context(io_ctx); - cache_context.stats = &stats; - cache_context.tablet_id = _tablet_id; - AsyncReadPlan plan(write_epoch, remaining_offset, remaining_offset + remaining_size - 1, - _cache->probe(_cache_hash, align_left, align_size, cache_context)); - g_cached_remote_reader_probe_total << 1; + AsyncReadPlan plan(write_epoch, remaining_offset, remaining_offset + remaining_size - 1); std::vector block_offsets; const size_t align_end = align_left + align_size; @@ -151,40 +144,57 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ } DORIS_CHECK(!plan.blocks.empty()); - auto* inflight_index = _cache->inflight_write_buffer_index(); - DORIS_CHECK(inflight_index != nullptr); - std::vector inflight_results; - if (config::enable_inflight_write_buffer_index) { - inflight_results = inflight_index->lookup_all(_cache_hash, block_offsets, write_epoch); + const bool inflight_index_enabled = config::enable_inflight_write_buffer_index; + bool all_blocks_inflight = inflight_index_enabled; + if (inflight_index_enabled) { + auto* inflight_index = _cache->inflight_write_buffer_index(); + DORIS_CHECK(inflight_index != nullptr); + auto inflight_results = inflight_index->lookup_all(_cache_hash, block_offsets, write_epoch); DORIS_CHECK(inflight_results.size() == plan.blocks.size()); + for (size_t index = 0; index < plan.blocks.size(); ++index) { + auto& entry = inflight_results[index].entry; + if (!entry) { + all_blocks_inflight = false; + ++stats.inflight_write_buffer_index_miss; + continue; + } + + auto& read_block = plan.blocks[index]; + DORIS_CHECK(entry->buffer != nullptr); + DORIS_CHECK(entry->buffer_offset <= read_block.range.left); + DORIS_CHECK(entry->buffer_offset + entry->buffer_size > read_block.range.right); + read_block.source = AsyncReadBlock::Source::INFLIGHT; + read_block.inflight_entry = std::move(entry); + ++stats.inflight_write_buffer_index_hit; + g_cached_remote_reader_inflight_hit << 1; + } + } + if (all_blocks_inflight) { + return plan; } + CacheContext cache_context(io_ctx); + cache_context.stats = &stats; + cache_context.tablet_id = _tablet_id; + plan.probe_result.emplace(_cache->probe(_cache_hash, align_left, align_size, cache_context)); + g_cached_remote_reader_probe_total << 1; + const auto& probe_result = *plan.probe_result; + const auto overlaps = [](const FileBlock::Range& lhs, const FileBlock::Range& rhs) { return lhs.left <= rhs.right && rhs.left <= lhs.right; }; for (size_t index = 0; index < plan.blocks.size(); ++index) { auto& read_block = plan.blocks[index]; - if (config::enable_inflight_write_buffer_index) { - auto& entry = inflight_results[index].entry; - if (entry) { - DORIS_CHECK(entry->buffer != nullptr); - DORIS_CHECK(entry->buffer_offset <= read_block.range.left); - DORIS_CHECK(entry->buffer_offset + entry->buffer_size > read_block.range.right); - read_block.source = AsyncReadBlock::Source::INFLIGHT; - read_block.inflight_entry = std::move(entry); - ++stats.inflight_write_buffer_index_hit; - g_cached_remote_reader_inflight_hit << 1; - continue; - } - ++stats.inflight_write_buffer_index_miss; + if (read_block.source == AsyncReadBlock::Source::INFLIGHT) { + continue; } bool has_miss = std::any_of( - plan.probe_result.gaps.begin(), plan.probe_result.gaps.end(), + probe_result.gaps.begin(), probe_result.gaps.end(), [&](const FileBlock::Range& gap) { return overlaps(gap, read_block.range); }); bool has_downloading = false; bool has_cache_block = false; - for (const auto& block : plan.probe_result.holder.file_blocks) { + for (const auto& block : probe_result.holder.file_blocks) { if (!overlaps(block->range(), read_block.range)) { continue; } @@ -253,9 +263,10 @@ bool CachedRemoteFileReader::_materialize_async_block( return true; } + DORIS_CHECK(plan.probe_result.has_value()); size_t local_read_bytes = 0; std::vector consumed_blocks; - for (const auto& block : plan.probe_result.holder.file_blocks) { + for (const auto& block : plan.probe_result->holder.file_blocks) { if (block->range().right < copy_left || block->range().left > copy_right) { continue; } diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index 03e30d762c5a32..313fa55fe879ff 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -266,9 +266,19 @@ TEST_F(BlockFileCacheTest, async_write_reuses_inflight_buffer_then_reads_downloa IOContext second_ctx; second_ctx.file_cache_stats = &second_stats; bytes_read = 0; - ASSERT_TRUE(reader->read_at(4096, Slice(second_page.data(), second_page.size()), &bytes_read, - &second_ctx) - .ok()); + std::future second_read; + { + // Full inflight coverage must complete while the BlockFileCache mutex is unavailable, + // proving that this fast path does not call BlockFileCache::probe. + std::lock_guard cache_lock(cache->_mutex); + second_read = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return reader->read_at(4096, Slice(second_page.data(), second_page.size()), &bytes_read, + &second_ctx); + }); + EXPECT_EQ(second_read.wait_for(std::chrono::seconds(5)), std::future_status::ready); + } + ASSERT_TRUE(second_read.get().ok()); EXPECT_EQ(bytes_read, second_page.size()); EXPECT_EQ(second_page, std::string(second_page.size(), '0')); EXPECT_EQ(second_stats.inflight_write_buffer_index_hit, 1); From 1be6a08430b21aa9eb44e1da2c1ba64451450b12 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 16 Jul 2026 16:31:40 +0800 Subject: [PATCH 04/16] [refactor](be) Decouple async cache writes from global config ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: AsyncCacheWriteService previously used a follow_global_config flag to switch between fixed test options and direct reads of mutable BE configuration. That made queue admission, batching, and watchdog behavior depend on global state that was not visible in the service interface. It also split online updates across two mechanisms: worker-count changes were forwarded by FileCacheFactory, while the remaining settings were read implicitly from worker and submission paths. Make configuration ownership explicit. A newly initialized BlockFileCache constructs a complete per-disk options snapshot, and FileCacheFactory registers update callbacks for all five mutable async-write settings. Each callback captures one complete configuration snapshot and forwards it through FileCacheFactory::update_async_write_options to AsyncCacheWriteService::update_options. The service validates the snapshot, applies the requested worker count, and atomically publishes immutable queue, batch, and watchdog settings. Submission and worker paths now consume service-owned snapshots and no longer include or reference common/config.h. Update unit tests to configure isolated services through the explicit interface, and add coverage proving that config::set_config propagates every mutable setting through the factory into an initialized per-disk service. ### Release note None ### Check List (For Author) - Test: - Unit Test: `./run-be-ut.sh --run --filter=AsyncCacheWriteServiceTest.*:BlockFileCacheTest.async_write_backpressure_rolls_back_inflight_entry -j 100` passed all 11 tests under ASAN - Build: `./build.sh --be -j100` passed - Style check: `build-support/check-format.sh` and `git diff --check` passed - Behavior changed: No. Online mutable settings keep their existing behavior but are propagated through explicit update interfaces. - Does this need documentation: No --- be/src/io/cache/async_cache_write_service.cpp | 94 +++++++-------- be/src/io/cache/async_cache_write_service.h | 27 +++-- be/src/io/cache/block_file_cache.cpp | 14 ++- be/src/io/cache/block_file_cache_factory.cpp | 54 ++++++++- be/src/io/cache/block_file_cache_factory.h | 6 +- .../cache/async_cache_write_service_test.cpp | 112 +++++++++++++++--- .../cache/cached_remote_file_reader_test.cpp | 6 +- 7 files changed, 223 insertions(+), 90 deletions(-) diff --git a/be/src/io/cache/async_cache_write_service.cpp b/be/src/io/cache/async_cache_write_service.cpp index 4afaac844798d2..1930f1822fd41f 100644 --- a/be/src/io/cache/async_cache_write_service.cpp +++ b/be/src/io/cache/async_cache_write_service.cpp @@ -22,7 +22,6 @@ #include #include -#include "common/config.h" #include "common/logging.h" #include "core/allocator.h" #include "cpp/sync_point.h" @@ -48,26 +47,17 @@ AsyncCacheWriteBuffer::~AsyncCacheWriteBuffer() { allocator.free(_data, _size); } -AsyncCacheWriteServiceOptions AsyncCacheWriteServiceOptions::from_config() { - return AsyncCacheWriteServiceOptions { - .worker_count = static_cast(config::async_file_cache_write_workers_per_disk), - .max_pending_tasks = - static_cast(config::async_file_cache_write_max_pending_tasks_per_disk), - .batch_size = static_cast(config::async_file_cache_write_batch_size), - .watchdog_warn_secs = config::async_file_cache_write_watchdog_warn_secs, - .watchdog_drop_secs = config::async_file_cache_write_watchdog_drop_secs, - .follow_global_config = true, - }; -} - AsyncCacheWriteService::AsyncCacheWriteService(BlockFileCache* cache, AsyncCacheWriteServiceOptions options) - : _cache(cache), _options(std::move(options)) { + : _cache(cache), + _options(std::make_shared(options)), + _desired_worker_count(options.worker_count) { DORIS_CHECK(_cache != nullptr); - DORIS_CHECK(_options.worker_count > 0); - DORIS_CHECK(_options.max_pending_tasks > 0); - DORIS_CHECK(_options.batch_size > 0); - DORIS_CHECK(_options.watchdog_drop_secs > _options.watchdog_warn_secs); + DORIS_CHECK(options.worker_count > 0); + DORIS_CHECK(options.max_pending_tasks > 0); + DORIS_CHECK(options.batch_size > 0); + DORIS_CHECK(options.watchdog_warn_secs >= 0); + DORIS_CHECK(options.watchdog_drop_secs > options.watchdog_warn_secs); const char* prefix = _cache->get_base_path().c_str(); _mem_tracker = MemTrackerLimiter::create_shared( @@ -119,7 +109,7 @@ Status AsyncCacheWriteService::start() { return Status::OK(); } - const size_t worker_count = _options.worker_count; + const size_t worker_count = _desired_worker_count.load(std::memory_order_acquire); RETURN_IF_ERROR( ThreadPoolBuilder(fmt::format("AsyncFileCacheWrite-{}", std::hash {}(_cache->get_base_path()))) @@ -144,7 +134,8 @@ bool AsyncCacheWriteService::try_submit(AsyncCacheWriteTask task) { return false; } - const size_t max_pending = _max_pending_tasks(); + const auto options = _options.load(std::memory_order_acquire); + const size_t max_pending = options->max_pending_tasks; size_t current = _pending_count.load(std::memory_order_relaxed); while (current < max_pending) { if (_pending_count.compare_exchange_weak(current, current + 1, std::memory_order_acq_rel, @@ -213,9 +204,9 @@ void AsyncCacheWriteService::_worker_loop(size_t worker_id) { } size_t processed = 0; - const size_t batch_size = _batch_size(); + const auto options = _options.load(std::memory_order_acquire); AsyncCacheWriteTask task; - while (processed < batch_size && _queue.try_dequeue(task)) { + while (processed < options->batch_size && _queue.try_dequeue(task)) { ++processed; Defer finish {[&]() { _finish_task(task); @@ -223,8 +214,8 @@ void AsyncCacheWriteService::_worker_loop(size_t worker_id) { }}; const int64_t age_us = MonotonicMicros() - task.submit_ts_us; - const int64_t warn_us = _watchdog_warn_secs() * 1000 * 1000; - const int64_t drop_us = _watchdog_drop_secs() * 1000 * 1000; + const int64_t warn_us = options->watchdog_warn_secs * 1000 * 1000; + const int64_t drop_us = options->watchdog_drop_secs * 1000 * 1000; if (age_us > warn_us) { *_watchdog_timeout_metric << 1; LOG(WARNING) << "Async file cache write task waited " << age_us @@ -361,7 +352,7 @@ Status AsyncCacheWriteService::resize_workers(size_t worker_count) { } std::lock_guard resize_lock(_resize_mutex); if (!_started.load(std::memory_order_acquire)) { - _options.worker_count = worker_count; + _desired_worker_count.store(worker_count, std::memory_order_release); return Status::OK(); } if (_shutdown_requested.load(std::memory_order_acquire)) { @@ -397,6 +388,35 @@ Status AsyncCacheWriteService::resize_workers(size_t worker_count) { return Status::OK(); } +Status AsyncCacheWriteService::update_options(const AsyncCacheWriteServiceOptions& options) { + if (options.worker_count == 0) { + return Status::InvalidArgument("async file cache write worker count must be positive"); + } + if (options.max_pending_tasks == 0) { + return Status::InvalidArgument( + "async file cache write pending task limit must be positive"); + } + if (options.batch_size == 0) { + return Status::InvalidArgument("async file cache write batch size must be positive"); + } + if (options.watchdog_warn_secs < 0 || + options.watchdog_drop_secs <= options.watchdog_warn_secs) { + return Status::InvalidArgument( + "async file cache write watchdog thresholds must satisfy 0 <= warn < drop"); + } + + auto next_options = std::make_shared(options); + RETURN_IF_ERROR(resize_workers(options.worker_count)); + _options.store(std::move(next_options), std::memory_order_release); + return Status::OK(); +} + +AsyncCacheWriteServiceOptions AsyncCacheWriteService::options() const { + AsyncCacheWriteServiceOptions result = *_options.load(std::memory_order_acquire); + result.worker_count = _desired_worker_count.load(std::memory_order_acquire); + return result; +} + void AsyncCacheWriteService::shutdown() { std::lock_guard resize_lock(_resize_mutex); if (!_accepting.exchange(false, std::memory_order_acq_rel)) { @@ -438,28 +458,4 @@ AsyncCacheWriteServiceStats AsyncCacheWriteService::stats() const { }; } -size_t AsyncCacheWriteService::_max_pending_tasks() const { - if (_options.follow_global_config) { - return static_cast(config::async_file_cache_write_max_pending_tasks_per_disk); - } - return _options.max_pending_tasks; -} - -size_t AsyncCacheWriteService::_batch_size() const { - if (_options.follow_global_config) { - return static_cast(config::async_file_cache_write_batch_size); - } - return _options.batch_size; -} - -int64_t AsyncCacheWriteService::_watchdog_warn_secs() const { - return _options.follow_global_config ? config::async_file_cache_write_watchdog_warn_secs - : _options.watchdog_warn_secs; -} - -int64_t AsyncCacheWriteService::_watchdog_drop_secs() const { - return _options.follow_global_config ? config::async_file_cache_write_watchdog_drop_secs - : _options.watchdog_drop_secs; -} - } // namespace doris::io diff --git a/be/src/io/cache/async_cache_write_service.h b/be/src/io/cache/async_cache_write_service.h index 9fac7927d005f3..545c71a6fb0d69 100644 --- a/be/src/io/cache/async_cache_write_service.h +++ b/be/src/io/cache/async_cache_write_service.h @@ -30,6 +30,7 @@ #include #include +#include "common/atomic_shared_ptr.h" #include "common/status.h" #include "io/cache/file_cache_common.h" #include "runtime/memory/mem_tracker_limiter.h" @@ -83,18 +84,14 @@ struct AsyncCacheWriteTask { std::function on_finalized; }; -/// Per-cache-disk worker and queue limits. Tests may supply fixed values; production instances set -/// `follow_global_config` so mutable limits are read when each task is handled. +/// Complete per-cache-disk worker, queue, batch, and watchdog settings. The service receives this +/// value explicitly at construction and through update_options(); it never reads global config. struct AsyncCacheWriteServiceOptions { size_t worker_count {1}; size_t max_pending_tasks {1}; size_t batch_size {1}; int64_t watchdog_warn_secs {30}; int64_t watchdog_drop_secs {120}; - bool follow_global_config {false}; - - /// Build production options from the current BE configuration. - static AsyncCacheWriteServiceOptions from_config(); }; /// Snapshot of service counters used by tests and cache-stat reporting. @@ -150,6 +147,16 @@ class AsyncCacheWriteService { /// @param worker_count Positive target worker count for this cache disk. Status resize_workers(size_t worker_count); + /// Replace all mutable service settings with one coherent snapshot. Configuration adapters + /// call this method explicitly; the service itself has no dependency on global config. + /// @param options Complete validated settings, including the desired worker count. + /// @return OK after the new snapshot is active; InvalidArgument for invalid limits, or a + /// worker-resize error when the requested concurrency cannot be applied. + Status update_options(const AsyncCacheWriteServiceOptions& options); + + /// Return the currently active settings as a value snapshot. + AsyncCacheWriteServiceOptions options() const; + /// Stop submissions, drain all accepted tasks, and join worker loops. Idempotent. void shutdown(); @@ -175,14 +182,8 @@ class AsyncCacheWriteService { /// Release one pending slot and invoke the inflight-index cleanup callback. void _finish_task(const AsyncCacheWriteTask& task); - /// Read queue/batch/watchdog limits from mutable config or fixed test options. - size_t _max_pending_tasks() const; - size_t _batch_size() const; - int64_t _watchdog_warn_secs() const; - int64_t _watchdog_drop_secs() const; - BlockFileCache* _cache; - AsyncCacheWriteServiceOptions _options; + atomic_shared_ptr _options; moodycamel::ConcurrentQueue _queue; std::atomic _pending_count {0}; std::condition_variable _cv; diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index c1dc89a4b4ff8c..15ea8b9dbb66e0 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -550,8 +550,18 @@ Status BlockFileCache::initialize_unlocked(std::lock_guard& cache_lo _inflight_write_buffer_index = std::make_unique( static_cast(config::inflight_write_buffer_index_shard_count), _cache_base_path); - _async_write_service = std::make_unique( - this, AsyncCacheWriteServiceOptions::from_config()); + // BlockFileCache is the configuration boundary for a newly created per-disk service. Pass a + // value snapshot so AsyncCacheWriteService does not need to know where its settings came from. + AsyncCacheWriteServiceOptions async_write_options { + .worker_count = static_cast(config::async_file_cache_write_workers_per_disk), + .max_pending_tasks = + static_cast(config::async_file_cache_write_max_pending_tasks_per_disk), + .batch_size = static_cast(config::async_file_cache_write_batch_size), + .watchdog_warn_secs = config::async_file_cache_write_watchdog_warn_secs, + .watchdog_drop_secs = config::async_file_cache_write_watchdog_drop_secs, + }; + _async_write_service = + std::make_unique(this, std::move(async_write_options)); RETURN_IF_ERROR(_async_write_service->start()); if (auto* fs_storage = dynamic_cast(_storage.get())) { diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index 060f023da60ed2..5fc248f69807fd 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -70,10 +70,10 @@ size_t FileCacheFactory::try_release(const std::string& base_path) { return 0; } -Status FileCacheFactory::resize_async_write_workers(size_t worker_count) { +Status FileCacheFactory::update_async_write_options(const AsyncCacheWriteServiceOptions& options) { std::lock_guard lock(_mtx); for (const auto& cache : _caches) { - RETURN_IF_ERROR(cache->async_write_service()->resize_workers(worker_count)); + RETURN_IF_ERROR(cache->async_write_service()->update_options(options)); } return Status::OK(); } @@ -405,7 +405,27 @@ void FileCacheFactory::get_cache_stats_block(Block* block) { namespace doris::config { -DEFINE_ON_UPDATE(async_file_cache_write_workers_per_disk, [](int32_t old_value, int32_t new_value) { +namespace { + +/// Capture all mutable async-write fields after a config update. Returning one complete value +/// keeps every per-disk service on a coherent set of queue, worker, batch, and watchdog settings. +io::AsyncCacheWriteServiceOptions load_async_write_options_from_config() { + return io::AsyncCacheWriteServiceOptions { + .worker_count = static_cast(async_file_cache_write_workers_per_disk), + .max_pending_tasks = + static_cast(async_file_cache_write_max_pending_tasks_per_disk), + .batch_size = static_cast(async_file_cache_write_batch_size), + .watchdog_warn_secs = async_file_cache_write_watchdog_warn_secs, + .watchdog_drop_secs = async_file_cache_write_watchdog_drop_secs, + }; +} + +/// Forward one changed config field through the explicit factory/service update interface. +/// @param config_name Name used only to identify failures in the log. +/// @param old_value Previous config value; equal values require no service update. +/// @param new_value Newly accepted config value. +template +void update_async_write_options(const char* config_name, T old_value, T new_value) { if (old_value == new_value) { return; } @@ -413,11 +433,33 @@ DEFINE_ON_UPDATE(async_file_cache_write_workers_per_disk, [](int32_t old_value, if (factory == nullptr) { return; } - Status status = factory->resize_async_write_workers(static_cast(new_value)); + Status status = factory->update_async_write_options(load_async_write_options_from_config()); if (!status.ok()) { - LOG(WARNING) << "Failed to resize async file cache write workers from " << old_value - << " to " << new_value << ": " << status.to_string(); + LOG(WARNING) << "Failed to apply async file cache write option " << config_name << " from " + << old_value << " to " << new_value << ": " << status.to_string(); } +} + +} // namespace + +DEFINE_ON_UPDATE(async_file_cache_write_workers_per_disk, [](int32_t old_value, int32_t new_value) { + update_async_write_options("async_file_cache_write_workers_per_disk", old_value, new_value); +}); +DEFINE_ON_UPDATE(async_file_cache_write_max_pending_tasks_per_disk, + [](int64_t old_value, int64_t new_value) { + update_async_write_options("async_file_cache_write_max_pending_tasks_per_disk", + old_value, new_value); + }); +DEFINE_ON_UPDATE(async_file_cache_write_batch_size, [](int32_t old_value, int32_t new_value) { + update_async_write_options("async_file_cache_write_batch_size", old_value, new_value); +}); +DEFINE_ON_UPDATE(async_file_cache_write_watchdog_warn_secs, [](int64_t old_value, + int64_t new_value) { + update_async_write_options("async_file_cache_write_watchdog_warn_secs", old_value, new_value); +}); +DEFINE_ON_UPDATE(async_file_cache_write_watchdog_drop_secs, [](int64_t old_value, + int64_t new_value) { + update_async_write_options("async_file_cache_write_watchdog_drop_secs", old_value, new_value); }); } // namespace doris::config diff --git a/be/src/io/cache/block_file_cache_factory.h b/be/src/io/cache/block_file_cache_factory.h index 8a986a94cba793..2aa024a6938594 100644 --- a/be/src/io/cache/block_file_cache_factory.h +++ b/be/src/io/cache/block_file_cache_factory.h @@ -95,8 +95,10 @@ class FileCacheFactory { std::vector get_base_paths(); - /// Apply a positive per-disk async-write worker count to every initialized cache instance. - Status resize_async_write_workers(size_t worker_count); + /// Explicitly replace async-write service settings on every initialized cache disk. + /// @param options Complete settings snapshot produced by the configuration adapter. + /// @return OK after every service is updated; otherwise the first service update error. + Status update_async_write_options(const AsyncCacheWriteServiceOptions& options); /** * Clears data of all file cache instances diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp index 114abe2be25199..ef4ddd52e53e34 100644 --- a/be/test/io/cache/async_cache_write_service_test.cpp +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -19,6 +19,7 @@ #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include +#include "common/config.h" #include "cpp/sync_point.h" #include "io/cache/block_file_cache_test_common.h" #include "util/defer_op.h" @@ -140,13 +142,12 @@ TEST_F(AsyncCacheWriteServiceTest, TaskWritesDownloadedBlockAndCleansInflightEnt } TEST_F(AsyncCacheWriteServiceTest, PendingLimitRejectsWhileWorkerIsActive) { - const int64_t old_max_pending = config::async_file_cache_write_max_pending_tasks_per_disk; - config::async_file_cache_write_max_pending_tasks_per_disk = 1; - Defer restore_config { - [&]() { config::async_file_cache_write_max_pending_tasks_per_disk = old_max_pending; }}; auto cache = create_cache("async_write_service_backpressure"); auto* service = cache->async_write_service(); ASSERT_NE(service, nullptr); + auto options = service->options(); + options.max_pending_tasks = 1; + ASSERT_TRUE(service->update_options(options).ok()); std::mutex mutex; std::condition_variable cv; @@ -222,19 +223,15 @@ TEST_F(AsyncCacheWriteServiceTest, PendingLimitRejectsWhileWorkerIsActive) { } TEST_F(AsyncCacheWriteServiceTest, WatchdogDropsExpiredTaskAndCleansInflightEntry) { - const int64_t old_warn_secs = config::async_file_cache_write_watchdog_warn_secs; - const int64_t old_drop_secs = config::async_file_cache_write_watchdog_drop_secs; - config::async_file_cache_write_watchdog_warn_secs = 0; - config::async_file_cache_write_watchdog_drop_secs = 1; - Defer restore_config {[&]() { - config::async_file_cache_write_watchdog_warn_secs = old_warn_secs; - config::async_file_cache_write_watchdog_drop_secs = old_drop_secs; - }}; auto cache = create_cache("async_write_service_watchdog"); auto* service = cache->async_write_service(); auto* index = cache->inflight_write_buffer_index(); ASSERT_NE(service, nullptr); ASSERT_NE(index, nullptr); + auto options = service->options(); + options.watchdog_warn_secs = 0; + options.watchdog_drop_secs = 1; + ASSERT_TRUE(service->update_options(options).ok()); const auto hash = BlockFileCache::hash("watchdog_drop"); AsyncCacheWriteBufferPtr buffer; @@ -493,17 +490,102 @@ TEST_F(AsyncCacheWriteServiceTest, EpochChangeAfterFirstCheckDropsTaskAndCleansE EXPECT_EQ(probe_result.gaps.front().right, 4095); } -TEST_F(AsyncCacheWriteServiceTest, ResizeWorkersAtRuntime) { +TEST_F(AsyncCacheWriteServiceTest, UpdateOptionsAtRuntime) { auto cache = create_cache("async_write_service_resize"); auto* service = cache->async_write_service(); ASSERT_NE(service, nullptr); - ASSERT_TRUE(service->resize_workers(3).ok()); + auto options = service->options(); + options.worker_count = 3; + options.max_pending_tasks = 7; + options.batch_size = 2; + options.watchdog_warn_secs = 1; + options.watchdog_drop_secs = 2; + ASSERT_TRUE(service->update_options(options).ok()); + const auto updated = service->options(); + EXPECT_EQ(updated.worker_count, 3); + EXPECT_EQ(updated.max_pending_tasks, 7); + EXPECT_EQ(updated.batch_size, 2); + EXPECT_EQ(updated.watchdog_warn_secs, 1); + EXPECT_EQ(updated.watchdog_drop_secs, 2); EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 3); - ASSERT_TRUE(service->resize_workers(1).ok()); + + options.worker_count = 1; + ASSERT_TRUE(service->update_options(options).ok()); EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 1); } +TEST_F(AsyncCacheWriteServiceTest, MutableConfigUpdatesServicesExplicitly) { + auto* factory = FileCacheFactory::instance(); + factory->_caches.clear(); + factory->_path_to_cache.clear(); + factory->_capacity = 0; + + const auto path = caches_dir / "async_write_service_config_update"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + ASSERT_TRUE(factory->create_file_cache(path.string(), async_write_cache_settings()).ok()); + auto* cache = factory->get_by_path(path.string()); + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + const int32_t old_workers = config::async_file_cache_write_workers_per_disk; + const int64_t old_max_pending = config::async_file_cache_write_max_pending_tasks_per_disk; + const int32_t old_batch_size = config::async_file_cache_write_batch_size; + const int64_t old_warn_secs = config::async_file_cache_write_watchdog_warn_secs; + const int64_t old_drop_secs = config::async_file_cache_write_watchdog_drop_secs; + Defer restore {[&]() { + EXPECT_TRUE(config::set_config("async_file_cache_write_workers_per_disk", + std::to_string(old_workers)) + .ok()); + EXPECT_TRUE(config::set_config("async_file_cache_write_max_pending_tasks_per_disk", + std::to_string(old_max_pending)) + .ok()); + EXPECT_TRUE(config::set_config("async_file_cache_write_batch_size", + std::to_string(old_batch_size)) + .ok()); + EXPECT_TRUE(config::set_config("async_file_cache_write_watchdog_warn_secs", + std::to_string(old_warn_secs)) + .ok()); + EXPECT_TRUE(config::set_config("async_file_cache_write_watchdog_drop_secs", + std::to_string(old_drop_secs)) + .ok()); + factory->_caches.clear(); + factory->_path_to_cache.clear(); + factory->_capacity = 0; + fs::remove_all(path, error); + }}; + + const int32_t new_workers = old_workers == 1 ? 2 : 1; + const int64_t new_max_pending = old_max_pending + 1; + const int32_t new_batch_size = old_batch_size + 1; + const int64_t new_warn_secs = old_warn_secs + 1; + const int64_t new_drop_secs = std::max(old_drop_secs + 1, new_warn_secs + 1); + ASSERT_TRUE(config::set_config("async_file_cache_write_workers_per_disk", + std::to_string(new_workers)) + .ok()); + ASSERT_TRUE(config::set_config("async_file_cache_write_max_pending_tasks_per_disk", + std::to_string(new_max_pending)) + .ok()); + ASSERT_TRUE( + config::set_config("async_file_cache_write_batch_size", std::to_string(new_batch_size)) + .ok()); + ASSERT_TRUE(config::set_config("async_file_cache_write_watchdog_drop_secs", + std::to_string(new_drop_secs)) + .ok()); + ASSERT_TRUE(config::set_config("async_file_cache_write_watchdog_warn_secs", + std::to_string(new_warn_secs)) + .ok()); + + const auto updated = cache->async_write_service()->options(); + EXPECT_EQ(updated.worker_count, new_workers); + EXPECT_EQ(updated.max_pending_tasks, new_max_pending); + EXPECT_EQ(updated.batch_size, new_batch_size); + EXPECT_EQ(updated.watchdog_warn_secs, new_warn_secs); + EXPECT_EQ(updated.watchdog_drop_secs, new_drop_secs); +} + TEST_F(AsyncCacheWriteServiceTest, ShutdownWaitsForRegisteredSubmitterAndRejectsItsTask) { auto cache = create_cache("async_write_service_shutdown"); auto* service = cache->async_write_service(); diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index 313fa55fe879ff..d697a79031feea 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -570,20 +570,17 @@ TEST_F(BlockFileCacheTest, async_write_backpressure_rolls_back_inflight_entry) { const bool old_enable_direct = config::enable_read_cache_file_directly; const bool old_enable_peer = config::enable_cache_read_from_peer; const int64_t old_block_size = config::file_cache_each_block_size; - const int64_t old_max_pending = config::async_file_cache_write_max_pending_tasks_per_disk; config::enable_async_file_cache_write = true; config::enable_inflight_write_buffer_index = true; config::enable_read_cache_file_directly = false; config::enable_cache_read_from_peer = false; config::file_cache_each_block_size = 1_mb; - config::async_file_cache_write_max_pending_tasks_per_disk = 1; Defer restore_config {[&]() { config::enable_async_file_cache_write = old_enable_async; config::enable_inflight_write_buffer_index = old_enable_inflight; config::enable_read_cache_file_directly = old_enable_direct; config::enable_cache_read_from_peer = old_enable_peer; config::file_cache_each_block_size = old_block_size; - config::async_file_cache_write_max_pending_tasks_per_disk = old_max_pending; }}; reset_async_reader_cache_factory(); @@ -602,6 +599,9 @@ TEST_F(BlockFileCacheTest, async_write_backpressure_rolls_back_inflight_entry) { auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; ASSERT_NE(cache, nullptr); wait_until_cache_ready(*cache); + auto async_write_options = cache->async_write_service()->options(); + async_write_options.max_pending_tasks = 1; + ASSERT_TRUE(cache->async_write_service()->update_options(async_write_options).ok()); std::mutex mutex; std::condition_variable cv; From 0714da7da8ede48f9c52b85f6b1f7b4491fe2975 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 16 Jul 2026 17:19:56 +0800 Subject: [PATCH 05/16] [improvement](be) Increase async file cache write concurrency ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The phase-one asynchronous file-cache write service used two persistent workers per cache disk. The synchronous path it replaces persisted cache blocks directly on scanner threads, so its effective per-disk write concurrency could scale with the external scanner concurrency, whose default per-context upper bound is 16, and could grow further across concurrent query contexts. A two-worker default therefore serialized writeback far more aggressively than the former path and could fill the bounded pending queue during ordinary scan fan-out. Increase the default to 16 workers per cache disk. Keep one MPMC queue and let each worker dequeue, revalidate, claim the FileBlock downloader, and write the block in the same thread. Splitting consumption and persistence into separate pools would add a full-task handoff without an independent processing stage, and claiming a downloader before that handoff would violate FileBlock's thread-bound ownership contract. Each worker now uses its own ConsumerToken so concurrent consumers maintain independent producer-stream cursors instead of rescanning streams for every task. Avoid creating 16 persistent per-disk worker loops while asynchronous writeback is disabled. The cache still constructs the service state and inflight index, but starts workers only when the feature is enabled. A false-to-true online configuration update explicitly starts all initialized services through the factory interface, while mutable service options continue to flow through the explicit factory/service update API. Service readiness is published only after all configured worker loops have been accepted, so query threads reject best-effort submissions instead of enqueueing work to an unready service. Add deterministic coverage for eight workers consuming distinct tasks concurrently, disabled-service rejection, online enablement, and the existing runtime resize, shutdown, watchdog, inflight cleanup, and reader backpressure rollback behavior. ### Release note Increase the default asynchronous file-cache write concurrency from 2 to 16 workers per cache disk. Worker threads are created only after asynchronous file-cache writeback is enabled. ### Check List (For Author) - Test: Unit Test - `./build.sh --be -j100` - `./run-be-ut.sh --run --filter=AsyncCacheWriteServiceTest.*:BlockFileCacheTest.async_write_backpressure_rolls_back_inflight_entry -j 100` (12 tests passed) - `build-support/check-format.sh` - `git diff --check` - Behavior changed: Yes. The default per-disk asynchronous write concurrency is 16, and disabled services no longer keep worker loops resident. - Does this need documentation: No --- be/src/common/config.cpp | 2 +- be/src/io/cache/async_cache_write_service.cpp | 55 +++++--- be/src/io/cache/async_cache_write_service.h | 5 +- be/src/io/cache/block_file_cache.cpp | 6 +- be/src/io/cache/block_file_cache_factory.cpp | 22 ++++ be/src/io/cache/block_file_cache_factory.h | 4 + .../cache/async_cache_write_service_test.cpp | 118 ++++++++++++++++-- 7 files changed, 185 insertions(+), 27 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 20d3fd4202dee8..a2be1ce77481d3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1262,7 +1262,7 @@ DEFINE_mInt64(file_cache_background_block_lru_update_qps_limit, "1000"); DEFINE_mInt64(file_cache_background_block_lru_update_queue_max_size, "500000"); DEFINE_mBool(enable_file_cache_async_touch_on_get_or_set, "false"); DEFINE_mBool(enable_async_file_cache_write, "false"); -DEFINE_mInt32(async_file_cache_write_workers_per_disk, "2"); +DEFINE_mInt32(async_file_cache_write_workers_per_disk, "16"); DEFINE_mInt64(async_file_cache_write_max_pending_tasks_per_disk, "256"); DEFINE_mInt32(async_file_cache_write_batch_size, "16"); DEFINE_mInt64(async_file_cache_write_watchdog_warn_secs, "30"); diff --git a/be/src/io/cache/async_cache_write_service.cpp b/be/src/io/cache/async_cache_write_service.cpp index 1930f1822fd41f..b62cdf7a3f1daa 100644 --- a/be/src/io/cache/async_cache_write_service.cpp +++ b/be/src/io/cache/async_cache_write_service.cpp @@ -104,24 +104,47 @@ AsyncCacheWriteService::~AsyncCacheWriteService() { } Status AsyncCacheWriteService::start() { - bool expected = false; - if (!_started.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + std::lock_guard resize_lock(_resize_mutex); + if (_shutdown_requested.load(std::memory_order_acquire) || + !_accepting.load(std::memory_order_acquire)) { + return Status::InternalError("async file cache write service is shutting down"); + } + if (_started.load(std::memory_order_acquire)) { return Status::OK(); } const size_t worker_count = _desired_worker_count.load(std::memory_order_acquire); - RETURN_IF_ERROR( - ThreadPoolBuilder(fmt::format("AsyncFileCacheWrite-{}", - std::hash {}(_cache->get_base_path()))) - .set_min_threads(0) - .set_max_threads(static_cast(worker_count)) - .set_max_queue_size(128) - .build(&_worker_pool)); - _desired_worker_count.store(worker_count, std::memory_order_release); - _worker_scheduled.resize(worker_count, false); + if (_worker_pool == nullptr) { + RETURN_IF_ERROR( + ThreadPoolBuilder(fmt::format("AsyncFileCacheWrite-{}", + std::hash {}(_cache->get_base_path()))) + .set_min_threads(0) + .set_max_threads(static_cast(worker_count)) + .set_max_queue_size(128) + .build(&_worker_pool)); + } else { + // A previous start may have created only part of the requested workers before returning an + // error. Reuse that pool and apply the latest desired size before filling the missing ids. + RETURN_IF_ERROR(_worker_pool->set_max_threads(static_cast(worker_count))); + } + { + std::lock_guard state_lock(_worker_state_mutex); + if (_worker_scheduled.size() < worker_count) { + _worker_scheduled.resize(worker_count, false); + } + } for (size_t worker_id = 0; worker_id < worker_count; ++worker_id) { - RETURN_IF_ERROR(_schedule_worker(worker_id)); + bool scheduled = false; + { + std::lock_guard state_lock(_worker_state_mutex); + scheduled = _worker_scheduled[worker_id]; + } + if (!scheduled) { + RETURN_IF_ERROR(_schedule_worker(worker_id)); + } } + // Publish readiness only after every configured worker loop has been accepted by the pool. + _started.store(true, std::memory_order_release); return Status::OK(); } @@ -129,7 +152,7 @@ bool AsyncCacheWriteService::try_submit(AsyncCacheWriteTask task) { _active_submitters.fetch_add(1, std::memory_order_acq_rel); Defer submitter_done {[&]() { _active_submitters.fetch_sub(1, std::memory_order_acq_rel); }}; TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::try_submit:after_register", &task); - if (!_accepting.load(std::memory_order_acquire)) { + if (!_started.load(std::memory_order_acquire) || !_accepting.load(std::memory_order_acquire)) { *_rejected_metric << 1; return false; } @@ -197,6 +220,10 @@ void AsyncCacheWriteService::_worker_loop(size_t worker_id) { _worker_state_cv.notify_all(); }}; + // Keep one consumer cursor per worker so concurrent consumers rotate across the queue's + // producer streams instead of rescanning them from scratch for every disk-write task. + moodycamel::ConsumerToken consumer_token(_queue); + while (true) { if (!_shutdown_requested.load(std::memory_order_acquire) && worker_id >= _desired_worker_count.load(std::memory_order_acquire)) { @@ -206,7 +233,7 @@ void AsyncCacheWriteService::_worker_loop(size_t worker_id) { size_t processed = 0; const auto options = _options.load(std::memory_order_acquire); AsyncCacheWriteTask task; - while (processed < options->batch_size && _queue.try_dequeue(task)) { + while (processed < options->batch_size && _queue.try_dequeue(consumer_token, task)) { ++processed; Defer finish {[&]() { _finish_task(task); diff --git a/be/src/io/cache/async_cache_write_service.h b/be/src/io/cache/async_cache_write_service.h index 545c71a6fb0d69..f12f36526e795f 100644 --- a/be/src/io/cache/async_cache_write_service.h +++ b/be/src/io/cache/async_cache_write_service.h @@ -124,8 +124,9 @@ class AsyncCacheWriteService { Status start(); /// Reserve one pending slot and enqueue `task` without blocking the query thread. - /// @return true if ownership was transferred to the queue; false on shutdown, backpressure, or - /// queue rejection. A rejected task's finalization callback is not invoked. + /// @return true if ownership was transferred to the queue; false when workers have not been + /// started, during shutdown, on backpressure, or on queue rejection. A rejected task's + /// finalization callback is not invoked. bool try_submit(AsyncCacheWriteTask task); /// Allocate `size` payload bytes charged to the service tracker and return them in `buffer`. diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index 15ea8b9dbb66e0..f21bd74ef460cb 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -562,7 +562,11 @@ Status BlockFileCache::initialize_unlocked(std::lock_guard& cache_lo }; _async_write_service = std::make_unique(this, std::move(async_write_options)); - RETURN_IF_ERROR(_async_write_service->start()); + // Do not create persistent per-disk workers while async writeback is disabled. The config + // update adapter starts existing services explicitly when the switch is enabled online. + if (config::enable_async_file_cache_write) { + RETURN_IF_ERROR(_async_write_service->start()); + } if (auto* fs_storage = dynamic_cast(_storage.get())) { if (auto* meta_store = fs_storage->get_meta_store()) { diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index 5fc248f69807fd..8720836d373884 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -78,6 +78,14 @@ Status FileCacheFactory::update_async_write_options(const AsyncCacheWriteService return Status::OK(); } +Status FileCacheFactory::start_async_write_services() { + std::lock_guard lock(_mtx); + for (const auto& cache : _caches) { + RETURN_IF_ERROR(cache->async_write_service()->start()); + } + return Status::OK(); +} + Status FileCacheFactory::create_file_cache(const std::string& cache_base_path, FileCacheSettings file_cache_settings) { if (file_cache_settings.storage == "memory") { @@ -442,6 +450,20 @@ void update_async_write_options(const char* config_name, T old_value, T new_valu } // namespace +DEFINE_ON_UPDATE(enable_async_file_cache_write, [](bool old_value, bool new_value) { + if (old_value == new_value || !new_value) { + return; + } + auto* factory = io::FileCacheFactory::instance(); + if (factory == nullptr) { + return; + } + Status status = factory->start_async_write_services(); + if (!status.ok()) { + LOG(WARNING) << "Failed to start async file cache write services: " << status.to_string(); + } +}); + DEFINE_ON_UPDATE(async_file_cache_write_workers_per_disk, [](int32_t old_value, int32_t new_value) { update_async_write_options("async_file_cache_write_workers_per_disk", old_value, new_value); }); diff --git a/be/src/io/cache/block_file_cache_factory.h b/be/src/io/cache/block_file_cache_factory.h index 2aa024a6938594..aa696e00175abc 100644 --- a/be/src/io/cache/block_file_cache_factory.h +++ b/be/src/io/cache/block_file_cache_factory.h @@ -100,6 +100,10 @@ class FileCacheFactory { /// @return OK after every service is updated; otherwise the first service update error. Status update_async_write_options(const AsyncCacheWriteServiceOptions& options); + /// Start async-write workers for every initialized cache disk. Repeated calls are idempotent. + /// @return OK after every service is ready; otherwise the first startup error. + Status start_async_write_services(); + /** * Clears data of all file cache instances * diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp index ef4ddd52e53e34..359f640aa32890 100644 --- a/be/test/io/cache/async_cache_write_service_test.cpp +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -62,6 +62,7 @@ class AsyncCacheWriteServiceTest : public BlockFileCacheTest { auto cache = std::make_unique(path.string(), async_write_cache_settings()); EXPECT_TRUE(cache->initialize().ok()); wait_until_cache_ready(*cache); + EXPECT_TRUE(cache->async_write_service()->start().ok()); return cache; } @@ -515,26 +516,97 @@ TEST_F(AsyncCacheWriteServiceTest, UpdateOptionsAtRuntime) { EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 1); } +TEST_F(AsyncCacheWriteServiceTest, MultipleWorkersConsumeQueueConcurrently) { + auto cache = create_cache("async_write_service_mpmc_consumers"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t worker_count = 8; + auto options = service->options(); + options.worker_count = worker_count; + options.max_pending_tasks = worker_count; + options.batch_size = 1; + ASSERT_TRUE(service->update_options(options).ok()); + + std::mutex mutex; + std::condition_variable cv; + size_t entered_workers = 0; + size_t finished_tasks = 0; + bool release_workers = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + ++entered_workers; + cv.notify_all(); + cv.wait(lock, [&]() { return release_workers; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_workers = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + for (size_t task_id = 0; task_id < worker_count; ++task_id) { + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), static_cast('a' + task_id), buffer->size()); + AsyncCacheWriteTask task { + .cache_hash = BlockFileCache::hash("mpmc_consumer_" + std::to_string(task_id)), + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&](const AsyncCacheWriteTask&) { + std::lock_guard lock(mutex); + ++finished_tasks; + cv.notify_all(); + }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + } + + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return entered_workers == worker_count; })); + EXPECT_EQ(service->pending_count(), worker_count); + release_workers = true; + } + cv.notify_all(); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return finished_tasks == worker_count; })); + } + EXPECT_EQ(service->pending_count(), 0); +} + TEST_F(AsyncCacheWriteServiceTest, MutableConfigUpdatesServicesExplicitly) { auto* factory = FileCacheFactory::instance(); factory->_caches.clear(); factory->_path_to_cache.clear(); factory->_capacity = 0; - const auto path = caches_dir / "async_write_service_config_update"; - std::error_code error; - fs::remove_all(path, error); - fs::create_directories(path); - ASSERT_TRUE(factory->create_file_cache(path.string(), async_write_cache_settings()).ok()); - auto* cache = factory->get_by_path(path.string()); - ASSERT_NE(cache, nullptr); - wait_until_cache_ready(*cache); - + const bool old_enable = config::enable_async_file_cache_write; const int32_t old_workers = config::async_file_cache_write_workers_per_disk; const int64_t old_max_pending = config::async_file_cache_write_max_pending_tasks_per_disk; const int32_t old_batch_size = config::async_file_cache_write_batch_size; const int64_t old_warn_secs = config::async_file_cache_write_watchdog_warn_secs; const int64_t old_drop_secs = config::async_file_cache_write_watchdog_drop_secs; + const auto path = caches_dir / "async_write_service_config_update"; + std::error_code error; Defer restore {[&]() { EXPECT_TRUE(config::set_config("async_file_cache_write_workers_per_disk", std::to_string(old_workers)) @@ -551,12 +623,38 @@ TEST_F(AsyncCacheWriteServiceTest, MutableConfigUpdatesServicesExplicitly) { EXPECT_TRUE(config::set_config("async_file_cache_write_watchdog_drop_secs", std::to_string(old_drop_secs)) .ok()); + EXPECT_TRUE( + config::set_config("enable_async_file_cache_write", old_enable ? "true" : "false") + .ok()); factory->_caches.clear(); factory->_path_to_cache.clear(); factory->_capacity = 0; fs::remove_all(path, error); }}; + ASSERT_TRUE(config::set_config("enable_async_file_cache_write", "false").ok()); + fs::remove_all(path, error); + fs::create_directories(path); + ASSERT_TRUE(factory->create_file_cache(path.string(), async_write_cache_settings()).ok()); + auto* cache = factory->get_by_path(path.string()); + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + ASSERT_FALSE(cache->async_write_service()->_started.load(std::memory_order_acquire)); + AsyncCacheWriteBufferPtr disabled_buffer; + ASSERT_TRUE(cache->async_write_service()->allocate_tracked_buffer(4096, &disabled_buffer).ok()); + AsyncCacheWriteTask disabled_task { + .cache_hash = BlockFileCache::hash("disabled_async_write_service"), + .file_offset = 0, + .file_size = disabled_buffer->size(), + .buffer = disabled_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = cache->async_write_service()->current_write_epoch(), + .on_finalized = nullptr, + }; + EXPECT_FALSE(cache->async_write_service()->try_submit(std::move(disabled_task))); + EXPECT_EQ(cache->async_write_service()->pending_count(), 0); + const int32_t new_workers = old_workers == 1 ? 2 : 1; const int64_t new_max_pending = old_max_pending + 1; const int32_t new_batch_size = old_batch_size + 1; @@ -577,8 +675,10 @@ TEST_F(AsyncCacheWriteServiceTest, MutableConfigUpdatesServicesExplicitly) { ASSERT_TRUE(config::set_config("async_file_cache_write_watchdog_warn_secs", std::to_string(new_warn_secs)) .ok()); + ASSERT_TRUE(config::set_config("enable_async_file_cache_write", "true").ok()); const auto updated = cache->async_write_service()->options(); + EXPECT_TRUE(cache->async_write_service()->_started.load(std::memory_order_acquire)); EXPECT_EQ(updated.worker_count, new_workers); EXPECT_EQ(updated.max_pending_tasks, new_max_pending); EXPECT_EQ(updated.batch_size, new_batch_size); From 8cbf21441a6eb6bcc3f243e672de658d40d8c887 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 16 Jul 2026 18:16:53 +0800 Subject: [PATCH 06/16] [test](be) Complete async file cache write unit coverage ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The phase-one async file-cache write implementation had strong happy-path coverage, but several correctness boundaries were not exercised through the complete component interactions. Missing coverage included cache-file disappearance and whole-key self-heal cleanup, wait timeout fallback, direct-read prefix preservation, final concurrent publication deduplication, tracked-buffer allocation failure, external-table cache reuse, worker ownership of existing or deleting cells, remove/write epoch races, runtime worker growth and shrink, and complete propagation of the new profile and synchronous cache-population semantics. Add compact scenario-oriented BE unit tests that drive the real reader, cache, inflight index, async service, worker, removal, downloader, and index-preload paths. The tests verify both returned data and persistent cache state, including metadata and physical file deletion. Narrow test-only sync points make allocation failures and race windows deterministic without changing normal runtime behavior. ### Release note None ### Check List (For Author) - Test: Unit Test - 51 related ASAN BE unit tests passed with run-be-ut.sh and -j100 - 7 focused changed-path tests passed with run-be-ut.sh and -j100 - BE build passed with build.sh --be -j100 - build-support/check-format.sh passed - Behavior changed: No; only test coverage and deterministic test injection points are added - Does this need documentation: No --- be/src/io/cache/async_cache_write_service.cpp | 7 + .../io/cache/block_file_cache_downloader.cpp | 2 + .../cached_remote_file_reader_async_write.cpp | 4 + .../cache/async_cache_write_service_test.cpp | 265 +++++++++- .../block_file_cache_downloader_test.cpp | 70 +++ .../io/cache/block_file_cache_probe_test.cpp | 75 +++ ...block_file_cache_profile_reporter_test.cpp | 23 + .../cache/cached_remote_file_reader_test.cpp | 476 +++++++++++++++++- ...cloud_file_cache_write_index_only_test.cpp | 2 + 9 files changed, 899 insertions(+), 25 deletions(-) diff --git a/be/src/io/cache/async_cache_write_service.cpp b/be/src/io/cache/async_cache_write_service.cpp index b62cdf7a3f1daa..85c3ceac68eaca 100644 --- a/be/src/io/cache/async_cache_write_service.cpp +++ b/be/src/io/cache/async_cache_write_service.cpp @@ -181,6 +181,13 @@ Status AsyncCacheWriteService::allocate_tracked_buffer(size_t size, AsyncCacheWriteBufferPtr* buffer) { DORIS_CHECK(buffer != nullptr); DORIS_CHECK(size > 0); + Status injected_status; + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::allocate_tracked_buffer:inject_failure", + &injected_status); + if (!injected_status.ok()) { + *_buffer_alloc_fail_metric << 1; + return injected_status; + } SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); Status status = Status::OK(); try { diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index be8373ef4d99b4..90554dbe1b1fb0 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -354,6 +354,8 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met // 1. Directly append buffer data to file cache // 2. Provide `FileReader::async_read()` interface DCHECK(meta.ctx.is_dryrun == config::enable_reader_dryrun_when_download_file_cache); + TEST_SYNC_POINT_CALLBACK("FileCacheBlockDownloader::download_segment_file:before_read", + &download_ctx); st = file_reader->read_at(offset, {buffer.get(), size}, &bytes_read, &download_ctx); if (!st.ok()) { LOG(WARNING) << "failed to download file path=" << meta.path << ", st=" << st; diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp index ef2a8e56c7b6bf..d426ae60ecc411 100644 --- a/be/src/io/cache/cached_remote_file_reader_async_write.cpp +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -27,6 +27,7 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" +#include "cpp/sync_point.h" #include "io/cache/async_cache_write_service.h" #include "io/cache/block_file_cache.h" #include "io/cache/cached_remote_file_reader.h" @@ -474,6 +475,9 @@ void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan entry = std::make_shared( tracked_buffer, read_block.range.left, read_block.range.size(), task.submit_ts_us, plan.write_epoch); + TEST_SYNC_POINT_CALLBACK( + "CachedRemoteFileReader::_submit_async_write_tasks:before_inflight_insert", + &task); auto existing = inflight_index->insert_if_absent(_cache_hash, read_block.range.left, entry); if (existing) { diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp index 359f640aa32890..d43f108e2409d7 100644 --- a/be/test/io/cache/async_cache_write_service_test.cpp +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -362,6 +362,157 @@ TEST_F(AsyncCacheWriteServiceTest, OneTaskWritesMultipleContainedCells) { EXPECT_EQ(second, std::string(cell_size, 'n')); } +TEST_F(AsyncCacheWriteServiceTest, ExistingAndDeletingCellsKeepTheirOwners) { + auto cache = create_cache("async_write_service_existing_states"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t cell_size = 4096; + constexpr size_t task_size = cell_size * 3; + const auto hash = BlockFileCache::hash("existing_states"); + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + auto holder = cache->get_or_set(hash, 0, task_size, context); + ASSERT_EQ(holder.file_blocks.size(), 3); + auto iterator = holder.file_blocks.begin(); + const auto downloaded_block = *iterator; + ++iterator; + const auto downloading_block = *iterator; + ++iterator; + const auto deleting_block = *iterator; + + ASSERT_EQ(downloaded_block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string original(cell_size, 'x'); + ASSERT_TRUE(downloaded_block->append(Slice(original.data(), original.size())).ok()); + ASSERT_TRUE(downloaded_block->finalize().ok()); + ASSERT_EQ(downloading_block->get_or_set_downloader(), FileBlock::get_caller_id()); + deleting_block->set_deleting(); + + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(task_size, &buffer).ok()); + memset(buffer->data(), 'y', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = task_size, + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_GE(service->stats().skip_downloaded, 1); + EXPECT_GE(service->stats().skip_downloading, 1); + EXPECT_GE(service->stats().skip_deleting, 1); + + std::string actual(cell_size, '\0'); + ASSERT_TRUE(downloaded_block->read(Slice(actual.data(), actual.size()), 0).ok()); + EXPECT_EQ(actual, original); + EXPECT_EQ(downloading_block->state(), FileBlock::State::DOWNLOADING); + EXPECT_EQ(downloading_block->get_downloader(), FileBlock::get_caller_id()); + EXPECT_TRUE(cache->is_block_deleting(deleting_block)); +} + +TEST_F(AsyncCacheWriteServiceTest, RemoveDuringAppendDoesNotLeaveResurrectedCacheData) { + auto cache = create_cache("async_write_service_remove_during_append"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + std::mutex mutex; + std::condition_variable cv; + bool before_append = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_append", + [&](auto&&) { + std::unique_lock lock(mutex); + before_append = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + const auto hash = BlockFileCache::hash("remove_during_append"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), 'r', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return before_append; })); + } + + fs::path cache_file; + { + ReadStatistics probe_stats; + CacheContext probe_context; + probe_context.stats = &probe_stats; + auto probe_result = cache->probe(hash, 0, 4096, probe_context); + ASSERT_EQ(probe_result.holder.file_blocks.size(), 1); + EXPECT_EQ(probe_result.holder.file_blocks.front()->state(), FileBlock::State::DOWNLOADING); + cache_file = probe_result.holder.file_blocks.front()->get_cache_file(); + } + const uint64_t old_epoch = service->current_write_epoch(); + cache->remove_if_cached_async(hash); + EXPECT_EQ(service->current_write_epoch(), old_epoch + 1); + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + + bool removed = false; + for (int attempt = 0; attempt < 5000; ++attempt) { + ReadStatistics probe_stats; + CacheContext probe_context; + probe_context.stats = &probe_stats; + bool metadata_removed = false; + { + auto probe_result = cache->probe(hash, 0, 4096, probe_context); + metadata_removed = + probe_result.holder.file_blocks.empty() && probe_result.gaps.size() == 1; + } + if (metadata_removed && !fs::exists(cache_file)) { + removed = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(removed); +} + TEST_F(AsyncCacheWriteServiceTest, PartialOverlapIsSkippedWithoutOutOfBoundsWrite) { auto cache = create_cache("async_write_service_partial_overlap"); auto* service = cache->async_write_service(); @@ -420,10 +571,13 @@ TEST_F(AsyncCacheWriteServiceTest, PartialOverlapIsSkippedWithoutOutOfBoundsWrit EXPECT_EQ(tail, std::string(task_offset, 'y')); } -TEST_F(AsyncCacheWriteServiceTest, EpochChangeAfterFirstCheckDropsTaskAndCleansEmptyCell) { +TEST_F(AsyncCacheWriteServiceTest, RemoveInvalidatesActiveAndQueuedTasksAndCleansEmptyCells) { auto cache = create_cache("async_write_service_epoch"); auto* service = cache->async_write_service(); ASSERT_NE(service, nullptr); + auto options = service->options(); + options.worker_count = 1; + ASSERT_TRUE(service->update_options(options).ok()); std::mutex mutex; std::condition_variable cv; @@ -451,52 +605,95 @@ TEST_F(AsyncCacheWriteServiceTest, EpochChangeAfterFirstCheckDropsTaskAndCleansE sync_point->clear_all_call_backs(); }}; - const auto hash = BlockFileCache::hash("epoch_drop"); - AsyncCacheWriteBufferPtr buffer; - ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); - memset(buffer->data(), 'c', buffer->size()); - std::promise finished; - auto finished_future = finished.get_future(); - AsyncCacheWriteTask task { - .cache_hash = hash, + const auto active_hash = BlockFileCache::hash("epoch_drop_active"); + AsyncCacheWriteBufferPtr active_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &active_buffer).ok()); + memset(active_buffer->data(), 'a', active_buffer->size()); + std::promise active_finished; + auto active_future = active_finished.get_future(); + AsyncCacheWriteTask active_task { + .cache_hash = active_hash, .file_offset = 0, - .file_size = buffer->size(), - .buffer = buffer, + .file_size = active_buffer->size(), + .buffer = active_buffer, .admission_ctx = {}, .submit_ts_us = MonotonicMicros(), .write_epoch = service->current_write_epoch(), - .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + .on_finalized = + [&active_finished](const AsyncCacheWriteTask&) { active_finished.set_value(); }, }; - ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_TRUE(service->try_submit(std::move(active_task))); { std::unique_lock lock(mutex); ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); } - service->invalidate_pending_writes(); + + const auto queued_hash = BlockFileCache::hash("epoch_drop_queued"); + AsyncCacheWriteBufferPtr queued_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &queued_buffer).ok()); + memset(queued_buffer->data(), 'q', queued_buffer->size()); + std::promise queued_finished; + auto queued_future = queued_finished.get_future(); + AsyncCacheWriteTask queued_task { + .cache_hash = queued_hash, + .file_offset = 0, + .file_size = queued_buffer->size(), + .buffer = queued_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&queued_finished](const AsyncCacheWriteTask&) { queued_finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(queued_task))); + ASSERT_EQ(service->pending_count(), 2); + + const uint64_t old_epoch = service->current_write_epoch(); + cache->remove_if_cached_async(active_hash); + EXPECT_EQ(service->current_write_epoch(), old_epoch + 1); { std::lock_guard lock(mutex); release_worker = true; } cv.notify_all(); - ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); - EXPECT_GE(service->stats().drop_stale_epoch, 1); + ASSERT_EQ(active_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + ASSERT_EQ(queued_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_GE(service->stats().drop_stale_epoch, 2); ReadStatistics read_stats; CacheContext context; context.stats = &read_stats; - auto probe_result = cache->probe(hash, 0, 4096, context); - EXPECT_TRUE(probe_result.holder.file_blocks.empty()); - ASSERT_EQ(probe_result.gaps.size(), 1); - EXPECT_EQ(probe_result.gaps.front().left, 0); - EXPECT_EQ(probe_result.gaps.front().right, 4095); + const auto expect_cache_gap = [&](const UInt128Wrapper& hash) { + auto probe_result = cache->probe(hash, 0, 4096, context); + EXPECT_TRUE(probe_result.holder.file_blocks.empty()); + ASSERT_EQ(probe_result.gaps.size(), 1); + EXPECT_EQ(probe_result.gaps.front().left, 0); + EXPECT_EQ(probe_result.gaps.front().right, 4095); + }; + expect_cache_gap(active_hash); + expect_cache_gap(queued_hash); } -TEST_F(AsyncCacheWriteServiceTest, UpdateOptionsAtRuntime) { +TEST_F(AsyncCacheWriteServiceTest, UpdateOptionsValidatesAndAppliesAtRuntime) { auto cache = create_cache("async_write_service_resize"); auto* service = cache->async_write_service(); ASSERT_NE(service, nullptr); auto options = service->options(); + auto invalid_options = options; + invalid_options.worker_count = 0; + EXPECT_TRUE(service->update_options(invalid_options).is()); + invalid_options = options; + invalid_options.max_pending_tasks = 0; + EXPECT_TRUE(service->update_options(invalid_options).is()); + invalid_options = options; + invalid_options.batch_size = 0; + EXPECT_TRUE(service->update_options(invalid_options).is()); + invalid_options = options; + invalid_options.watchdog_drop_secs = invalid_options.watchdog_warn_secs; + EXPECT_TRUE(service->update_options(invalid_options).is()); + options.worker_count = 3; options.max_pending_tasks = 7; options.batch_size = 2; @@ -516,17 +713,19 @@ TEST_F(AsyncCacheWriteServiceTest, UpdateOptionsAtRuntime) { EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 1); } -TEST_F(AsyncCacheWriteServiceTest, MultipleWorkersConsumeQueueConcurrently) { +TEST_F(AsyncCacheWriteServiceTest, RuntimeGrowAndShrinkPreserveConcurrentTasks) { auto cache = create_cache("async_write_service_mpmc_consumers"); auto* service = cache->async_write_service(); ASSERT_NE(service, nullptr); constexpr size_t worker_count = 8; auto options = service->options(); - options.worker_count = worker_count; + options.worker_count = 1; options.max_pending_tasks = worker_count; options.batch_size = 1; ASSERT_TRUE(service->update_options(options).ok()); + options.worker_count = worker_count; + ASSERT_TRUE(service->update_options(options).ok()); std::mutex mutex; std::condition_variable cv; @@ -582,6 +781,22 @@ TEST_F(AsyncCacheWriteServiceTest, MultipleWorkersConsumeQueueConcurrently) { ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return entered_workers == worker_count; })); EXPECT_EQ(service->pending_count(), worker_count); + } + + auto shrink_options = service->options(); + shrink_options.worker_count = 1; + auto shrink_future = std::async(std::launch::async, [service, shrink_options]() { + return service->update_options(shrink_options); + }); + for (int attempt = 0; + attempt < 5000 && service->_desired_worker_count.load(std::memory_order_acquire) != 1; + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 1); + EXPECT_EQ(shrink_future.wait_for(std::chrono::milliseconds(0)), std::future_status::timeout); + { + std::lock_guard lock(mutex); release_workers = true; } cv.notify_all(); @@ -590,7 +805,9 @@ TEST_F(AsyncCacheWriteServiceTest, MultipleWorkersConsumeQueueConcurrently) { ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return finished_tasks == worker_count; })); } + ASSERT_TRUE(shrink_future.get().ok()); EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(service->options().worker_count, 1); } TEST_F(AsyncCacheWriteServiceTest, MutableConfigUpdatesServicesExplicitly) { diff --git a/be/test/io/cache/block_file_cache_downloader_test.cpp b/be/test/io/cache/block_file_cache_downloader_test.cpp index 79974ef4860ad1..c74e9f0e9b9505 100644 --- a/be/test/io/cache/block_file_cache_downloader_test.cpp +++ b/be/test/io/cache/block_file_cache_downloader_test.cpp @@ -26,6 +26,10 @@ #include "cloud/cloud_storage_engine.h" #include "common/config.h" +#include "cpp/sync_point.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "util/defer_op.h" namespace doris::io { @@ -237,4 +241,70 @@ TEST_F(FileCacheBlockDownloaderTest, TestCheckDownloadTaskInflightStatus) { EXPECT_TRUE(done) << "Task should complete with decremented inflight count"; } +TEST_F(FileCacheBlockDownloaderTest, DownloadSegmentForcesSynchronousCacheWriteForBothDryRunModes) { + const Path directory = "ut_dir/block_file_cache_downloader_sync_override"; + const Path file = directory / "segment.dat"; + static_cast(global_local_filesystem()->delete_file(file)); + static_cast(global_local_filesystem()->delete_directory(directory)); + ASSERT_TRUE(global_local_filesystem()->create_directory(directory).ok()); + Defer cleanup {[&]() { + static_cast(global_local_filesystem()->delete_file(file)); + static_cast(global_local_filesystem()->delete_directory(directory)); + }}; + FileWriterPtr writer; + ASSERT_TRUE(global_local_filesystem()->create_file(file, &writer).ok()); + ASSERT_TRUE(writer->append(Slice("x", 1)).ok()); + ASSERT_TRUE(writer->close().ok()); + + const bool old_dryrun_config = config::enable_reader_dryrun_when_download_file_cache; + const int64_t old_buffer_size = config::s3_write_buffer_size; + Defer restore_config {[&]() { + config::enable_reader_dryrun_when_download_file_cache = old_dryrun_config; + config::s3_write_buffer_size = old_buffer_size; + }}; + config::s3_write_buffer_size = 1; + + std::vector> observed_contexts; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "FileCacheBlockDownloader::download_segment_file:before_read", + [&](auto&& values) { + auto* context = try_any_cast(values.back()); + ASSERT_TRUE(context->cache_write_mode_override.has_value()); + observed_contexts.emplace_back(context->is_dryrun, + *context->cache_write_mode_override); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + FileCacheBlockDownloader downloader(_engine); + for (bool is_dryrun : {false, true}) { + config::enable_reader_dryrun_when_download_file_cache = is_dryrun; + Status completion_status = Status::InternalError("download completion was not called"); + DownloadFileMeta meta { + .path = file, + .file_size = 1, + .offset = 0, + .download_size = 1, + .file_system = global_local_filesystem(), + .ctx = IOContext {.is_dryrun = is_dryrun}, + .download_done = [&](Status status) { completion_status = std::move(status); }, + .tablet_id = 10086, + }; + DownloadTask task(std::move(meta)); + downloader.download_blocks(task); + EXPECT_TRUE(completion_status.ok()); + } + + EXPECT_EQ(observed_contexts, (std::vector> { + {false, CacheWriteMode::SYNC_WRITE}, + {true, CacheWriteMode::SYNC_WRITE}, + })); +} + } // namespace doris::io diff --git a/be/test/io/cache/block_file_cache_probe_test.cpp b/be/test/io/cache/block_file_cache_probe_test.cpp index ba1982bcd72bf4..4c01415589fb6a 100644 --- a/be/test/io/cache/block_file_cache_probe_test.cpp +++ b/be/test/io/cache/block_file_cache_probe_test.cpp @@ -113,6 +113,81 @@ TEST_F(BlockFileCacheTest, ProbeReturnsDownloadedBlockAndIndependentGap) { fs::remove_all(path, error); } +TEST_F(BlockFileCacheTest, ProbeTouchAndRemovalRevalidateTheRetainedBlock) { + const auto path = caches_dir / "block_file_cache_probe_touch_remove"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + const bool old_async_touch = config::enable_file_cache_async_touch_on_get_or_set; + config::enable_file_cache_async_touch_on_get_or_set = false; + Defer restore_config { + [&]() { config::enable_file_cache_async_touch_on_get_or_set = old_async_touch; }}; + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_touch_remove"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + { + auto holder = cache.get_or_set(hash, 0, 4096, context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + ASSERT_EQ(block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(4096, 't'); + ASSERT_TRUE(block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(block->finalize().ok()); + } + { + std::lock_guard cache_lock(cache._mutex); + cache._files.at(hash).begin()->second.atime = 123; + } + + fs::path cache_file; + const uint64_t old_epoch = cache.async_write_service()->current_write_epoch(); + { + auto probe_result = cache.probe(hash, 0, 4096, context); + ASSERT_EQ(probe_result.holder.file_blocks.size(), 1); + const auto& block = probe_result.holder.file_blocks.front(); + cache_file = block->get_cache_file(); + { + std::lock_guard cache_lock(cache._mutex); + EXPECT_EQ(cache._files.at(hash).begin()->second.atime, 123); + } + + EXPECT_FALSE(cache.is_block_deleting(block)); + cache.touch_probe_block_if_cached(block, context); + { + std::lock_guard cache_lock(cache._mutex); + EXPECT_NE(cache._files.at(hash).begin()->second.atime, 123); + } + + cache.remove_if_cached(hash); + EXPECT_EQ(cache.async_write_service()->current_write_epoch(), old_epoch + 1); + EXPECT_TRUE(cache.is_block_deleting(block)); + cache.touch_probe_block_if_cached(block, context); + } + + bool removed = false; + for (int attempt = 0; attempt < 5000; ++attempt) { + bool metadata_removed = false; + { + auto probe_result = cache.probe(hash, 0, 4096, context); + metadata_removed = + probe_result.holder.file_blocks.empty() && probe_result.gaps.size() == 1; + } + if (metadata_removed && !fs::exists(cache_file)) { + removed = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(removed); + } + fs::remove_all(path, error); +} + TEST_F(BlockFileCacheTest, ProbeObservesDownloadingAndEmptyBlocksWithoutTakingDownloader) { const auto path = caches_dir / "block_file_cache_probe_states"; std::error_code error; diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index bbbea1fe2f6600..107da8fc5abd6c 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -141,6 +141,17 @@ TEST(FileCacheProfileReporterTest, DiffReturnsZeroWithoutNewData) { make_file_cache_stats(0)); } +TEST(FileCacheProfileReporterTest, MergeIncludesEveryAsyncReadAndWriteField) { + auto aggregate = make_file_cache_stats(2); + aggregate.merge_from(make_file_cache_stats(3)); + + auto expected = make_file_cache_stats(5); + // These two pre-existing fields merge by OR/max instead of addition. + expected.remote_only_on_miss_triggered = true; + expected.remote_only_on_miss_threshold_bytes = 90; + expect_file_cache_stats_eq(aggregate, expected); +} + TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTotals) { auto profile = std::make_unique("test_profile"); io::FileCacheProfileReporter reporter(profile.get()); @@ -167,9 +178,21 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot after_second_report.async_cache_write_submitted); EXPECT_EQ(profile->get_counter("AsyncCacheWriteRejected")->value(), after_second_report.async_cache_write_rejected); + EXPECT_EQ(profile->get_counter("AsyncCacheWriteBufferAllocFail")->value(), + after_second_report.async_cache_write_buffer_alloc_fail); + EXPECT_EQ(profile->get_counter("AsyncCacheWriteDropStaleEpoch")->value(), + after_second_report.async_cache_write_drop_stale_epoch); EXPECT_EQ(profile->get_counter("InflightWriteBufferIndexHit")->value(), after_second_report.inflight_write_buffer_index_hit); + EXPECT_EQ(profile->get_counter("InflightWriteBufferIndexMiss")->value(), + after_second_report.inflight_write_buffer_index_miss); + EXPECT_EQ(profile->get_counter("ProbeDownloadedHit")->value(), + after_second_report.probe_downloaded_hit); + EXPECT_EQ(profile->get_counter("ProbeDownloadingHit")->value(), + after_second_report.probe_downloading_hit); EXPECT_EQ(profile->get_counter("ProbeMiss")->value(), after_second_report.probe_miss); + EXPECT_EQ(profile->get_counter("BlockWaitSuccess")->value(), + after_second_report.block_wait_success); EXPECT_EQ(profile->get_counter("BlockWaitTimeout")->value(), after_second_report.block_wait_timeout); } diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index d697a79031feea..b9695fd3b1fd7f 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -73,8 +73,463 @@ class CountingFileReader final : public FileReader { size_t _last_size {0}; }; +// Provides one isolated cache disk with phase-one async-read settings for concise end-to-end +// reader tests. Every test gets a fresh FileCacheFactory entry and restores global knobs afterward. +class AsyncCachedRemoteFileReaderTest : public BlockFileCacheTest { +protected: + void SetUp() override { + _old_enable_async = config::enable_async_file_cache_write; + _old_enable_inflight = config::enable_inflight_write_buffer_index; + _old_enable_direct = config::enable_read_cache_file_directly; + _old_enable_peer = config::enable_cache_read_from_peer; + _old_block_size = config::file_cache_each_block_size; + _old_wait_timeout_ms = config::block_cache_wait_timeout_ms; + + config::enable_async_file_cache_write = true; + config::enable_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + reset_async_reader_cache_factory(); + } + + void TearDown() override { + reset_async_reader_cache_factory(); + if (!_cache_path.empty()) { + std::error_code error; + fs::remove_all(_cache_path, error); + } + config::enable_async_file_cache_write = _old_enable_async; + config::enable_inflight_write_buffer_index = _old_enable_inflight; + config::enable_read_cache_file_directly = _old_enable_direct; + config::enable_cache_read_from_peer = _old_enable_peer; + config::file_cache_each_block_size = _old_block_size; + config::block_cache_wait_timeout_ms = _old_wait_timeout_ms; + } + + // Create and initialize the single cache disk used by the current test. + void create_cache(std::string_view name) { + _cache_path = caches_dir / name; + std::error_code error; + fs::remove_all(_cache_path, error); + fs::create_directories(_cache_path); + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(_cache_path.string(), async_reader_cache_settings()) + .ok()); + _cache = FileCacheFactory::instance()->_path_to_cache[_cache_path.string()]; + ASSERT_NE(_cache, nullptr); + wait_until_cache_ready(*_cache); + } + + // Open the deterministic 10MB fixture file used as the remote source in cache tests. + FileReaderSPtr open_remote_file() { + FileReaderSPtr reader; + EXPECT_TRUE(global_local_filesystem()->open_file(tmp_file, &reader).ok()); + return reader; + } + + // Wrap a supplied remote source with the normal file-block-cache reader options. + std::shared_ptr create_reader(FileReaderSPtr remote_reader) { + FileReaderOptions options; + options.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + options.is_doris_table = true; + options.tablet_id = 10086; + return std::make_shared(std::move(remote_reader), options); + } + + // Wait until accepted writes and their inflight payload entries have both been finalized. + void wait_for_async_writes() { + ASSERT_NE(_cache, nullptr); + for (int attempt = 0; + attempt < 5000 && (_cache->async_write_service()->pending_count() != 0 || + _cache->inflight_write_buffer_index()->size() != 0); + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(_cache->async_write_service()->pending_count(), 0); + EXPECT_EQ(_cache->inflight_write_buffer_index()->size(), 0); + } + + BlockFileCache* cache() const { return _cache; } + +private: + fs::path _cache_path; + BlockFileCache* _cache = nullptr; + bool _old_enable_async = false; + bool _old_enable_inflight = false; + bool _old_enable_direct = false; + bool _old_enable_peer = false; + int64_t _old_block_size = 0; + int64_t _old_wait_timeout_ms = 0; +}; + } // namespace +TEST_F(AsyncCachedRemoteFileReaderTest, + missing_cache_file_falls_back_and_removes_the_complete_cache_hash) { + create_cache("cached_remote_reader_async_self_heal"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + std::string initial_result(2_mb, '\0'); + FileCacheStatistics initial_stats; + IOContext initial_context; + initial_context.file_cache_stats = &initial_stats; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, Slice(initial_result.data(), initial_result.size()), &bytes_read, + &initial_context) + .ok()); + ASSERT_EQ(bytes_read, initial_result.size()); + wait_for_async_writes(); + + std::vector cache_files; + { + ReadStatistics probe_stats; + CacheContext probe_context; + probe_context.stats = &probe_stats; + auto probe_result = cache()->probe(reader->_cache_hash, 0, 2_mb, probe_context); + ASSERT_EQ(probe_result.holder.file_blocks.size(), 2); + for (const auto& block : probe_result.holder.file_blocks) { + ASSERT_EQ(block->state(), FileBlock::State::DOWNLOADED); + cache_files.emplace_back(block->get_cache_file()); + } + } + ASSERT_EQ(cache_files.size(), 2); + ASSERT_NE(cache_files[0], cache_files[1]); + std::error_code remove_error; + ASSERT_TRUE(fs::remove(cache_files[0], remove_error)); + ASSERT_FALSE(remove_error); + + const uint64_t epoch_before_self_heal = cache()->async_write_service()->current_write_epoch(); + std::string fallback_result(4096, '\0'); + FileCacheStatistics fallback_stats; + IOContext fallback_context; + fallback_context.file_cache_stats = &fallback_stats; + bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, Slice(fallback_result.data(), fallback_result.size()), + &bytes_read, &fallback_context) + .ok()); + EXPECT_EQ(bytes_read, fallback_result.size()); + EXPECT_EQ(fallback_result, std::string(fallback_result.size(), '0')); + EXPECT_EQ(counting_reader->read_count(), 2); + EXPECT_EQ(fallback_stats.bytes_read_from_remote, fallback_result.size()); + EXPECT_EQ(fallback_stats.async_cache_write_submitted, 0); + EXPECT_EQ(cache()->async_write_service()->current_write_epoch(), epoch_before_self_heal + 1); + + bool hash_removed = false; + for (int attempt = 0; attempt < 5000; ++attempt) { + ReadStatistics probe_stats; + CacheContext probe_context; + probe_context.stats = &probe_stats; + bool metadata_removed = false; + { + auto probe_result = cache()->probe(reader->_cache_hash, 0, 2_mb, probe_context); + metadata_removed = probe_result.holder.file_blocks.empty() && + probe_result.gaps.size() == 1 && + probe_result.gaps.front().left == 0 && + probe_result.gaps.front().right == 2_mb - 1; + } + const bool files_removed = + std::none_of(cache_files.begin(), cache_files.end(), + [](const fs::path& path) { return fs::exists(path); }); + if (metadata_removed && files_removed) { + hash_removed = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(hash_removed); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + downloading_cache_block_timeout_falls_back_without_duplicate_submission) { + create_cache("cached_remote_reader_async_wait_timeout"); + config::block_cache_wait_timeout_ms = 10; + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + std::mutex mutex; + std::condition_variable cv; + bool downloader_ready = false; + bool release_downloader = false; + std::thread downloader([&]() { + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache()->get_or_set(reader->_cache_hash, 0, 1_mb, cache_context); + DORIS_CHECK(holder.file_blocks.size() == 1); + DORIS_CHECK(holder.file_blocks.front()->get_or_set_downloader() == + FileBlock::get_caller_id()); + { + std::lock_guard lock(mutex); + downloader_ready = true; + } + cv.notify_all(); + std::unique_lock lock(mutex); + cv.wait(lock, [&]() { return release_downloader; }); + }); + Defer stop_downloader {[&]() { + { + std::lock_guard lock(mutex); + release_downloader = true; + } + cv.notify_all(); + if (downloader.joinable()) { + downloader.join(); + } + }}; + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return downloader_ready; })); + } + + std::string result(4096, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(result, std::string(result.size(), '0')); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 0); + EXPECT_EQ(counting_reader->last_size(), 1_mb); + EXPECT_EQ(stats.probe_downloading_hit, 1); + EXPECT_EQ(stats.block_wait_success, 0); + EXPECT_EQ(stats.block_wait_timeout, 1); + EXPECT_EQ(stats.bytes_read_from_remote, result.size()); + EXPECT_EQ(stats.async_cache_write_submitted, 0); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + direct_cache_prefix_is_preserved_while_async_path_fills_the_unread_suffix) { + create_cache("cached_remote_reader_async_direct_prefix"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache()->get_or_set(reader->_cache_hash, 0, 4096, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& prefix_block = holder.file_blocks.front(); + ASSERT_EQ(prefix_block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string cached_prefix(4096, 'x'); + ASSERT_TRUE(prefix_block->append(Slice(cached_prefix.data(), cached_prefix.size())).ok()); + ASSERT_TRUE(prefix_block->finalize().ok()); + config::enable_read_cache_file_directly = true; + reader->_insert_file_reader(prefix_block); + + std::string result(8192, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result.substr(0, 4096), cached_prefix); + EXPECT_EQ(result.substr(4096), std::string(4096, '0')); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 0); + EXPECT_EQ(counting_reader->last_size(), 1_mb); + EXPECT_EQ(stats.bytes_read_from_local, 4096); + EXPECT_EQ(stats.bytes_read_from_remote, 4096); + EXPECT_EQ(stats.async_cache_write_submitted, 1); + wait_for_async_writes(); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, concurrent_cold_reads_publish_only_one_async_write_task) { + create_cache("cached_remote_reader_async_concurrent_dedup"); + auto first_counting_reader = std::make_shared(open_remote_file()); + auto second_counting_reader = std::make_shared(open_remote_file()); + auto first_reader = create_reader(first_counting_reader); + auto second_reader = create_reader(second_counting_reader); + + std::mutex insert_mutex; + std::condition_variable insert_cv; + size_t insert_arrivals = 0; + bool release_inserters = false; + std::mutex worker_mutex; + std::condition_variable worker_cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard insert_guard; + SyncPoint::CallbackGuard worker_guard; + sync_point->set_call_back( + "CachedRemoteFileReader::_submit_async_write_tasks:before_inflight_insert", + [&](auto&&) { + std::unique_lock lock(insert_mutex); + ++insert_arrivals; + if (insert_arrivals == 2) { + release_inserters = true; + insert_cv.notify_all(); + } + insert_cv.wait(lock, [&]() { return release_inserters; }); + }, + &insert_guard); + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(worker_mutex); + worker_entered = true; + worker_cv.notify_all(); + worker_cv.wait(lock, [&]() { return release_worker; }); + }, + &worker_guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(insert_mutex); + release_inserters = true; + } + insert_cv.notify_all(); + { + std::lock_guard lock(worker_mutex); + release_worker = true; + } + worker_cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + std::string first_result(4096, '\0'); + std::string second_result(4096, '\0'); + FileCacheStatistics first_stats; + FileCacheStatistics second_stats; + IOContext first_context; + IOContext second_context; + first_context.file_cache_stats = &first_stats; + second_context.file_cache_stats = &second_stats; + size_t first_bytes_read = 0; + size_t second_bytes_read = 0; + auto first_future = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return first_reader->read_at(0, Slice(first_result.data(), first_result.size()), + &first_bytes_read, &first_context); + }); + auto second_future = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return second_reader->read_at(0, Slice(second_result.data(), second_result.size()), + &second_bytes_read, &second_context); + }); + const auto first_status = first_future.wait_for(std::chrono::seconds(5)); + const auto second_status = second_future.wait_for(std::chrono::seconds(5)); + if (first_status != std::future_status::ready || second_status != std::future_status::ready) { + { + std::lock_guard lock(insert_mutex); + release_inserters = true; + } + insert_cv.notify_all(); + { + std::lock_guard lock(worker_mutex); + release_worker = true; + } + worker_cv.notify_all(); + } + ASSERT_EQ(first_status, std::future_status::ready); + ASSERT_EQ(second_status, std::future_status::ready); + ASSERT_TRUE(first_future.get().ok()); + ASSERT_TRUE(second_future.get().ok()); + EXPECT_EQ(first_result, std::string(first_result.size(), '0')); + EXPECT_EQ(second_result, std::string(second_result.size(), '0')); + EXPECT_EQ(first_counting_reader->read_count(), 1); + EXPECT_EQ(second_counting_reader->read_count(), 1); + EXPECT_EQ(first_stats.async_cache_write_submitted + second_stats.async_cache_write_submitted, + 1); + EXPECT_EQ(cache()->async_write_service()->pending_count(), 1); + EXPECT_EQ(cache()->inflight_write_buffer_index()->size(), 1); + { + std::unique_lock lock(worker_mutex); + ASSERT_TRUE(worker_cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return worker_entered; })); + release_worker = true; + } + worker_cv.notify_all(); + wait_for_async_writes(); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + async_buffer_allocation_failure_returns_remote_data_without_publishing_state) { + create_cache("cached_remote_reader_async_allocation_failure"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::allocate_tracked_buffer:inject_failure", + [&](auto&& values) { + auto* status = try_any_cast(values.back()); + *status = Status::MemoryAllocFailed("injected async write buffer failure"); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + std::string result(4096, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(result, std::string(result.size(), '0')); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(stats.async_cache_write_buffer_alloc_fail, 1); + EXPECT_EQ(stats.async_cache_write_submitted, 0); + EXPECT_EQ(cache()->async_write_service()->pending_count(), 0); + EXPECT_EQ(cache()->inflight_write_buffer_index()->size(), 0); + EXPECT_GE(cache()->async_write_service()->stats().buffer_alloc_fail, 1); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + external_table_reader_uses_async_write_and_reuses_downloaded_cache) { + create_cache("cached_remote_reader_async_external_table"); + auto first_remote = std::make_shared(open_remote_file()); + FileReaderOptions options; + options.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + options.cache_base_path = cache()->get_base_path(); + options.mtime = 123; + auto first_reader = std::make_shared(first_remote, options); + + std::string first_result(4096, '\0'); + FileCacheStatistics first_stats; + IOContext first_context; + first_context.file_cache_stats = &first_stats; + size_t bytes_read = 0; + ASSERT_TRUE(first_reader + ->read_at(0, Slice(first_result.data(), first_result.size()), &bytes_read, + &first_context) + .ok()); + EXPECT_EQ(first_result, std::string(first_result.size(), '0')); + EXPECT_EQ(first_remote->read_count(), 1); + EXPECT_EQ(first_stats.async_cache_write_submitted, 1); + wait_for_async_writes(); + + auto second_remote = std::make_shared(open_remote_file()); + auto second_reader = std::make_shared(second_remote, options); + std::string second_result(4096, '\0'); + FileCacheStatistics second_stats; + IOContext second_context; + second_context.file_cache_stats = &second_stats; + bytes_read = 0; + ASSERT_TRUE(second_reader + ->read_at(4096, Slice(second_result.data(), second_result.size()), + &bytes_read, &second_context) + .ok()); + EXPECT_EQ(second_result, std::string(second_result.size(), '0')); + EXPECT_EQ(second_remote->read_count(), 0); + EXPECT_EQ(second_stats.probe_downloaded_hit, 1); + EXPECT_EQ(second_stats.async_cache_write_submitted, 0); +} + TEST_F(BlockFileCacheTest, direct_partial_hit_with_downloaded_remainder_should_not_read_remote_again) { std::string local_cache_base_path = @@ -534,6 +989,10 @@ TEST_F(BlockFileCacheTest, async_write_middle_hit_uses_one_remote_span_and_submi const std::string payload(1_mb, '1'); ASSERT_TRUE(middle_block->append(Slice(payload.data(), payload.size())).ok()); ASSERT_TRUE(middle_block->finalize().ok()); + { + std::lock_guard cache_lock(cache->_mutex); + cache->_files.at(reader->_cache_hash).at(1_mb).atime = 123; + } std::string result(3_mb, '\0'); FileCacheStatistics read_stats; @@ -554,6 +1013,10 @@ TEST_F(BlockFileCacheTest, async_write_middle_hit_uses_one_remote_span_and_submi EXPECT_EQ(read_stats.probe_downloaded_hit, 1); EXPECT_EQ(read_stats.probe_miss, 2); EXPECT_EQ(read_stats.async_cache_write_submitted, 2); + { + std::lock_guard cache_lock(cache->_mutex); + EXPECT_EQ(cache->_files.at(reader->_cache_hash).at(1_mb).atime, 123); + } for (int attempt = 0; attempt < 5000 && (cache->async_write_service()->pending_count() != 0 || cache->inflight_write_buffer_index()->size() != 0); @@ -687,10 +1150,12 @@ TEST_F(BlockFileCacheTest, async_write_backpressure_rolls_back_inflight_entry) { TEST_F(BlockFileCacheTest, cache_write_mode_is_resolved_for_each_read_context) { const bool old_enable_async = config::enable_async_file_cache_write; const bool old_enable_peer = config::enable_cache_read_from_peer; + const std::string old_deploy_mode = config::deploy_mode; config::enable_cache_read_from_peer = false; Defer restore_config {[&]() { config::enable_async_file_cache_write = old_enable_async; config::enable_cache_read_from_peer = old_enable_peer; + config::deploy_mode = old_deploy_mode; }}; reset_async_reader_cache_factory(); @@ -723,7 +1188,7 @@ TEST_F(BlockFileCacheTest, cache_write_mode_is_resolved_for_each_read_context) { context.cache_write_mode_override = CacheWriteMode::SYNC_WRITE; EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); - context.cache_write_mode_override.reset(); + context.cache_write_mode_override = CacheWriteMode::ASYNC_WRITE; context.is_dryrun = true; EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); context.is_dryrun = false; @@ -731,6 +1196,15 @@ TEST_F(BlockFileCacheTest, cache_write_mode_is_resolved_for_each_read_context) { EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); context.is_warmup = false; + config::deploy_mode = "cloud"; + config::enable_cache_read_from_peer = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + context.bypass_peer_read = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::ASYNC_WRITE); + + context.bypass_peer_read = false; + config::enable_cache_read_from_peer = false; + context.cache_write_mode_override.reset(); opts.cache_write_mode = CacheWriteMode::SYNC_WRITE; auto sync_reader = std::make_shared(local_reader, opts); EXPECT_EQ(sync_reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); diff --git a/be/test/storage/cloud_file_cache_write_index_only_test.cpp b/be/test/storage/cloud_file_cache_write_index_only_test.cpp index c330b93fee489b..49f43e80cf16b0 100644 --- a/be/test/storage/cloud_file_cache_write_index_only_test.cpp +++ b/be/test/storage/cloud_file_cache_write_index_only_test.cpp @@ -404,6 +404,8 @@ class CloudFileCacheWriteIndexOnlyTest : public testing::Test { EXPECT_TRUE(io_ctx->is_index_data); EXPECT_TRUE(io_ctx->is_dryrun); EXPECT_FALSE(io_ctx->is_warmup); + ASSERT_TRUE(io_ctx->cache_write_mode_override.has_value()); + EXPECT_EQ(*io_ctx->cache_write_mode_override, io::CacheWriteMode::SYNC_WRITE); observed->push_back(ObservedIndexPreload { .reason = ctx->reason, .segment_id = ctx->segment_id, From 97ff8ed069fa0e10bf209e36cd91ac84665126d3 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 16 Jul 2026 19:49:17 +0800 Subject: [PATCH 07/16] [refactor](be) Align file cache probe results with read blocks ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The async cache-write planner queried one block-aligned range, but `BlockFileCache::probe` returned a `FileBlocksHolder` of cache hits plus an independent gap list. The planner then had to scan all hits and gaps for every logical read block even though the read-plan blocks and probe slots use the same block boundaries. This obscured the alignment invariant and introduced unnecessary nested matching logic on the query read path. Change the probe contract to return one ordered nullable `FileBlock` pointer per aligned input block. A non-null slot is asserted to have the exact corresponding range, except that the final block may end at EOF, while a null slot directly represents a cache miss. The planner now preserves its inflight-first fast path and joins probe results to plan blocks by index; materialization also reads the matching slot directly. Remove `FileBlocksHolder` and the independent gaps from `FileBlocksProbeResult`. Preserve the existing deferred cleanup semantics for EMPTY and deleting cache blocks through a shared cache-user reference release helper rather than embedding a holder in the probe result. Update focused and end-to-end tests for hit/miss slots, a short final block, retained block states, self-heal cleanup, and an aligned direct-cache prefix followed by an async-written suffix. ### Release note None ### Check List (For Author) - Test: Unit Test - 51 related ASAN BE unit tests passed with `run-be-ut.sh` and `-j100` - 2 focused probe/direct-prefix ASAN BE unit tests passed with `run-be-ut.sh` and `-j100` - `build-support/check-format.sh` passed - Behavior changed: No; this simplifies an internal probe/planning contract without changing user-visible cache semantics - Does this need documentation: No --- be/src/io/cache/block_file_cache.cpp | 64 +++----- be/src/io/cache/block_file_cache.h | 17 +- be/src/io/cache/cached_remote_file_reader.h | 17 +- .../cached_remote_file_reader_async_write.cpp | 155 +++++++++--------- be/src/io/cache/file_block.cpp | 61 ++++--- be/src/io/cache/file_block.h | 6 + .../cache/async_cache_write_service_test.cpp | 21 +-- .../io/cache/block_file_cache_probe_test.cpp | 51 +++--- .../cache/cached_remote_file_reader_test.cpp | 27 +-- 9 files changed, 212 insertions(+), 207 deletions(-) diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index f21bd74ef460cb..3568f8816bc0b8 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -845,9 +845,9 @@ Status BlockFileCache::get_downloaded_blocks_if_fully_covered(const UInt128Wrapp FileBlocksProbeResult BlockFileCache::probe(const UInt128Wrapper& hash, size_t offset, size_t size, const CacheContext& context) { DORIS_CHECK(size > 0); - const FileBlock::Range range(offset, offset + size - 1); - FileBlocks blocks; - std::vector gaps; + DORIS_CHECK(_max_file_block_size > 0); + const size_t end = offset + size; + DORIS_CHECK(end > offset); std::lock_guard cache_lock(_mutex); auto file_iterator = _files.find(hash); @@ -860,46 +860,36 @@ FileBlocksProbeResult BlockFileCache::probe(const UInt128Wrapper& hash, size_t o _storage->load_blocks_directly_unlocked(this, key, cache_lock); file_iterator = _files.find(hash); } - if (file_iterator == _files.end()) { - gaps.emplace_back(range); - return FileBlocksProbeResult(std::move(blocks), std::move(gaps)); - } - auto& file_blocks = file_iterator->second; - DORIS_CHECK(!file_blocks.empty()); - auto block_iterator = file_blocks.lower_bound(range.left); - if (block_iterator != file_blocks.begin()) { - auto previous = std::prev(block_iterator); - if (previous->second.file_block->range().right >= range.left) { - block_iterator = previous; + FileBlocksByOffset* cached_blocks = nullptr; + FileBlocksByOffset::iterator cached_block; + if (file_iterator != _files.end()) { + DORIS_CHECK(!file_iterator->second.empty()); + cached_blocks = &file_iterator->second; + cached_block = cached_blocks->lower_bound(offset); + if (cached_block != cached_blocks->begin()) { + const auto& previous_range = std::prev(cached_block)->second.file_block->range(); + DORIS_CHECK(previous_range.right < offset); } } - size_t current = range.left; - for (; block_iterator != file_blocks.end(); ++block_iterator) { - const auto& block = block_iterator->second.file_block; - const auto& block_range = block->range(); - if (block_range.left > range.right) { - break; - } - if (block_range.right < range.left) { - continue; + std::vector result; + result.reserve(size / _max_file_block_size + (size % _max_file_block_size != 0)); + for (size_t block_offset = offset; block_offset < end;) { + const size_t block_size = std::min(_max_file_block_size, end - block_offset); + const FileBlock::Range expected_range(block_offset, block_offset + block_size - 1); + FileBlockSPtr file_block; + if (cached_blocks != nullptr && cached_block != cached_blocks->end() && + cached_block->second.file_block->range().left <= expected_range.right) { + file_block = cached_block->second.file_block; + DORIS_CHECK(file_block->range().left == expected_range.left); + DORIS_CHECK(file_block->range().right == expected_range.right); + ++cached_block; } - const size_t clipped_left = std::max(block_range.left, range.left); - const size_t clipped_right = std::min(block_range.right, range.right); - if (current < clipped_left) { - gaps.emplace_back(current, clipped_left - 1); - } - blocks.emplace_back(block); - current = clipped_right + 1; - if (current > range.right) { - break; - } - } - if (current <= range.right) { - gaps.emplace_back(current, range.right); + result.emplace_back(std::move(file_block)); + block_offset += block_size; } - return FileBlocksProbeResult(std::move(blocks), std::move(gaps)); + return FileBlocksProbeResult(std::move(result)); } void BlockFileCache::touch_probe_block_if_cached(const FileBlockSPtr& block, diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index c52129242d0dac..8324b77d770ba5 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -83,15 +84,17 @@ class LockScopedTimer { class FSFileCacheStorage; struct FileBlocksProbeResult { - FileBlocksProbeResult(FileBlocks blocks, std::vector gaps_) - : holder(std::move(blocks)), gaps(std::move(gaps_)) {} + explicit FileBlocksProbeResult(std::vector file_blocks_) + : file_blocks(std::move(file_blocks_)) {} FileBlocksProbeResult(FileBlocksProbeResult&&) noexcept = default; FileBlocksProbeResult& operator=(FileBlocksProbeResult&&) noexcept = delete; FileBlocksProbeResult(const FileBlocksProbeResult&) = delete; FileBlocksProbeResult& operator=(const FileBlocksProbeResult&) = delete; + ~FileBlocksProbeResult(); - FileBlocksHolder holder; - std::vector gaps; + /// One entry per cache-block-sized input slot, in offset order. A null entry is a cache miss; + /// a non-null entry has the exact slot range, except that the final slot may end at EOF. + std::vector file_blocks; }; // NeedUpdateLRUBlocks keeps FileBlockSPtr entries that require LRU updates in a @@ -254,8 +257,10 @@ class BlockFileCache { FileBlocksHolder get_or_set(const UInt128Wrapper& hash, size_t offset, size_t size, CacheContext& context); - /// Return existing blocks and uncovered gaps for `[offset, offset + size)` without creating - /// cache cells or touching LRU state. `context` supplies cache-type visibility rules. + /// Probe the block-aligned `[offset, offset + size)` range without creating cache cells or + /// touching LRU state. The result contains one ordered slot per cache block; each slot is null + /// on miss or owns the exactly aligned existing block. `context` supplies cache metadata when + /// lazy loading is required. FileBlocksProbeResult probe(const UInt128Wrapper& hash, size_t offset, size_t size, const CacheContext& context); diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h index 364bad55ec9653..c74c95ffb7f5fa 100644 --- a/be/src/io/cache/cached_remote_file_reader.h +++ b/be/src/io/cache/cached_remote_file_reader.h @@ -170,13 +170,14 @@ class CachedRemoteFileReader final : public FileReader, /// Build block-aligned source coverage for the unread suffix. It first performs one inflight /// batch lookup; a fully covered request returns without taking the BlockFileCache lock, while - /// incomplete coverage triggers one read-only cache probe for the whole aligned range. + /// incomplete coverage triggers one read-only whole-range probe with one result per aligned + /// block. /// @param[in] remaining_offset First user byte not filled by the direct-cache path. /// @param[in] remaining_size Number of unread user bytes. /// @param[in] write_epoch Epoch captured before any lookup or remote IO. /// @param[in] io_ctx Context used to build the cache admission/probe context. /// @param[in,out] stats Lookup and probe counters updated during planning. - /// @return Plan that owns any required probe holder and first-to-last remote range. + /// @return Plan that owns any retained probe blocks and first-to-last remote range. AsyncReadPlan _build_async_read_plan(size_t remaining_offset, size_t remaining_size, uint64_t write_epoch, const IOContext* io_ctx, ReadStatistics& stats); @@ -184,8 +185,8 @@ class CachedRemoteFileReader final : public FileReader, /// Copy one block already available from an inflight buffer or downloaded cache file. Cache /// state is revalidated before IO; a race is reported to the caller as a simple full-range /// remote fallback. - /// @param[in] plan Plan owning the probe holder and user boundaries. - /// @param[in] read_block Aligned block classified as INFLIGHT, CACHE, or DOWNLOADING. + /// @param[in] plan Plan owning the probed blocks and user boundaries. + /// @param[in] block_index Index of the aligned block and its matching probe result. /// @param[in] user_offset Original user request offset used to locate the destination slice. /// @param[out] result Destination buffer for the complete user request. /// @param[in] cache_context Context used only when a successful local read touches LRU. @@ -193,10 +194,10 @@ class CachedRemoteFileReader final : public FileReader, /// @param[in,out] materialized_bytes User bytes copied from cache or inflight memory. /// @param[in,out] need_self_heal Set when a cache file disappears during a local read. /// @return true when the block was copied; false when the caller should use remote fallback. - bool _materialize_async_block(const AsyncReadPlan& plan, const AsyncReadBlock& read_block, - size_t user_offset, Slice result, - const CacheContext& cache_context, ReadStatistics& stats, - size_t* materialized_bytes, bool* need_self_heal); + bool _materialize_async_block(const AsyncReadPlan& plan, size_t block_index, size_t user_offset, + Slice result, const CacheContext& cache_context, + ReadStatistics& stats, size_t* materialized_bytes, + bool* need_self_heal); /// Copy only blocks before the first and after the last REMOTE block. When no REMOTE block /// exists, copy the entire request. DOWNLOADING blocks outside the remote span keep the diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp index d426ae60ecc411..4a9b3f65448194 100644 --- a/be/src/io/cache/cached_remote_file_reader_async_write.cpp +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -122,8 +122,7 @@ CacheWriteMode CachedRemoteFileReader::_resolve_cache_write_mode(const IOContext // Build a deliberately small plan. The inflight index is checked first so a fully covered read // avoids BlockFileCache::probe and its cache mutex. Only an incomplete inflight lookup performs one -// whole-range cache probe; the per-block scans favor readability because read_at normally spans -// very few cache blocks. +// whole-range probe whose result entries map directly to the logical plan blocks. CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_plan( size_t remaining_offset, size_t remaining_size, uint64_t write_epoch, const IOContext* io_ctx, ReadStatistics& stats) { @@ -140,7 +139,8 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ block_offset += cache_block_size) { const size_t block_size = std::min(cache_block_size, size() - block_offset); DORIS_CHECK(block_size > 0); - plan.blocks.emplace_back(FileBlock::Range(block_offset, block_offset + block_size - 1)); + const FileBlock::Range block_range(block_offset, block_offset + block_size - 1); + plan.blocks.emplace_back(block_range); block_offsets.emplace_back(block_offset); } DORIS_CHECK(!plan.blocks.empty()); @@ -180,42 +180,42 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ plan.probe_result.emplace(_cache->probe(_cache_hash, align_left, align_size, cache_context)); g_cached_remote_reader_probe_total << 1; const auto& probe_result = *plan.probe_result; + DORIS_CHECK(probe_result.file_blocks.size() == plan.blocks.size()); - const auto overlaps = [](const FileBlock::Range& lhs, const FileBlock::Range& rhs) { - return lhs.left <= rhs.right && rhs.left <= lhs.right; - }; for (size_t index = 0; index < plan.blocks.size(); ++index) { auto& read_block = plan.blocks[index]; if (read_block.source == AsyncReadBlock::Source::INFLIGHT) { continue; } - bool has_miss = std::any_of( - probe_result.gaps.begin(), probe_result.gaps.end(), - [&](const FileBlock::Range& gap) { return overlaps(gap, read_block.range); }); - bool has_downloading = false; - bool has_cache_block = false; - for (const auto& block : probe_result.holder.file_blocks) { - if (!overlaps(block->range(), read_block.range)) { - continue; - } - has_cache_block = true; - if (_cache->is_block_deleting(block)) { - has_miss = true; + const auto& file_block = probe_result.file_blocks[index]; + bool is_miss = file_block == nullptr; + bool is_downloading = false; + if (file_block != nullptr) { + DORIS_CHECK(file_block->range().left == read_block.range.left); + DORIS_CHECK(file_block->range().right == read_block.range.right); + if (_cache->is_block_deleting(file_block)) { + is_miss = true; } else { - const FileBlock::State state = block->state(); - has_miss = has_miss || state == FileBlock::State::EMPTY || - state == FileBlock::State::SKIP_CACHE; - has_downloading = has_downloading || state == FileBlock::State::DOWNLOADING; + switch (file_block->state()) { + case FileBlock::State::DOWNLOADED: + break; + case FileBlock::State::DOWNLOADING: + is_downloading = true; + break; + case FileBlock::State::EMPTY: + case FileBlock::State::SKIP_CACHE: + is_miss = true; + break; + } } } - DORIS_CHECK(has_miss || has_cache_block); - if (has_miss) { + if (is_miss) { read_block.submit_write = true; ++stats.probe_miss; g_cached_remote_reader_probe_miss << 1; - } else if (has_downloading) { + } else if (is_downloading) { read_block.source = AsyncReadBlock::Source::DOWNLOADING; ++stats.probe_downloading_hit; g_cached_remote_reader_probe_downloading << 1; @@ -238,10 +238,14 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ // Copy one block available from inflight memory or cache. DOWNLOADING blocks outside the remote // span retain the existing wait semantics. Any race or read failure asks the caller to replace the // whole planned request with one remote read instead of incrementally repairing the range. -bool CachedRemoteFileReader::_materialize_async_block( - const AsyncReadPlan& plan, const AsyncReadBlock& read_block, size_t user_offset, - Slice result, const CacheContext& cache_context, ReadStatistics& stats, - size_t* materialized_bytes, bool* need_self_heal) { +bool CachedRemoteFileReader::_materialize_async_block(const AsyncReadPlan& plan, size_t block_index, + size_t user_offset, Slice result, + const CacheContext& cache_context, + ReadStatistics& stats, + size_t* materialized_bytes, + bool* need_self_heal) { + DORIS_CHECK(block_index < plan.blocks.size()); + const auto& read_block = plan.blocks[block_index]; DORIS_CHECK(read_block.source != AsyncReadBlock::Source::REMOTE); DORIS_CHECK(materialized_bytes != nullptr); DORIS_CHECK(need_self_heal != nullptr); @@ -265,64 +269,54 @@ bool CachedRemoteFileReader::_materialize_async_block( } DORIS_CHECK(plan.probe_result.has_value()); - size_t local_read_bytes = 0; - std::vector consumed_blocks; - for (const auto& block : plan.probe_result->holder.file_blocks) { - if (block->range().right < copy_left || block->range().left > copy_right) { - continue; - } - if (_cache->is_block_deleting(block)) { - return false; - } + DORIS_CHECK(block_index < plan.probe_result->file_blocks.size()); + const auto& file_block = plan.probe_result->file_blocks[block_index]; + DORIS_CHECK(file_block != nullptr); + DORIS_CHECK(file_block->range().left == read_block.range.left); + DORIS_CHECK(file_block->range().right == read_block.range.right); + if (_cache->is_block_deleting(file_block)) { + return false; + } - FileBlock::State state = block->state(); - if (state == FileBlock::State::DOWNLOADING) { - DORIS_CHECK(read_block.source == AsyncReadBlock::Source::DOWNLOADING); - { - SCOPED_RAW_TIMER(&stats.remote_wait_timer); - state = block->wait(); - } - if (state != FileBlock::State::DOWNLOADED) { - ++stats.block_wait_timeout; - g_cached_remote_reader_block_wait_timeout << 1; - return false; - } - ++stats.block_wait_success; - g_cached_remote_reader_block_wait << 1; + FileBlock::State state = file_block->state(); + if (state == FileBlock::State::DOWNLOADING) { + DORIS_CHECK(read_block.source == AsyncReadBlock::Source::DOWNLOADING); + { + SCOPED_RAW_TIMER(&stats.remote_wait_timer); + state = file_block->wait(); } if (state != FileBlock::State::DOWNLOADED) { + ++stats.block_wait_timeout; + g_cached_remote_reader_block_wait_timeout << 1; return false; } + ++stats.block_wait_success; + g_cached_remote_reader_block_wait << 1; + } + if (state != FileBlock::State::DOWNLOADED) { + return false; + } - const size_t read_left = std::max(block->range().left, copy_left); - const size_t read_right = std::min(block->range().right, copy_right); - const size_t read_size = read_right - read_left + 1; - Status status; - { - SCOPED_RAW_TIMER(&stats.local_read_timer); - status = block->read(Slice(result.data + (read_left - user_offset), read_size), - read_left - block->range().left); - } - if (!status.ok()) { - if (status.is()) { - *need_self_heal = true; - g_read_cache_self_heal_on_not_found << 1; - } - LOG_EVERY_N(WARNING, 100) - << "Read probed file cache block failed, falling back to remote. path=" - << path().native() << ", hash=" << _cache_hash.to_string() - << ", offset=" << block->offset() << ", status=" << status; - return false; + Status status; + { + SCOPED_RAW_TIMER(&stats.local_read_timer); + status = file_block->read(Slice(result.data + (copy_left - user_offset), copy_size), + copy_left - file_block->range().left); + } + if (!status.ok()) { + if (status.is()) { + *need_self_heal = true; + g_read_cache_self_heal_on_not_found << 1; } - local_read_bytes += read_size; - consumed_blocks.emplace_back(block); + LOG_EVERY_N(WARNING, 100) + << "Read probed file cache block failed, falling back to remote. path=" + << path().native() << ", hash=" << _cache_hash.to_string() + << ", offset=" << file_block->offset() << ", status=" << status; + return false; } - DORIS_CHECK(local_read_bytes == copy_size); - for (const auto& block : consumed_blocks) { - _cache->touch_probe_block_if_cached(block, cache_context); - } - *materialized_bytes += local_read_bytes; + _cache->touch_probe_block_if_cached(file_block, cache_context); + *materialized_bytes += copy_size; return true; } @@ -339,9 +333,8 @@ bool CachedRemoteFileReader::_materialize_async_cached_sides( size_t materialized_bytes = 0; const auto materialize_range = [&](size_t begin, size_t end) { for (size_t index = begin; index < end; ++index) { - if (!_materialize_async_block(plan, plan.blocks[index], user_offset, result, - cache_context, stats, &materialized_bytes, - need_self_heal)) { + if (!_materialize_async_block(plan, index, user_offset, result, cache_context, stats, + &materialized_bytes, need_self_heal)) { return false; } } diff --git a/be/src/io/cache/file_block.cpp b/be/src/io/cache/file_block.cpp index 944e93baa419bb..b7ffa7a3418c3a 100644 --- a/be/src/io/cache/file_block.cpp +++ b/be/src/io/cache/file_block.cpp @@ -331,36 +331,47 @@ std::string FileBlock::get_cache_file() const { return _mgr->_storage->get_local_file(this->_key); } -FileBlocksHolder::~FileBlocksHolder() { - for (auto file_block_it = file_blocks.begin(); file_block_it != file_blocks.end();) { - auto current_file_block_it = file_block_it; - auto& file_block = *current_file_block_it; - BlockFileCache* _mgr = file_block->_mgr; +void FileBlock::release_cache_user_reference(std::shared_ptr& file_block) { + if (!file_block) { + return; + } + BlockFileCache* mgr = file_block->_mgr; + { + bool should_remove = false; { - bool should_remove = false; - { - std::lock_guard block_lock(file_block->_mutex); - file_block->complete_unlocked(block_lock); - if (file_block.use_count() == 2 && - (file_block->is_deleting() || - file_block->state_unlock(block_lock) == FileBlock::State::EMPTY)) { - should_remove = true; - } + std::lock_guard block_lock(file_block->_mutex); + file_block->complete_unlocked(block_lock); + if (file_block.use_count() == 2 && + (file_block->is_deleting() || + file_block->state_unlock(block_lock) == FileBlock::State::EMPTY)) { + should_remove = true; } - if (should_remove) { - SCOPED_CACHE_LOCK(_mgr->_mutex, _mgr); - std::lock_guard block_lock(file_block->_mutex); - if (file_block.use_count() == 2) { - DCHECK(file_block->state_unlock(block_lock) != FileBlock::State::DOWNLOADING); - // one in cache, one in here - if (file_block->is_deleting() || - file_block->state_unlock(block_lock) == FileBlock::State::EMPTY) { - _mgr->remove(file_block, cache_lock, block_lock, false); - } + } + if (should_remove) { + SCOPED_CACHE_LOCK(mgr->_mutex, mgr); + std::lock_guard block_lock(file_block->_mutex); + if (file_block.use_count() == 2) { + DCHECK(file_block->state_unlock(block_lock) != FileBlock::State::DOWNLOADING); + // one in cache, one held by the cache user + if (file_block->is_deleting() || + file_block->state_unlock(block_lock) == FileBlock::State::EMPTY) { + mgr->remove(file_block, cache_lock, block_lock, false); } } } - file_block_it = file_blocks.erase(current_file_block_it); + } + file_block.reset(); +} + +FileBlocksHolder::~FileBlocksHolder() { + for (auto& file_block : file_blocks) { + FileBlock::release_cache_user_reference(file_block); + } +} + +FileBlocksProbeResult::~FileBlocksProbeResult() { + for (auto& file_block : file_blocks) { + FileBlock::release_cache_user_reference(file_block); } } diff --git a/be/src/io/cache/file_block.h b/be/src/io/cache/file_block.h index 6ddd78e717e00c..08b4d3be3afb27 100644 --- a/be/src/io/cache/file_block.h +++ b/be/src/io/cache/file_block.h @@ -40,11 +40,13 @@ namespace doris { namespace io { struct FileBlocksHolder; +struct FileBlocksProbeResult; class BlockFileCache; struct FileBlockCell; class FileBlock { friend struct FileBlocksHolder; + friend struct FileBlocksProbeResult; friend class BlockFileCache; friend class CachedRemoteFileReader; friend struct FileBlockCell; @@ -163,6 +165,10 @@ class FileBlock { void reset_downloader_impl(std::lock_guard& block_lock); + /// Release one cache-user reference and complete deferred EMPTY/deleting block cleanup when it + /// is the last reference outside the cache map. + static void release_cache_user_reference(std::shared_ptr& file_block); + Range _block_range; State _download_state; diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp index d43f108e2409d7..e8764cb2f2798f 100644 --- a/be/test/io/cache/async_cache_write_service_test.cpp +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -268,10 +268,8 @@ TEST_F(AsyncCacheWriteServiceTest, WatchdogDropsExpiredTaskAndCleansInflightEntr CacheContext context; context.stats = &read_stats; auto probe_result = cache->probe(hash, 0, 4096, context); - EXPECT_TRUE(probe_result.holder.file_blocks.empty()); - ASSERT_EQ(probe_result.gaps.size(), 1); - EXPECT_EQ(probe_result.gaps.front().left, 0); - EXPECT_EQ(probe_result.gaps.front().right, 4095); + ASSERT_EQ(probe_result.file_blocks.size(), 1); + EXPECT_EQ(probe_result.file_blocks[0], nullptr); } TEST_F(AsyncCacheWriteServiceTest, ShutdownDrainsAcceptedTask) { @@ -478,9 +476,10 @@ TEST_F(AsyncCacheWriteServiceTest, RemoveDuringAppendDoesNotLeaveResurrectedCach CacheContext probe_context; probe_context.stats = &probe_stats; auto probe_result = cache->probe(hash, 0, 4096, probe_context); - ASSERT_EQ(probe_result.holder.file_blocks.size(), 1); - EXPECT_EQ(probe_result.holder.file_blocks.front()->state(), FileBlock::State::DOWNLOADING); - cache_file = probe_result.holder.file_blocks.front()->get_cache_file(); + ASSERT_EQ(probe_result.file_blocks.size(), 1); + ASSERT_NE(probe_result.file_blocks[0], nullptr); + EXPECT_EQ(probe_result.file_blocks[0]->state(), FileBlock::State::DOWNLOADING); + cache_file = probe_result.file_blocks[0]->get_cache_file(); } const uint64_t old_epoch = service->current_write_epoch(); cache->remove_if_cached_async(hash); @@ -502,7 +501,7 @@ TEST_F(AsyncCacheWriteServiceTest, RemoveDuringAppendDoesNotLeaveResurrectedCach { auto probe_result = cache->probe(hash, 0, 4096, probe_context); metadata_removed = - probe_result.holder.file_blocks.empty() && probe_result.gaps.size() == 1; + probe_result.file_blocks.size() == 1 && probe_result.file_blocks[0] == nullptr; } if (metadata_removed && !fs::exists(cache_file)) { removed = true; @@ -666,10 +665,8 @@ TEST_F(AsyncCacheWriteServiceTest, RemoveInvalidatesActiveAndQueuedTasksAndClean context.stats = &read_stats; const auto expect_cache_gap = [&](const UInt128Wrapper& hash) { auto probe_result = cache->probe(hash, 0, 4096, context); - EXPECT_TRUE(probe_result.holder.file_blocks.empty()); - ASSERT_EQ(probe_result.gaps.size(), 1); - EXPECT_EQ(probe_result.gaps.front().left, 0); - EXPECT_EQ(probe_result.gaps.front().right, 4095); + ASSERT_EQ(probe_result.file_blocks.size(), 1); + EXPECT_EQ(probe_result.file_blocks[0], nullptr); }; expect_cache_gap(active_hash); expect_cache_gap(queued_hash); diff --git a/be/test/io/cache/block_file_cache_probe_test.cpp b/be/test/io/cache/block_file_cache_probe_test.cpp index 4c01415589fb6a..165281d036e528 100644 --- a/be/test/io/cache/block_file_cache_probe_test.cpp +++ b/be/test/io/cache/block_file_cache_probe_test.cpp @@ -56,11 +56,9 @@ TEST_F(BlockFileCacheTest, ProbeMissDoesNotCreateCacheCell) { context.stats = &stats; const size_t cache_size_before = cache._cur_cache_size; - auto result = cache.probe(hash, 1024, 4096, context); - EXPECT_TRUE(result.holder.file_blocks.empty()); - ASSERT_EQ(result.gaps.size(), 1); - EXPECT_EQ(result.gaps.front().left, 1024); - EXPECT_EQ(result.gaps.front().right, 5119); + auto result = cache.probe(hash, 0, 4096, context); + ASSERT_EQ(result.file_blocks.size(), 1); + EXPECT_EQ(result.file_blocks[0], nullptr); EXPECT_EQ(cache._cur_cache_size, cache_size_before); std::lock_guard cache_lock(cache._mutex); EXPECT_EQ(cache._files.find(hash), cache._files.end()); @@ -68,7 +66,7 @@ TEST_F(BlockFileCacheTest, ProbeMissDoesNotCreateCacheCell) { fs::remove_all(path, error); } -TEST_F(BlockFileCacheTest, ProbeReturnsDownloadedBlockAndIndependentGap) { +TEST_F(BlockFileCacheTest, ProbeReturnsDownloadedBlockAndNullMissSlot) { const auto path = caches_dir / "block_file_cache_probe_mixed"; std::error_code error; fs::remove_all(path, error); @@ -97,13 +95,12 @@ TEST_F(BlockFileCacheTest, ProbeReturnsDownloadedBlockAndIndependentGap) { } auto result = cache.probe(hash, 0, 8192, context); - ASSERT_EQ(result.holder.file_blocks.size(), 1); - EXPECT_EQ(result.holder.file_blocks.front()->state(), FileBlock::State::DOWNLOADED); - EXPECT_EQ(result.holder.file_blocks.front()->range().left, 0); - EXPECT_EQ(result.holder.file_blocks.front()->range().right, 4095); - ASSERT_EQ(result.gaps.size(), 1); - EXPECT_EQ(result.gaps.front().left, 4096); - EXPECT_EQ(result.gaps.front().right, 8191); + ASSERT_EQ(result.file_blocks.size(), 2); + ASSERT_NE(result.file_blocks[0], nullptr); + EXPECT_EQ(result.file_blocks[0]->state(), FileBlock::State::DOWNLOADED); + EXPECT_EQ(result.file_blocks[0]->range().left, 0); + EXPECT_EQ(result.file_blocks[0]->range().right, 4095); + EXPECT_EQ(result.file_blocks[1], nullptr); EXPECT_EQ(cache._cur_cache_size, cache_size_before); { std::lock_guard cache_lock(cache._mutex); @@ -148,8 +145,9 @@ TEST_F(BlockFileCacheTest, ProbeTouchAndRemovalRevalidateTheRetainedBlock) { const uint64_t old_epoch = cache.async_write_service()->current_write_epoch(); { auto probe_result = cache.probe(hash, 0, 4096, context); - ASSERT_EQ(probe_result.holder.file_blocks.size(), 1); - const auto& block = probe_result.holder.file_blocks.front(); + ASSERT_EQ(probe_result.file_blocks.size(), 1); + const auto& block = probe_result.file_blocks[0]; + ASSERT_NE(block, nullptr); cache_file = block->get_cache_file(); { std::lock_guard cache_lock(cache._mutex); @@ -174,8 +172,8 @@ TEST_F(BlockFileCacheTest, ProbeTouchAndRemovalRevalidateTheRetainedBlock) { bool metadata_removed = false; { auto probe_result = cache.probe(hash, 0, 4096, context); - metadata_removed = - probe_result.holder.file_blocks.empty() && probe_result.gaps.size() == 1; + metadata_removed = probe_result.file_blocks.size() == 1 && + probe_result.file_blocks[0] == nullptr; } if (metadata_removed && !fs::exists(cache_file)) { removed = true; @@ -210,7 +208,7 @@ TEST_F(BlockFileCacheTest, ProbeObservesDownloadingAndEmptyBlocksWithoutTakingDo ReadStatistics downloader_stats; CacheContext downloader_context; downloader_context.stats = &downloader_stats; - auto holder = cache.get_or_set(hash, 0, 8192, downloader_context); + auto holder = cache.get_or_set(hash, 0, 6144, downloader_context); DORIS_CHECK(holder.file_blocks.size() == 2); DORIS_CHECK(holder.file_blocks.front()->get_or_set_downloader() == FileBlock::get_caller_id()); @@ -242,13 +240,16 @@ TEST_F(BlockFileCacheTest, ProbeObservesDownloadingAndEmptyBlocksWithoutTakingDo } { - auto result = cache.probe(hash, 0, 8192, context); - ASSERT_EQ(result.holder.file_blocks.size(), 2); - auto iterator = result.holder.file_blocks.begin(); - EXPECT_EQ((*iterator)->state(), FileBlock::State::DOWNLOADING); - ++iterator; - EXPECT_EQ((*iterator)->state(), FileBlock::State::EMPTY); - EXPECT_TRUE(result.gaps.empty()); + auto result = cache.probe(hash, 0, 6144, context); + ASSERT_EQ(result.file_blocks.size(), 2); + ASSERT_NE(result.file_blocks[0], nullptr); + ASSERT_NE(result.file_blocks[1], nullptr); + EXPECT_EQ(result.file_blocks[0]->state(), FileBlock::State::DOWNLOADING); + EXPECT_EQ(result.file_blocks[0]->range().left, 0); + EXPECT_EQ(result.file_blocks[0]->range().right, 4095); + EXPECT_EQ(result.file_blocks[1]->state(), FileBlock::State::EMPTY); + EXPECT_EQ(result.file_blocks[1]->range().left, 4096); + EXPECT_EQ(result.file_blocks[1]->range().right, 6143); } } fs::remove_all(path, error); diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index b9695fd3b1fd7f..e702aa93910305 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -188,8 +188,9 @@ TEST_F(AsyncCachedRemoteFileReaderTest, CacheContext probe_context; probe_context.stats = &probe_stats; auto probe_result = cache()->probe(reader->_cache_hash, 0, 2_mb, probe_context); - ASSERT_EQ(probe_result.holder.file_blocks.size(), 2); - for (const auto& block : probe_result.holder.file_blocks) { + ASSERT_EQ(probe_result.file_blocks.size(), 2); + for (const auto& block : probe_result.file_blocks) { + ASSERT_NE(block, nullptr); ASSERT_EQ(block->state(), FileBlock::State::DOWNLOADED); cache_files.emplace_back(block->get_cache_file()); } @@ -224,10 +225,10 @@ TEST_F(AsyncCachedRemoteFileReaderTest, bool metadata_removed = false; { auto probe_result = cache()->probe(reader->_cache_hash, 0, 2_mb, probe_context); - metadata_removed = probe_result.holder.file_blocks.empty() && - probe_result.gaps.size() == 1 && - probe_result.gaps.front().left == 0 && - probe_result.gaps.front().right == 2_mb - 1; + metadata_removed = + probe_result.file_blocks.size() == 2 && + std::all_of(probe_result.file_blocks.begin(), probe_result.file_blocks.end(), + [](const auto& block) { return block == nullptr; }); } const bool files_removed = std::none_of(cache_files.begin(), cache_files.end(), @@ -311,17 +312,17 @@ TEST_F(AsyncCachedRemoteFileReaderTest, ReadStatistics cache_stats; CacheContext cache_context; cache_context.stats = &cache_stats; - auto holder = cache()->get_or_set(reader->_cache_hash, 0, 4096, cache_context); + auto holder = cache()->get_or_set(reader->_cache_hash, 0, 1_mb, cache_context); ASSERT_EQ(holder.file_blocks.size(), 1); const auto& prefix_block = holder.file_blocks.front(); ASSERT_EQ(prefix_block->get_or_set_downloader(), FileBlock::get_caller_id()); - const std::string cached_prefix(4096, 'x'); + const std::string cached_prefix(1_mb, 'x'); ASSERT_TRUE(prefix_block->append(Slice(cached_prefix.data(), cached_prefix.size())).ok()); ASSERT_TRUE(prefix_block->finalize().ok()); config::enable_read_cache_file_directly = true; reader->_insert_file_reader(prefix_block); - std::string result(8192, '\0'); + std::string result(1_mb + 4096, '\0'); FileCacheStatistics stats; IOContext context; context.file_cache_stats = &stats; @@ -329,12 +330,12 @@ TEST_F(AsyncCachedRemoteFileReaderTest, ASSERT_TRUE( reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); EXPECT_EQ(bytes_read, result.size()); - EXPECT_EQ(result.substr(0, 4096), cached_prefix); - EXPECT_EQ(result.substr(4096), std::string(4096, '0')); + EXPECT_EQ(result.substr(0, 1_mb), cached_prefix); + EXPECT_EQ(result.substr(1_mb), std::string(4096, '1')); EXPECT_EQ(counting_reader->read_count(), 1); - EXPECT_EQ(counting_reader->last_offset(), 0); + EXPECT_EQ(counting_reader->last_offset(), 1_mb); EXPECT_EQ(counting_reader->last_size(), 1_mb); - EXPECT_EQ(stats.bytes_read_from_local, 4096); + EXPECT_EQ(stats.bytes_read_from_local, 1_mb); EXPECT_EQ(stats.bytes_read_from_remote, 4096); EXPECT_EQ(stats.async_cache_write_submitted, 1); wait_for_async_writes(); From 22ea1b034df5a3e85e3e8db90b4bbabba9f80a69 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 17 Jul 2026 10:40:05 +0800 Subject: [PATCH 08/16] [test](be) Add dynamic async write backpressure coverage ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: Existing async file-cache write tests covered a single pending-limit rejection and runtime worker resizing independently, but they did not exercise the complete dynamic backpressure lifecycle. Without that coverage, regressions could allow the MPMC backlog to exceed its bound, lose submissions under producer concurrency, fail to expose sustained rejection at capacity, or leave accepted work stranded after consumers are scaled up. Add one deterministic service-level BE unit test that stalls the initial worker before cache mutation and drives four producers in controlled waves. The test verifies that the actual queued backlog grows through 4, 8, 12, and 16 tasks, that pending count includes the blocked active task, and that a subsequent 48-task producer burst is rejected without changing the bounded queue or accepted-task count. After producers stop, the test increases worker concurrency from one to four and batch size from one to four, then releases the artificial write delay. It samples the MPMC backlog independently from pending count, verifies an intermediate lower watermark and an empty queue, and confirms that all accepted tasks finalize with pending count returning to zero. The synchronization point makes both phases deterministic without adding a production-only observation API. ### Release note None ### Check List (For Author) - Test: Unit Test - New dynamic MPMC backpressure ASAN BE unit test passed with run-be-ut.sh and -j100 - All 14 AsyncCacheWriteServiceTest ASAN BE unit tests passed with run-be-ut.sh and -j100 - build-support/check-format.sh passed - Behavior changed: No; this adds deterministic test coverage only - Does this need documentation: No --- .../cache/async_cache_write_service_test.cpp | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp index e8764cb2f2798f..5e65dca3536044 100644 --- a/be/test/io/cache/async_cache_write_service_test.cpp +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include "common/config.h" #include "cpp/sync_point.h" @@ -807,6 +809,168 @@ TEST_F(AsyncCacheWriteServiceTest, RuntimeGrowAndShrinkPreserveConcurrentTasks) EXPECT_EQ(service->options().worker_count, 1); } +TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { + auto cache = create_cache("async_write_service_dynamic_backpressure"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t producer_count = 4; + constexpr size_t fill_wave_count = 4; + constexpr size_t accepted_producer_tasks = producer_count * fill_wave_count; + constexpr size_t rejected_tasks_per_producer = 12; + constexpr size_t rejected_producer_tasks = producer_count * rejected_tasks_per_producer; + constexpr size_t total_producer_tasks = accepted_producer_tasks + rejected_producer_tasks; + constexpr size_t max_pending_tasks = accepted_producer_tasks + 1; + constexpr size_t fast_worker_count = 4; + constexpr size_t block_size = 4096; + + auto options = service->options(); + options.worker_count = 1; + options.max_pending_tasks = max_pending_tasks; + options.batch_size = 1; + ASSERT_TRUE(service->update_options(options).ok()); + + std::mutex mutex; + std::condition_variable cv; + size_t consumer_entries = 0; + size_t finished_tasks = 0; + bool slow_consumers = true; + bool record_drain_samples = false; + std::vector drain_queue_sizes; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + ++consumer_entries; + cv.notify_all(); + cv.wait(lock, [&]() { return !slow_consumers; }); + if (record_drain_samples) { + // This excludes active tasks, unlike pending_count(), and therefore observes + // the actual MPMC backlog after each dequeue. + drain_queue_sizes.emplace_back(service->_queue.size_approx()); + } + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + slow_consumers = false; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + service->shutdown(); + }}; + + std::vector tasks; + tasks.reserve(total_producer_tasks + 1); + for (size_t task_id = 0; task_id <= total_producer_tasks; ++task_id) { + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(block_size, &buffer).ok()); + memset(buffer->data(), static_cast('a' + task_id % 26), buffer->size()); + tasks.emplace_back(AsyncCacheWriteTask { + .cache_hash = + BlockFileCache::hash("dynamic_backpressure_" + std::to_string(task_id)), + .file_offset = 0, + .file_size = buffer->size(), + .buffer = std::move(buffer), + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&](const AsyncCacheWriteTask&) { + std::lock_guard lock(mutex); + ++finished_tasks; + cv.notify_all(); + }, + }); + } + + const auto baseline_stats = service->stats(); + ASSERT_TRUE(service->try_submit(std::move(tasks[0]))); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return consumer_entries == 1; })); + } + + std::atomic producer_accepts {0}; + std::atomic producer_rejects {0}; + // Each producer owns a contiguous task slice beginning at first_task; tasks_per_producer + // controls whether this invocation is one fill wave or the final rejection burst. + const auto submit_wave = [&](size_t first_task, size_t tasks_per_producer) { + std::vector producers; + producers.reserve(producer_count); + for (size_t producer_id = 0; producer_id < producer_count; ++producer_id) { + producers.emplace_back([&, producer_id]() { + const size_t producer_first = first_task + producer_id * tasks_per_producer; + for (size_t task_offset = 0; task_offset < tasks_per_producer; ++task_offset) { + if (service->try_submit(std::move(tasks[producer_first + task_offset]))) { + producer_accepts.fetch_add(1, std::memory_order_relaxed); + } else { + producer_rejects.fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + for (auto& producer : producers) { + producer.join(); + } + }; + + // With the only consumer blocked, each producer wave adds four queued tasks. The stable + // snapshots make backlog growth deterministic instead of depending on polling timing. + for (size_t wave = 0; wave < fill_wave_count; ++wave) { + submit_wave(1 + wave * producer_count, 1); + EXPECT_EQ(service->_queue.size_approx(), (wave + 1) * producer_count); + EXPECT_EQ(service->pending_count(), 1 + (wave + 1) * producer_count); + } + + // The queue is now bounded by max_pending_tasks. A larger concurrent burst must be dropped by + // backpressure without changing either the queued or active task count. + submit_wave(1 + accepted_producer_tasks, rejected_tasks_per_producer); + EXPECT_EQ(producer_accepts.load(std::memory_order_relaxed), accepted_producer_tasks); + EXPECT_EQ(producer_rejects.load(std::memory_order_relaxed), rejected_producer_tasks); + EXPECT_GT(producer_rejects.load(std::memory_order_relaxed), + producer_accepts.load(std::memory_order_relaxed)); + EXPECT_EQ(service->_queue.size_approx(), accepted_producer_tasks); + EXPECT_EQ(service->pending_count(), max_pending_tasks); + const auto pressured_stats = service->stats(); + EXPECT_EQ(pressured_stats.submitted - baseline_stats.submitted, max_pending_tasks); + EXPECT_EQ(pressured_stats.rejected - baseline_stats.rejected, rejected_producer_tasks); + + // Producers have stopped. Grow from one to four consumers while writes remain blocked, then + // remove the artificial delay and increase the batch size to model faster disk consumption. + options.worker_count = fast_worker_count; + options.batch_size = 4; + ASSERT_TRUE(service->update_options(options).ok()); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return consumer_entries == fast_worker_count; })); + EXPECT_EQ(service->_queue.size_approx(), accepted_producer_tasks - (fast_worker_count - 1)); + record_drain_samples = true; + slow_consumers = false; + } + cv.notify_all(); + + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(10), + [&]() { return finished_tasks == max_pending_tasks; })); + } + EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(service->_queue.size_approx(), 0); + EXPECT_TRUE(std::any_of(drain_queue_sizes.begin(), drain_queue_sizes.end(), [&](size_t size) { + return size > 0 && size <= accepted_producer_tasks / 2; + })); + EXPECT_TRUE(std::any_of(drain_queue_sizes.begin(), drain_queue_sizes.end(), + [](size_t size) { return size == 0; })); +} + TEST_F(AsyncCacheWriteServiceTest, MutableConfigUpdatesServicesExplicitly) { auto* factory = FileCacheFactory::instance(); factory->_caches.clear(); From 0be7eb7c269d72df205903c67bcb16bdee486919 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 17 Jul 2026 11:31:49 +0800 Subject: [PATCH 09/16] [refactor](be) Group async file cache write configs ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The async file cache write settings were mixed into the broader block file cache configuration section, while the inflight write buffer index settings did not carry the feature name. This made the feature difficult to locate as one configuration group and made name-based filtering incomplete. Move all async file cache write declarations, definitions, and validators into a dedicated contiguous section. Keep the primary enable_async_file_cache_write switch unchanged, rename only the inflight index enable and shard-count settings with the async_file_cache_write prefix, and update runtime consumers plus BE and regression test configuration. Defaults, mutability, validation, and runtime behavior remain unchanged. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=AsyncCachedRemoteFileReaderTest.*:BlockFileCacheTest.*async_write*:BlockFileCacheTest.cache_write_mode_is_resolved_for_each_read_context:AsyncCacheWriteServiceTest.* -j100 (26 tests passed) - build-support/check-format.sh - Behavior changed: No - Does this need documentation: No --- be/src/common/config.cpp | 41 ++++++++++--------- be/src/common/config.h | 18 ++++---- be/src/io/cache/block_file_cache.cpp | 4 +- .../cached_remote_file_reader_async_write.cpp | 5 ++- .../cache/cached_remote_file_reader_test.cpp | 41 +++++++++++-------- .../cache/test_async_file_cache_write.groovy | 2 +- 6 files changed, 62 insertions(+), 49 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index a2be1ce77481d3..99f542e7c57930 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1261,25 +1261,6 @@ DEFINE_mInt64(file_cache_background_block_lru_update_interval_ms, "5000"); DEFINE_mInt64(file_cache_background_block_lru_update_qps_limit, "1000"); DEFINE_mInt64(file_cache_background_block_lru_update_queue_max_size, "500000"); DEFINE_mBool(enable_file_cache_async_touch_on_get_or_set, "false"); -DEFINE_mBool(enable_async_file_cache_write, "false"); -DEFINE_mInt32(async_file_cache_write_workers_per_disk, "16"); -DEFINE_mInt64(async_file_cache_write_max_pending_tasks_per_disk, "256"); -DEFINE_mInt32(async_file_cache_write_batch_size, "16"); -DEFINE_mInt64(async_file_cache_write_watchdog_warn_secs, "30"); -DEFINE_mInt64(async_file_cache_write_watchdog_drop_secs, "120"); -DEFINE_mBool(enable_inflight_write_buffer_index, "true"); -DEFINE_Int32(inflight_write_buffer_index_shard_count, "64"); -DEFINE_Validator(async_file_cache_write_workers_per_disk, - [](int32_t value) { return value > 0 && value <= 128; }); -DEFINE_Validator(async_file_cache_write_max_pending_tasks_per_disk, - [](int64_t value) { return value > 0; }); -DEFINE_Validator(async_file_cache_write_batch_size, [](int32_t value) { return value > 0; }); -DEFINE_Validator(async_file_cache_write_watchdog_warn_secs, [](int64_t value) { - return value >= 0 && value < async_file_cache_write_watchdog_drop_secs; -}); -DEFINE_Validator(async_file_cache_write_watchdog_drop_secs, - [](int64_t value) { return value > async_file_cache_write_watchdog_warn_secs; }); -DEFINE_Validator(inflight_write_buffer_index_shard_count, [](int32_t value) { return value > 0; }); DEFINE_mBool(enable_reader_dryrun_when_download_file_cache, "true"); DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000"); DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "180000"); @@ -1298,6 +1279,28 @@ DEFINE_mBool(file_cache_enable_only_warm_up_idx, "false"); DEFINE_Int32(file_cache_downloader_thread_num_min, "32"); DEFINE_Int32(file_cache_downloader_thread_num_max, "32"); +// async file cache write +DEFINE_mBool(enable_async_file_cache_write, "false"); +DEFINE_mInt32(async_file_cache_write_workers_per_disk, "16"); +DEFINE_mInt64(async_file_cache_write_max_pending_tasks_per_disk, "256"); +DEFINE_mInt32(async_file_cache_write_batch_size, "16"); +DEFINE_mInt64(async_file_cache_write_watchdog_warn_secs, "30"); +DEFINE_mInt64(async_file_cache_write_watchdog_drop_secs, "120"); +DEFINE_mBool(enable_async_file_cache_write_inflight_write_buffer_index, "true"); +DEFINE_Int32(async_file_cache_write_inflight_write_buffer_index_shard_count, "64"); +DEFINE_Validator(async_file_cache_write_workers_per_disk, + [](int32_t value) { return value > 0 && value <= 128; }); +DEFINE_Validator(async_file_cache_write_max_pending_tasks_per_disk, + [](int64_t value) { return value > 0; }); +DEFINE_Validator(async_file_cache_write_batch_size, [](int32_t value) { return value > 0; }); +DEFINE_Validator(async_file_cache_write_watchdog_warn_secs, [](int64_t value) { + return value >= 0 && value < async_file_cache_write_watchdog_drop_secs; +}); +DEFINE_Validator(async_file_cache_write_watchdog_drop_secs, + [](int64_t value) { return value > async_file_cache_write_watchdog_warn_secs; }); +DEFINE_Validator(async_file_cache_write_inflight_write_buffer_index_shard_count, + [](int32_t value) { return value > 0; }); + DEFINE_mInt32(index_cache_entry_stay_time_after_lookup_s, "1800"); DEFINE_mInt32(inverted_index_cache_stale_sweep_time_sec, "600"); DEFINE_mBool(enable_write_index_searcher_cache, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 987aad42fad568..2a50e571614d74 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1301,14 +1301,6 @@ DECLARE_mInt64(file_cache_background_block_lru_update_interval_ms); DECLARE_mInt64(file_cache_background_block_lru_update_qps_limit); DECLARE_mInt64(file_cache_background_block_lru_update_queue_max_size); DECLARE_mBool(enable_file_cache_async_touch_on_get_or_set); -DECLARE_mBool(enable_async_file_cache_write); -DECLARE_mInt32(async_file_cache_write_workers_per_disk); -DECLARE_mInt64(async_file_cache_write_max_pending_tasks_per_disk); -DECLARE_mInt32(async_file_cache_write_batch_size); -DECLARE_mInt64(async_file_cache_write_watchdog_warn_secs); -DECLARE_mInt64(async_file_cache_write_watchdog_drop_secs); -DECLARE_mBool(enable_inflight_write_buffer_index); -DECLARE_Int32(inflight_write_buffer_index_shard_count); DECLARE_mBool(enable_reader_dryrun_when_download_file_cache); DECLARE_mInt64(file_cache_background_monitor_interval_ms); DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms); @@ -1327,6 +1319,16 @@ DECLARE_mBool(enable_evaluate_shadow_queue_diff); DECLARE_mBool(file_cache_enable_only_warm_up_idx); +// async file cache write +DECLARE_mBool(enable_async_file_cache_write); +DECLARE_mInt32(async_file_cache_write_workers_per_disk); +DECLARE_mInt64(async_file_cache_write_max_pending_tasks_per_disk); +DECLARE_mInt32(async_file_cache_write_batch_size); +DECLARE_mInt64(async_file_cache_write_watchdog_warn_secs); +DECLARE_mInt64(async_file_cache_write_watchdog_drop_secs); +DECLARE_mBool(enable_async_file_cache_write_inflight_write_buffer_index); +DECLARE_Int32(async_file_cache_write_inflight_write_buffer_index_shard_count); + // inverted index searcher cache // cache entry stay time after lookup DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s); diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index 3568f8816bc0b8..ef5527bacabe14 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -549,7 +549,9 @@ Status BlockFileCache::initialize_unlocked(std::lock_guard& cache_lo RETURN_IF_ERROR(_storage->init(this)); _inflight_write_buffer_index = std::make_unique( - static_cast(config::inflight_write_buffer_index_shard_count), _cache_base_path); + static_cast( + config::async_file_cache_write_inflight_write_buffer_index_shard_count), + _cache_base_path); // BlockFileCache is the configuration boundary for a newly created per-disk service. Pass a // value snapshot so AsyncCacheWriteService does not need to know where its settings came from. AsyncCacheWriteServiceOptions async_write_options { diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp index 4a9b3f65448194..740e029f79ddae 100644 --- a/be/src/io/cache/cached_remote_file_reader_async_write.cpp +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -145,7 +145,8 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ } DORIS_CHECK(!plan.blocks.empty()); - const bool inflight_index_enabled = config::enable_inflight_write_buffer_index; + const bool inflight_index_enabled = + config::enable_async_file_cache_write_inflight_write_buffer_index; bool all_blocks_inflight = inflight_index_enabled; if (inflight_index_enabled) { auto* inflight_index = _cache->inflight_write_buffer_index(); @@ -464,7 +465,7 @@ void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan .on_finalized = nullptr, }; std::shared_ptr entry; - if (config::enable_inflight_write_buffer_index) { + if (config::enable_async_file_cache_write_inflight_write_buffer_index) { entry = std::make_shared( tracked_buffer, read_block.range.left, read_block.range.size(), task.submit_ts_us, plan.write_epoch); diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index e702aa93910305..9b859666d035d8 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -79,14 +79,14 @@ class AsyncCachedRemoteFileReaderTest : public BlockFileCacheTest { protected: void SetUp() override { _old_enable_async = config::enable_async_file_cache_write; - _old_enable_inflight = config::enable_inflight_write_buffer_index; + _old_enable_inflight = config::enable_async_file_cache_write_inflight_write_buffer_index; _old_enable_direct = config::enable_read_cache_file_directly; _old_enable_peer = config::enable_cache_read_from_peer; _old_block_size = config::file_cache_each_block_size; _old_wait_timeout_ms = config::block_cache_wait_timeout_ms; config::enable_async_file_cache_write = true; - config::enable_inflight_write_buffer_index = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; config::enable_read_cache_file_directly = false; config::enable_cache_read_from_peer = false; config::file_cache_each_block_size = 1_mb; @@ -100,7 +100,7 @@ class AsyncCachedRemoteFileReaderTest : public BlockFileCacheTest { fs::remove_all(_cache_path, error); } config::enable_async_file_cache_write = _old_enable_async; - config::enable_inflight_write_buffer_index = _old_enable_inflight; + config::enable_async_file_cache_write_inflight_write_buffer_index = _old_enable_inflight; config::enable_read_cache_file_directly = _old_enable_direct; config::enable_cache_read_from_peer = _old_enable_peer; config::file_cache_each_block_size = _old_block_size; @@ -636,16 +636,17 @@ TEST_F(BlockFileCacheTest, TEST_F(BlockFileCacheTest, async_write_reuses_inflight_buffer_then_reads_downloaded_block) { const bool old_enable_async = config::enable_async_file_cache_write; - const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; const bool old_enable_direct = config::enable_read_cache_file_directly; const bool old_enable_peer = config::enable_cache_read_from_peer; config::enable_async_file_cache_write = true; - config::enable_inflight_write_buffer_index = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; config::enable_read_cache_file_directly = false; config::enable_cache_read_from_peer = false; Defer restore_config {[&]() { config::enable_async_file_cache_write = old_enable_async; - config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; config::enable_read_cache_file_directly = old_enable_direct; config::enable_cache_read_from_peer = old_enable_peer; }}; @@ -773,18 +774,19 @@ TEST_F(BlockFileCacheTest, async_write_reuses_inflight_buffer_then_reads_downloa TEST_F(BlockFileCacheTest, async_write_waits_for_downloading_block_outside_remote_span) { const bool old_enable_async = config::enable_async_file_cache_write; - const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; const bool old_enable_direct = config::enable_read_cache_file_directly; const bool old_enable_peer = config::enable_cache_read_from_peer; const int64_t old_block_size = config::file_cache_each_block_size; config::enable_async_file_cache_write = true; - config::enable_inflight_write_buffer_index = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; config::enable_read_cache_file_directly = false; config::enable_cache_read_from_peer = false; config::file_cache_each_block_size = 1_mb; Defer restore_config {[&]() { config::enable_async_file_cache_write = old_enable_async; - config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; config::enable_read_cache_file_directly = old_enable_direct; config::enable_cache_read_from_peer = old_enable_peer; config::file_cache_each_block_size = old_block_size; @@ -852,18 +854,19 @@ TEST_F(BlockFileCacheTest, async_write_waits_for_downloading_block_outside_remot TEST_F(BlockFileCacheTest, async_write_reads_only_middle_span_between_cached_sides) { const bool old_enable_async = config::enable_async_file_cache_write; - const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; const bool old_enable_direct = config::enable_read_cache_file_directly; const bool old_enable_peer = config::enable_cache_read_from_peer; const int64_t old_block_size = config::file_cache_each_block_size; config::enable_async_file_cache_write = true; - config::enable_inflight_write_buffer_index = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; config::enable_read_cache_file_directly = false; config::enable_cache_read_from_peer = false; config::file_cache_each_block_size = 1_mb; Defer restore_config {[&]() { config::enable_async_file_cache_write = old_enable_async; - config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; config::enable_read_cache_file_directly = old_enable_direct; config::enable_cache_read_from_peer = old_enable_peer; config::file_cache_each_block_size = old_block_size; @@ -937,18 +940,19 @@ TEST_F(BlockFileCacheTest, async_write_reads_only_middle_span_between_cached_sid TEST_F(BlockFileCacheTest, async_write_middle_hit_uses_one_remote_span_and_submits_only_misses) { const bool old_enable_async = config::enable_async_file_cache_write; - const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; const bool old_enable_direct = config::enable_read_cache_file_directly; const bool old_enable_peer = config::enable_cache_read_from_peer; const int64_t old_block_size = config::file_cache_each_block_size; config::enable_async_file_cache_write = true; - config::enable_inflight_write_buffer_index = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; config::enable_read_cache_file_directly = false; config::enable_cache_read_from_peer = false; config::file_cache_each_block_size = 1_mb; Defer restore_config {[&]() { config::enable_async_file_cache_write = old_enable_async; - config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; config::enable_read_cache_file_directly = old_enable_direct; config::enable_cache_read_from_peer = old_enable_peer; config::file_cache_each_block_size = old_block_size; @@ -1030,18 +1034,19 @@ TEST_F(BlockFileCacheTest, async_write_middle_hit_uses_one_remote_span_and_submi TEST_F(BlockFileCacheTest, async_write_backpressure_rolls_back_inflight_entry) { const bool old_enable_async = config::enable_async_file_cache_write; - const bool old_enable_inflight = config::enable_inflight_write_buffer_index; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; const bool old_enable_direct = config::enable_read_cache_file_directly; const bool old_enable_peer = config::enable_cache_read_from_peer; const int64_t old_block_size = config::file_cache_each_block_size; config::enable_async_file_cache_write = true; - config::enable_inflight_write_buffer_index = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; config::enable_read_cache_file_directly = false; config::enable_cache_read_from_peer = false; config::file_cache_each_block_size = 1_mb; Defer restore_config {[&]() { config::enable_async_file_cache_write = old_enable_async; - config::enable_inflight_write_buffer_index = old_enable_inflight; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; config::enable_read_cache_file_directly = old_enable_direct; config::enable_cache_read_from_peer = old_enable_peer; config::file_cache_each_block_size = old_block_size; diff --git a/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy b/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy index 44a215f39b90a2..e78c339883125f 100644 --- a/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy +++ b/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy @@ -26,7 +26,7 @@ suite("test_async_file_cache_write", "docker") { options.beConfigs += [ "enable_file_cache=true", "enable_async_file_cache_write=true", - "enable_inflight_write_buffer_index=true", + "enable_async_file_cache_write_inflight_write_buffer_index=true", "enable_read_cache_file_directly=false", "enable_cache_read_from_peer=false", "disable_storage_page_cache=true", From 18db3668b5032a47c553f85f55e9ee2084d3e62c Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 17 Jul 2026 11:32:07 +0800 Subject: [PATCH 10/16] [test](be) Strengthen async write MPMC pressure coverage ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The dynamic async-write backpressure test created four producer threads during queue growth, but each producer submitted only one task per fill wave. That exercised simultaneous entry only weakly and did not model sustained concurrent production before backpressure. Start every producer in a wave through a barrier and let each producer submit four consecutive tasks. The test now observes deterministic queue growth through 16, 32, 48, and 64 queued tasks, verifies a subsequent 128-task concurrent burst is rejected at the pending limit, then confirms the enlarged and accelerated consumer side drains the queue to zero. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=AsyncCachedRemoteFileReaderTest.*:BlockFileCacheTest.*async_write*:BlockFileCacheTest.cache_write_mode_is_resolved_for_each_read_context:AsyncCacheWriteServiceTest.* -j100 (26 tests passed) - build-support/check-format.sh - Behavior changed: No - Does this need documentation: No --- .../cache/async_cache_write_service_test.cpp | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp index 5e65dca3536044..cc1f13b06cfd89 100644 --- a/be/test/io/cache/async_cache_write_service_test.cpp +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -816,8 +817,10 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { constexpr size_t producer_count = 4; constexpr size_t fill_wave_count = 4; - constexpr size_t accepted_producer_tasks = producer_count * fill_wave_count; - constexpr size_t rejected_tasks_per_producer = 12; + constexpr size_t fill_tasks_per_producer = 4; + constexpr size_t accepted_producer_tasks = + producer_count * fill_wave_count * fill_tasks_per_producer; + constexpr size_t rejected_tasks_per_producer = 32; constexpr size_t rejected_producer_tasks = producer_count * rejected_tasks_per_producer; constexpr size_t total_producer_tasks = accepted_producer_tasks + rejected_producer_tasks; constexpr size_t max_pending_tasks = accepted_producer_tasks + 1; @@ -904,8 +907,10 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { const auto submit_wave = [&](size_t first_task, size_t tasks_per_producer) { std::vector producers; producers.reserve(producer_count); + std::barrier producer_start(producer_count); for (size_t producer_id = 0; producer_id < producer_count; ++producer_id) { producers.emplace_back([&, producer_id]() { + producer_start.arrive_and_wait(); const size_t producer_first = first_task + producer_id * tasks_per_producer; for (size_t task_offset = 0; task_offset < tasks_per_producer; ++task_offset) { if (service->try_submit(std::move(tasks[producer_first + task_offset]))) { @@ -921,12 +926,15 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { } }; - // With the only consumer blocked, each producer wave adds four queued tasks. The stable - // snapshots make backlog growth deterministic instead of depending on polling timing. + // With the only consumer blocked, every producer continuously submits several tasks per wave. + // The stable snapshots make backlog growth deterministic instead of depending on polling + // timing. for (size_t wave = 0; wave < fill_wave_count; ++wave) { - submit_wave(1 + wave * producer_count, 1); - EXPECT_EQ(service->_queue.size_approx(), (wave + 1) * producer_count); - EXPECT_EQ(service->pending_count(), 1 + (wave + 1) * producer_count); + submit_wave(1 + wave * producer_count * fill_tasks_per_producer, fill_tasks_per_producer); + EXPECT_EQ(service->_queue.size_approx(), + (wave + 1) * producer_count * fill_tasks_per_producer); + EXPECT_EQ(service->pending_count(), + 1 + (wave + 1) * producer_count * fill_tasks_per_producer); } // The queue is now bounded by max_pending_tasks. A larger concurrent burst must be dropped by From e8bd2ffc54dcf9a405634fb5a3c86efda841afe7 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 17 Jul 2026 15:44:58 +0800 Subject: [PATCH 11/16] [improvement](be) Add async file cache write observability ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: The async file cache write path exposed only an aggregate pending count and a limited set of outcome counters. When throughput degraded, operators could not distinguish producer backpressure, MPMC queue buildup, unavailable workers, inflight-index lock contention, BlockFileCache metadata contention, append/finalize latency, or watchdog and stale-epoch drops. Add exact atomically maintained queue, active-task, running-worker, configured-capacity, and active-stage gauges. Add reason-specific rejection and watchdog counters, submitted and persisted byte/block throughput, and latency recorders for submission, allocation, queue wait, worker processing, get-or-set, append, finalize, probing, read-plan construction, and write submission. Instrument inflight shard lock wait and hold time, and route probe/get-or-set locking through the existing BlockFileCache lock-wait metric. Keep queue monitoring passive and exact instead of adding a sampling thread. Remove the unused AsyncCacheWriteService::stats snapshot API and use the service state and metrics directly in tests. Do not expose an inflight index metadata memory estimate because it can be mistaken for total payload memory; async_cache_write_buffer_memory_bytes remains the payload-memory metric. Extend the BE unit tests to verify live queue and stage gauges, metric counters and latency samples, lock instrumentation, and the concurrent MPMC growth, rejection, scale-up, and drain flow. ### Release note Add monitoring metrics for async file cache write queue pressure, worker activity, stage latency, throughput, rejection causes, and lock contention. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh -j100 --run --filter=AsyncCacheWriteServiceTest.*:InflightWriteBufferIndexTest.*:BlockFileCacheTest.Probe*:AsyncCachedRemoteFileReaderTest.* (29 tests passed under ASAN_UT) - Behavior changed: Yes. Adds observability only; async cache read and write semantics are unchanged. - Does this need documentation: No --- be/src/io/cache/async_cache_write_service.cpp | 203 +++++++++++++++--- be/src/io/cache/async_cache_write_service.h | 62 ++++-- be/src/io/cache/block_file_cache.cpp | 12 +- be/src/io/cache/block_file_cache.h | 1 + .../cached_remote_file_reader_async_write.cpp | 14 ++ .../io/cache/inflight_write_buffer_index.cpp | 76 ++++--- be/src/io/cache/inflight_write_buffer_index.h | 9 +- .../cache/async_cache_write_service_test.cpp | 72 +++++-- .../io/cache/block_file_cache_probe_test.cpp | 4 + .../cache/cached_remote_file_reader_test.cpp | 2 +- .../inflight_write_buffer_index_test.cpp | 4 + 11 files changed, 348 insertions(+), 111 deletions(-) diff --git a/be/src/io/cache/async_cache_write_service.cpp b/be/src/io/cache/async_cache_write_service.cpp index 85c3ceac68eaca..e959d29bf48b47 100644 --- a/be/src/io/cache/async_cache_write_service.cpp +++ b/be/src/io/cache/async_cache_write_service.cpp @@ -34,6 +34,27 @@ namespace doris::io { using AsyncCacheWriteAllocator = Allocator; +namespace { + +/// Keep an in-progress phase gauge balanced across every return path. +class ScopedActiveCounter { +public: + explicit ScopedActiveCounter(std::atomic* counter) : _counter(counter) { + DORIS_CHECK(_counter != nullptr); + _counter->fetch_add(1, std::memory_order_relaxed); + } + + ~ScopedActiveCounter() { + const size_t old_count = _counter->fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_count > 0); + } + +private: + std::atomic* _counter; +}; + +} // namespace + AsyncCacheWriteBuffer::AsyncCacheWriteBuffer(size_t size, std::shared_ptr tracker) : _size(size), _tracker(std::move(tracker)) { @@ -69,6 +90,60 @@ AsyncCacheWriteService::AsyncCacheWriteService(BlockFileCache* cache, return static_cast(service)->pending_count(); }, this); + _queued_count_metric = std::make_shared>( + prefix, "async_cache_write_queue_size", + [](void* service) { + return static_cast(service)->queued_count(); + }, + this); + _active_task_count_metric = std::make_shared>( + prefix, "async_cache_write_active_tasks", + [](void* service) { + return static_cast(service)->active_task_count(); + }, + this); + _running_worker_count_metric = std::make_shared>( + prefix, "async_cache_write_running_workers", + [](void* service) { + return static_cast(service)->running_worker_count(); + }, + this); + _configured_worker_count_metric = std::make_shared>( + prefix, "async_cache_write_configured_workers", + [](void* service) { + return static_cast(service)->_desired_worker_count.load( + std::memory_order_relaxed); + }, + this); + _max_pending_tasks_metric = std::make_shared>( + prefix, "async_cache_write_max_pending_tasks", + [](void* service) { + return static_cast(service) + ->_options.load(std::memory_order_acquire) + ->max_pending_tasks; + }, + this); + _active_get_or_set_count_metric = std::make_shared>( + prefix, "async_cache_write_active_get_or_set", + [](void* service) { + return static_cast(service)->_active_get_or_set_count.load( + std::memory_order_relaxed); + }, + this); + _active_append_count_metric = std::make_shared>( + prefix, "async_cache_write_active_append", + [](void* service) { + return static_cast(service)->_active_append_count.load( + std::memory_order_relaxed); + }, + this); + _active_finalize_count_metric = std::make_shared>( + prefix, "async_cache_write_active_finalize", + [](void* service) { + return static_cast(service)->_active_finalize_count.load( + std::memory_order_relaxed); + }, + this); _buffer_memory_metric = std::make_shared>( prefix, "async_cache_write_buffer_memory_bytes", [](void* service) { @@ -77,12 +152,34 @@ AsyncCacheWriteService::AsyncCacheWriteService(BlockFileCache* cache, this); _submitted_metric = std::make_shared>(prefix, "async_cache_write_submitted_total"); + _submitted_bytes_metric = std::make_shared>( + prefix, "async_cache_write_submitted_bytes_total"); + _finished_metric = + std::make_shared>(prefix, "async_cache_write_finished_total"); _rejected_metric = std::make_shared>(prefix, "async_cache_write_rejected_total"); + _reject_not_running_metric = std::make_shared>( + prefix, "async_cache_write_reject_not_running_total"); + _reject_backpressure_metric = std::make_shared>( + prefix, "async_cache_write_reject_backpressure_total"); + _reject_enqueue_failure_metric = std::make_shared>( + prefix, "async_cache_write_reject_enqueue_failure_total"); _buffer_alloc_fail_metric = std::make_shared>( prefix, "async_cache_write_buffer_alloc_fail_total"); - _latency_metric = - std::make_shared(prefix, "async_cache_write_latency_us"); + _submit_latency_metric = + std::make_shared(prefix, "async_cache_write_submit_latency_us"); + _buffer_alloc_latency_metric = std::make_shared( + prefix, "async_cache_write_buffer_alloc_latency_us"); + _queue_wait_latency_metric = std::make_shared( + prefix, "async_cache_write_queue_wait_latency_us"); + _worker_task_latency_metric = std::make_shared( + prefix, "async_cache_write_worker_task_latency_us"); + _get_or_set_latency_metric = std::make_shared( + prefix, "async_cache_write_get_or_set_latency_us"); + _append_latency_metric = + std::make_shared(prefix, "async_cache_write_append_latency_us"); + _finalize_latency_metric = std::make_shared( + prefix, "async_cache_write_finalize_latency_us"); _skip_downloaded_metric = std::make_shared>( prefix, "async_cache_write_skip_downloaded_total"); _skip_downloading_metric = std::make_shared>( @@ -95,8 +192,16 @@ AsyncCacheWriteService::AsyncCacheWriteService(BlockFileCache* cache, prefix, "async_cache_write_skip_deleting_total"); _append_fail_metric = std::make_shared>(prefix, "async_cache_write_append_fail_total"); - _watchdog_timeout_metric = std::make_shared>( - prefix, "async_cache_write_watchdog_timeout_total"); + _finalize_fail_metric = std::make_shared>( + prefix, "async_cache_write_finalize_fail_total"); + _persisted_blocks_metric = std::make_shared>( + prefix, "async_cache_write_persisted_blocks_total"); + _persisted_bytes_metric = std::make_shared>( + prefix, "async_cache_write_persisted_bytes_total"); + _watchdog_warn_metric = std::make_shared>( + prefix, "async_cache_write_watchdog_warn_total"); + _watchdog_drop_metric = std::make_shared>( + prefix, "async_cache_write_watchdog_drop_total"); } AsyncCacheWriteService::~AsyncCacheWriteService() { @@ -149,31 +254,43 @@ Status AsyncCacheWriteService::start() { } bool AsyncCacheWriteService::try_submit(AsyncCacheWriteTask task) { + const int64_t submit_start_us = MonotonicMicros(); + Defer record_submit_latency { + [&]() { *_submit_latency_metric << (MonotonicMicros() - submit_start_us); }}; _active_submitters.fetch_add(1, std::memory_order_acq_rel); Defer submitter_done {[&]() { _active_submitters.fetch_sub(1, std::memory_order_acq_rel); }}; TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::try_submit:after_register", &task); if (!_started.load(std::memory_order_acquire) || !_accepting.load(std::memory_order_acquire)) { *_rejected_metric << 1; + *_reject_not_running_metric << 1; return false; } const auto options = _options.load(std::memory_order_acquire); const size_t max_pending = options->max_pending_tasks; + const size_t task_bytes = task.file_size; size_t current = _pending_count.load(std::memory_order_relaxed); while (current < max_pending) { if (_pending_count.compare_exchange_weak(current, current + 1, std::memory_order_acq_rel, std::memory_order_relaxed)) { + _queued_count.fetch_add(1, std::memory_order_release); if (_queue.enqueue(std::move(task))) { *_submitted_metric << 1; + *_submitted_bytes_metric << task_bytes; _cv.notify_one(); return true; } - _pending_count.fetch_sub(1, std::memory_order_acq_rel); + const size_t old_queued = _queued_count.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(old_queued > 0); + const size_t old_pending = _pending_count.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(old_pending > 0); *_rejected_metric << 1; + *_reject_enqueue_failure_metric << 1; return false; } } *_rejected_metric << 1; + *_reject_backpressure_metric << 1; return false; } @@ -181,6 +298,9 @@ Status AsyncCacheWriteService::allocate_tracked_buffer(size_t size, AsyncCacheWriteBufferPtr* buffer) { DORIS_CHECK(buffer != nullptr); DORIS_CHECK(size > 0); + const int64_t allocation_start_us = MonotonicMicros(); + Defer record_allocation_latency { + [&]() { *_buffer_alloc_latency_metric << (MonotonicMicros() - allocation_start_us); }}; Status injected_status; TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::allocate_tracked_buffer:inject_failure", &injected_status); @@ -221,7 +341,10 @@ Status AsyncCacheWriteService::_schedule_worker(size_t worker_id) { } void AsyncCacheWriteService::_worker_loop(size_t worker_id) { + _running_worker_count.fetch_add(1, std::memory_order_relaxed); Defer mark_stopped {[&]() { + const size_t old_running = _running_worker_count.fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_running > 0); std::lock_guard lock(_worker_state_mutex); _worker_scheduled[worker_id] = false; _worker_state_cv.notify_all(); @@ -242,22 +365,27 @@ void AsyncCacheWriteService::_worker_loop(size_t worker_id) { AsyncCacheWriteTask task; while (processed < options->batch_size && _queue.try_dequeue(consumer_token, task)) { ++processed; + const size_t old_queued = _queued_count.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(old_queued > 0); + _active_task_count.fetch_add(1, std::memory_order_relaxed); Defer finish {[&]() { _finish_task(task); task = AsyncCacheWriteTask {}; }}; const int64_t age_us = MonotonicMicros() - task.submit_ts_us; + *_queue_wait_latency_metric << age_us; const int64_t warn_us = options->watchdog_warn_secs * 1000 * 1000; const int64_t drop_us = options->watchdog_drop_secs * 1000 * 1000; if (age_us > warn_us) { - *_watchdog_timeout_metric << 1; + *_watchdog_warn_metric << 1; LOG(WARNING) << "Async file cache write task waited " << age_us << " us, cache=" << _cache->get_base_path() << ", hash=" << task.cache_hash.to_string() << ", offset=" << task.file_offset; } if (age_us > drop_us) { + *_watchdog_drop_metric << 1; continue; } if (!is_current_write_epoch(task.write_epoch)) { @@ -267,7 +395,7 @@ void AsyncCacheWriteService::_worker_loop(size_t worker_id) { const int64_t start_us = MonotonicMicros(); Status status = _write_one(task); - *_latency_metric << (MonotonicMicros() - start_us); + *_worker_task_latency_metric << (MonotonicMicros() - start_us); if (!status.ok()) { LOG(WARNING) << "Async file cache write failed, cache=" << _cache->get_base_path() << ", hash=" << task.cache_hash.to_string() @@ -283,7 +411,7 @@ void AsyncCacheWriteService::_worker_loop(size_t worker_id) { if (processed == 0) { std::unique_lock lock(_cv_mutex); _cv.wait_for(lock, std::chrono::milliseconds(1), [&]() { - return _queue.size_approx() != 0 || + return _queued_count.load(std::memory_order_acquire) != 0 || _shutdown_requested.load(std::memory_order_acquire) || worker_id >= _desired_worker_count.load(std::memory_order_acquire); }); @@ -299,7 +427,6 @@ Status AsyncCacheWriteService::_write_one(const AsyncCacheWriteTask& task) { return Status::OK(); } - TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:before_get_or_set", &task); ReadStatistics dummy_stats; CacheContext context; context.query_id = task.admission_ctx.query_id; @@ -308,8 +435,17 @@ Status AsyncCacheWriteService::_write_one(const AsyncCacheWriteTask& task) { context.tablet_id = task.admission_ctx.tablet_id; context.is_warmup = task.admission_ctx.is_warmup; context.stats = &dummy_stats; - auto holder = _cache->get_or_set(task.cache_hash, task.file_offset, task.file_size, context); - TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:after_get_or_set", &task); + auto holder = [&]() { + ScopedActiveCounter active_get_or_set(&_active_get_or_set_count); + const int64_t start_us = MonotonicMicros(); + Defer record_latency { + [&]() { *_get_or_set_latency_metric << (MonotonicMicros() - start_us); }}; + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:before_get_or_set", &task); + auto result = + _cache->get_or_set(task.cache_hash, task.file_offset, task.file_size, context); + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:after_get_or_set", &task); + return result; + }(); if (!is_current_write_epoch(task.write_epoch)) { *_drop_stale_epoch_metric << 1; @@ -349,9 +485,15 @@ Status AsyncCacheWriteService::_write_one(const AsyncCacheWriteTask& task) { continue; } const size_t buffer_offset = block->range().left - task.file_offset; - TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:before_append", &task); - Status status = - block->append(Slice(task.buffer->data() + buffer_offset, block->range().size())); + Status status; + { + ScopedActiveCounter active_append(&_active_append_count); + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:before_append", &task); + const int64_t start_us = MonotonicMicros(); + status = block->append( + Slice(task.buffer->data() + buffer_offset, block->range().size())); + *_append_latency_metric << (MonotonicMicros() - start_us); + } if (!status.ok()) { *_append_fail_metric << 1; LOG(WARNING) << "Append async file cache block failed, cache=" @@ -360,21 +502,32 @@ Status AsyncCacheWriteService::_write_one(const AsyncCacheWriteTask& task) { << ", status=" << status; continue; } - status = block->finalize(); + { + ScopedActiveCounter active_finalize(&_active_finalize_count); + const int64_t start_us = MonotonicMicros(); + status = block->finalize(); + *_finalize_latency_metric << (MonotonicMicros() - start_us); + } if (!status.ok()) { - *_append_fail_metric << 1; + *_finalize_fail_metric << 1; LOG(WARNING) << "Finalize async file cache block failed, cache=" << _cache->get_base_path() << ", hash=" << task.cache_hash.to_string() << ", offset=" << block->offset() << ", size=" << block->range().size() << ", status=" << status; + continue; } + *_persisted_blocks_metric << 1; + *_persisted_bytes_metric << block->range().size(); } return Status::OK(); } void AsyncCacheWriteService::_finish_task(const AsyncCacheWriteTask& task) { + const size_t old_active = _active_task_count.fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_active > 0); const size_t old_pending = _pending_count.fetch_sub(1, std::memory_order_acq_rel); DORIS_CHECK(old_pending > 0); + *_finished_metric << 1; if (task.on_finalized) { task.on_finalized(task); } @@ -472,24 +625,10 @@ void AsyncCacheWriteService::shutdown() { }); } DORIS_CHECK(_pending_count.load(std::memory_order_acquire) == 0); + DORIS_CHECK(_queued_count.load(std::memory_order_acquire) == 0); + DORIS_CHECK(_active_task_count.load(std::memory_order_acquire) == 0); _worker_pool->shutdown(); } } -AsyncCacheWriteServiceStats AsyncCacheWriteService::stats() const { - return AsyncCacheWriteServiceStats { - .pending_count = pending_count(), - .submitted = _submitted_metric->get_value(), - .rejected = _rejected_metric->get_value(), - .buffer_alloc_fail = _buffer_alloc_fail_metric->get_value(), - .skip_downloaded = _skip_downloaded_metric->get_value(), - .skip_downloading = _skip_downloading_metric->get_value(), - .skip_partial_overlap = _skip_partial_overlap_metric->get_value(), - .drop_stale_epoch = _drop_stale_epoch_metric->get_value(), - .skip_deleting = _skip_deleting_metric->get_value(), - .append_fail = _append_fail_metric->get_value(), - .watchdog_timeout = _watchdog_timeout_metric->get_value(), - }; -} - } // namespace doris::io diff --git a/be/src/io/cache/async_cache_write_service.h b/be/src/io/cache/async_cache_write_service.h index f12f36526e795f..de191eeb043cf6 100644 --- a/be/src/io/cache/async_cache_write_service.h +++ b/be/src/io/cache/async_cache_write_service.h @@ -94,21 +94,6 @@ struct AsyncCacheWriteServiceOptions { int64_t watchdog_drop_secs {120}; }; -/// Snapshot of service counters used by tests and cache-stat reporting. -struct AsyncCacheWriteServiceStats { - size_t pending_count {0}; - uint64_t submitted {0}; - uint64_t rejected {0}; - uint64_t buffer_alloc_fail {0}; - uint64_t skip_downloaded {0}; - uint64_t skip_downloading {0}; - uint64_t skip_partial_overlap {0}; - uint64_t drop_stale_epoch {0}; - uint64_t skip_deleting {0}; - uint64_t append_fail {0}; - uint64_t watchdog_timeout {0}; -}; - /// Owns the bounded async-write queue and workers for one BlockFileCache (one cache disk). /// /// The referenced cache must outlive this service. Shutdown stops new producers, waits registered @@ -161,12 +146,20 @@ class AsyncCacheWriteService { /// Stop submissions, drain all accepted tasks, and join worker loops. Idempotent. void shutdown(); - /// Return a point-in-time counter snapshot. - AsyncCacheWriteServiceStats stats() const; - /// Return accepted tasks that have not yet completed finalization. size_t pending_count() const { return _pending_count.load(std::memory_order_relaxed); } + /// Return accepted tasks still waiting in the MPMC queue, excluding active workers. + size_t queued_count() const { return _queued_count.load(std::memory_order_relaxed); } + + /// Return tasks currently owned by workers, including watchdog and write processing. + size_t active_task_count() const { return _active_task_count.load(std::memory_order_relaxed); } + + /// Return worker loops that are currently alive. + size_t running_worker_count() const { + return _running_worker_count.load(std::memory_order_relaxed); + } + /// Return bytes currently held by tracked task buffers. int64_t buffer_memory_bytes() const { return _mem_tracker->consumption(); } @@ -187,6 +180,12 @@ class AsyncCacheWriteService { atomic_shared_ptr _options; moodycamel::ConcurrentQueue _queue; std::atomic _pending_count {0}; + std::atomic _queued_count {0}; + std::atomic _active_task_count {0}; + std::atomic _running_worker_count {0}; + std::atomic _active_get_or_set_count {0}; + std::atomic _active_append_count {0}; + std::atomic _active_finalize_count {0}; std::condition_variable _cv; std::mutex _cv_mutex; std::atomic _accepting {true}; @@ -204,18 +203,41 @@ class AsyncCacheWriteService { std::vector _worker_scheduled; std::shared_ptr> _pending_count_metric; + std::shared_ptr> _queued_count_metric; + std::shared_ptr> _active_task_count_metric; + std::shared_ptr> _running_worker_count_metric; + std::shared_ptr> _configured_worker_count_metric; + std::shared_ptr> _max_pending_tasks_metric; + std::shared_ptr> _active_get_or_set_count_metric; + std::shared_ptr> _active_append_count_metric; + std::shared_ptr> _active_finalize_count_metric; std::shared_ptr> _buffer_memory_metric; std::shared_ptr> _submitted_metric; + std::shared_ptr> _submitted_bytes_metric; + std::shared_ptr> _finished_metric; std::shared_ptr> _rejected_metric; + std::shared_ptr> _reject_not_running_metric; + std::shared_ptr> _reject_backpressure_metric; + std::shared_ptr> _reject_enqueue_failure_metric; std::shared_ptr> _buffer_alloc_fail_metric; - std::shared_ptr _latency_metric; + std::shared_ptr _submit_latency_metric; + std::shared_ptr _buffer_alloc_latency_metric; + std::shared_ptr _queue_wait_latency_metric; + std::shared_ptr _worker_task_latency_metric; + std::shared_ptr _get_or_set_latency_metric; + std::shared_ptr _append_latency_metric; + std::shared_ptr _finalize_latency_metric; std::shared_ptr> _skip_downloaded_metric; std::shared_ptr> _skip_downloading_metric; std::shared_ptr> _skip_partial_overlap_metric; std::shared_ptr> _drop_stale_epoch_metric; std::shared_ptr> _skip_deleting_metric; std::shared_ptr> _append_fail_metric; - std::shared_ptr> _watchdog_timeout_metric; + std::shared_ptr> _finalize_fail_metric; + std::shared_ptr> _persisted_blocks_metric; + std::shared_ptr> _persisted_bytes_metric; + std::shared_ptr> _watchdog_warn_metric; + std::shared_ptr> _watchdog_drop_metric; }; } // namespace doris::io diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index ef5527bacabe14..3fa8a974ad3962 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -385,6 +385,8 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path, _cache_base_path.c_str(), "file_cache_cache_lock_wait_time_us"); _get_or_set_latency_us = std::make_shared( _cache_base_path.c_str(), "file_cache_get_or_set_latency_us"); + _probe_latency_us = std::make_shared(_cache_base_path.c_str(), + "file_cache_probe_latency_us"); _storage_sync_remove_latency_us = std::make_shared( _cache_base_path.c_str(), "file_cache_storage_sync_remove_latency_us"); _storage_retry_sync_remove_latency_us = std::make_shared( @@ -850,7 +852,8 @@ FileBlocksProbeResult BlockFileCache::probe(const UInt128Wrapper& hash, size_t o DORIS_CHECK(_max_file_block_size > 0); const size_t end = offset + size; DORIS_CHECK(end > offset); - std::lock_guard cache_lock(_mutex); + const int64_t probe_start_us = MonotonicMicros(); + SCOPED_CACHE_LOCK(_mutex, this); auto file_iterator = _files.find(hash); if (file_iterator == _files.end() && !_async_open_done) { @@ -891,6 +894,7 @@ FileBlocksProbeResult BlockFileCache::probe(const UInt128Wrapper& hash, size_t o result.emplace_back(std::move(file_block)); block_offset += block_size; } + *_probe_latency_us << (MonotonicMicros() - probe_start_us); return FileBlocksProbeResult(std::move(result)); } @@ -899,7 +903,7 @@ void BlockFileCache::touch_probe_block_if_cached(const FileBlockSPtr& block, DORIS_CHECK(block != nullptr); FileBlockSPtr async_touch; { - std::lock_guard cache_lock(_mutex); + SCOPED_CACHE_LOCK(_mutex, this); auto file_iterator = _files.find(block->get_hash_value()); if (file_iterator == _files.end()) { return; @@ -933,7 +937,7 @@ void BlockFileCache::touch_probe_block_if_cached(const FileBlockSPtr& block, bool BlockFileCache::is_block_deleting(const FileBlockSPtr& block) const { DORIS_CHECK(block != nullptr); - std::lock_guard cache_lock(_mutex); + SCOPED_CACHE_LOCK(_mutex, this); auto file_iterator = _files.find(block->get_hash_value()); if (file_iterator == _files.end()) { return true; @@ -1151,7 +1155,7 @@ FileBlocksHolder BlockFileCache::get_or_set(const UInt128Wrapper& hash, size_t o int64_t duration = 0; { ConcurrencyStatsManager::instance().cached_remote_reader_get_or_set_wait_lock->increment(); - std::lock_guard cache_lock(_mutex); + SCOPED_CACHE_LOCK(_mutex, this); ConcurrencyStatsManager::instance().cached_remote_reader_get_or_set_wait_lock->decrement(); stats->lock_wait_timer += sw.elapsed_time(); SCOPED_RAW_TIMER(&duration); diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 8324b77d770ba5..0426431f15bd43 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -667,6 +667,7 @@ class BlockFileCache { std::shared_ptr _cache_lock_wait_time_us; std::shared_ptr _get_or_set_latency_us; + std::shared_ptr _probe_latency_us; std::shared_ptr _storage_sync_remove_latency_us; std::shared_ptr _storage_retry_sync_remove_latency_us; std::shared_ptr _storage_async_remove_latency_us; diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp index 740e029f79ddae..421190cf31d096 100644 --- a/be/src/io/cache/cached_remote_file_reader_async_write.cpp +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -34,6 +34,7 @@ #include "io/cache/inflight_write_buffer_index.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" +#include "util/defer_op.h" #include "util/time.h" namespace doris::io { @@ -68,6 +69,10 @@ bvar::Adder g_cached_remote_reader_middle_span_read_bytes( "cached_remote_file_reader_middle_span_read_bytes"); bvar::Adder g_cached_remote_reader_middle_span_miss_bytes( "cached_remote_file_reader_middle_span_miss_bytes"); +bvar::LatencyRecorder g_cached_remote_reader_async_read_plan_latency( + "cached_remote_file_reader_async_read_plan_latency_us"); +bvar::LatencyRecorder g_cached_remote_reader_async_write_submission_latency( + "cached_remote_file_reader_async_write_submission_latency_us"); } // namespace @@ -126,6 +131,10 @@ CacheWriteMode CachedRemoteFileReader::_resolve_cache_write_mode(const IOContext CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_plan( size_t remaining_offset, size_t remaining_size, uint64_t write_epoch, const IOContext* io_ctx, ReadStatistics& stats) { + const int64_t plan_start_us = MonotonicMicros(); + Defer record_plan_latency {[&]() { + g_cached_remote_reader_async_read_plan_latency << (MonotonicMicros() - plan_start_us); + }}; DORIS_CHECK(remaining_size > 0); const auto [align_left, align_size] = s_align_size(remaining_offset, remaining_size, size()); const size_t cache_block_size = static_cast(config::file_cache_each_block_size); @@ -416,6 +425,11 @@ void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan const std::unique_ptr& remote_buffer, const IOContext* io_ctx, ReadStatistics& stats) { + const int64_t submission_start_us = MonotonicMicros(); + Defer record_submission_latency {[&]() { + g_cached_remote_reader_async_write_submission_latency + << (MonotonicMicros() - submission_start_us); + }}; auto* service = _cache->async_write_service(); auto* inflight_index = _cache->inflight_write_buffer_index(); DORIS_CHECK(service != nullptr); diff --git a/be/src/io/cache/inflight_write_buffer_index.cpp b/be/src/io/cache/inflight_write_buffer_index.cpp index b80a5f684c523a..b3d7af24c4309a 100644 --- a/be/src/io/cache/inflight_write_buffer_index.cpp +++ b/be/src/io/cache/inflight_write_buffer_index.cpp @@ -20,9 +20,45 @@ #include #include "common/logging.h" +#include "util/time.h" namespace doris::io { +namespace { + +/// Acquire one index shard while recording both contention and critical-section duration. +class TimedShardLock { +public: + TimedShardLock(std::mutex& mutex, bvar::LatencyRecorder* wait_latency, + bvar::LatencyRecorder* hold_latency) + : _lock(mutex, std::defer_lock), + _wait_latency(wait_latency), + _hold_latency(hold_latency) { + DORIS_CHECK(wait_latency != nullptr); + DORIS_CHECK(_hold_latency != nullptr); + const int64_t wait_start_us = MonotonicMicros(); + _lock.lock(); + _acquired_at_us = MonotonicMicros(); + _wait_us = _acquired_at_us - wait_start_us; + } + + ~TimedShardLock() { + const int64_t hold_us = MonotonicMicros() - _acquired_at_us; + _lock.unlock(); + *_wait_latency << _wait_us; + *_hold_latency << hold_us; + } + +private: + std::unique_lock _lock; + bvar::LatencyRecorder* _wait_latency; + bvar::LatencyRecorder* _hold_latency; + int64_t _acquired_at_us {0}; + int64_t _wait_us {0}; +}; + +} // namespace + InflightWriteBufferIndex::InflightWriteBufferIndex(size_t shard_count, std::string metric_prefix) { DORIS_CHECK(shard_count > 0); _shards.reserve(shard_count); @@ -35,12 +71,6 @@ InflightWriteBufferIndex::InflightWriteBufferIndex(size_t shard_count, std::stri prefix, "inflight_write_buffer_index_size", [](void* index) { return static_cast(index)->size(); }, this); - _memory_metric = std::make_shared>( - prefix, "inflight_write_buffer_index_memory_bytes", - [](void* index) { - return static_cast(index)->memory_usage(); - }, - this); _lookup_metric = std::make_shared>( prefix, "inflight_write_buffer_index_lookup_total"); _hit_metric = std::make_shared>(prefix, @@ -61,6 +91,10 @@ InflightWriteBufferIndex::InflightWriteBufferIndex(size_t shard_count, std::stri prefix, "inflight_write_buffer_index_stale_epoch_replace_total"); _rollback_on_backpressure_metric = std::make_shared>( prefix, "inflight_write_buffer_index_rollback_on_backpressure_total"); + _lock_wait_latency_metric = std::make_shared( + prefix, "inflight_write_buffer_index_lock_wait_latency_us"); + _lock_hold_latency_metric = std::make_shared( + prefix, "inflight_write_buffer_index_lock_hold_latency_us"); } std::shared_ptr InflightWriteBufferIndex::insert_if_absent( @@ -71,7 +105,8 @@ std::shared_ptr InflightWriteBufferIndex::insert_if_ab auto& shard = *_shards[_shard_index(key)]; bool inserted = false; { - std::lock_guard lock(shard.mutex); + TimedShardLock lock(shard.mutex, _lock_wait_latency_metric.get(), + _lock_hold_latency_metric.get()); auto iterator = shard.entries.find(key); if (iterator == shard.entries.end()) { shard.entries.emplace(key, std::move(entry)); @@ -99,7 +134,8 @@ std::shared_ptr InflightWriteBufferIndex::lookup( Key key {.cache_hash = cache_hash, .block_offset = block_offset}; auto& shard = *_shards[_shard_index(key)]; { - std::lock_guard lock(shard.mutex); + TimedShardLock lock(shard.mutex, _lock_wait_latency_metric.get(), + _lock_hold_latency_metric.get()); auto iterator = shard.entries.find(key); if (iterator == shard.entries.end()) { *_miss_metric << 1; @@ -142,7 +178,8 @@ bool InflightWriteBufferIndex::remove_if( auto& shard = *_shards[_shard_index(key)]; bool removed = false; { - std::lock_guard lock(shard.mutex); + TimedShardLock lock(shard.mutex, _lock_wait_latency_metric.get(), + _lock_hold_latency_metric.get()); auto iterator = shard.entries.find(key); if (iterator != shard.entries.end() && iterator->second == expected) { shard.entries.erase(iterator); @@ -159,25 +196,4 @@ bool InflightWriteBufferIndex::remove_if( return false; } -void InflightWriteBufferIndex::for_each_entry( - const std::function& visitor) const { - std::vector> entries; - entries.reserve(size()); - for (const auto& shard : _shards) { - std::lock_guard lock(shard->mutex); - for (const auto& [_, entry] : shard->entries) { - entries.emplace_back(entry); - } - } - for (const auto& entry : entries) { - visitor(*entry); - } -} - -size_t InflightWriteBufferIndex::memory_usage() const { - constexpr size_t entry_overhead = sizeof(Key) + sizeof(InflightWriteBufferEntry) + - sizeof(std::shared_ptr) + 32; - return size() * entry_overhead; -} - } // namespace doris::io diff --git a/be/src/io/cache/inflight_write_buffer_index.h b/be/src/io/cache/inflight_write_buffer_index.h index 1c6e13662791d7..67fe535b47e6e0 100644 --- a/be/src/io/cache/inflight_write_buffer_index.h +++ b/be/src/io/cache/inflight_write_buffer_index.h @@ -89,15 +89,9 @@ class InflightWriteBufferIndex { bool remove_if(const UInt128Wrapper& cache_hash, size_t block_offset, const std::shared_ptr& expected); - /// Visit a retained snapshot of entries without holding shard locks during callbacks. - void for_each_entry(const std::function& visitor) const; - /// Return the number of indexed block payloads. size_t size() const { return _size.load(std::memory_order_relaxed); } - /// Estimate index metadata usage; shared payload bytes are tracked by the write service. - size_t memory_usage() const; - /// Record removal of an inserted entry because queue submission hit backpressure. void record_backpressure_rollback() { *_rollback_on_backpressure_metric << 1; } @@ -128,7 +122,6 @@ class InflightWriteBufferIndex { std::atomic _size {0}; std::shared_ptr> _size_metric; - std::shared_ptr> _memory_metric; std::shared_ptr> _lookup_metric; std::shared_ptr> _hit_metric; std::shared_ptr> _miss_metric; @@ -139,6 +132,8 @@ class InflightWriteBufferIndex { std::shared_ptr> _stale_epoch_miss_metric; std::shared_ptr> _stale_epoch_replace_metric; std::shared_ptr> _rollback_on_backpressure_metric; + std::shared_ptr _lock_wait_latency_metric; + std::shared_ptr _lock_hold_latency_metric; }; } // namespace doris::io diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp index cc1f13b06cfd89..8eaad8c41aa980 100644 --- a/be/test/io/cache/async_cache_write_service_test.cpp +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -86,6 +86,20 @@ TEST_F(AsyncCacheWriteServiceTest, TaskWritesDownloadedBlockAndCleansInflightEnt auto* index = cache->inflight_write_buffer_index(); ASSERT_NE(service, nullptr); ASSERT_NE(index, nullptr); + const uint64_t baseline_submitted = service->_submitted_metric->get_value(); + const uint64_t baseline_submitted_bytes = service->_submitted_bytes_metric->get_value(); + const uint64_t baseline_finished = service->_finished_metric->get_value(); + const uint64_t baseline_persisted_blocks = service->_persisted_blocks_metric->get_value(); + const uint64_t baseline_persisted_bytes = service->_persisted_bytes_metric->get_value(); + const int64_t baseline_submit_latency_count = service->_submit_latency_metric->count(); + const int64_t baseline_buffer_alloc_latency_count = + service->_buffer_alloc_latency_metric->count(); + const int64_t baseline_queue_wait_latency_count = service->_queue_wait_latency_metric->count(); + const int64_t baseline_worker_task_latency_count = + service->_worker_task_latency_metric->count(); + const int64_t baseline_get_or_set_latency_count = service->_get_or_set_latency_metric->count(); + const int64_t baseline_append_latency_count = service->_append_latency_metric->count(); + const int64_t baseline_finalize_latency_count = service->_finalize_latency_metric->count(); constexpr size_t block_size = 4096; const auto hash = BlockFileCache::hash("async_single_task"); @@ -119,7 +133,23 @@ TEST_F(AsyncCacheWriteServiceTest, TaskWritesDownloadedBlockAndCleansInflightEnt ASSERT_TRUE(service->try_submit(std::move(task))); ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(service->queued_count(), 0); + EXPECT_EQ(service->active_task_count(), 0); EXPECT_EQ(index->lookup(hash, 0, epoch), nullptr); + EXPECT_EQ(service->_submitted_metric->get_value() - baseline_submitted, 1); + EXPECT_EQ(service->_submitted_bytes_metric->get_value() - baseline_submitted_bytes, block_size); + EXPECT_EQ(service->_finished_metric->get_value() - baseline_finished, 1); + EXPECT_EQ(service->_persisted_blocks_metric->get_value() - baseline_persisted_blocks, 1); + EXPECT_EQ(service->_persisted_bytes_metric->get_value() - baseline_persisted_bytes, block_size); + EXPECT_EQ(service->_submit_latency_metric->count() - baseline_submit_latency_count, 1); + EXPECT_EQ(service->_buffer_alloc_latency_metric->count() - baseline_buffer_alloc_latency_count, + 1); + EXPECT_EQ(service->_queue_wait_latency_metric->count() - baseline_queue_wait_latency_count, 1); + EXPECT_EQ(service->_worker_task_latency_metric->count() - baseline_worker_task_latency_count, + 1); + EXPECT_EQ(service->_get_or_set_latency_metric->count() - baseline_get_or_set_latency_count, 1); + EXPECT_EQ(service->_append_latency_metric->count() - baseline_append_latency_count, 1); + EXPECT_EQ(service->_finalize_latency_metric->count() - baseline_finalize_latency_count, 1); ReadStatistics read_stats; CacheContext context; @@ -215,7 +245,10 @@ TEST_F(AsyncCacheWriteServiceTest, PendingLimitRejectsWhileWorkerIsActive) { }; EXPECT_FALSE(service->try_submit(std::move(rejected_task))); EXPECT_EQ(service->pending_count(), 1); - EXPECT_GE(service->stats().rejected, 1); + EXPECT_EQ(service->queued_count(), 0); + EXPECT_EQ(service->active_task_count(), 1); + EXPECT_EQ(service->_active_get_or_set_count.load(std::memory_order_relaxed), 1); + EXPECT_GE(service->_reject_backpressure_metric->get_value(), 1); { std::lock_guard lock(mutex); @@ -265,7 +298,8 @@ TEST_F(AsyncCacheWriteServiceTest, WatchdogDropsExpiredTaskAndCleansInflightEntr ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); EXPECT_EQ(service->pending_count(), 0); EXPECT_EQ(index->lookup(hash, 0, epoch), nullptr); - EXPECT_GE(service->stats().watchdog_timeout, 1); + EXPECT_GE(service->_watchdog_warn_metric->get_value(), 1); + EXPECT_GE(service->_watchdog_drop_metric->get_value(), 1); ReadStatistics read_stats; CacheContext context; @@ -408,9 +442,9 @@ TEST_F(AsyncCacheWriteServiceTest, ExistingAndDeletingCellsKeepTheirOwners) { ASSERT_TRUE(service->try_submit(std::move(task))); ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); EXPECT_EQ(service->pending_count(), 0); - EXPECT_GE(service->stats().skip_downloaded, 1); - EXPECT_GE(service->stats().skip_downloading, 1); - EXPECT_GE(service->stats().skip_deleting, 1); + EXPECT_GE(service->_skip_downloaded_metric->get_value(), 1); + EXPECT_GE(service->_skip_downloading_metric->get_value(), 1); + EXPECT_GE(service->_skip_deleting_metric->get_value(), 1); std::string actual(cell_size, '\0'); ASSERT_TRUE(downloaded_block->read(Slice(actual.data(), actual.size()), 0).ok()); @@ -472,6 +506,7 @@ TEST_F(AsyncCacheWriteServiceTest, RemoveDuringAppendDoesNotLeaveResurrectedCach std::unique_lock lock(mutex); ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return before_append; })); } + EXPECT_EQ(service->_active_append_count.load(std::memory_order_relaxed), 1); fs::path cache_file; { @@ -494,6 +529,7 @@ TEST_F(AsyncCacheWriteServiceTest, RemoveDuringAppendDoesNotLeaveResurrectedCach cv.notify_all(); ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(service->_active_append_count.load(std::memory_order_relaxed), 0); bool removed = false; for (int attempt = 0; attempt < 5000; ++attempt) { @@ -554,7 +590,7 @@ TEST_F(AsyncCacheWriteServiceTest, PartialOverlapIsSkippedWithoutOutOfBoundsWrit }; ASSERT_TRUE(service->try_submit(std::move(task))); ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); - EXPECT_GE(service->stats().skip_partial_overlap, 1); + EXPECT_GE(service->_skip_partial_overlap_metric->get_value(), 1); FileBlocks blocks; bool fully_covered = false; @@ -661,7 +697,7 @@ TEST_F(AsyncCacheWriteServiceTest, RemoveInvalidatesActiveAndQueuedTasksAndClean ASSERT_EQ(active_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); ASSERT_EQ(queued_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); EXPECT_EQ(service->pending_count(), 0); - EXPECT_GE(service->stats().drop_stale_epoch, 2); + EXPECT_GE(service->_drop_stale_epoch_metric->get_value(), 2); ReadStatistics read_stats; CacheContext context; @@ -852,7 +888,7 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { if (record_drain_samples) { // This excludes active tasks, unlike pending_count(), and therefore observes // the actual MPMC backlog after each dequeue. - drain_queue_sizes.emplace_back(service->_queue.size_approx()); + drain_queue_sizes.emplace_back(service->queued_count()); } }, &guard); @@ -892,7 +928,9 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { }); } - const auto baseline_stats = service->stats(); + const uint64_t baseline_submitted = service->_submitted_metric->get_value(); + const uint64_t baseline_rejected = service->_rejected_metric->get_value(); + const uint64_t baseline_backpressure = service->_reject_backpressure_metric->get_value(); ASSERT_TRUE(service->try_submit(std::move(tasks[0]))); { std::unique_lock lock(mutex); @@ -931,8 +969,7 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { // timing. for (size_t wave = 0; wave < fill_wave_count; ++wave) { submit_wave(1 + wave * producer_count * fill_tasks_per_producer, fill_tasks_per_producer); - EXPECT_EQ(service->_queue.size_approx(), - (wave + 1) * producer_count * fill_tasks_per_producer); + EXPECT_EQ(service->queued_count(), (wave + 1) * producer_count * fill_tasks_per_producer); EXPECT_EQ(service->pending_count(), 1 + (wave + 1) * producer_count * fill_tasks_per_producer); } @@ -944,11 +981,12 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { EXPECT_EQ(producer_rejects.load(std::memory_order_relaxed), rejected_producer_tasks); EXPECT_GT(producer_rejects.load(std::memory_order_relaxed), producer_accepts.load(std::memory_order_relaxed)); - EXPECT_EQ(service->_queue.size_approx(), accepted_producer_tasks); + EXPECT_EQ(service->queued_count(), accepted_producer_tasks); EXPECT_EQ(service->pending_count(), max_pending_tasks); - const auto pressured_stats = service->stats(); - EXPECT_EQ(pressured_stats.submitted - baseline_stats.submitted, max_pending_tasks); - EXPECT_EQ(pressured_stats.rejected - baseline_stats.rejected, rejected_producer_tasks); + EXPECT_EQ(service->_submitted_metric->get_value() - baseline_submitted, max_pending_tasks); + EXPECT_EQ(service->_rejected_metric->get_value() - baseline_rejected, rejected_producer_tasks); + EXPECT_EQ(service->_reject_backpressure_metric->get_value() - baseline_backpressure, + rejected_producer_tasks); // Producers have stopped. Grow from one to four consumers while writes remain blocked, then // remove the artificial delay and increase the batch size to model faster disk consumption. @@ -959,7 +997,7 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { std::unique_lock lock(mutex); ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return consumer_entries == fast_worker_count; })); - EXPECT_EQ(service->_queue.size_approx(), accepted_producer_tasks - (fast_worker_count - 1)); + EXPECT_EQ(service->queued_count(), accepted_producer_tasks - (fast_worker_count - 1)); record_drain_samples = true; slow_consumers = false; } @@ -971,7 +1009,7 @@ TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { [&]() { return finished_tasks == max_pending_tasks; })); } EXPECT_EQ(service->pending_count(), 0); - EXPECT_EQ(service->_queue.size_approx(), 0); + EXPECT_EQ(service->queued_count(), 0); EXPECT_TRUE(std::any_of(drain_queue_sizes.begin(), drain_queue_sizes.end(), [&](size_t size) { return size > 0 && size <= accepted_producer_tasks / 2; })); diff --git a/be/test/io/cache/block_file_cache_probe_test.cpp b/be/test/io/cache/block_file_cache_probe_test.cpp index 165281d036e528..cca0a1b6c7aad2 100644 --- a/be/test/io/cache/block_file_cache_probe_test.cpp +++ b/be/test/io/cache/block_file_cache_probe_test.cpp @@ -55,11 +55,15 @@ TEST_F(BlockFileCacheTest, ProbeMissDoesNotCreateCacheCell) { CacheContext context; context.stats = &stats; const size_t cache_size_before = cache._cur_cache_size; + const int64_t probe_latency_count = cache._probe_latency_us->count(); + const int64_t cache_lock_wait_count = cache._cache_lock_wait_time_us->count(); auto result = cache.probe(hash, 0, 4096, context); ASSERT_EQ(result.file_blocks.size(), 1); EXPECT_EQ(result.file_blocks[0], nullptr); EXPECT_EQ(cache._cur_cache_size, cache_size_before); + EXPECT_EQ(cache._probe_latency_us->count() - probe_latency_count, 1); + EXPECT_EQ(cache._cache_lock_wait_time_us->count() - cache_lock_wait_count, 1); std::lock_guard cache_lock(cache._mutex); EXPECT_EQ(cache._files.find(hash), cache._files.end()); } diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index 9b859666d035d8..b904d04b359add 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -487,7 +487,7 @@ TEST_F(AsyncCachedRemoteFileReaderTest, EXPECT_EQ(stats.async_cache_write_submitted, 0); EXPECT_EQ(cache()->async_write_service()->pending_count(), 0); EXPECT_EQ(cache()->inflight_write_buffer_index()->size(), 0); - EXPECT_GE(cache()->async_write_service()->stats().buffer_alloc_fail, 1); + EXPECT_GE(cache()->async_write_service()->_buffer_alloc_fail_metric->get_value(), 1); } TEST_F(AsyncCachedRemoteFileReaderTest, diff --git a/be/test/io/cache/inflight_write_buffer_index_test.cpp b/be/test/io/cache/inflight_write_buffer_index_test.cpp index 4445a690632e95..bd5bfdf45c1da2 100644 --- a/be/test/io/cache/inflight_write_buffer_index_test.cpp +++ b/be/test/io/cache/inflight_write_buffer_index_test.cpp @@ -40,6 +40,8 @@ TEST(InflightWriteBufferIndexTest, InsertLookupAndConditionalRemove) { InflightWriteBufferIndex index(8, "inflight_index_basic_test"); const auto hash = BlockFileCache::hash("basic"); auto entry = make_entry(7); + const int64_t lock_wait_count = index._lock_wait_latency_metric->count(); + const int64_t lock_hold_count = index._lock_hold_latency_metric->count(); EXPECT_EQ(index.insert_if_absent(hash, 0, entry), nullptr); EXPECT_EQ(index.size(), 1); @@ -51,6 +53,8 @@ TEST(InflightWriteBufferIndexTest, InsertLookupAndConditionalRemove) { EXPECT_TRUE(index.remove_if(hash, 0, entry)); EXPECT_EQ(index.size(), 0); EXPECT_EQ(index.lookup(hash, 0, 7), nullptr); + EXPECT_EQ(index._lock_wait_latency_metric->count() - lock_wait_count, 6); + EXPECT_EQ(index._lock_hold_latency_metric->count() - lock_hold_count, 6); } TEST(InflightWriteBufferIndexTest, LookupAllSupportsPartialHit) { From fbc5bcc3042cfa5dbe7566ec16d713c4c7275d47 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 17 Jul 2026 17:20:14 +0800 Subject: [PATCH 12/16] [test](be) Add async file cache write microbenchmark ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The existing file-cache microbenchmark either includes object-store and network latency through CachedRemoteFileReader or stops at BlockFileCache::get_or_set. It cannot isolate phase-1 asynchronous writeback, distinguish caller return time from background drain, or expose queue saturation and inflight-index contention. Add a standalone Release microbenchmark beside the existing tool. It combines a deterministic in-memory remote reader with a real filesystem-backed BlockFileCache and covers three layers: synchronous versus asynchronous cold-miss reader latency with complete-range persistence verification; producer admission, bounded MPMC queue behavior, worker scaling, backpressure, and real get_or_set/append/finalize persistence; and sharded miss/hit versus single-hot-key InflightWriteBufferIndex contention. Each case emits machine-readable latency percentiles, throughput, accepted/rejected/persisted counts, and pending/queued/inflight high-water marks. Benchmark data defaults to output/ so it remains untracked and uses the larger workspace disk. ### Release note None ### Check List (For Author) - Test: Manual test - ./build.sh --be --file-cache-microbench -j100 - Existing file_cache_get_or_set benchmark with 1 and 32 threads - Full async benchmark in all mode with 16 producers and worker counts 1,4,16 - build-support/check-format.sh - git diff --check - Behavior changed: No (benchmark tooling only) - Does this need documentation: No (tool README updated) --- be/src/io/tools/CMakeLists.txt | 9 + .../async_file_cache_write_microbench.cpp | 1011 +++++++++++++++++ be/src/io/tools/readme.md | 32 + build.sh | 1 + 4 files changed, 1053 insertions(+) create mode 100644 be/src/io/tools/async_file_cache_write_microbench.cpp diff --git a/be/src/io/tools/CMakeLists.txt b/be/src/io/tools/CMakeLists.txt index 40088892a97280..c956e81211bc2e 100644 --- a/be/src/io/tools/CMakeLists.txt +++ b/be/src/io/tools/CMakeLists.txt @@ -53,6 +53,10 @@ add_executable(file_cache_microbench ${PROTO_SRCS} ) +add_executable(async_file_cache_write_microbench + async_file_cache_write_microbench.cpp +) + # 添加proto生成文件的包含路径 target_include_directories(file_cache_microbench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/build/proto @@ -64,5 +68,10 @@ target_link_libraries(file_cache_microbench protobuf ) +target_link_libraries(async_file_cache_write_microbench + ${DORIS_LINK_LIBS} +) + # 安装规则 install(TARGETS file_cache_microbench DESTINATION ${OUTPUT_DIR}/lib/) +install(TARGETS async_file_cache_write_microbench DESTINATION ${OUTPUT_DIR}/lib/) diff --git a/be/src/io/tools/async_file_cache_write_microbench.cpp b/be/src/io/tools/async_file_cache_write_microbench.cpp new file mode 100644 index 00000000000000..035c2ad519e401 --- /dev/null +++ b/be/src/io/tools/async_file_cache_write_microbench.cpp @@ -0,0 +1,1011 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/file_cache_common.h" + +#if defined(BE_TEST) && defined(BUILD_FILE_CACHE_MICROBENCH_TOOL) + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cloud/config.h" +#include "common/config.h" +#include "common/status.h" +#include "io/cache/async_cache_write_service.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_factory.h" +#include "io/cache/cached_remote_file_reader.h" +#include "io/cache/fs_file_cache_storage.h" +#include "io/cache/inflight_write_buffer_index.h" +#include "io/fs/file_reader.h" +#include "io/fs/path.h" +#include "runtime/exec_env.h" +#include "runtime/thread_context.h" +#include "util/cpu_info.h" +#include "util/disk_info.h" +#include "util/mem_info.h" +#include "util/slice.h" +#include "util/time.h" + +DEFINE_string(benchmark_mode, "all", + "Comma-separated benchmark groups: reader, service, index, or all"); +DEFINE_string(cache_path, "./output/async_file_cache_write_microbench", + "Directory used by the real filesystem-backed BlockFileCache"); +DEFINE_uint64(block_size, 1024 * 1024, + "File-cache block size and stride used by cold reader misses"); +DEFINE_uint64(request_size, 64 * 1024, "Bytes returned to the caller by each reader operation"); +DEFINE_uint64(reader_operations, 128, "Cold read operations in each sync/async reader case"); +DEFINE_uint64(service_task_size, 1024 * 1024, "Payload bytes in each direct service task"); +DEFINE_uint64(service_operations, 256, "Attempted tasks in each direct service case"); +DEFINE_uint64(service_key_count, 64, "Logical remote files spread across direct service tasks"); +DEFINE_uint64(index_operations_per_thread, 100000, + "Inflight-index lookups performed by each producer thread"); +DEFINE_uint64(index_key_count, 4096, "Keys used by the representative sharded index case"); +DEFINE_int32(producer_threads, 16, "Concurrent foreground readers or task producers"); +DEFINE_int32(reader_workers, 16, "Async write workers used by the reader comparison"); +DEFINE_string(worker_counts, "1,4,16", + "Comma-separated async write worker counts used by service scaling cases"); +DEFINE_uint64(backpressure_pending_tasks, 64, + "Pending-task limit used by the saturated service case"); +DEFINE_uint64(queue_sample_interval_us, 50, + "Sampling interval for pending, queued, and inflight peak values"); +DEFINE_uint64(timeout_seconds, 120, "Maximum time to wait for one benchmark case to drain"); +DEFINE_bool(keep_cache, false, "Keep benchmark cache files after the process exits"); + +namespace doris::io { +namespace { + +using Clock = std::chrono::steady_clock; +using Nanoseconds = std::chrono::nanoseconds; + +constexpr size_t kMiB = 1024 * 1024; + +/// Compact percentile summary for foreground operation latency. +struct LatencySummary { + double average_us {0}; + double p50_us {0}; + double p95_us {0}; + double p99_us {0}; + double maximum_us {0}; +}; + +/// Return one nearest-rank percentile from an already sorted nanosecond sample set. +/// @param sorted_ns Ascending latency samples in nanoseconds. +/// @param percentile Requested percentile in the inclusive range [0, 1]. +double percentile_us(const std::vector& sorted_ns, double percentile) { + DORIS_CHECK(!sorted_ns.empty()); + DORIS_CHECK(percentile > 0 && percentile <= 1); + const size_t index = + static_cast(std::ceil(percentile * static_cast(sorted_ns.size()))) - 1; + return static_cast(sorted_ns[std::min(index, sorted_ns.size() - 1)]) / 1000.0; +} + +/// Merge per-thread samples and calculate stable latency percentiles. +/// @param per_thread_ns Independently owned samples, one vector per producer. +LatencySummary summarize_latencies(const std::vector>& per_thread_ns) { + size_t sample_count = 0; + for (const auto& samples : per_thread_ns) { + sample_count += samples.size(); + } + DORIS_CHECK(sample_count > 0); + + std::vector sorted_ns; + sorted_ns.reserve(sample_count); + for (const auto& samples : per_thread_ns) { + sorted_ns.insert(sorted_ns.end(), samples.begin(), samples.end()); + } + std::sort(sorted_ns.begin(), sorted_ns.end()); + const int64_t total_ns = std::accumulate(sorted_ns.begin(), sorted_ns.end(), int64_t {0}); + return LatencySummary { + .average_us = + static_cast(total_ns) / static_cast(sample_count) / 1000.0, + .p50_us = percentile_us(sorted_ns, 0.50), + .p95_us = percentile_us(sorted_ns, 0.95), + .p99_us = percentile_us(sorted_ns, 0.99), + .maximum_us = static_cast(sorted_ns.back()) / 1000.0, + }; +} + +/// Parse a comma-separated positive integer list used for worker scaling. +/// @param text Raw gflag value such as "1,4,16". +/// @param values Parsed worker counts in input order. +Status parse_positive_integer_list(std::string_view text, std::vector* values) { + DORIS_CHECK(values != nullptr); + values->clear(); + std::stringstream stream {std::string(text)}; + std::string token; + while (std::getline(stream, token, ',')) { + try { + const long long value = std::stoll(token); + if (value <= 0) { + return Status::InvalidArgument("worker count must be positive: {}", token); + } + values->push_back(static_cast(value)); + } catch (const std::exception& error) { + return Status::InvalidArgument("invalid worker count '{}': {}", token, error.what()); + } + } + if (values->empty()) { + return Status::InvalidArgument("worker_counts cannot be empty"); + } + return Status::OK(); +} + +/// Split the selected benchmark groups while preserving a small command-line surface. +/// @param text Raw mode string. +std::vector parse_modes(std::string_view text) { + std::stringstream stream {std::string(text)}; + std::vector modes; + std::string mode; + while (std::getline(stream, mode, ',')) { + if (!mode.empty()) { + modes.emplace_back(std::move(mode)); + } + } + return modes; +} + +/// Return whether a benchmark group was requested explicitly or through "all". +/// @param modes Parsed benchmark groups. +/// @param target Group to test. +bool mode_enabled(const std::vector& modes, std::string_view target) { + return std::find(modes.begin(), modes.end(), "all") != modes.end() || + std::find(modes.begin(), modes.end(), target) != modes.end(); +} + +/// Validate sizes and concurrency before allocating cache capacity or starting threads. +Status validate_flags(const std::vector& modes, + const std::vector& worker_counts) { + if (modes.empty()) { + return Status::InvalidArgument("benchmark_mode cannot be empty"); + } + for (const auto& mode : modes) { + if (mode != "all" && mode != "reader" && mode != "service" && mode != "index") { + return Status::InvalidArgument("unsupported benchmark mode: {}", mode); + } + } + DORIS_CHECK(!worker_counts.empty()); + if (FLAGS_producer_threads <= 0 || FLAGS_reader_workers <= 0) { + return Status::InvalidArgument("producer_threads and reader_workers must be positive"); + } + if (FLAGS_block_size == 0 || FLAGS_request_size == 0 || FLAGS_request_size > FLAGS_block_size) { + return Status::InvalidArgument("sizes must satisfy 0 < request_size <= block_size"); + } + if (FLAGS_reader_operations < static_cast(FLAGS_producer_threads) || + FLAGS_service_operations < static_cast(FLAGS_producer_threads)) { + return Status::InvalidArgument( + "reader_operations and service_operations must be at least producer_threads"); + } + if (FLAGS_service_task_size == 0 || FLAGS_service_task_size > FLAGS_block_size || + FLAGS_service_key_count == 0 || FLAGS_index_operations_per_thread == 0 || + FLAGS_index_key_count == 0 || FLAGS_backpressure_pending_tasks == 0 || + FLAGS_queue_sample_interval_us == 0 || FLAGS_timeout_seconds == 0) { + return Status::InvalidArgument( + "operation counts, timeouts, and 0 < service_task_size <= block_size are required"); + } + return Status::OK(); +} + +/// Produce deterministic data without adding network or object-store latency to reader results. +class SyntheticRemoteFileReader final : public FileReader { +public: + /// @param path Stable logical path used to derive the file-cache hash. + /// @param file_size Virtual file length; no payload is allocated for it. + /// @param block_size Pattern granularity used to validate copied bytes. + SyntheticRemoteFileReader(Path path, size_t file_size, size_t block_size) + : _path(std::move(path)), _file_size(file_size), _block_size(block_size) {} + + Status close() override { + _closed.store(true, std::memory_order_release); + return Status::OK(); + } + + const Path& path() const override { return _path; } + size_t size() const override { return _file_size; } + bool closed() const override { return _closed.load(std::memory_order_acquire); } + int64_t mtime() const override { return 0; } + + /// Return the byte value expected at an aligned benchmark block. + /// @param offset File offset inside the virtual source. + char expected_byte(size_t offset) const { + return static_cast((offset / _block_size) % 251); + } + +protected: + /// Fill the requested span from a deterministic virtual file. + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override { + DORIS_CHECK(bytes_read != nullptr); + if (offset > _file_size || result.size > _file_size - offset) { + return Status::InvalidArgument("synthetic read [{}, {}) exceeds file size {}", offset, + offset + result.size, _file_size); + } + size_t copied = 0; + while (copied < result.size) { + const size_t current_offset = offset + copied; + const size_t block_end = + std::min(_file_size, (current_offset / _block_size + 1) * _block_size); + const size_t bytes = std::min(result.size - copied, block_end - current_offset); + std::memset(result.data + copied, expected_byte(current_offset), bytes); + copied += bytes; + } + *bytes_read = result.size; + return Status::OK(); + } + +private: + Path _path; + size_t _file_size; + size_t _block_size; + std::atomic _closed {false}; +}; + +/// Record the first worker-thread error without obscuring the performance hot path. +class ConcurrentError { +public: + /// Store the first non-OK status observed by any producer. + void set(Status status) { + if (status.ok()) { + return; + } + bool expected = false; + if (_failed.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + std::lock_guard lock(_mutex); + _status = std::move(status); + } + } + + bool failed() const { return _failed.load(std::memory_order_acquire); } + + /// Return the stored error after producer threads have joined. + Status status() const { + std::lock_guard lock(_mutex); + return _status; + } + +private: + std::atomic _failed {false}; + mutable std::mutex _mutex; + Status _status; +}; + +/// Sample public queue gauges so short benchmark cases still report a meaningful high-water mark. +class QueuePeakSampler { +public: + /// @param service Service whose pending and queued gauges are sampled. + /// @param index Inflight index paired with the service. + QueuePeakSampler(AsyncCacheWriteService* service, InflightWriteBufferIndex* index) + : _service(service), _index(index) { + DORIS_CHECK(_service != nullptr); + DORIS_CHECK(_index != nullptr); + } + + /// Start the sampling thread. The sampler does not mutate benchmark state. + void start() { + _running.store(true, std::memory_order_release); + _thread = std::thread([this]() { + while (_running.load(std::memory_order_acquire)) { + update_max(&_peak_pending, _service->pending_count()); + update_max(&_peak_queued, _service->queued_count()); + update_max(&_peak_inflight, _index->size()); + std::this_thread::sleep_for( + std::chrono::microseconds(FLAGS_queue_sample_interval_us)); + } + update_max(&_peak_pending, _service->pending_count()); + update_max(&_peak_queued, _service->queued_count()); + update_max(&_peak_inflight, _index->size()); + }); + } + + /// Stop sampling and join before reading peak values. + void stop() { + _running.store(false, std::memory_order_release); + if (_thread.joinable()) { + _thread.join(); + } + } + + ~QueuePeakSampler() { stop(); } + + size_t peak_pending() const { return _peak_pending.load(std::memory_order_relaxed); } + size_t peak_queued() const { return _peak_queued.load(std::memory_order_relaxed); } + size_t peak_inflight() const { return _peak_inflight.load(std::memory_order_relaxed); } + +private: + /// Atomically preserve the largest sampled gauge value. + static void update_max(std::atomic* maximum, size_t value) { + size_t current = maximum->load(std::memory_order_relaxed); + while (current < value && + !maximum->compare_exchange_weak(current, value, std::memory_order_relaxed)) { + } + } + + AsyncCacheWriteService* _service; + InflightWriteBufferIndex* _index; + std::atomic _running {false}; + std::thread _thread; + std::atomic _peak_pending {0}; + std::atomic _peak_queued {0}; + std::atomic _peak_inflight {0}; +}; + +/// Poll one asynchronous completion predicate with a bounded failure mode. +/// @param predicate Returns true after the expected state is reached. +/// @param description Included in timeout diagnostics. +template +Status wait_until(Predicate&& predicate, std::string_view description) { + const auto deadline = Clock::now() + std::chrono::seconds(FLAGS_timeout_seconds); + while (!predicate()) { + if (Clock::now() >= deadline) { + return Status::TimedOut("timed out waiting for {}", description); + } + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + return Status::OK(); +} + +/// Own one real filesystem cache and install its factory into the otherwise minimal ExecEnv. +class BenchmarkEnvironment { +public: + BenchmarkEnvironment() = default; + + ~BenchmarkEnvironment() { + if (_factory) { + ExecEnv::GetInstance()->set_file_cache_factory(nullptr); + _factory.reset(); + } + if (!FLAGS_keep_cache) { + std::error_code error; + std::filesystem::remove_all(FLAGS_cache_path, error); + } + } + + /// Configure globals, create the cache, and wait for its asynchronous metadata open. + /// @param cache_capacity Bytes reserved for the benchmark's normal queue. + Status initialize(size_t cache_capacity) { + config::enable_async_file_cache_write = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::clear_file_cache = true; + config::enable_evict_file_cache_in_advance = false; + config::file_cache_enter_disk_resource_limit_mode_percent = 99; + config::file_cache_each_block_size = static_cast(FLAGS_block_size); + // Benchmark reads are external-table style with tablet_id=0, so they do not register + // tablet-scoped TTL work. Short intervals only bound cache teardown latency; production + // defaults let the otherwise idle TTL threads sleep for three minutes before join(). + config::file_cache_background_ttl_gc_interval_ms = 100; + config::file_cache_background_ttl_info_update_interval_ms = 100; + config::file_cache_background_tablet_id_flush_interval_ms = 100; + config::async_file_cache_write_workers_per_disk = FLAGS_reader_workers; + config::async_file_cache_write_max_pending_tasks_per_disk = + static_cast(std::max(FLAGS_reader_operations, FLAGS_service_operations)); + config::async_file_cache_write_batch_size = 16; + + DORIS_CHECK(ExecEnv::GetInstance()->file_cache_factory() == nullptr); + ExecEnv::GetInstance()->set_file_cache_open_fd_cache(std::make_unique()); + _factory = std::make_unique(); + ExecEnv::GetInstance()->set_file_cache_factory(_factory.get()); + + std::error_code error; + std::filesystem::remove_all(FLAGS_cache_path, error); + std::filesystem::create_directories(FLAGS_cache_path, error); + if (error) { + return Status::IOError("failed to create cache path {}: {}", FLAGS_cache_path, + error.message()); + } + + FileCacheSettings settings; + settings.capacity = cache_capacity; + settings.max_file_block_size = FLAGS_block_size; + settings.max_query_cache_size = 0; + const size_t auxiliary_queue_size = FLAGS_block_size; + settings.disposable_queue_size = auxiliary_queue_size; + settings.disposable_queue_elements = 128; + settings.index_queue_size = auxiliary_queue_size; + settings.index_queue_elements = 128; + settings.ttl_queue_size = auxiliary_queue_size; + settings.ttl_queue_elements = 128; + settings.query_queue_size = cache_capacity - 3 * auxiliary_queue_size; + settings.query_queue_elements = std::max( + 1024, 2 * std::max(FLAGS_reader_operations, FLAGS_service_operations)); + + RETURN_IF_ERROR(_factory->create_file_cache(FLAGS_cache_path, settings)); + _cache = _factory->get_by_path(FLAGS_cache_path); + DORIS_CHECK(_cache != nullptr); + RETURN_IF_ERROR(wait_until([&]() { return _cache->get_async_open_success(); }, + "file cache initialization")); + return Status::OK(); + } + + /// Drain accepted writes, then synchronously invalidate and clear all cached blocks. + Status clear_cache() { + RETURN_IF_ERROR(wait_for_idle()); + std::string clear_result; + RETURN_IF_ERROR(_factory->clear_file_caches(true, &clear_result)); + return wait_for_idle(); + } + + /// Apply one explicit worker/queue snapshot to the production service. + /// @param workers Active background writer count. + /// @param max_pending Maximum accepted tasks, including active workers. + Status configure_service(size_t workers, size_t max_pending) { + auto options = service()->options(); + options.worker_count = workers; + options.max_pending_tasks = max_pending; + options.batch_size = 16; + options.watchdog_warn_secs = static_cast(FLAGS_timeout_seconds); + options.watchdog_drop_secs = static_cast(FLAGS_timeout_seconds * 2); + return service()->update_options(options); + } + + /// Wait until both queue ownership and reader-visible inflight payloads are gone. + Status wait_for_idle() { + return wait_until( + [&]() { + return service()->pending_count() == 0 && service()->queued_count() == 0 && + index()->size() == 0; + }, + "async write queue and inflight index to drain"); + } + + BlockFileCache* cache() const { return _cache; } + AsyncCacheWriteService* service() const { return _cache->async_write_service(); } + InflightWriteBufferIndex* index() const { return _cache->inflight_write_buffer_index(); } + +private: + std::unique_ptr _factory; + BlockFileCache* _cache {nullptr}; +}; + +/// Verify that a complete range can be resolved from the final cache state. +/// @param cache Target cache. +/// @param hash Logical file hash. +/// @param offset Range offset to verify. +/// @param size Expected persisted bytes. +Status verify_cached_range(BlockFileCache* cache, const UInt128Wrapper& hash, size_t offset, + size_t size) { + DORIS_CHECK(cache != nullptr); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + FileBlocks blocks; + bool fully_covered = false; + RETURN_IF_ERROR(cache->get_downloaded_blocks_if_fully_covered(hash, offset, size, context, + &blocks, &fully_covered)); + if (!fully_covered) { + return Status::InternalError("cache range [{}, {}) was not persisted", offset, + offset + size); + } + return Status::OK(); +} + +/// Shared result fields printed for reader and direct-service cases. +struct AsyncWriteResult { + std::string benchmark; + std::string variant; + size_t producers {0}; + size_t workers {0}; + size_t operations {0}; + size_t accepted {0}; + size_t rejected {0}; + size_t persisted {0}; + size_t bytes_per_operation {0}; + double foreground_seconds {0}; + double drain_seconds {0}; + double total_seconds {0}; + size_t peak_pending {0}; + size_t peak_queued {0}; + size_t peak_inflight {0}; + LatencySummary latency; +}; + +/// Print one machine-readable line without hiding the foreground/drain distinction. +/// @param result Completed benchmark result. +void print_async_write_result(const AsyncWriteResult& result) { + const double foreground_ops_per_sec = + static_cast(result.operations) / result.foreground_seconds; + const double persisted_mib_per_sec = + result.total_seconds > 0 + ? static_cast(result.persisted * result.bytes_per_operation) / + static_cast(kMiB) / result.total_seconds + : 0; + std::cout << std::fixed << std::setprecision(3) << "RESULT" + << " benchmark=" << result.benchmark << " variant=" << result.variant + << " producers=" << result.producers << " workers=" << result.workers + << " operations=" << result.operations << " accepted=" << result.accepted + << " rejected=" << result.rejected << " persisted=" << result.persisted + << " bytes_per_operation=" << result.bytes_per_operation + << " foreground_seconds=" << result.foreground_seconds + << " drain_seconds=" << result.drain_seconds + << " total_seconds=" << result.total_seconds + << " foreground_ops_per_sec=" << foreground_ops_per_sec + << " persisted_mib_per_sec=" << persisted_mib_per_sec + << " avg_us=" << result.latency.average_us << " p50_us=" << result.latency.p50_us + << " p95_us=" << result.latency.p95_us << " p99_us=" << result.latency.p99_us + << " max_us=" << result.latency.maximum_us << " peak_pending=" << result.peak_pending + << " peak_queued=" << result.peak_queued << " peak_inflight=" << result.peak_inflight + << '\n'; +} + +/// Compare cold-miss caller latency with synchronous and asynchronous cache persistence. +/// @param environment Shared real cache, cleared before the case. +/// @param mode Explicit write policy applied to every CachedRemoteFileReader. +/// @param variant Stable output label. +Status run_reader_case(BenchmarkEnvironment* environment, CacheWriteMode mode, + std::string variant) { + DORIS_CHECK(environment != nullptr); + RETURN_IF_ERROR(environment->clear_cache()); + RETURN_IF_ERROR(environment->configure_service( + static_cast(FLAGS_reader_workers), + static_cast(FLAGS_reader_operations + FLAGS_reader_workers))); + + const size_t producer_count = static_cast(FLAGS_producer_threads); + const size_t operation_count = static_cast(FLAGS_reader_operations); + const size_t file_size = operation_count * FLAGS_block_size; + const std::string file_name = "async_write_reader_" + variant + ".bin"; + const Path path("/synthetic/" + file_name); + + std::vector producers; + producers.reserve(producer_count); + std::vector> latencies(producer_count); + std::vector statistics(producer_count); + std::barrier start_barrier(static_cast(producer_count + 1)); + ConcurrentError error; + QueuePeakSampler sampler(environment->service(), environment->index()); + sampler.start(); + + for (size_t producer = 0; producer < producer_count; ++producer) { + producers.emplace_back([&, producer]() { + SCOPED_INIT_THREAD_CONTEXT(); + auto remote = + std::make_shared(path, file_size, FLAGS_block_size); + FileReaderOptions options; + options.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + options.cache_write_mode = mode; + options.cache_base_path = FLAGS_cache_path; + options.tablet_id = 0; + auto reader = std::make_shared(remote, options); + IOContext io_context; + io_context.file_cache_stats = &statistics[producer]; + io_context.bypass_peer_read = true; + std::string buffer(FLAGS_request_size, '\0'); + latencies[producer].reserve((operation_count + producer_count - 1) / producer_count); + + start_barrier.arrive_and_wait(); + for (size_t operation = producer; operation < operation_count; + operation += producer_count) { + if (error.failed()) { + break; + } + const size_t offset = operation * FLAGS_block_size; + size_t bytes_read = 0; + const auto start = Clock::now(); + Status status = reader->read_at(offset, Slice(buffer.data(), buffer.size()), + &bytes_read, &io_context); + const auto elapsed = std::chrono::duration_cast(Clock::now() - start); + latencies[producer].push_back(elapsed.count()); + if (!status.ok()) { + error.set(std::move(status)); + break; + } + if (bytes_read != buffer.size() || + buffer.front() != remote->expected_byte(offset) || + buffer.back() != remote->expected_byte(offset + buffer.size() - 1)) { + error.set(Status::InternalError( + "reader result mismatch at operation {}, bytes_read={}", operation, + bytes_read)); + break; + } + } + }); + } + + const auto foreground_start = Clock::now(); + start_barrier.arrive_and_wait(); + for (auto& producer : producers) { + producer.join(); + } + const auto foreground_end = Clock::now(); + if (error.failed()) { + sampler.stop(); + return error.status(); + } + + RETURN_IF_ERROR(environment->wait_for_idle()); + const auto drain_end = Clock::now(); + sampler.stop(); + + FileCacheStatistics total_stats; + for (const auto& local_stats : statistics) { + total_stats.merge_from(local_stats); + } + const auto hash = BlockFileCache::hash(path.native() + ":0"); + RETURN_IF_ERROR(verify_cached_range(environment->cache(), hash, 0, file_size)); + + const double foreground_seconds = + std::chrono::duration(foreground_end - foreground_start).count(); + const double drain_seconds = std::chrono::duration(drain_end - foreground_end).count(); + AsyncWriteResult result { + .benchmark = "reader", + .variant = std::move(variant), + .producers = producer_count, + .workers = mode == CacheWriteMode::ASYNC_WRITE + ? static_cast(FLAGS_reader_workers) + : 0, + .operations = operation_count, + .accepted = mode == CacheWriteMode::ASYNC_WRITE + ? static_cast(total_stats.async_cache_write_submitted) + : 0, + .rejected = static_cast(total_stats.async_cache_write_rejected), + .persisted = operation_count, + .bytes_per_operation = FLAGS_block_size, + .foreground_seconds = foreground_seconds, + .drain_seconds = drain_seconds, + .total_seconds = foreground_seconds + drain_seconds, + .peak_pending = sampler.peak_pending(), + .peak_queued = sampler.peak_queued(), + .peak_inflight = sampler.peak_inflight(), + .latency = summarize_latencies(latencies), + }; + print_async_write_result(result); + return Status::OK(); +} + +/// One accepted direct-service task retained for post-timing persistence verification. +struct ServiceTaskRecord { + UInt128Wrapper hash; + size_t offset {0}; + size_t size {0}; +}; + +/// Exercise producer admission, inflight publication, MPMC consumption, and real persistence. +/// @param environment Shared real cache, cleared before the case. +/// @param variant Stable output label. +/// @param workers Active service consumers. +/// @param max_pending Bounded pending-task limit. +Status run_service_case(BenchmarkEnvironment* environment, std::string variant, size_t workers, + size_t max_pending) { + DORIS_CHECK(environment != nullptr); + RETURN_IF_ERROR(environment->clear_cache()); + RETURN_IF_ERROR(environment->configure_service(workers, max_pending)); + + const size_t producer_count = static_cast(FLAGS_producer_threads); + const size_t operation_count = static_cast(FLAGS_service_operations); + std::vector producers; + producers.reserve(producer_count); + std::vector> latencies(producer_count); + std::vector> accepted_records(producer_count); + std::barrier start_barrier(static_cast(producer_count + 1)); + std::atomic accepted {0}; + std::atomic rejected {0}; + std::atomic completed {0}; + ConcurrentError error; + QueuePeakSampler sampler(environment->service(), environment->index()); + sampler.start(); + + for (size_t producer = 0; producer < producer_count; ++producer) { + producers.emplace_back([&, producer]() { + SCOPED_INIT_THREAD_CONTEXT(); + latencies[producer].reserve((operation_count + producer_count - 1) / producer_count); + accepted_records[producer].reserve((operation_count + producer_count - 1) / + producer_count); + start_barrier.arrive_and_wait(); + + for (size_t operation = producer; operation < operation_count; + operation += producer_count) { + if (error.failed()) { + break; + } + const size_t key_index = operation % FLAGS_service_key_count; + const size_t block_index = operation / FLAGS_service_key_count; + const auto hash = BlockFileCache::hash("async_write_service_" + variant + "_" + + std::to_string(key_index)); + const size_t offset = block_index * FLAGS_block_size; + const auto start = Clock::now(); + + AsyncCacheWriteBufferPtr buffer; + Status status = environment->service()->allocate_tracked_buffer( + FLAGS_service_task_size, &buffer); + if (!status.ok()) { + error.set(std::move(status)); + break; + } + std::memset(buffer->data(), static_cast(operation % 251), buffer->size()); + const uint64_t epoch = environment->service()->current_write_epoch(); + auto entry = std::make_shared( + buffer, offset, buffer->size(), MonotonicMicros(), epoch); + auto existing = environment->index()->insert_if_absent(hash, offset, entry); + if (existing != nullptr) { + error.set(Status::InternalError( + "unexpected duplicate inflight owner at operation {}", operation)); + break; + } + + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = offset, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = epoch, + .on_finalized = + [index = environment->index(), hash, offset, entry, + &completed](const AsyncCacheWriteTask&) { + index->remove_if(hash, offset, entry); + completed.fetch_add(1, std::memory_order_release); + }, + }; + const bool submitted = environment->service()->try_submit(std::move(task)); + const auto elapsed = std::chrono::duration_cast(Clock::now() - start); + latencies[producer].push_back(elapsed.count()); + if (submitted) { + accepted.fetch_add(1, std::memory_order_relaxed); + accepted_records[producer].push_back(ServiceTaskRecord { + .hash = hash, .offset = offset, .size = FLAGS_service_task_size}); + } else { + environment->index()->remove_if(hash, offset, entry); + environment->index()->record_backpressure_rollback(); + rejected.fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + + const auto foreground_start = Clock::now(); + start_barrier.arrive_and_wait(); + for (auto& producer : producers) { + producer.join(); + } + const auto foreground_end = Clock::now(); + if (error.failed()) { + sampler.stop(); + return error.status(); + } + + RETURN_IF_ERROR(wait_until( + [&]() { + return environment->service()->pending_count() == 0 && + completed.load(std::memory_order_acquire) == + accepted.load(std::memory_order_acquire) && + environment->index()->size() == 0; + }, + "direct service tasks to complete")); + const auto drain_end = Clock::now(); + sampler.stop(); + + size_t persisted = 0; + for (const auto& records : accepted_records) { + for (const auto& record : records) { + Status status = verify_cached_range(environment->cache(), record.hash, record.offset, + record.size); + if (status.ok()) { + ++persisted; + } + } + } + if (persisted != accepted.load(std::memory_order_relaxed)) { + return Status::InternalError("only {} of {} accepted service tasks persisted", persisted, + accepted.load(std::memory_order_relaxed)); + } + + const double foreground_seconds = + std::chrono::duration(foreground_end - foreground_start).count(); + const double drain_seconds = std::chrono::duration(drain_end - foreground_end).count(); + AsyncWriteResult result { + .benchmark = "service", + .variant = std::move(variant), + .producers = producer_count, + .workers = workers, + .operations = operation_count, + .accepted = accepted.load(std::memory_order_relaxed), + .rejected = rejected.load(std::memory_order_relaxed), + .persisted = persisted, + .bytes_per_operation = FLAGS_service_task_size, + .foreground_seconds = foreground_seconds, + .drain_seconds = drain_seconds, + .total_seconds = foreground_seconds + drain_seconds, + .peak_pending = sampler.peak_pending(), + .peak_queued = sampler.peak_queued(), + .peak_inflight = sampler.peak_inflight(), + .latency = summarize_latencies(latencies), + }; + print_async_write_result(result); + return Status::OK(); +} + +/// Print one inflight-index result using the same latency field names as write cases. +void print_index_result(std::string_view variant, size_t operations, double elapsed_seconds, + const LatencySummary& latency) { + std::cout << std::fixed << std::setprecision(3) << "RESULT benchmark=index" + << " variant=" << variant << " producers=" << FLAGS_producer_threads + << " operations=" << operations << " elapsed_seconds=" << elapsed_seconds + << " operations_per_sec=" << static_cast(operations) / elapsed_seconds + << " avg_us=" << latency.average_us << " p50_us=" << latency.p50_us + << " p95_us=" << latency.p95_us << " p99_us=" << latency.p99_us + << " max_us=" << latency.maximum_us << '\n'; +} + +/// Measure either representative sharded misses or worst-case single-key hit contention. +/// @param variant Stable output label. +/// @param key_count Number of distinct cache hashes used by the workload. +/// @param populate Whether every lookup should hit a pre-populated entry. +Status run_index_case(std::string variant, size_t key_count, bool populate) { + const size_t producer_count = static_cast(FLAGS_producer_threads); + const size_t operations_per_thread = static_cast(FLAGS_index_operations_per_thread); + InflightWriteBufferIndex index(64, FLAGS_cache_path + "_index_" + variant); + std::vector hashes; + hashes.reserve(key_count); + const uint64_t epoch = 1; + auto entry = + std::make_shared(nullptr, 0, 1, MonotonicMicros(), epoch); + for (size_t key = 0; key < key_count; ++key) { + hashes.emplace_back( + BlockFileCache::hash("inflight_index_" + variant + "_" + std::to_string(key))); + if (populate) { + DORIS_CHECK(index.insert_if_absent(hashes.back(), 0, entry) == nullptr); + } + } + + std::vector producers; + producers.reserve(producer_count); + std::vector> latencies(producer_count); + std::barrier start_barrier(static_cast(producer_count + 1)); + ConcurrentError error; + for (size_t producer = 0; producer < producer_count; ++producer) { + producers.emplace_back([&, producer]() { + latencies[producer].reserve(operations_per_thread); + start_barrier.arrive_and_wait(); + for (size_t operation = 0; operation < operations_per_thread; ++operation) { + const size_t key = (operation * 2654435761ULL + producer) % key_count; + const auto start = Clock::now(); + auto result = index.lookup(hashes[key], 0, epoch); + const auto elapsed = std::chrono::duration_cast(Clock::now() - start); + latencies[producer].push_back(elapsed.count()); + if ((result != nullptr) != populate) { + error.set(Status::InternalError("unexpected index lookup result")); + break; + } + } + }); + } + + const auto start = Clock::now(); + start_barrier.arrive_and_wait(); + for (auto& producer : producers) { + producer.join(); + } + const auto end = Clock::now(); + if (error.failed()) { + return error.status(); + } + + const size_t total_operations = producer_count * operations_per_thread; + print_index_result(variant, total_operations, + std::chrono::duration(end - start).count(), + summarize_latencies(latencies)); + return Status::OK(); +} + +/// Run the selected benchmark groups in a fixed, directly comparable order. +Status run_benchmarks(const std::vector& modes, + const std::vector& worker_counts) { + const size_t reader_bytes = FLAGS_reader_operations * FLAGS_block_size; + const size_t service_bytes = FLAGS_service_operations * FLAGS_service_task_size; + const size_t workload_bytes = std::max(reader_bytes, service_bytes); + const size_t cache_capacity = std::max(256 * kMiB, workload_bytes * 4); + if (cache_capacity <= 3 * FLAGS_block_size) { + return Status::InvalidArgument("calculated cache capacity is too small"); + } + + BenchmarkEnvironment environment; + RETURN_IF_ERROR(environment.initialize(cache_capacity)); + std::cout << "CONFIG cache_path=" << FLAGS_cache_path << " cache_capacity=" << cache_capacity + << " block_size=" << FLAGS_block_size << " request_size=" << FLAGS_request_size + << " producer_threads=" << FLAGS_producer_threads << '\n'; + + if (mode_enabled(modes, "reader")) { + RETURN_IF_ERROR(run_reader_case(&environment, CacheWriteMode::SYNC_WRITE, "sync_write")); + RETURN_IF_ERROR(run_reader_case(&environment, CacheWriteMode::ASYNC_WRITE, "async_write")); + } + if (mode_enabled(modes, "service")) { + for (size_t workers : worker_counts) { + RETURN_IF_ERROR(run_service_case( + &environment, "drain_workers_" + std::to_string(workers), workers, + static_cast(FLAGS_service_operations + workers))); + } + RETURN_IF_ERROR(run_service_case( + &environment, "backpressure", worker_counts.back(), + std::min(FLAGS_backpressure_pending_tasks, FLAGS_service_operations))); + } + if (mode_enabled(modes, "index")) { + RETURN_IF_ERROR(run_index_case("sharded_miss", FLAGS_index_key_count, false)); + RETURN_IF_ERROR(run_index_case("sharded_hit", FLAGS_index_key_count, true)); + RETURN_IF_ERROR(run_index_case("hot_key_hit", 1, true)); + } + return Status::OK(); +} + +} // namespace +} // namespace doris::io + +int main(int argc, char** argv) { + google::ParseCommandLineFlags(&argc, &argv, true); + FLAGS_logtostderr = true; + google::InitGoogleLogging(argv[0]); + + // Doris configuration defaults contain ${DORIS_HOME}; a standalone benchmark has no launcher + // to provide it, and only needs a valid expansion rather than a configuration file. + if (std::getenv("DORIS_HOME") == nullptr) { + const std::string current_directory = std::filesystem::current_path().string(); + if (::setenv("DORIS_HOME", current_directory.c_str(), 0) != 0) { + std::cerr << "Benchmark failed: cannot initialize DORIS_HOME\n"; + return 1; + } + } + if (!doris::config::init(nullptr, true)) { + std::cerr << "Benchmark failed: cannot initialize Doris configuration defaults\n"; + return 1; + } + doris::CpuInfo::init(); + doris::DiskInfo::init(); + doris::MemInfo::init(); + SCOPED_INIT_THREAD_CONTEXT(); + doris::ExecEnv::GetInstance()->init_mem_tracker(); + DORIS_CHECK(doris::thread_context()->thread_mem_tracker_mgr->init()); + + std::vector worker_counts; + doris::Status status = + doris::io::parse_positive_integer_list(FLAGS_worker_counts, &worker_counts); + const auto modes = doris::io::parse_modes(FLAGS_benchmark_mode); + if (status.ok()) { + status = doris::io::validate_flags(modes, worker_counts); + } + if (status.ok()) { + status = doris::io::run_benchmarks(modes, worker_counts); + } + if (!status.ok()) { + std::cerr << "Benchmark failed: " << status.to_string() << '\n'; + return 1; + } + return 0; +} + +#endif diff --git a/be/src/io/tools/readme.md b/be/src/io/tools/readme.md index a5804958da07b9..1e5ea613790e7b 100644 --- a/be/src/io/tools/readme.md +++ b/be/src/io/tools/readme.md @@ -9,6 +9,8 @@ To compile the project, run the following command: ``` This will generate the `file_cache_microbench` executable in the `apache_doris/output/be/lib` directory. +It also generates `async_file_cache_write_microbench`, a standalone benchmark for the +asynchronous file-cache write path. ## Usage @@ -131,3 +133,33 @@ curl "http://localhost:{port}/MicrobenchService/get_help" ### Version Information: you can see it in get_help return msg +## Asynchronous write benchmark + +`async_file_cache_write_microbench` uses a real filesystem-backed `BlockFileCache` but a +deterministic in-memory remote reader. This keeps object-store latency out of the measurements while +retaining the production inflight index, bounded MPMC queue, worker pool, `get_or_set`, `append`, +and `finalize` implementation. + +The benchmark groups are: + +- `reader`: compares cold-miss foreground latency for forced synchronous and asynchronous cache + writes, then reports asynchronous drain time separately. +- `service`: measures end-to-end persistence while scaling worker count, followed by a bounded + queue case that reports producer-side rejection and queue high-water marks. +- `index`: separates representative sharded miss/hit lookup cost from worst-case hot-key hit + contention in `InflightWriteBufferIndex`. + +Run all groups with the default production-sized 1 MiB cache block: + +```bash +./output/be/lib/async_file_cache_write_microbench \ + --benchmark_mode=all \ + --cache_path=./output/async_file_cache_write_microbench \ + --producer_threads=16 \ + --worker_counts=1,4,16 +``` + +Every result is printed on one `RESULT` line. Reader and service results distinguish +`foreground_seconds` from `drain_seconds`, include operation latency percentiles, accepted and +rejected task counts, verified persisted tasks, and sampled peaks for pending, queued, and inflight +work. Use `--help` to adjust operation counts, request/task sizes, queue limit, and worker counts. diff --git a/build.sh b/build.sh index 7bcd0dc7439000..38501f37439d3f 100755 --- a/build.sh +++ b/build.sh @@ -1138,6 +1138,7 @@ EOF if [[ "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" = "ON" ]]; then cp -r -p "${DORIS_HOME}/be/output/lib/file_cache_microbench" "${DORIS_OUTPUT}/be/lib"/ + cp -r -p "${DORIS_HOME}/be/output/lib/async_file_cache_write_microbench" "${DORIS_OUTPUT}/be/lib"/ fi if [[ "${BUILD_INDEX_TOOL}" = "ON" ]]; then From 733ee939fbcfff40622a4378796519f126ebe686 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 17 Jul 2026 17:45:25 +0800 Subject: [PATCH 13/16] [test](be) Repeat and document async cache write benchmark ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: The asynchronous file-cache write microbenchmark previously emitted only one sample per case and did not establish the storage baseline of the cache filesystem. These short concurrent cases are sensitive to scheduler activity, page-cache state, filesystem metadata, and background writeback, so a single number can hide material variance and make worker-scaling conclusions unreliable. Run every selected reader, service, and inflight-index case five times by default and add the one-based repetition to each machine-readable RESULT line. Add an installed runner that can measure direct 1 MiB sequential QD1 and random QD16 writes on the same filesystem before starting the benchmark. The fio behavior is explicit and does not make fio a mandatory dependency: | RUN_FIO | fio available | Behavior | | --- | --- | --- | | auto (default) | Yes | Run both disk baselines, then run the cache benchmark | | auto (default) | No | Print DISK_BASELINE skipped and continue directly with the cache benchmark | | 1 | No | Fail because the caller explicitly required fio | | 0 | Any | Skip fio and run the cache benchmark | The runner uses a unique sibling directory under the selected cache path, unlinks fio data, and keeps direct I/O out of the page cache. The benchmark rejects non-empty cache paths instead of recursively clearing them, and suppresses INFO logging so merged stdout and stderr cannot corrupt RESULT records. Expand the tool README with the component flow, coverage and non-goals of each group, default workload, field semantics, fio controls, cache-path ownership, repetition methodology, and interpretation guidance. Median is the primary value and the observed minimum and maximum are retained. Release experiment configuration: /dev/nvme11n1 ext4, 1 MiB blocks, 64 KiB caller reads, 16 producers, 128 reader operations, 256 service attempts, and five repetitions. fio direct-I/O baseline: | Workload | Bandwidth | p95 completion latency | | --- | ---: | ---: | | 1 MiB sequential write, QD1 | 2513 MiB/s | 161 us | | 1 MiB random write, QD16 | 3106 MiB/s | 10.552 ms | CachedRemoteFileReader foreground results: | Write mode | Median ops/s | Minimum ops/s | Maximum ops/s | Median average latency | | --- | ---: | ---: | ---: | ---: | | Synchronous | 5645 | 5124 | 7649 | 1459 us | | Asynchronous | 6739 | 4739 | 8298 | 912 us | The asynchronous median was 19.4% higher in throughput and 37.5% lower in average latency. The overlapping ranges are retained because they show why a single run is insufficient. AsyncCacheWriteService verified completion results: | Workers | Median MiB/s | Minimum MiB/s | Maximum MiB/s | Median drain time | | ---: | ---: | ---: | ---: | ---: | | 1 | 798 | 730 | 968 | 0.300 s | | 4 | 1562 | 1260 | 1751 | 0.153 s | | 16 | 7825 | 5255 | 13203 | 0.014 s | These values measure buffered append and finalize completion without fsync. They are not durable-media throughput and are not directly comparable with the direct-I/O fio baseline. Bounded backpressure results: | Metric | Median | Minimum | Maximum | | --- | ---: | ---: | ---: | | Accepted tasks | 76 | 64 | 101 | | Rejected tasks | 180 | 155 | 192 | | Peak pending | 64 | 64 | 64 | | Peak queued | 48 | 48 | 48 | | Peak inflight | 65 | 65 | 67 | Every accepted task was verified as persisted, and peak pending stayed at the configured limit. InflightWriteBufferIndex lookup results: | Workload | Median ops/s | Minimum ops/s | Maximum ops/s | Median average latency | | --- | ---: | ---: | ---: | ---: | | Sharded miss | 5.531M | 4.028M | 6.621M | 2.688 us | | Sharded hit | 4.198M | 3.270M | 6.036M | 3.171 us | | Hot-key hit | 1.104M | 1.090M | 1.268M | 13.701 us | All 45 RESULT records were complete and parseable. All reader ranges and all accepted service tasks passed final BlockFileCache coverage verification. ### Release note None ### Check List (For Author) - Test: Manual test - ./build.sh --be --file-cache-microbench -j100 (Release) - ./output/be/bin/run-async-file-cache-write-microbench.sh --benchmark_mode=all --cache_path=./output/async_file_cache_write_microbench_repeat_5_clean --producer_threads=16 --reader_workers=16 --worker_counts=1,4,16 --repetitions=5 - Non-empty cache-path rejection with sentinel preservation - build-support/clang-format.sh - build-support/check-format.sh - bash -n and shellcheck for the runner - git diff --check - Behavior changed: No (benchmark tooling only) - Does this need documentation: No (tool README updated) --- .../async_file_cache_write_microbench.cpp | 149 +++++++++----- be/src/io/tools/readme.md | 187 ++++++++++++++++-- bin/run-async-file-cache-write-microbench.sh | 140 +++++++++++++ build.sh | 1 + 4 files changed, 414 insertions(+), 63 deletions(-) create mode 100755 bin/run-async-file-cache-write-microbench.sh diff --git a/be/src/io/tools/async_file_cache_write_microbench.cpp b/be/src/io/tools/async_file_cache_write_microbench.cpp index 035c2ad519e401..13d7ba51dcedd6 100644 --- a/be/src/io/tools/async_file_cache_write_microbench.cpp +++ b/be/src/io/tools/async_file_cache_write_microbench.cpp @@ -81,6 +81,7 @@ DEFINE_int32(producer_threads, 16, "Concurrent foreground readers or task produc DEFINE_int32(reader_workers, 16, "Async write workers used by the reader comparison"); DEFINE_string(worker_counts, "1,4,16", "Comma-separated async write worker counts used by service scaling cases"); +DEFINE_uint64(repetitions, 5, "Measured repetitions of every selected benchmark case"); DEFINE_uint64(backpressure_pending_tasks, 64, "Pending-task limit used by the saturated service case"); DEFINE_uint64(queue_sample_interval_us, 50, @@ -215,7 +216,8 @@ Status validate_flags(const std::vector& modes, if (FLAGS_service_task_size == 0 || FLAGS_service_task_size > FLAGS_block_size || FLAGS_service_key_count == 0 || FLAGS_index_operations_per_thread == 0 || FLAGS_index_key_count == 0 || FLAGS_backpressure_pending_tasks == 0 || - FLAGS_queue_sample_interval_us == 0 || FLAGS_timeout_seconds == 0) { + FLAGS_queue_sample_interval_us == 0 || FLAGS_timeout_seconds == 0 || + FLAGS_repetitions == 0) { return Status::InvalidArgument( "operation counts, timeouts, and 0 < service_task_size <= block_size are required"); } @@ -390,7 +392,7 @@ class BenchmarkEnvironment { ExecEnv::GetInstance()->set_file_cache_factory(nullptr); _factory.reset(); } - if (!FLAGS_keep_cache) { + if (_owns_cache_path && !FLAGS_keep_cache) { std::error_code error; std::filesystem::remove_all(FLAGS_cache_path, error); } @@ -399,6 +401,55 @@ class BenchmarkEnvironment { /// Configure globals, create the cache, and wait for its asynchronous metadata open. /// @param cache_capacity Bytes reserved for the benchmark's normal queue. Status initialize(size_t cache_capacity) { + std::error_code error; + const auto cache_path = + std::filesystem::absolute(FLAGS_cache_path, error).lexically_normal(); + if (error) { + return Status::IOError("failed to resolve cache path {}: {}", FLAGS_cache_path, + error.message()); + } + const auto current_path = std::filesystem::current_path(error); + if (error) { + return Status::IOError("failed to resolve current directory: {}", error.message()); + } + if (cache_path == cache_path.root_path() || cache_path == current_path) { + return Status::InvalidArgument("cache path must be a dedicated subdirectory: {}", + cache_path.string()); + } + const bool cache_path_exists = std::filesystem::exists(cache_path, error); + if (error) { + return Status::IOError("failed to inspect cache path {}: {}", cache_path.string(), + error.message()); + } + if (cache_path_exists) { + const bool cache_path_is_directory = std::filesystem::is_directory(cache_path, error); + if (error) { + return Status::IOError("failed to inspect cache path {}: {}", cache_path.string(), + error.message()); + } + if (!cache_path_is_directory) { + return Status::InvalidArgument("cache path is not a directory: {}", + cache_path.string()); + } + const bool cache_path_is_empty = std::filesystem::is_empty(cache_path, error); + if (error) { + return Status::IOError("failed to inspect cache path {}: {}", cache_path.string(), + error.message()); + } + if (!cache_path_is_empty) { + return Status::InvalidArgument( + "cache path must not exist or must be an empty directory: {}", + cache_path.string()); + } + } else { + std::filesystem::create_directories(cache_path, error); + if (error) { + return Status::IOError("failed to create cache path {}: {}", cache_path.string(), + error.message()); + } + } + _owns_cache_path = true; + config::enable_async_file_cache_write = true; config::enable_async_file_cache_write_inflight_write_buffer_index = true; config::enable_read_cache_file_directly = false; @@ -423,14 +474,6 @@ class BenchmarkEnvironment { _factory = std::make_unique(); ExecEnv::GetInstance()->set_file_cache_factory(_factory.get()); - std::error_code error; - std::filesystem::remove_all(FLAGS_cache_path, error); - std::filesystem::create_directories(FLAGS_cache_path, error); - if (error) { - return Status::IOError("failed to create cache path {}: {}", FLAGS_cache_path, - error.message()); - } - FileCacheSettings settings; settings.capacity = cache_capacity; settings.max_file_block_size = FLAGS_block_size; @@ -492,6 +535,7 @@ class BenchmarkEnvironment { private: std::unique_ptr _factory; BlockFileCache* _cache {nullptr}; + bool _owns_cache_path {false}; }; /// Verify that a complete range can be resolved from the final cache state. @@ -538,7 +582,8 @@ struct AsyncWriteResult { /// Print one machine-readable line without hiding the foreground/drain distinction. /// @param result Completed benchmark result. -void print_async_write_result(const AsyncWriteResult& result) { +/// @param repetition One-based repetition index. +void print_async_write_result(const AsyncWriteResult& result, size_t repetition) { const double foreground_ops_per_sec = static_cast(result.operations) / result.foreground_seconds; const double persisted_mib_per_sec = @@ -548,9 +593,10 @@ void print_async_write_result(const AsyncWriteResult& result) { : 0; std::cout << std::fixed << std::setprecision(3) << "RESULT" << " benchmark=" << result.benchmark << " variant=" << result.variant - << " producers=" << result.producers << " workers=" << result.workers - << " operations=" << result.operations << " accepted=" << result.accepted - << " rejected=" << result.rejected << " persisted=" << result.persisted + << " repetition=" << repetition << " producers=" << result.producers + << " workers=" << result.workers << " operations=" << result.operations + << " accepted=" << result.accepted << " rejected=" << result.rejected + << " persisted=" << result.persisted << " bytes_per_operation=" << result.bytes_per_operation << " foreground_seconds=" << result.foreground_seconds << " drain_seconds=" << result.drain_seconds @@ -568,8 +614,9 @@ void print_async_write_result(const AsyncWriteResult& result) { /// @param environment Shared real cache, cleared before the case. /// @param mode Explicit write policy applied to every CachedRemoteFileReader. /// @param variant Stable output label. -Status run_reader_case(BenchmarkEnvironment* environment, CacheWriteMode mode, - std::string variant) { +/// @param repetition One-based repetition index included in output. +Status run_reader_case(BenchmarkEnvironment* environment, CacheWriteMode mode, std::string variant, + size_t repetition) { DORIS_CHECK(environment != nullptr); RETURN_IF_ERROR(environment->clear_cache()); RETURN_IF_ERROR(environment->configure_service( @@ -684,7 +731,7 @@ Status run_reader_case(BenchmarkEnvironment* environment, CacheWriteMode mode, .peak_inflight = sampler.peak_inflight(), .latency = summarize_latencies(latencies), }; - print_async_write_result(result); + print_async_write_result(result, repetition); return Status::OK(); } @@ -700,8 +747,9 @@ struct ServiceTaskRecord { /// @param variant Stable output label. /// @param workers Active service consumers. /// @param max_pending Bounded pending-task limit. +/// @param repetition One-based repetition index included in output. Status run_service_case(BenchmarkEnvironment* environment, std::string variant, size_t workers, - size_t max_pending) { + size_t max_pending, size_t repetition) { DORIS_CHECK(environment != nullptr); RETURN_IF_ERROR(environment->clear_cache()); RETURN_IF_ERROR(environment->configure_service(workers, max_pending)); @@ -847,16 +895,18 @@ Status run_service_case(BenchmarkEnvironment* environment, std::string variant, .peak_inflight = sampler.peak_inflight(), .latency = summarize_latencies(latencies), }; - print_async_write_result(result); + print_async_write_result(result, repetition); return Status::OK(); } /// Print one inflight-index result using the same latency field names as write cases. -void print_index_result(std::string_view variant, size_t operations, double elapsed_seconds, - const LatencySummary& latency) { +/// @param repetition One-based repetition index. +void print_index_result(std::string_view variant, size_t repetition, size_t operations, + double elapsed_seconds, const LatencySummary& latency) { std::cout << std::fixed << std::setprecision(3) << "RESULT benchmark=index" - << " variant=" << variant << " producers=" << FLAGS_producer_threads - << " operations=" << operations << " elapsed_seconds=" << elapsed_seconds + << " variant=" << variant << " repetition=" << repetition + << " producers=" << FLAGS_producer_threads << " operations=" << operations + << " elapsed_seconds=" << elapsed_seconds << " operations_per_sec=" << static_cast(operations) / elapsed_seconds << " avg_us=" << latency.average_us << " p50_us=" << latency.p50_us << " p95_us=" << latency.p95_us << " p99_us=" << latency.p99_us @@ -867,10 +917,12 @@ void print_index_result(std::string_view variant, size_t operations, double elap /// @param variant Stable output label. /// @param key_count Number of distinct cache hashes used by the workload. /// @param populate Whether every lookup should hit a pre-populated entry. -Status run_index_case(std::string variant, size_t key_count, bool populate) { +/// @param repetition One-based repetition index included in output. +Status run_index_case(std::string variant, size_t key_count, bool populate, size_t repetition) { const size_t producer_count = static_cast(FLAGS_producer_threads); const size_t operations_per_thread = static_cast(FLAGS_index_operations_per_thread); - InflightWriteBufferIndex index(64, FLAGS_cache_path + "_index_" + variant); + InflightWriteBufferIndex index( + 64, FLAGS_cache_path + "_index_" + variant + "_" + std::to_string(repetition)); std::vector hashes; hashes.reserve(key_count); const uint64_t epoch = 1; @@ -918,7 +970,7 @@ Status run_index_case(std::string variant, size_t key_count, bool populate) { } const size_t total_operations = producer_count * operations_per_thread; - print_index_result(variant, total_operations, + print_index_result(variant, repetition, total_operations, std::chrono::duration(end - start).count(), summarize_latencies(latencies)); return Status::OK(); @@ -939,26 +991,33 @@ Status run_benchmarks(const std::vector& modes, RETURN_IF_ERROR(environment.initialize(cache_capacity)); std::cout << "CONFIG cache_path=" << FLAGS_cache_path << " cache_capacity=" << cache_capacity << " block_size=" << FLAGS_block_size << " request_size=" << FLAGS_request_size - << " producer_threads=" << FLAGS_producer_threads << '\n'; - - if (mode_enabled(modes, "reader")) { - RETURN_IF_ERROR(run_reader_case(&environment, CacheWriteMode::SYNC_WRITE, "sync_write")); - RETURN_IF_ERROR(run_reader_case(&environment, CacheWriteMode::ASYNC_WRITE, "async_write")); - } - if (mode_enabled(modes, "service")) { - for (size_t workers : worker_counts) { + << " producer_threads=" << FLAGS_producer_threads + << " repetitions=" << FLAGS_repetitions << '\n'; + + for (size_t repetition = 1; repetition <= FLAGS_repetitions; ++repetition) { + if (mode_enabled(modes, "reader")) { + RETURN_IF_ERROR(run_reader_case(&environment, CacheWriteMode::SYNC_WRITE, "sync_write", + repetition)); + RETURN_IF_ERROR(run_reader_case(&environment, CacheWriteMode::ASYNC_WRITE, + "async_write", repetition)); + } + if (mode_enabled(modes, "service")) { + for (size_t workers : worker_counts) { + RETURN_IF_ERROR(run_service_case( + &environment, "drain_workers_" + std::to_string(workers), workers, + static_cast(FLAGS_service_operations + workers), repetition)); + } RETURN_IF_ERROR(run_service_case( - &environment, "drain_workers_" + std::to_string(workers), workers, - static_cast(FLAGS_service_operations + workers))); + &environment, "backpressure", worker_counts.back(), + std::min(FLAGS_backpressure_pending_tasks, FLAGS_service_operations), + repetition)); + } + if (mode_enabled(modes, "index")) { + RETURN_IF_ERROR( + run_index_case("sharded_miss", FLAGS_index_key_count, false, repetition)); + RETURN_IF_ERROR(run_index_case("sharded_hit", FLAGS_index_key_count, true, repetition)); + RETURN_IF_ERROR(run_index_case("hot_key_hit", 1, true, repetition)); } - RETURN_IF_ERROR(run_service_case( - &environment, "backpressure", worker_counts.back(), - std::min(FLAGS_backpressure_pending_tasks, FLAGS_service_operations))); - } - if (mode_enabled(modes, "index")) { - RETURN_IF_ERROR(run_index_case("sharded_miss", FLAGS_index_key_count, false)); - RETURN_IF_ERROR(run_index_case("sharded_hit", FLAGS_index_key_count, true)); - RETURN_IF_ERROR(run_index_case("hot_key_hit", 1, true)); } return Status::OK(); } @@ -969,6 +1028,8 @@ Status run_benchmarks(const std::vector& modes, int main(int argc, char** argv) { google::ParseCommandLineFlags(&argc, &argv, true); FLAGS_logtostderr = true; + // Keep RESULT lines parseable when the documented runner merges stdout and stderr. + FLAGS_minloglevel = google::GLOG_ERROR; google::InitGoogleLogging(argv[0]); // Doris configuration defaults contain ${DORIS_HOME}; a standalone benchmark has no launcher diff --git a/be/src/io/tools/readme.md b/be/src/io/tools/readme.md index 1e5ea613790e7b..803e02a4c2b161 100644 --- a/be/src/io/tools/readme.md +++ b/be/src/io/tools/readme.md @@ -5,12 +5,13 @@ To compile the project, run the following command: ```bash -./build.sh --clean --file-cache-microbench --be +./build.sh --be --file-cache-microbench -j100 ``` This will generate the `file_cache_microbench` executable in the `apache_doris/output/be/lib` directory. It also generates `async_file_cache_write_microbench`, a standalone benchmark for the -asynchronous file-cache write path. +asynchronous file-cache write path, and installs its runner as +`output/be/bin/run-async-file-cache-write-microbench.sh`. ## Usage @@ -135,31 +136,179 @@ you can see it in get_help return msg ## Asynchronous write benchmark -`async_file_cache_write_microbench` uses a real filesystem-backed `BlockFileCache` but a -deterministic in-memory remote reader. This keeps object-store latency out of the measurements while -retaining the production inflight index, bounded MPMC queue, worker pool, `get_or_set`, `append`, -and `finalize` implementation. +### Scope + +`async_file_cache_write_microbench` measures the asynchronous cache-write components added below +`CachedRemoteFileReader`. It uses a real filesystem-backed `BlockFileCache` and production +`InflightWriteBufferIndex`, admission control, bounded MPMC queue, worker pool, `get_or_set`, +`append`, and `finalize` implementations. Its deterministic in-memory remote reader removes S3 and +network latency from the comparison. + +This complements the existing `file_cache_microbench`: + +- The existing S3 read mode is useful for a complete remote-read environment, but its results also + contain object-store and network latency. +- The existing direct `get_or_set` mode measures the lower-level cache lookup path, but does not + exercise asynchronous buffer publication, queue admission, background workers, or persistence. +- The asynchronous benchmark connects those production components and verifies the final cache + state, while keeping the source read deterministic. + +The measured reader flow is: + +```text +SyntheticRemoteFileReader + -> CachedRemoteFileReader + -> inflight-buffer lookup + -> BlockFileCache probe + -> foreground remote fill + -> InflightWriteBufferIndex publication + -> AsyncCacheWriteService admission and MPMC queue + -> worker get_or_set, append, and finalize + -> inflight entry cleanup +``` + +### Benchmark groups The benchmark groups are: -- `reader`: compares cold-miss foreground latency for forced synchronous and asynchronous cache - writes, then reports asynchronous drain time separately. -- `service`: measures end-to-end persistence while scaling worker count, followed by a bounded - queue case that reports producer-side rejection and queue high-water marks. -- `index`: separates representative sharded miss/hit lookup cost from worst-case hot-key hit - contention in `InflightWriteBufferIndex`. +- `reader` + - Starts every case with an empty cache. + - Issues concurrent, non-overlapping cold reads through `CachedRemoteFileReader`. + - Compares forced synchronous writes with asynchronous writes using the same source data and + requested ranges. + - Validates returned bytes and verifies that the complete aligned block range is readable from + the final cache state. + - Separates caller-visible foreground time from the time needed to drain accepted asynchronous + writes. +- `service` + - Submits unique real buffers directly to `AsyncCacheWriteService` from concurrent producers. + - Measures 1, 4, and 16 workers by default, including allocation, inflight publication, queue + admission, worker consumption, `get_or_set`, `append`, `finalize`, and completion cleanup. + - Verifies every accepted task in the final `BlockFileCache`. + - Includes a deliberately bounded backpressure case. `accepted` and `rejected` show admission + behavior, while peak gauges show where work accumulated. +- `index` + - Measures `InflightWriteBufferIndex::lookup` independently from disk writes. + - `sharded_miss` spreads absent keys across shards. + - `sharded_hit` spreads pre-populated keys across shards. + - `hot_key_hit` sends all producers to one key to expose the upper bound of lock contention. + +The integrated reader and service groups cover index insertion and conditional removal. The index +group intentionally measures only lookup contention so it does not duplicate those flows. + +### Disk baseline + +Run the installed wrapper instead of invoking the binary directly when comparing machines. With +the default `RUN_FIO=auto`, the wrapper checks whether `fio` is installed and, if available, runs +these baselines on the same filesystem before starting the cache benchmark: + +- `seqwrite_qd1`: direct 1 MiB sequential writes at queue depth 1, showing single-stream device + throughput and latency. +- `randwrite_qd16`: direct 1 MiB random writes at queue depth 16, showing concurrent device + throughput and latency for a workload closer to writes spread across cache files. + +The fio files are created in a unique directory next to `--cache_path`, use `--unlink=1`, and are +not placed in `/tmp`. Direct I/O avoids filling the page cache immediately before the cache +benchmark. These fio results describe the storage environment; they are not numerically equivalent +to cache `persisted_mib_per_sec`, because production cache files use buffered writes without an +`fsync` or `fdatasync` in the measured completion path. + +Environment controls for the wrapper are: + +| Variable | Default | Meaning | +| --- | --- | --- | +| `RUN_FIO` | `auto` | Run when fio exists. Use `1` to require fio or `0` to skip it. | +| `FIO_SIZE` | `1G` | Address space used by each direct-I/O fio case. | +| `FIO_RUNTIME` | `5` | Measured seconds for each fio case, after a one-second ramp. | + +### Defaults and repetitions + +The defaults represent small reads that cause full 1 MiB cache-block writes: + +| Setting | Default | Purpose | +| --- | ---: | --- | +| `block_size` | 1 MiB | Cache alignment and persisted bytes per reader miss | +| `request_size` | 64 KiB | Bytes returned by each caller read | +| `reader_operations` | 128 | Cold blocks in each synchronous or asynchronous reader case | +| `service_task_size` | 1 MiB | Payload in each direct service task | +| `service_operations` | 256 | Attempts in each service case | +| `producer_threads` | 16 | Concurrent readers or submitters | +| `reader_workers` | 16 | Workers in the asynchronous reader comparison | +| `worker_counts` | 1, 4, 16 | Worker scaling points in the service group | +| `backpressure_pending_tasks` | 64 | Pending-task limit in the rejection case | +| `index_operations_per_thread` | 100,000 | Lookups performed by each index producer | +| `index_key_count` | 4,096 | Keys in each sharded index case | +| `repetitions` | 5 | Repeated executions of every selected case | + +Each repetition clears and drains the real cache before every reader or service case. The process +prints the one-based `repetition` on every `RESULT` line. Use the median across repetitions as the +primary comparison and retain the minimum-to-maximum range to expose scheduler, filesystem, page +cache, and background writeback noise. The first repetition is intentionally retained instead of +being silently treated as warm-up. + +For formal comparisons, use the same Release build, host, cache filesystem, arguments, and idle +machine state. Five repetitions are the default for a quick comparison; increase +`--repetitions` when the min-to-max spread is large. + +### Build and run + +Performance numbers should come from a Release build. Build both microbenchmarks and the wrapper +with: + +```bash +./build.sh --be --file-cache-microbench -j100 +``` -Run all groups with the default production-sized 1 MiB cache block: +Run all groups, the fio baseline, and five repetitions with: ```bash -./output/be/lib/async_file_cache_write_microbench \ +./output/be/bin/run-async-file-cache-write-microbench.sh \ --benchmark_mode=all \ --cache_path=./output/async_file_cache_write_microbench \ --producer_threads=16 \ - --worker_counts=1,4,16 + --worker_counts=1,4,16 \ + --repetitions=5 2>&1 | tee ./output/async_file_cache_write_microbench.log ``` -Every result is printed on one `RESULT` line. Reader and service results distinguish -`foreground_seconds` from `drain_seconds`, include operation latency percentiles, accepted and -rejected task counts, verified persisted tasks, and sampled peaks for pending, queued, and inflight -work. Use `--help` to adjust operation counts, request/task sizes, queue limit, and worker counts. +> `--cache_path` is benchmark-owned and must not exist or must be an empty directory. The benchmark +> rejects a non-empty path instead of clearing it, and removes only the directory it accepted on +> exit unless `--keep_cache` is set. Always use a dedicated path under `output/`; never point it at +> an existing cache or data directory. + +Invoke `output/be/lib/async_file_cache_write_microbench` directly only when the fio baseline is not +wanted. Use `--help` to adjust operation counts, request and task sizes, queue limit, worker counts, +repetitions, and cache retention. + +### Result fields + +Each measured case emits one machine-readable `RESULT` line: + +- Identity and shape: `benchmark`, `variant`, `repetition`, `producers`, `workers`, and + `operations`. +- Admission and correctness: `accepted`, `rejected`, and `persisted`. A case fails instead of + printing a successful result if returned data or final cache coverage is incorrect. +- Timing: `foreground_seconds` ends when producer or reader calls return; `drain_seconds` is the + remaining background completion time; `total_seconds` includes both. +- Rates: `foreground_ops_per_sec` measures caller or submitter completion. + `persisted_mib_per_sec` divides verified bytes by total time and does not claim durable-media + completion. +- Foreground latency: `avg_us`, `p50_us`, `p95_us`, `p99_us`, and `max_us`. +- Queue shape: `peak_pending`, `peak_queued`, and `peak_inflight` are sampled high-water marks. + `pending` includes queued and active accepted tasks. `inflight` can briefly exceed `pending` + because a producer publishes its buffer before admission and conditionally removes it after a + rejection. +- Index-only rates: `elapsed_seconds` and `operations_per_sec` replace write-specific rate fields. + +Use reader `foreground_ops_per_sec` and latency to quantify caller benefit from asynchronous +writes. Use service `persisted_mib_per_sec`, `drain_seconds`, and peak gauges together to determine +whether workers or admission are limiting progress. Use the backpressure acceptance ratio to +validate bounded overload behavior, and compare sharded versus hot-key index results to quantify +lock-contention sensitivity. + +### Out of scope + +This benchmark does not measure S3 or network latency, complete scanner/query throughput, +cache-hit read throughput, restart recovery, multiple-cache-disk balancing, or durable `fsync` +throughput. It is also not a replacement for BE unit and regression tests: it validates data and +final cache coverage to reject invalid performance samples, but its primary purpose is controlled +performance comparison. diff --git a/bin/run-async-file-cache-write-microbench.sh b/bin/run-async-file-cache-write-microbench.sh new file mode 100755 index 00000000000000..a4d06c3410e1e0 --- /dev/null +++ b/bin/run-async-file-cache-write-microbench.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +doris_home="$(cd "${curdir}/.." && pwd)" +benchmark="${doris_home}/lib/async_file_cache_write_microbench" +if [[ ! -x "${benchmark}" ]]; then + benchmark="${doris_home}/output/be/lib/async_file_cache_write_microbench" +fi +if [[ ! -x "${benchmark}" ]]; then + echo "Cannot find async_file_cache_write_microbench under ${doris_home}." >&2 + exit 1 +fi + +cache_path="./output/async_file_cache_write_microbench" +benchmark_args=("$@") +for ((argument_index = 0; argument_index < ${#benchmark_args[@]}; ++argument_index)); do + argument="${benchmark_args[argument_index]}" + case "${argument}" in + --cache_path=*) + cache_path="${argument#*=}" + ;; + --cache_path) + argument_index=$((argument_index + 1)) + if [[ "${argument_index}" -ge "${#benchmark_args[@]}" ]]; then + echo "--cache_path requires a value." >&2 + exit 1 + fi + cache_path="${benchmark_args[argument_index]}" + ;; + esac +done +cache_path="${cache_path%/}" +if [[ -z "${cache_path}" ]]; then + echo "--cache_path cannot be empty or root." >&2 + exit 1 +fi + +run_fio="${RUN_FIO:-auto}" +case "${run_fio}" in +auto) + if command -v fio &>/dev/null; then + run_fio=1 + else + run_fio=0 + echo "DISK_BASELINE skipped: fio is not installed." + fi + ;; +1 | true | on) + if ! command -v fio &>/dev/null; then + echo "RUN_FIO=${run_fio}, but fio is not installed." >&2 + exit 1 + fi + run_fio=1 + ;; +0 | false | off) + run_fio=0 + ;; +*) + echo "RUN_FIO must be auto, 1, or 0." >&2 + exit 1 + ;; +esac + +if [[ "${run_fio}" -eq 1 ]]; then + fio_size="${FIO_SIZE:-1G}" + fio_runtime="${FIO_RUNTIME:-5}" + if [[ ! "${fio_runtime}" =~ ^[1-9][0-9]*$ ]]; then + echo "FIO_RUNTIME must be a positive integer number of seconds." >&2 + exit 1 + fi + + cache_parent="$(dirname "${cache_path}")" + mkdir -p "${cache_parent}" + fio_dir="$(mktemp -d "${cache_path}_fio.XXXXXX")" + echo "DISK_BASELINE path=${fio_dir} size=${fio_size} runtime_seconds=${fio_runtime}" + df -hT "${fio_dir}" + + fio --name=seqwrite_qd1 \ + --filename="${fio_dir}/seqwrite_qd1.data" \ + --size="${fio_size}" \ + --runtime="${fio_runtime}" \ + --ramp_time=1 \ + --time_based=1 \ + --rw=write \ + --bs=1m \ + --ioengine=libaio \ + --iodepth=1 \ + --numjobs=1 \ + --direct=1 \ + --invalidate=1 \ + --refill_buffers=1 \ + --group_reporting=1 \ + --eta=never \ + --unlink=1 + + fio --name=randwrite_qd16 \ + --filename="${fio_dir}/randwrite_qd16.data" \ + --size="${fio_size}" \ + --runtime="${fio_runtime}" \ + --ramp_time=1 \ + --time_based=1 \ + --rw=randwrite \ + --bs=1m \ + --ioengine=libaio \ + --iodepth=16 \ + --numjobs=1 \ + --direct=1 \ + --invalidate=1 \ + --refill_buffers=1 \ + --randrepeat=1 \ + --group_reporting=1 \ + --eta=never \ + --unlink=1 + + rmdir "${fio_dir}" +fi + +if [[ -n "${JAVA_HOME:-}" && -d "${JAVA_HOME}/lib/server" ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH:-}" +fi +exec "${benchmark}" "$@" diff --git a/build.sh b/build.sh index 38501f37439d3f..133d2ce24bd0e8 100755 --- a/build.sh +++ b/build.sh @@ -1139,6 +1139,7 @@ EOF if [[ "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" = "ON" ]]; then cp -r -p "${DORIS_HOME}/be/output/lib/file_cache_microbench" "${DORIS_OUTPUT}/be/lib"/ cp -r -p "${DORIS_HOME}/be/output/lib/async_file_cache_write_microbench" "${DORIS_OUTPUT}/be/lib"/ + cp -r -p "${DORIS_HOME}/bin/run-async-file-cache-write-microbench.sh" "${DORIS_OUTPUT}/be/bin"/ fi if [[ "${BUILD_INDEX_TOOL}" = "ON" ]]; then From 184797aece3c16a94d53ad34137b7fdfe4ac4425 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 17 Jul 2026 18:42:38 +0800 Subject: [PATCH 14/16] test --- be/src/common/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 99f542e7c57930..4fcf595ec9e671 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1280,7 +1280,7 @@ DEFINE_Int32(file_cache_downloader_thread_num_min, "32"); DEFINE_Int32(file_cache_downloader_thread_num_max, "32"); // async file cache write -DEFINE_mBool(enable_async_file_cache_write, "false"); +DEFINE_mBool(enable_async_file_cache_write, "true"); DEFINE_mInt32(async_file_cache_write_workers_per_disk, "16"); DEFINE_mInt64(async_file_cache_write_max_pending_tasks_per_disk, "256"); DEFINE_mInt32(async_file_cache_write_batch_size, "16"); From a741249c45e4c62dc54c2bd5649d92260de07755 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 20 Jul 2026 11:37:26 +0800 Subject: [PATCH 15/16] [fix](be) Handle preallocated cache blocks at file tail ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: File writers allocate cache cells at the normal full block size before the final upload buffer size is known. The final partial block is shrunk to its actual byte count only during FileBlock::finalize(). An async cache read racing with that interval builds a logical tail block ending at file EOF, but BlockFileCache::probe required the cached right boundary to match it exactly. The mismatch aborted the BE in probe; the async read plan and materialization path also carried the same exact-range assumption. Allow the final short probe slot to be covered by a larger preallocated cache block while preserving exact-size assertions for complete slots. Keep the async reader stricter by allowing the larger boundary only for the logical block that ends at the real file EOF. Add a focused probe unit test that reproduced the original fatal assertion and an async CachedRemoteFileReader test that exercises the complete EOF read flow. The reproducer aborted before the fix and both tests pass after it. ### Release note Fix a BE crash when an async file-cache read races with preallocation of a partial final file block. ### Check List (For Author) - Test: Unit Test - ./run-be-ut.sh --run --filter=BlockFileCacheTest.ProbeAcceptsPreallocatedBlockCoveringFileTail:AsyncCachedRemoteFileReaderTest.preallocated_cache_block_can_cover_the_short_file_tail -j100 (ASAN, 2 tests passed) - Behavior changed: Yes. Async cache reads now accept a full-size preallocated cache block that covers the short EOF block. - Does this need documentation: No --- be/src/io/cache/block_file_cache.cpp | 6 +++- be/src/io/cache/block_file_cache.h | 8 +++-- .../cached_remote_file_reader_async_write.cpp | 16 ++++++--- .../io/cache/block_file_cache_probe_test.cpp | 32 ++++++++++++++++++ .../cache/cached_remote_file_reader_test.cpp | 33 +++++++++++++++++++ 5 files changed, 87 insertions(+), 8 deletions(-) diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index 3fa8a974ad3962..38a3442738dc63 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -888,7 +888,11 @@ FileBlocksProbeResult BlockFileCache::probe(const UInt128Wrapper& hash, size_t o cached_block->second.file_block->range().left <= expected_range.right) { file_block = cached_block->second.file_block; DORIS_CHECK(file_block->range().left == expected_range.left); - DORIS_CHECK(file_block->range().right == expected_range.right); + DORIS_CHECK(file_block->range().right >= expected_range.right); + // File writers allocate full-size cache blocks before the final buffer size is known. + // Only the last short probe slot can therefore have a larger cached right boundary. + DORIS_CHECK(file_block->range().right == expected_range.right || + block_size < _max_file_block_size); ++cached_block; } result.emplace_back(std::move(file_block)); diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 0426431f15bd43..4dce662dd49c09 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -93,7 +93,8 @@ struct FileBlocksProbeResult { ~FileBlocksProbeResult(); /// One entry per cache-block-sized input slot, in offset order. A null entry is a cache miss; - /// a non-null entry has the exact slot range, except that the final slot may end at EOF. + /// a non-null entry covers the whole slot. Its right boundary can exceed the final short slot + /// while a file writer still owns a full-size preallocated tail block. std::vector file_blocks; }; @@ -259,8 +260,9 @@ class BlockFileCache { /// Probe the block-aligned `[offset, offset + size)` range without creating cache cells or /// touching LRU state. The result contains one ordered slot per cache block; each slot is null - /// on miss or owns the exactly aligned existing block. `context` supplies cache metadata when - /// lazy loading is required. + /// on miss or owns an existing block that starts at and covers the slot. A final short slot can + /// be covered by a full-size block preallocated by a file writer. `context` supplies cache + /// metadata when lazy loading is required. FileBlocksProbeResult probe(const UInt128Wrapper& hash, size_t offset, size_t size, const CacheContext& context); diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp index 421190cf31d096..b678b7e42ffd8c 100644 --- a/be/src/io/cache/cached_remote_file_reader_async_write.cpp +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -74,6 +74,16 @@ bvar::LatencyRecorder g_cached_remote_reader_async_read_plan_latency( bvar::LatencyRecorder g_cached_remote_reader_async_write_submission_latency( "cached_remote_file_reader_async_write_submission_latency_us"); +// Verify the one-to-one mapping between a logical read-plan block and its probed cache block. +// `file_size` identifies the only logical block whose cached right boundary may be larger: a short +// EOF block that a concurrent file writer preallocated at the normal full block size. +void check_probe_block_range(const FileBlock::Range& cached_range, + const FileBlock::Range& logical_range, size_t file_size) { + DORIS_CHECK(cached_range.left == logical_range.left); + DORIS_CHECK(cached_range.right >= logical_range.right); + DORIS_CHECK(cached_range.right == logical_range.right || logical_range.right == file_size - 1); +} + } // namespace // One aligned cache block in the simplified read plan. REMOTE blocks delimit the single remote @@ -202,8 +212,7 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ bool is_miss = file_block == nullptr; bool is_downloading = false; if (file_block != nullptr) { - DORIS_CHECK(file_block->range().left == read_block.range.left); - DORIS_CHECK(file_block->range().right == read_block.range.right); + check_probe_block_range(file_block->range(), read_block.range, size()); if (_cache->is_block_deleting(file_block)) { is_miss = true; } else { @@ -282,8 +291,7 @@ bool CachedRemoteFileReader::_materialize_async_block(const AsyncReadPlan& plan, DORIS_CHECK(block_index < plan.probe_result->file_blocks.size()); const auto& file_block = plan.probe_result->file_blocks[block_index]; DORIS_CHECK(file_block != nullptr); - DORIS_CHECK(file_block->range().left == read_block.range.left); - DORIS_CHECK(file_block->range().right == read_block.range.right); + check_probe_block_range(file_block->range(), read_block.range, size()); if (_cache->is_block_deleting(file_block)) { return false; } diff --git a/be/test/io/cache/block_file_cache_probe_test.cpp b/be/test/io/cache/block_file_cache_probe_test.cpp index cca0a1b6c7aad2..7ac0d2fa7eb528 100644 --- a/be/test/io/cache/block_file_cache_probe_test.cpp +++ b/be/test/io/cache/block_file_cache_probe_test.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -114,6 +115,37 @@ TEST_F(BlockFileCacheTest, ProbeReturnsDownloadedBlockAndNullMissSlot) { fs::remove_all(path, error); } +TEST_F(BlockFileCacheTest, ProbeAcceptsPreallocatedBlockCoveringFileTail) { + const auto path = caches_dir / "block_file_cache_probe_preallocated_file_tail"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_preallocated_file_tail"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + + // File writers allocate a full block before the final buffer size is known. During that + // window, the last cache block can extend beyond the actual file tail passed to probe(). + auto holder = cache.get_or_set(hash, 0, 8192, context); + ASSERT_EQ(holder.file_blocks.size(), 2); + const auto& preallocated_tail = *std::next(holder.file_blocks.begin()); + ASSERT_EQ(preallocated_tail->range().left, 4096); + ASSERT_EQ(preallocated_tail->range().right, 8191); + + auto result = cache.probe(hash, 0, 6144, context); + ASSERT_EQ(result.file_blocks.size(), 2); + ASSERT_EQ(result.file_blocks[1], preallocated_tail); + EXPECT_EQ(result.file_blocks[1]->range().left, 4096); + EXPECT_EQ(result.file_blocks[1]->range().right, 8191); + } + fs::remove_all(path, error); +} + TEST_F(BlockFileCacheTest, ProbeTouchAndRemovalRevalidateTheRetainedBlock) { const auto path = caches_dir / "block_file_cache_probe_touch_remove"; std::error_code error; diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index b904d04b359add..2a5e25d066eee9 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -165,6 +165,39 @@ class AsyncCachedRemoteFileReaderTest : public BlockFileCacheTest { } // namespace +TEST_F(AsyncCachedRemoteFileReaderTest, preallocated_cache_block_can_cover_the_short_file_tail) { + create_cache("cached_remote_reader_async_preallocated_file_tail"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + ASSERT_EQ(reader->size(), 10_mb + 1); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + const size_t file_tail_offset = reader->size() - 1; + auto holder = cache()->get_or_set(reader->_cache_hash, file_tail_offset, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& preallocated_tail = holder.file_blocks.front(); + ASSERT_EQ(preallocated_tail->range().left, file_tail_offset); + ASSERT_EQ(preallocated_tail->range().right, file_tail_offset + 1_mb - 1); + + std::string result(1, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(file_tail_offset, Slice(result.data(), result.size()), &bytes_read, + &context) + .ok()); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result, "0"); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 9_mb); + EXPECT_EQ(counting_reader->last_size(), 1_mb + 1); + EXPECT_EQ(stats.probe_miss, 2); + wait_for_async_writes(); +} + TEST_F(AsyncCachedRemoteFileReaderTest, missing_cache_file_falls_back_and_removes_the_complete_cache_hash) { create_cache("cached_remote_reader_async_self_heal"); From bce089a63b40c44705fd24416bb83544fffb2dab Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 20 Jul 2026 12:35:40 +0800 Subject: [PATCH 16/16] [fix](be) Preserve downloader ownership across read-only cache probes ### What problem does this PR solve? Issue Number: None Related PR: #65658 Problem Summary: A systematic review after the file-tail crash found that FileBlocksProbeResult reused FileBlocksHolder cleanup semantics. Destroying a read-only probe result on the same thread as an independently owned downloader therefore called complete_unlocked(), reset a valid DOWNLOADING block to EMPTY, and cleared its downloader. A focused BEUT reproduced that state transition before the fix. Give holder and probe references explicit cleanup roles: holders still complete downloader ownership acquired through get_or_set(), while probes only retain blocks and perform the existing deferred EMPTY/deleting-cell cleanup. Also stop re-reading the mutable FileBlock range after probe() has validated slot coverage under the cache mutex; a concurrent file writer may shrink a preallocated EOF block during finalize(), so the async reader now consistently uses its immutable logical plan range for cache offsets and diagnostics. The review added concise end-to-end coverage for a DOWNLOADING preallocated tail that finalizes while a reader waits, mixed existing-cache and inflight coverage, and operation with the optional inflight index disabled. The fixture now resets the process-wide FD cache together with FileCacheFactory because its key omits the per-test cache path; without that isolation, newly added cases exposed stale descriptors from earlier cases. ### Release note Fix async file-cache read races involving read-only probe lifetime and concurrent finalization of a preallocated file-tail block. ### Check List (For Author) - Test: Unit Test - Pre-fix reproduction: BlockFileCacheTest.ProbeResultDoesNotCompleteDownloaderOwnedByCaller failed because the block became EMPTY and its downloader was cleared - Targeted ASAN BEUT: 6 focused probe/EOF/cache-inflight/external-table cases passed with -j100 - Relevant ASAN BEUT sweep: 58 of 60 passed and exposed two cross-case FDCache isolation failures; after the isolation fix, the complete affected AsyncCachedRemoteFileReaderTest suite passed 9 of 9 with -j100, while the other 51 relevant tests had already passed in the sweep - build-support/clang-format.sh, build-support/check-format.sh, and git diff --check passed - Behavior changed: Yes. Read-only probes no longer complete downloader ownership, and async reads remain valid while a preallocated EOF block is finalized and shrunk. - Does this need documentation: No --- be/src/io/cache/block_file_cache.h | 3 +- .../cached_remote_file_reader_async_write.cpp | 20 +-- be/src/io/cache/file_block.cpp | 11 +- be/src/io/cache/file_block.h | 15 +- .../io/cache/block_file_cache_probe_test.cpp | 34 +++++ .../cache/cached_remote_file_reader_test.cpp | 139 ++++++++++++++++++ 6 files changed, 200 insertions(+), 22 deletions(-) diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 4dce662dd49c09..f4d2f912ac9957 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -94,7 +94,8 @@ struct FileBlocksProbeResult { /// One entry per cache-block-sized input slot, in offset order. A null entry is a cache miss; /// a non-null entry covers the whole slot. Its right boundary can exceed the final short slot - /// while a file writer still owns a full-size preallocated tail block. + /// while a file writer still owns a full-size preallocated tail block. Retaining and releasing + /// a probe result never acquires or completes downloader ownership. std::vector file_blocks; }; diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp index b678b7e42ffd8c..f57535407864d8 100644 --- a/be/src/io/cache/cached_remote_file_reader_async_write.cpp +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -74,16 +74,6 @@ bvar::LatencyRecorder g_cached_remote_reader_async_read_plan_latency( bvar::LatencyRecorder g_cached_remote_reader_async_write_submission_latency( "cached_remote_file_reader_async_write_submission_latency_us"); -// Verify the one-to-one mapping between a logical read-plan block and its probed cache block. -// `file_size` identifies the only logical block whose cached right boundary may be larger: a short -// EOF block that a concurrent file writer preallocated at the normal full block size. -void check_probe_block_range(const FileBlock::Range& cached_range, - const FileBlock::Range& logical_range, size_t file_size) { - DORIS_CHECK(cached_range.left == logical_range.left); - DORIS_CHECK(cached_range.right >= logical_range.right); - DORIS_CHECK(cached_range.right == logical_range.right || logical_range.right == file_size - 1); -} - } // namespace // One aligned cache block in the simplified read plan. REMOTE blocks delimit the single remote @@ -202,6 +192,9 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ const auto& probe_result = *plan.probe_result; DORIS_CHECK(probe_result.file_blocks.size() == plan.blocks.size()); + // probe() validates slot coverage while holding the cache mutex. Use the immutable logical + // plan ranges after it returns: a concurrent file writer may shrink a preallocated EOF block + // during finalize(). for (size_t index = 0; index < plan.blocks.size(); ++index) { auto& read_block = plan.blocks[index]; if (read_block.source == AsyncReadBlock::Source::INFLIGHT) { @@ -212,7 +205,6 @@ CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_ bool is_miss = file_block == nullptr; bool is_downloading = false; if (file_block != nullptr) { - check_probe_block_range(file_block->range(), read_block.range, size()); if (_cache->is_block_deleting(file_block)) { is_miss = true; } else { @@ -291,7 +283,6 @@ bool CachedRemoteFileReader::_materialize_async_block(const AsyncReadPlan& plan, DORIS_CHECK(block_index < plan.probe_result->file_blocks.size()); const auto& file_block = plan.probe_result->file_blocks[block_index]; DORIS_CHECK(file_block != nullptr); - check_probe_block_range(file_block->range(), read_block.range, size()); if (_cache->is_block_deleting(file_block)) { return false; } @@ -299,6 +290,7 @@ bool CachedRemoteFileReader::_materialize_async_block(const AsyncReadPlan& plan, FileBlock::State state = file_block->state(); if (state == FileBlock::State::DOWNLOADING) { DORIS_CHECK(read_block.source == AsyncReadBlock::Source::DOWNLOADING); + TEST_SYNC_POINT("CachedRemoteFileReader::_materialize_async_block:before_wait"); { SCOPED_RAW_TIMER(&stats.remote_wait_timer); state = file_block->wait(); @@ -319,7 +311,7 @@ bool CachedRemoteFileReader::_materialize_async_block(const AsyncReadPlan& plan, { SCOPED_RAW_TIMER(&stats.local_read_timer); status = file_block->read(Slice(result.data + (copy_left - user_offset), copy_size), - copy_left - file_block->range().left); + copy_left - read_block.range.left); } if (!status.ok()) { if (status.is()) { @@ -329,7 +321,7 @@ bool CachedRemoteFileReader::_materialize_async_block(const AsyncReadPlan& plan, LOG_EVERY_N(WARNING, 100) << "Read probed file cache block failed, falling back to remote. path=" << path().native() << ", hash=" << _cache_hash.to_string() - << ", offset=" << file_block->offset() << ", status=" << status; + << ", offset=" << read_block.range.left << ", status=" << status; return false; } diff --git a/be/src/io/cache/file_block.cpp b/be/src/io/cache/file_block.cpp index b7ffa7a3418c3a..d9d57b08bf9c20 100644 --- a/be/src/io/cache/file_block.cpp +++ b/be/src/io/cache/file_block.cpp @@ -331,7 +331,8 @@ std::string FileBlock::get_cache_file() const { return _mgr->_storage->get_local_file(this->_key); } -void FileBlock::release_cache_user_reference(std::shared_ptr& file_block) { +void FileBlock::release_cache_reference(std::shared_ptr& file_block, + CacheReferenceRole role) { if (!file_block) { return; } @@ -340,7 +341,9 @@ void FileBlock::release_cache_user_reference(std::shared_ptr& file_bl bool should_remove = false; { std::lock_guard block_lock(file_block->_mutex); - file_block->complete_unlocked(block_lock); + if (role == CacheReferenceRole::HOLDER) { + file_block->complete_unlocked(block_lock); + } if (file_block.use_count() == 2 && (file_block->is_deleting() || file_block->state_unlock(block_lock) == FileBlock::State::EMPTY)) { @@ -365,13 +368,13 @@ void FileBlock::release_cache_user_reference(std::shared_ptr& file_bl FileBlocksHolder::~FileBlocksHolder() { for (auto& file_block : file_blocks) { - FileBlock::release_cache_user_reference(file_block); + FileBlock::release_cache_reference(file_block, FileBlock::CacheReferenceRole::HOLDER); } } FileBlocksProbeResult::~FileBlocksProbeResult() { for (auto& file_block : file_blocks) { - FileBlock::release_cache_user_reference(file_block); + FileBlock::release_cache_reference(file_block, FileBlock::CacheReferenceRole::PROBE); } } diff --git a/be/src/io/cache/file_block.h b/be/src/io/cache/file_block.h index 08b4d3be3afb27..3e29f8ddb6a035 100644 --- a/be/src/io/cache/file_block.h +++ b/be/src/io/cache/file_block.h @@ -156,6 +156,11 @@ class FileBlock { false}; // pocessed by CachedRemoteFileReader::_cache_file_readers private: + enum class CacheReferenceRole { + HOLDER, + PROBE, + }; + std::string get_info_for_log_impl(std::lock_guard& block_lock) const; [[nodiscard]] Status set_downloaded(std::lock_guard& block_lock); @@ -165,9 +170,13 @@ class FileBlock { void reset_downloader_impl(std::lock_guard& block_lock); - /// Release one cache-user reference and complete deferred EMPTY/deleting block cleanup when it - /// is the last reference outside the cache map. - static void release_cache_user_reference(std::shared_ptr& file_block); + /// Release one holder/probe reference and complete deferred EMPTY/deleting block cleanup when + /// it is the last reference outside the cache map. + /// @param[in,out] file_block Reference to release. + /// @param[in] role A holder completes downloader ownership acquired through get_or_set; a + /// read-only probe never changes downloader state. + static void release_cache_reference(std::shared_ptr& file_block, + CacheReferenceRole role); Range _block_range; diff --git a/be/test/io/cache/block_file_cache_probe_test.cpp b/be/test/io/cache/block_file_cache_probe_test.cpp index 7ac0d2fa7eb528..11c88f9d5dd9d0 100644 --- a/be/test/io/cache/block_file_cache_probe_test.cpp +++ b/be/test/io/cache/block_file_cache_probe_test.cpp @@ -291,5 +291,39 @@ TEST_F(BlockFileCacheTest, ProbeObservesDownloadingAndEmptyBlocksWithoutTakingDo fs::remove_all(path, error); } +TEST_F(BlockFileCacheTest, ProbeResultDoesNotCompleteDownloaderOwnedByCaller) { + const auto path = caches_dir / "block_file_cache_probe_preserves_downloader"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_preserves_downloader"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + auto holder = cache.get_or_set(hash, 0, 4096, context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + const uint64_t downloader = FileBlock::get_caller_id(); + ASSERT_EQ(block->get_or_set_downloader(), downloader); + + { + auto result = cache.probe(hash, 0, 4096, context); + ASSERT_EQ(result.file_blocks.size(), 1); + ASSERT_EQ(result.file_blocks[0], block); + EXPECT_EQ(result.file_blocks[0]->state(), FileBlock::State::DOWNLOADING); + } + + // A probe only retains the block against deletion. It must not complete downloader + // ownership established independently by a FileBlocksHolder, even on the same thread. + EXPECT_EQ(block->state(), FileBlock::State::DOWNLOADING); + EXPECT_EQ(block->get_downloader(), downloader); + } + fs::remove_all(path, error); +} + } // namespace } // namespace doris::io diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index 2a5e25d066eee9..911cb1869ba8c6 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. +#include #include #include "block_file_cache_test_common.h" #include "cloud/config.h" +#include "util/time.h" namespace doris::io { namespace { @@ -91,6 +93,10 @@ class AsyncCachedRemoteFileReaderTest : public BlockFileCacheTest { config::enable_cache_read_from_peer = false; config::file_cache_each_block_size = 1_mb; reset_async_reader_cache_factory(); + // FDCache is process-wide and keyed by cache hash plus offset, while these tests recreate + // the same logical file on a different cache path for every case. Reset it together with + // FileCacheFactory so one case cannot read an open descriptor retained by another case. + ExecEnv::GetInstance()->set_file_cache_open_fd_cache(std::make_unique()); } void TearDown() override { @@ -198,6 +204,134 @@ TEST_F(AsyncCachedRemoteFileReaderTest, preallocated_cache_block_can_cover_the_s wait_for_async_writes(); } +TEST_F(AsyncCachedRemoteFileReaderTest, + downloading_preallocated_tail_can_finalize_short_while_reader_waits) { + create_cache("cached_remote_reader_async_downloading_preallocated_tail"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + ASSERT_EQ(reader->size(), 10_mb + 1); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + const size_t file_tail_offset = reader->size() - 1; + auto holder = cache()->get_or_set(reader->_cache_hash, file_tail_offset, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& downloading_tail = holder.file_blocks.front(); + ASSERT_EQ(downloading_tail->range().right, file_tail_offset + 1_mb - 1); + ASSERT_EQ(downloading_tail->get_or_set_downloader(), FileBlock::get_caller_id()); + + std::mutex wait_mutex; + std::condition_variable wait_cv; + bool reader_reached_wait = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "CachedRemoteFileReader::_materialize_async_block:before_wait", + [&](auto&&) { + { + std::lock_guard lock(wait_mutex); + reader_reached_wait = true; + } + wait_cv.notify_all(); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + std::string result(1, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + auto read_future = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return reader->read_at(file_tail_offset, Slice(result.data(), result.size()), &bytes_read, + &context); + }); + { + std::unique_lock lock(wait_mutex); + ASSERT_TRUE(wait_cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return reader_reached_wait; })); + } + EXPECT_EQ(read_future.wait_for(std::chrono::milliseconds(100)), std::future_status::timeout); + + const std::string tail_payload(1, '0'); + ASSERT_TRUE(downloading_tail->append(Slice(tail_payload.data(), tail_payload.size())).ok()); + ASSERT_TRUE(downloading_tail->finalize().ok()); + ASSERT_EQ(read_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + ASSERT_TRUE(read_future.get().ok()); + + EXPECT_EQ(downloading_tail->range().left, file_tail_offset); + EXPECT_EQ(downloading_tail->range().right, file_tail_offset); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result, tail_payload); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 9_mb); + EXPECT_EQ(counting_reader->last_size(), 1_mb); + EXPECT_EQ(stats.bytes_read_from_local, 1); + EXPECT_EQ(stats.bytes_read_from_remote, 0); + EXPECT_EQ(stats.probe_miss, 1); + EXPECT_EQ(stats.probe_downloading_hit, 1); + EXPECT_EQ(stats.block_wait_success, 1); + EXPECT_EQ(stats.async_cache_write_submitted, 1); + wait_for_async_writes(); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + partial_inflight_coverage_combines_with_existing_cache_block) { + create_cache("cached_remote_reader_async_partial_inflight_and_cache"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache()->get_or_set(reader->_cache_hash, 0, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& cached_block = holder.file_blocks.front(); + ASSERT_EQ(cached_block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string cached_payload(1_mb, '0'); + ASSERT_TRUE(cached_block->append(Slice(cached_payload.data(), cached_payload.size())).ok()); + ASSERT_TRUE(cached_block->finalize().ok()); + + auto* service = cache()->async_write_service(); + auto* inflight_index = cache()->inflight_write_buffer_index(); + AsyncCacheWriteBufferPtr inflight_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(1_mb, &inflight_buffer).ok()); + const std::string inflight_payload(1_mb, '1'); + memcpy(inflight_buffer->data(), inflight_payload.data(), inflight_payload.size()); + auto inflight_entry = std::make_shared( + inflight_buffer, 1_mb, 1_mb, MonotonicMicros(), service->current_write_epoch()); + ASSERT_EQ(inflight_index->insert_if_absent(reader->_cache_hash, 1_mb, inflight_entry), nullptr); + Defer remove_inflight {[&]() { + static_cast(inflight_index->remove_if(reader->_cache_hash, 1_mb, inflight_entry)); + }}; + + std::string result(2_mb, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result.substr(0, 1_mb), cached_payload); + EXPECT_EQ(result.substr(1_mb), inflight_payload); + EXPECT_EQ(counting_reader->read_count(), 0); + EXPECT_EQ(stats.bytes_read_from_local, 2_mb); + EXPECT_EQ(stats.bytes_read_from_remote, 0); + EXPECT_EQ(stats.inflight_write_buffer_index_hit, 1); + EXPECT_EQ(stats.inflight_write_buffer_index_miss, 1); + EXPECT_EQ(stats.probe_downloaded_hit, 1); + EXPECT_EQ(stats.probe_miss, 0); + EXPECT_EQ(stats.async_cache_write_submitted, 0); +} + TEST_F(AsyncCachedRemoteFileReaderTest, missing_cache_file_falls_back_and_removes_the_complete_cache_hash) { create_cache("cached_remote_reader_async_self_heal"); @@ -526,6 +660,9 @@ TEST_F(AsyncCachedRemoteFileReaderTest, TEST_F(AsyncCachedRemoteFileReaderTest, external_table_reader_uses_async_write_and_reuses_downloaded_cache) { create_cache("cached_remote_reader_async_external_table"); + // Disabling the optional inflight index must only remove memory reuse/deduplication; accepted + // tasks still persist through AsyncCacheWriteService and become readable cache blocks. + config::enable_async_file_cache_write_inflight_write_buffer_index = false; auto first_remote = std::make_shared(open_remote_file()); FileReaderOptions options; options.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; @@ -545,6 +682,8 @@ TEST_F(AsyncCachedRemoteFileReaderTest, EXPECT_EQ(first_result, std::string(first_result.size(), '0')); EXPECT_EQ(first_remote->read_count(), 1); EXPECT_EQ(first_stats.async_cache_write_submitted, 1); + EXPECT_EQ(first_stats.inflight_write_buffer_index_hit, 0); + EXPECT_EQ(first_stats.inflight_write_buffer_index_miss, 0); wait_for_async_writes(); auto second_remote = std::make_shared(open_remote_file());