diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index ca20cb63990447..4fcf595ec9e671 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1279,6 +1279,28 @@ DEFINE_mBool(file_cache_enable_only_warm_up_idx, "false"); DEFINE_Int32(file_cache_downloader_thread_num_min, "32"); DEFINE_Int32(file_cache_downloader_thread_num_max, "32"); +// async file cache write +DEFINE_mBool(enable_async_file_cache_write, "true"); +DEFINE_mInt32(async_file_cache_write_workers_per_disk, "16"); +DEFINE_mInt64(async_file_cache_write_max_pending_tasks_per_disk, "256"); +DEFINE_mInt32(async_file_cache_write_batch_size, "16"); +DEFINE_mInt64(async_file_cache_write_watchdog_warn_secs, "30"); +DEFINE_mInt64(async_file_cache_write_watchdog_drop_secs, "120"); +DEFINE_mBool(enable_async_file_cache_write_inflight_write_buffer_index, "true"); +DEFINE_Int32(async_file_cache_write_inflight_write_buffer_index_shard_count, "64"); +DEFINE_Validator(async_file_cache_write_workers_per_disk, + [](int32_t value) { return value > 0 && value <= 128; }); +DEFINE_Validator(async_file_cache_write_max_pending_tasks_per_disk, + [](int64_t value) { return value > 0; }); +DEFINE_Validator(async_file_cache_write_batch_size, [](int32_t value) { return value > 0; }); +DEFINE_Validator(async_file_cache_write_watchdog_warn_secs, [](int64_t value) { + return value >= 0 && value < async_file_cache_write_watchdog_drop_secs; +}); +DEFINE_Validator(async_file_cache_write_watchdog_drop_secs, + [](int64_t value) { return value > async_file_cache_write_watchdog_warn_secs; }); +DEFINE_Validator(async_file_cache_write_inflight_write_buffer_index_shard_count, + [](int32_t value) { return value > 0; }); + DEFINE_mInt32(index_cache_entry_stay_time_after_lookup_s, "1800"); DEFINE_mInt32(inverted_index_cache_stale_sweep_time_sec, "600"); DEFINE_mBool(enable_write_index_searcher_cache, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 2a6b21ccf8c0e3..2a50e571614d74 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1319,6 +1319,16 @@ DECLARE_mBool(enable_evaluate_shadow_queue_diff); DECLARE_mBool(file_cache_enable_only_warm_up_idx); +// async file cache write +DECLARE_mBool(enable_async_file_cache_write); +DECLARE_mInt32(async_file_cache_write_workers_per_disk); +DECLARE_mInt64(async_file_cache_write_max_pending_tasks_per_disk); +DECLARE_mInt32(async_file_cache_write_batch_size); +DECLARE_mInt64(async_file_cache_write_watchdog_warn_secs); +DECLARE_mInt64(async_file_cache_write_watchdog_drop_secs); +DECLARE_mBool(enable_async_file_cache_write_inflight_write_buffer_index); +DECLARE_Int32(async_file_cache_write_inflight_write_buffer_index_shard_count); + // inverted index searcher cache // cache entry stay time after lookup DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s); diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 834ee012737ecd..8dc619619711bf 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -146,7 +146,14 @@ static bool has_file_cache_statistics(const io::FileCacheStatistics& stats) { stats.write_cache_io_timer != 0 || stats.bytes_write_into_cache != 0 || stats.num_skip_cache_io_total != 0 || stats.read_cache_file_directly_timer != 0 || stats.cache_get_or_set_timer != 0 || stats.lock_wait_timer != 0 || - stats.get_timer != 0 || stats.set_timer != 0 || + stats.get_timer != 0 || stats.set_timer != 0 || stats.async_cache_write_submitted != 0 || + stats.async_cache_write_rejected != 0 || + stats.async_cache_write_buffer_alloc_fail != 0 || + stats.async_cache_write_drop_stale_epoch != 0 || + stats.inflight_write_buffer_index_hit != 0 || + stats.inflight_write_buffer_index_miss != 0 || stats.probe_downloaded_hit != 0 || + stats.probe_downloading_hit != 0 || stats.probe_miss != 0 || + stats.block_wait_success != 0 || stats.block_wait_timeout != 0 || stats.inverted_index_num_local_io_total != 0 || stats.inverted_index_num_remote_io_total != 0 || stats.inverted_index_num_peer_io_total != 0 || diff --git a/be/src/io/cache/async_cache_write_service.cpp b/be/src/io/cache/async_cache_write_service.cpp new file mode 100644 index 00000000000000..e959d29bf48b47 --- /dev/null +++ b/be/src/io/cache/async_cache_write_service.cpp @@ -0,0 +1,634 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/async_cache_write_service.h" + +#include +#include +#include +#include + +#include "common/logging.h" +#include "core/allocator.h" +#include "cpp/sync_point.h" +#include "io/cache/block_file_cache.h" +#include "runtime/thread_context.h" +#include "util/defer_op.h" +#include "util/time.h" + +namespace doris::io { + +using AsyncCacheWriteAllocator = Allocator; + +namespace { + +/// Keep an in-progress phase gauge balanced across every return path. +class ScopedActiveCounter { +public: + explicit ScopedActiveCounter(std::atomic* counter) : _counter(counter) { + DORIS_CHECK(_counter != nullptr); + _counter->fetch_add(1, std::memory_order_relaxed); + } + + ~ScopedActiveCounter() { + const size_t old_count = _counter->fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_count > 0); + } + +private: + std::atomic* _counter; +}; + +} // namespace + +AsyncCacheWriteBuffer::AsyncCacheWriteBuffer(size_t size, + std::shared_ptr tracker) + : _size(size), _tracker(std::move(tracker)) { + AsyncCacheWriteAllocator allocator; + _data = reinterpret_cast(allocator.alloc(_size)); +} + +AsyncCacheWriteBuffer::~AsyncCacheWriteBuffer() { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_tracker); + AsyncCacheWriteAllocator allocator; + allocator.free(_data, _size); +} + +AsyncCacheWriteService::AsyncCacheWriteService(BlockFileCache* cache, + AsyncCacheWriteServiceOptions options) + : _cache(cache), + _options(std::make_shared(options)), + _desired_worker_count(options.worker_count) { + DORIS_CHECK(_cache != nullptr); + DORIS_CHECK(options.worker_count > 0); + DORIS_CHECK(options.max_pending_tasks > 0); + DORIS_CHECK(options.batch_size > 0); + DORIS_CHECK(options.watchdog_warn_secs >= 0); + DORIS_CHECK(options.watchdog_drop_secs > options.watchdog_warn_secs); + + const char* prefix = _cache->get_base_path().c_str(); + _mem_tracker = MemTrackerLimiter::create_shared( + MemTrackerLimiter::Type::CACHE, + fmt::format("AsyncFileCacheWrite:{}", _cache->get_base_path())); + _pending_count_metric = std::make_shared>( + prefix, "async_cache_write_pending_count", + [](void* service) { + return static_cast(service)->pending_count(); + }, + this); + _queued_count_metric = std::make_shared>( + prefix, "async_cache_write_queue_size", + [](void* service) { + return static_cast(service)->queued_count(); + }, + this); + _active_task_count_metric = std::make_shared>( + prefix, "async_cache_write_active_tasks", + [](void* service) { + return static_cast(service)->active_task_count(); + }, + this); + _running_worker_count_metric = std::make_shared>( + prefix, "async_cache_write_running_workers", + [](void* service) { + return static_cast(service)->running_worker_count(); + }, + this); + _configured_worker_count_metric = std::make_shared>( + prefix, "async_cache_write_configured_workers", + [](void* service) { + return static_cast(service)->_desired_worker_count.load( + std::memory_order_relaxed); + }, + this); + _max_pending_tasks_metric = std::make_shared>( + prefix, "async_cache_write_max_pending_tasks", + [](void* service) { + return static_cast(service) + ->_options.load(std::memory_order_acquire) + ->max_pending_tasks; + }, + this); + _active_get_or_set_count_metric = std::make_shared>( + prefix, "async_cache_write_active_get_or_set", + [](void* service) { + return static_cast(service)->_active_get_or_set_count.load( + std::memory_order_relaxed); + }, + this); + _active_append_count_metric = std::make_shared>( + prefix, "async_cache_write_active_append", + [](void* service) { + return static_cast(service)->_active_append_count.load( + std::memory_order_relaxed); + }, + this); + _active_finalize_count_metric = std::make_shared>( + prefix, "async_cache_write_active_finalize", + [](void* service) { + return static_cast(service)->_active_finalize_count.load( + std::memory_order_relaxed); + }, + this); + _buffer_memory_metric = std::make_shared>( + prefix, "async_cache_write_buffer_memory_bytes", + [](void* service) { + return static_cast(service)->buffer_memory_bytes(); + }, + this); + _submitted_metric = + std::make_shared>(prefix, "async_cache_write_submitted_total"); + _submitted_bytes_metric = std::make_shared>( + prefix, "async_cache_write_submitted_bytes_total"); + _finished_metric = + std::make_shared>(prefix, "async_cache_write_finished_total"); + _rejected_metric = + std::make_shared>(prefix, "async_cache_write_rejected_total"); + _reject_not_running_metric = std::make_shared>( + prefix, "async_cache_write_reject_not_running_total"); + _reject_backpressure_metric = std::make_shared>( + prefix, "async_cache_write_reject_backpressure_total"); + _reject_enqueue_failure_metric = std::make_shared>( + prefix, "async_cache_write_reject_enqueue_failure_total"); + _buffer_alloc_fail_metric = std::make_shared>( + prefix, "async_cache_write_buffer_alloc_fail_total"); + _submit_latency_metric = + std::make_shared(prefix, "async_cache_write_submit_latency_us"); + _buffer_alloc_latency_metric = std::make_shared( + prefix, "async_cache_write_buffer_alloc_latency_us"); + _queue_wait_latency_metric = std::make_shared( + prefix, "async_cache_write_queue_wait_latency_us"); + _worker_task_latency_metric = std::make_shared( + prefix, "async_cache_write_worker_task_latency_us"); + _get_or_set_latency_metric = std::make_shared( + prefix, "async_cache_write_get_or_set_latency_us"); + _append_latency_metric = + std::make_shared(prefix, "async_cache_write_append_latency_us"); + _finalize_latency_metric = std::make_shared( + prefix, "async_cache_write_finalize_latency_us"); + _skip_downloaded_metric = std::make_shared>( + prefix, "async_cache_write_skip_downloaded_total"); + _skip_downloading_metric = std::make_shared>( + prefix, "async_cache_write_skip_downloading_total"); + _skip_partial_overlap_metric = std::make_shared>( + prefix, "async_cache_write_skip_partial_overlap_total"); + _drop_stale_epoch_metric = std::make_shared>( + prefix, "async_cache_write_drop_stale_epoch_total"); + _skip_deleting_metric = std::make_shared>( + prefix, "async_cache_write_skip_deleting_total"); + _append_fail_metric = + std::make_shared>(prefix, "async_cache_write_append_fail_total"); + _finalize_fail_metric = std::make_shared>( + prefix, "async_cache_write_finalize_fail_total"); + _persisted_blocks_metric = std::make_shared>( + prefix, "async_cache_write_persisted_blocks_total"); + _persisted_bytes_metric = std::make_shared>( + prefix, "async_cache_write_persisted_bytes_total"); + _watchdog_warn_metric = std::make_shared>( + prefix, "async_cache_write_watchdog_warn_total"); + _watchdog_drop_metric = std::make_shared>( + prefix, "async_cache_write_watchdog_drop_total"); +} + +AsyncCacheWriteService::~AsyncCacheWriteService() { + shutdown(); +} + +Status AsyncCacheWriteService::start() { + std::lock_guard resize_lock(_resize_mutex); + if (_shutdown_requested.load(std::memory_order_acquire) || + !_accepting.load(std::memory_order_acquire)) { + return Status::InternalError("async file cache write service is shutting down"); + } + if (_started.load(std::memory_order_acquire)) { + return Status::OK(); + } + + const size_t worker_count = _desired_worker_count.load(std::memory_order_acquire); + if (_worker_pool == nullptr) { + RETURN_IF_ERROR( + ThreadPoolBuilder(fmt::format("AsyncFileCacheWrite-{}", + std::hash {}(_cache->get_base_path()))) + .set_min_threads(0) + .set_max_threads(static_cast(worker_count)) + .set_max_queue_size(128) + .build(&_worker_pool)); + } else { + // A previous start may have created only part of the requested workers before returning an + // error. Reuse that pool and apply the latest desired size before filling the missing ids. + RETURN_IF_ERROR(_worker_pool->set_max_threads(static_cast(worker_count))); + } + { + std::lock_guard state_lock(_worker_state_mutex); + if (_worker_scheduled.size() < worker_count) { + _worker_scheduled.resize(worker_count, false); + } + } + for (size_t worker_id = 0; worker_id < worker_count; ++worker_id) { + bool scheduled = false; + { + std::lock_guard state_lock(_worker_state_mutex); + scheduled = _worker_scheduled[worker_id]; + } + if (!scheduled) { + RETURN_IF_ERROR(_schedule_worker(worker_id)); + } + } + // Publish readiness only after every configured worker loop has been accepted by the pool. + _started.store(true, std::memory_order_release); + return Status::OK(); +} + +bool AsyncCacheWriteService::try_submit(AsyncCacheWriteTask task) { + const int64_t submit_start_us = MonotonicMicros(); + Defer record_submit_latency { + [&]() { *_submit_latency_metric << (MonotonicMicros() - submit_start_us); }}; + _active_submitters.fetch_add(1, std::memory_order_acq_rel); + Defer submitter_done {[&]() { _active_submitters.fetch_sub(1, std::memory_order_acq_rel); }}; + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::try_submit:after_register", &task); + if (!_started.load(std::memory_order_acquire) || !_accepting.load(std::memory_order_acquire)) { + *_rejected_metric << 1; + *_reject_not_running_metric << 1; + return false; + } + + const auto options = _options.load(std::memory_order_acquire); + const size_t max_pending = options->max_pending_tasks; + const size_t task_bytes = task.file_size; + size_t current = _pending_count.load(std::memory_order_relaxed); + while (current < max_pending) { + if (_pending_count.compare_exchange_weak(current, current + 1, std::memory_order_acq_rel, + std::memory_order_relaxed)) { + _queued_count.fetch_add(1, std::memory_order_release); + if (_queue.enqueue(std::move(task))) { + *_submitted_metric << 1; + *_submitted_bytes_metric << task_bytes; + _cv.notify_one(); + return true; + } + const size_t old_queued = _queued_count.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(old_queued > 0); + const size_t old_pending = _pending_count.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(old_pending > 0); + *_rejected_metric << 1; + *_reject_enqueue_failure_metric << 1; + return false; + } + } + *_rejected_metric << 1; + *_reject_backpressure_metric << 1; + return false; +} + +Status AsyncCacheWriteService::allocate_tracked_buffer(size_t size, + AsyncCacheWriteBufferPtr* buffer) { + DORIS_CHECK(buffer != nullptr); + DORIS_CHECK(size > 0); + const int64_t allocation_start_us = MonotonicMicros(); + Defer record_allocation_latency { + [&]() { *_buffer_alloc_latency_metric << (MonotonicMicros() - allocation_start_us); }}; + Status injected_status; + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::allocate_tracked_buffer:inject_failure", + &injected_status); + if (!injected_status.ok()) { + *_buffer_alloc_fail_metric << 1; + return injected_status; + } + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_mem_tracker); + Status status = Status::OK(); + try { + *buffer = AsyncCacheWriteBufferPtr(new AsyncCacheWriteBuffer(size, _mem_tracker)); + } catch (const std::exception& e) { + status = Status::MemoryAllocFailed("allocate async file cache write buffer failed: {}", + e.what()); + } + if (!status.ok()) { + *_buffer_alloc_fail_metric << 1; + } + return status; +} + +Status AsyncCacheWriteService::_schedule_worker(size_t worker_id) { + { + std::lock_guard lock(_worker_state_mutex); + if (_worker_scheduled.size() <= worker_id) { + _worker_scheduled.resize(worker_id + 1, false); + } + DORIS_CHECK(!_worker_scheduled[worker_id]); + _worker_scheduled[worker_id] = true; + } + Status status = _worker_pool->submit_func([this, worker_id]() { _worker_loop(worker_id); }); + if (!status.ok()) { + std::lock_guard lock(_worker_state_mutex); + _worker_scheduled[worker_id] = false; + _worker_state_cv.notify_all(); + } + return status; +} + +void AsyncCacheWriteService::_worker_loop(size_t worker_id) { + _running_worker_count.fetch_add(1, std::memory_order_relaxed); + Defer mark_stopped {[&]() { + const size_t old_running = _running_worker_count.fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_running > 0); + std::lock_guard lock(_worker_state_mutex); + _worker_scheduled[worker_id] = false; + _worker_state_cv.notify_all(); + }}; + + // Keep one consumer cursor per worker so concurrent consumers rotate across the queue's + // producer streams instead of rescanning them from scratch for every disk-write task. + moodycamel::ConsumerToken consumer_token(_queue); + + while (true) { + if (!_shutdown_requested.load(std::memory_order_acquire) && + worker_id >= _desired_worker_count.load(std::memory_order_acquire)) { + return; + } + + size_t processed = 0; + const auto options = _options.load(std::memory_order_acquire); + AsyncCacheWriteTask task; + while (processed < options->batch_size && _queue.try_dequeue(consumer_token, task)) { + ++processed; + const size_t old_queued = _queued_count.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(old_queued > 0); + _active_task_count.fetch_add(1, std::memory_order_relaxed); + Defer finish {[&]() { + _finish_task(task); + task = AsyncCacheWriteTask {}; + }}; + + const int64_t age_us = MonotonicMicros() - task.submit_ts_us; + *_queue_wait_latency_metric << age_us; + const int64_t warn_us = options->watchdog_warn_secs * 1000 * 1000; + const int64_t drop_us = options->watchdog_drop_secs * 1000 * 1000; + if (age_us > warn_us) { + *_watchdog_warn_metric << 1; + LOG(WARNING) << "Async file cache write task waited " << age_us + << " us, cache=" << _cache->get_base_path() + << ", hash=" << task.cache_hash.to_string() + << ", offset=" << task.file_offset; + } + if (age_us > drop_us) { + *_watchdog_drop_metric << 1; + continue; + } + if (!is_current_write_epoch(task.write_epoch)) { + *_drop_stale_epoch_metric << 1; + continue; + } + + const int64_t start_us = MonotonicMicros(); + Status status = _write_one(task); + *_worker_task_latency_metric << (MonotonicMicros() - start_us); + if (!status.ok()) { + LOG(WARNING) << "Async file cache write failed, cache=" << _cache->get_base_path() + << ", hash=" << task.cache_hash.to_string() + << ", offset=" << task.file_offset << ", size=" << task.file_size + << ", status=" << status; + } + } + + if (_shutdown_requested.load(std::memory_order_acquire) && + _pending_count.load(std::memory_order_acquire) == 0) { + return; + } + if (processed == 0) { + std::unique_lock lock(_cv_mutex); + _cv.wait_for(lock, std::chrono::milliseconds(1), [&]() { + return _queued_count.load(std::memory_order_acquire) != 0 || + _shutdown_requested.load(std::memory_order_acquire) || + worker_id >= _desired_worker_count.load(std::memory_order_acquire); + }); + } + } +} + +Status AsyncCacheWriteService::_write_one(const AsyncCacheWriteTask& task) { + DORIS_CHECK(task.buffer != nullptr); + DORIS_CHECK(task.file_size == task.buffer->size()); + if (!is_current_write_epoch(task.write_epoch)) { + *_drop_stale_epoch_metric << 1; + return Status::OK(); + } + + ReadStatistics dummy_stats; + CacheContext context; + context.query_id = task.admission_ctx.query_id; + context.cache_type = task.admission_ctx.cache_type; + context.expiration_time = task.admission_ctx.expiration_time; + context.tablet_id = task.admission_ctx.tablet_id; + context.is_warmup = task.admission_ctx.is_warmup; + context.stats = &dummy_stats; + auto holder = [&]() { + ScopedActiveCounter active_get_or_set(&_active_get_or_set_count); + const int64_t start_us = MonotonicMicros(); + Defer record_latency { + [&]() { *_get_or_set_latency_metric << (MonotonicMicros() - start_us); }}; + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:before_get_or_set", &task); + auto result = + _cache->get_or_set(task.cache_hash, task.file_offset, task.file_size, context); + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:after_get_or_set", &task); + return result; + }(); + + if (!is_current_write_epoch(task.write_epoch)) { + *_drop_stale_epoch_metric << 1; + return Status::OK(); + } + + const size_t task_end = task.file_offset + task.file_size; + for (const auto& block : holder.file_blocks) { + if (block->range().left < task.file_offset || block->range().right >= task_end) { + *_skip_partial_overlap_metric << 1; + continue; + } + if (!is_current_write_epoch(task.write_epoch)) { + *_drop_stale_epoch_metric << 1; + return Status::OK(); + } + if (_cache->is_block_deleting(block)) { + *_skip_deleting_metric << 1; + continue; + } + + switch (block->state()) { + case FileBlock::State::DOWNLOADED: + *_skip_downloaded_metric << 1; + continue; + case FileBlock::State::DOWNLOADING: + *_skip_downloading_metric << 1; + continue; + case FileBlock::State::SKIP_CACHE: + continue; + case FileBlock::State::EMPTY: + break; + } + + if (block->get_or_set_downloader() != FileBlock::get_caller_id()) { + *_skip_downloading_metric << 1; + continue; + } + const size_t buffer_offset = block->range().left - task.file_offset; + Status status; + { + ScopedActiveCounter active_append(&_active_append_count); + TEST_SYNC_POINT_CALLBACK("AsyncCacheWriteService::_write_one:before_append", &task); + const int64_t start_us = MonotonicMicros(); + status = block->append( + Slice(task.buffer->data() + buffer_offset, block->range().size())); + *_append_latency_metric << (MonotonicMicros() - start_us); + } + if (!status.ok()) { + *_append_fail_metric << 1; + LOG(WARNING) << "Append async file cache block failed, cache=" + << _cache->get_base_path() << ", hash=" << task.cache_hash.to_string() + << ", offset=" << block->offset() << ", size=" << block->range().size() + << ", status=" << status; + continue; + } + { + ScopedActiveCounter active_finalize(&_active_finalize_count); + const int64_t start_us = MonotonicMicros(); + status = block->finalize(); + *_finalize_latency_metric << (MonotonicMicros() - start_us); + } + if (!status.ok()) { + *_finalize_fail_metric << 1; + LOG(WARNING) << "Finalize async file cache block failed, cache=" + << _cache->get_base_path() << ", hash=" << task.cache_hash.to_string() + << ", offset=" << block->offset() << ", size=" << block->range().size() + << ", status=" << status; + continue; + } + *_persisted_blocks_metric << 1; + *_persisted_bytes_metric << block->range().size(); + } + return Status::OK(); +} + +void AsyncCacheWriteService::_finish_task(const AsyncCacheWriteTask& task) { + const size_t old_active = _active_task_count.fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_active > 0); + const size_t old_pending = _pending_count.fetch_sub(1, std::memory_order_acq_rel); + DORIS_CHECK(old_pending > 0); + *_finished_metric << 1; + if (task.on_finalized) { + task.on_finalized(task); + } +} + +Status AsyncCacheWriteService::resize_workers(size_t worker_count) { + if (worker_count == 0) { + return Status::InvalidArgument("async file cache write worker count must be positive"); + } + std::lock_guard resize_lock(_resize_mutex); + if (!_started.load(std::memory_order_acquire)) { + _desired_worker_count.store(worker_count, std::memory_order_release); + return Status::OK(); + } + if (_shutdown_requested.load(std::memory_order_acquire)) { + return Status::InternalError("async file cache write service is shutting down"); + } + + const size_t old_count = + _desired_worker_count.exchange(worker_count, std::memory_order_acq_rel); + if (old_count == worker_count) { + return Status::OK(); + } + _cv.notify_all(); + + if (worker_count < old_count) { + std::unique_lock state_lock(_worker_state_mutex); + _worker_state_cv.wait(state_lock, [&]() { + for (size_t id = worker_count; id < old_count; ++id) { + if (id < _worker_scheduled.size() && _worker_scheduled[id]) { + return false; + } + } + return true; + }); + state_lock.unlock(); + RETURN_IF_ERROR(_worker_pool->set_max_threads(static_cast(worker_count))); + return Status::OK(); + } + + RETURN_IF_ERROR(_worker_pool->set_max_threads(static_cast(worker_count))); + for (size_t worker_id = old_count; worker_id < worker_count; ++worker_id) { + RETURN_IF_ERROR(_schedule_worker(worker_id)); + } + return Status::OK(); +} + +Status AsyncCacheWriteService::update_options(const AsyncCacheWriteServiceOptions& options) { + if (options.worker_count == 0) { + return Status::InvalidArgument("async file cache write worker count must be positive"); + } + if (options.max_pending_tasks == 0) { + return Status::InvalidArgument( + "async file cache write pending task limit must be positive"); + } + if (options.batch_size == 0) { + return Status::InvalidArgument("async file cache write batch size must be positive"); + } + if (options.watchdog_warn_secs < 0 || + options.watchdog_drop_secs <= options.watchdog_warn_secs) { + return Status::InvalidArgument( + "async file cache write watchdog thresholds must satisfy 0 <= warn < drop"); + } + + auto next_options = std::make_shared(options); + RETURN_IF_ERROR(resize_workers(options.worker_count)); + _options.store(std::move(next_options), std::memory_order_release); + return Status::OK(); +} + +AsyncCacheWriteServiceOptions AsyncCacheWriteService::options() const { + AsyncCacheWriteServiceOptions result = *_options.load(std::memory_order_acquire); + result.worker_count = _desired_worker_count.load(std::memory_order_acquire); + return result; +} + +void AsyncCacheWriteService::shutdown() { + std::lock_guard resize_lock(_resize_mutex); + if (!_accepting.exchange(false, std::memory_order_acq_rel)) { + return; + } + TEST_SYNC_POINT("AsyncCacheWriteService::shutdown:after_stop_accepting"); + while (_active_submitters.load(std::memory_order_acquire) != 0) { + std::this_thread::yield(); + } + _shutdown_requested.store(true, std::memory_order_release); + _cv.notify_all(); + + if (_worker_pool) { + { + std::unique_lock lock(_worker_state_mutex); + _worker_state_cv.wait(lock, [&]() { + return std::none_of(_worker_scheduled.begin(), _worker_scheduled.end(), + [](bool scheduled) { return scheduled; }); + }); + } + DORIS_CHECK(_pending_count.load(std::memory_order_acquire) == 0); + DORIS_CHECK(_queued_count.load(std::memory_order_acquire) == 0); + DORIS_CHECK(_active_task_count.load(std::memory_order_acquire) == 0); + _worker_pool->shutdown(); + } +} + +} // namespace doris::io diff --git a/be/src/io/cache/async_cache_write_service.h b/be/src/io/cache/async_cache_write_service.h new file mode 100644 index 00000000000000..de191eeb043cf6 --- /dev/null +++ b/be/src/io/cache/async_cache_write_service.h @@ -0,0 +1,243 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/atomic_shared_ptr.h" +#include "common/status.h" +#include "io/cache/file_cache_common.h" +#include "runtime/memory/mem_tracker_limiter.h" +#include "util/threadpool.h" + +namespace doris::io { + +class BlockFileCache; + +/// Cache admission attributes captured on the query thread and replayed by a write worker. +struct CacheAdmissionContext { + /// Query identity used by per-query cache admission and accounting. + TUniqueId query_id; + FileCacheType cache_type {FileCacheType::NORMAL}; + int64_t expiration_time {0}; + int64_t tablet_id {0}; + bool is_warmup {false}; +}; + +/// Reference-counted payload whose allocation is charged to the async-write memory tracker. +class AsyncCacheWriteBuffer { +public: + ~AsyncCacheWriteBuffer(); + + char* data() { return _data; } + const char* data() const { return _data; } + size_t size() const { return _size; } + +private: + friend class AsyncCacheWriteService; + + AsyncCacheWriteBuffer(size_t size, std::shared_ptr tracker); + + char* _data = nullptr; + size_t _size = 0; + std::shared_ptr _tracker; +}; + +using AsyncCacheWriteBufferPtr = std::shared_ptr; + +/// One block-aligned cache write. `buffer` contains exactly `file_size` bytes starting at +/// `file_offset`; `write_epoch` prevents a worker from resurrecting data after cache invalidation. +struct AsyncCacheWriteTask { + UInt128Wrapper cache_hash; + size_t file_offset {0}; + size_t file_size {0}; + AsyncCacheWriteBufferPtr buffer; + CacheAdmissionContext admission_ctx; + int64_t submit_ts_us {0}; + uint64_t write_epoch {0}; + std::function on_finalized; +}; + +/// Complete per-cache-disk worker, queue, batch, and watchdog settings. The service receives this +/// value explicitly at construction and through update_options(); it never reads global config. +struct AsyncCacheWriteServiceOptions { + size_t worker_count {1}; + size_t max_pending_tasks {1}; + size_t batch_size {1}; + int64_t watchdog_warn_secs {30}; + int64_t watchdog_drop_secs {120}; +}; + +/// Owns the bounded async-write queue and workers for one BlockFileCache (one cache disk). +/// +/// The referenced cache must outlive this service. Shutdown stops new producers, waits registered +/// producers, and drains all accepted tasks before worker resources are released. +class AsyncCacheWriteService { +public: + /// @param cache Non-owning target cache; it must outlive this service. + /// @param options Initial worker, queue, batch, and watchdog limits. + AsyncCacheWriteService(BlockFileCache* cache, AsyncCacheWriteServiceOptions options); + ~AsyncCacheWriteService(); + + /// Create the worker pool and schedule the configured long-running workers. Idempotent. + Status start(); + + /// Reserve one pending slot and enqueue `task` without blocking the query thread. + /// @return true if ownership was transferred to the queue; false when workers have not been + /// started, during shutdown, on backpressure, or on queue rejection. A rejected task's + /// finalization callback is not invoked. + bool try_submit(AsyncCacheWriteTask task); + + /// Allocate `size` payload bytes charged to the service tracker and return them in `buffer`. + Status allocate_tracked_buffer(size_t size, AsyncCacheWriteBufferPtr* buffer); + + /// Return the epoch accepted by workers and inflight-buffer readers. + uint64_t current_write_epoch() const { return _write_epoch.load(std::memory_order_acquire); } + + /// Test whether `epoch` still belongs to the current cache contents. + bool is_current_write_epoch(uint64_t epoch) const { return epoch == current_write_epoch(); } + + /// Advance the epoch so queued/inflight writes captured before invalidation become stale. + /// @return The newly active epoch. + uint64_t invalidate_pending_writes() { + return _write_epoch.fetch_add(1, std::memory_order_acq_rel) + 1; + } + + /// Resize the number of active workers. A shrink waits only for retiring worker loops. + /// @param worker_count Positive target worker count for this cache disk. + Status resize_workers(size_t worker_count); + + /// Replace all mutable service settings with one coherent snapshot. Configuration adapters + /// call this method explicitly; the service itself has no dependency on global config. + /// @param options Complete validated settings, including the desired worker count. + /// @return OK after the new snapshot is active; InvalidArgument for invalid limits, or a + /// worker-resize error when the requested concurrency cannot be applied. + Status update_options(const AsyncCacheWriteServiceOptions& options); + + /// Return the currently active settings as a value snapshot. + AsyncCacheWriteServiceOptions options() const; + + /// Stop submissions, drain all accepted tasks, and join worker loops. Idempotent. + void shutdown(); + + /// Return accepted tasks that have not yet completed finalization. + size_t pending_count() const { return _pending_count.load(std::memory_order_relaxed); } + + /// Return accepted tasks still waiting in the MPMC queue, excluding active workers. + size_t queued_count() const { return _queued_count.load(std::memory_order_relaxed); } + + /// Return tasks currently owned by workers, including watchdog and write processing. + size_t active_task_count() const { return _active_task_count.load(std::memory_order_relaxed); } + + /// Return worker loops that are currently alive. + size_t running_worker_count() const { + return _running_worker_count.load(std::memory_order_relaxed); + } + + /// Return bytes currently held by tracked task buffers. + int64_t buffer_memory_bytes() const { return _mem_tracker->consumption(); } + +private: + /// Mark `worker_id` active and submit its persistent loop to the thread pool. + Status _schedule_worker(size_t worker_id); + + /// Drain batches until shutdown or until this worker id is retired by a resize. + void _worker_loop(size_t worker_id); + + /// Revalidate epoch/cache state and persist the task's still-empty complete blocks. + Status _write_one(const AsyncCacheWriteTask& task); + + /// Release one pending slot and invoke the inflight-index cleanup callback. + void _finish_task(const AsyncCacheWriteTask& task); + + BlockFileCache* _cache; + atomic_shared_ptr _options; + moodycamel::ConcurrentQueue _queue; + std::atomic _pending_count {0}; + std::atomic _queued_count {0}; + std::atomic _active_task_count {0}; + std::atomic _running_worker_count {0}; + std::atomic _active_get_or_set_count {0}; + std::atomic _active_append_count {0}; + std::atomic _active_finalize_count {0}; + std::condition_variable _cv; + std::mutex _cv_mutex; + std::atomic _accepting {true}; + std::atomic _active_submitters {0}; + std::atomic _shutdown_requested {false}; + std::atomic _started {false}; + std::atomic _write_epoch {1}; + + std::shared_ptr _mem_tracker; + std::unique_ptr _worker_pool; + std::atomic _desired_worker_count {0}; + std::mutex _resize_mutex; + std::mutex _worker_state_mutex; + std::condition_variable _worker_state_cv; + std::vector _worker_scheduled; + + std::shared_ptr> _pending_count_metric; + std::shared_ptr> _queued_count_metric; + std::shared_ptr> _active_task_count_metric; + std::shared_ptr> _running_worker_count_metric; + std::shared_ptr> _configured_worker_count_metric; + std::shared_ptr> _max_pending_tasks_metric; + std::shared_ptr> _active_get_or_set_count_metric; + std::shared_ptr> _active_append_count_metric; + std::shared_ptr> _active_finalize_count_metric; + std::shared_ptr> _buffer_memory_metric; + std::shared_ptr> _submitted_metric; + std::shared_ptr> _submitted_bytes_metric; + std::shared_ptr> _finished_metric; + std::shared_ptr> _rejected_metric; + std::shared_ptr> _reject_not_running_metric; + std::shared_ptr> _reject_backpressure_metric; + std::shared_ptr> _reject_enqueue_failure_metric; + std::shared_ptr> _buffer_alloc_fail_metric; + std::shared_ptr _submit_latency_metric; + std::shared_ptr _buffer_alloc_latency_metric; + std::shared_ptr _queue_wait_latency_metric; + std::shared_ptr _worker_task_latency_metric; + std::shared_ptr _get_or_set_latency_metric; + std::shared_ptr _append_latency_metric; + std::shared_ptr _finalize_latency_metric; + std::shared_ptr> _skip_downloaded_metric; + std::shared_ptr> _skip_downloading_metric; + std::shared_ptr> _skip_partial_overlap_metric; + std::shared_ptr> _drop_stale_epoch_metric; + std::shared_ptr> _skip_deleting_metric; + std::shared_ptr> _append_fail_metric; + std::shared_ptr> _finalize_fail_metric; + std::shared_ptr> _persisted_blocks_metric; + std::shared_ptr> _persisted_bytes_metric; + std::shared_ptr> _watchdog_warn_metric; + std::shared_ptr> _watchdog_drop_metric; +}; + +} // namespace doris::io diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index b44679a94fa150..38a3442738dc63 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -385,6 +385,8 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path, _cache_base_path.c_str(), "file_cache_cache_lock_wait_time_us"); _get_or_set_latency_us = std::make_shared( _cache_base_path.c_str(), "file_cache_get_or_set_latency_us"); + _probe_latency_us = std::make_shared(_cache_base_path.c_str(), + "file_cache_probe_latency_us"); _storage_sync_remove_latency_us = std::make_shared( _cache_base_path.c_str(), "file_cache_storage_sync_remove_latency_us"); _storage_retry_sync_remove_latency_us = std::make_shared( @@ -548,6 +550,28 @@ Status BlockFileCache::initialize_unlocked(std::lock_guard& cache_lo } RETURN_IF_ERROR(_storage->init(this)); + _inflight_write_buffer_index = std::make_unique( + static_cast( + config::async_file_cache_write_inflight_write_buffer_index_shard_count), + _cache_base_path); + // BlockFileCache is the configuration boundary for a newly created per-disk service. Pass a + // value snapshot so AsyncCacheWriteService does not need to know where its settings came from. + AsyncCacheWriteServiceOptions async_write_options { + .worker_count = static_cast(config::async_file_cache_write_workers_per_disk), + .max_pending_tasks = + static_cast(config::async_file_cache_write_max_pending_tasks_per_disk), + .batch_size = static_cast(config::async_file_cache_write_batch_size), + .watchdog_warn_secs = config::async_file_cache_write_watchdog_warn_secs, + .watchdog_drop_secs = config::async_file_cache_write_watchdog_drop_secs, + }; + _async_write_service = + std::make_unique(this, std::move(async_write_options)); + // Do not create persistent per-disk workers while async writeback is disabled. The config + // update adapter starts existing services explicitly when the switch is enabled online. + if (config::enable_async_file_cache_write) { + RETURN_IF_ERROR(_async_write_service->start()); + } + if (auto* fs_storage = dynamic_cast(_storage.get())) { if (auto* meta_store = fs_storage->get_meta_store()) { _ttl_mgr = std::make_unique(this, meta_store); @@ -822,6 +846,114 @@ Status BlockFileCache::get_downloaded_blocks_if_fully_covered(const UInt128Wrapp return Status::OK(); } +FileBlocksProbeResult BlockFileCache::probe(const UInt128Wrapper& hash, size_t offset, size_t size, + const CacheContext& context) { + DORIS_CHECK(size > 0); + DORIS_CHECK(_max_file_block_size > 0); + const size_t end = offset + size; + DORIS_CHECK(end > offset); + const int64_t probe_start_us = MonotonicMicros(); + SCOPED_CACHE_LOCK(_mutex, this); + + auto file_iterator = _files.find(hash); + if (file_iterator == _files.end() && !_async_open_done) { + FileCacheKey key; + key.hash = hash; + key.meta.type = context.cache_type; + key.meta.expiration_time = context.expiration_time; + key.meta.tablet_id = context.tablet_id; + _storage->load_blocks_directly_unlocked(this, key, cache_lock); + file_iterator = _files.find(hash); + } + + FileBlocksByOffset* cached_blocks = nullptr; + FileBlocksByOffset::iterator cached_block; + if (file_iterator != _files.end()) { + DORIS_CHECK(!file_iterator->second.empty()); + cached_blocks = &file_iterator->second; + cached_block = cached_blocks->lower_bound(offset); + if (cached_block != cached_blocks->begin()) { + const auto& previous_range = std::prev(cached_block)->second.file_block->range(); + DORIS_CHECK(previous_range.right < offset); + } + } + + std::vector result; + result.reserve(size / _max_file_block_size + (size % _max_file_block_size != 0)); + for (size_t block_offset = offset; block_offset < end;) { + const size_t block_size = std::min(_max_file_block_size, end - block_offset); + const FileBlock::Range expected_range(block_offset, block_offset + block_size - 1); + FileBlockSPtr file_block; + if (cached_blocks != nullptr && cached_block != cached_blocks->end() && + cached_block->second.file_block->range().left <= expected_range.right) { + file_block = cached_block->second.file_block; + DORIS_CHECK(file_block->range().left == expected_range.left); + DORIS_CHECK(file_block->range().right >= expected_range.right); + // File writers allocate full-size cache blocks before the final buffer size is known. + // Only the last short probe slot can therefore have a larger cached right boundary. + DORIS_CHECK(file_block->range().right == expected_range.right || + block_size < _max_file_block_size); + ++cached_block; + } + result.emplace_back(std::move(file_block)); + block_offset += block_size; + } + *_probe_latency_us << (MonotonicMicros() - probe_start_us); + return FileBlocksProbeResult(std::move(result)); +} + +void BlockFileCache::touch_probe_block_if_cached(const FileBlockSPtr& block, + const CacheContext& context) { + DORIS_CHECK(block != nullptr); + FileBlockSPtr async_touch; + { + SCOPED_CACHE_LOCK(_mutex, this); + auto file_iterator = _files.find(block->get_hash_value()); + if (file_iterator == _files.end()) { + return; + } + auto cell_iterator = file_iterator->second.find(block->offset()); + if (cell_iterator == file_iterator->second.end() || + cell_iterator->second.file_block != block) { + return; + } + { + std::lock_guard block_lock(block->_mutex); + if (block->_is_deleting) { + return; + } + } + auto& cell = cell_iterator->second; + const bool move_iter = need_to_move(block->cache_type(), context.cache_type); + if (config::enable_file_cache_async_touch_on_get_or_set) { + cell.update_atime(); + if (move_iter) { + async_touch = block; + } + } else { + use_cell(cell, nullptr, move_iter, cache_lock); + } + } + if (async_touch) { + add_need_update_lru_block(std::move(async_touch)); + } +} + +bool BlockFileCache::is_block_deleting(const FileBlockSPtr& block) const { + DORIS_CHECK(block != nullptr); + SCOPED_CACHE_LOCK(_mutex, this); + auto file_iterator = _files.find(block->get_hash_value()); + if (file_iterator == _files.end()) { + return true; + } + auto cell_iterator = file_iterator->second.find(block->offset()); + if (cell_iterator == file_iterator->second.end() || cell_iterator->second.file_block != block) { + return true; + } + std::lock_guard block_lock(block->_mutex); + return block->_is_deleting; +} + std::string BlockFileCache::clear_file_cache_async() { return clear_file_cache_impl(false); } @@ -831,6 +963,8 @@ std::string BlockFileCache::clear_file_cache_sync() { } std::string BlockFileCache::clear_file_cache_impl(bool sync_remove) { + DORIS_CHECK(_async_write_service != nullptr); + _async_write_service->invalidate_pending_writes(); const char* action = sync_remove ? "clear_file_cache_sync" : "clear_file_cache_async"; LOG(INFO) << "start " << action << ", path=" << _cache_base_path; _lru_dumper->remove_lru_dump_files(); @@ -1025,7 +1159,7 @@ FileBlocksHolder BlockFileCache::get_or_set(const UInt128Wrapper& hash, size_t o int64_t duration = 0; { ConcurrencyStatsManager::instance().cached_remote_reader_get_or_set_wait_lock->increment(); - std::lock_guard cache_lock(_mutex); + SCOPED_CACHE_LOCK(_mutex, this); ConcurrencyStatsManager::instance().cached_remote_reader_get_or_set_wait_lock->decrement(); stats->lock_wait_timer += sw.elapsed_time(); SCOPED_RAW_TIMER(&duration); @@ -1370,6 +1504,8 @@ void BlockFileCache::try_evict_in_advance(size_t size, std::lock_guardinvalidate_pending_writes(); std::string reason = "remove_if_cached"; SCOPED_CACHE_LOCK(_mutex, this); auto iter = _files.find(file_key); @@ -1390,6 +1526,8 @@ void BlockFileCache::remove_if_cached(const UInt128Wrapper& file_key) { // cache meta is deleted synchronously if not in use, and the block file is deleted asynchronously // if in use, cache meta will be deleted after use and the block file is then deleted asynchronously void BlockFileCache::remove_if_cached_async(const UInt128Wrapper& file_key) { + DORIS_CHECK(_async_write_service != nullptr); + _async_write_service->invalidate_pending_writes(); std::string reason = "remove_if_cached_async"; SCOPED_CACHE_LOCK(_mutex, this); diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 9a01d07d136a62..f4d2f912ac9957 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -33,11 +34,13 @@ #include #include +#include "io/cache/async_cache_write_service.h" #include "io/cache/block_file_cache_ttl_mgr.h" #include "io/cache/cache_lru_dumper.h" #include "io/cache/file_block.h" #include "io/cache/file_cache_common.h" #include "io/cache/file_cache_storage.h" +#include "io/cache/inflight_write_buffer_index.h" #include "io/cache/lru_queue_recorder.h" #include "runtime/runtime_profile.h" #include "util/threadpool.h" @@ -80,6 +83,22 @@ class LockScopedTimer { class FSFileCacheStorage; +struct FileBlocksProbeResult { + explicit FileBlocksProbeResult(std::vector file_blocks_) + : file_blocks(std::move(file_blocks_)) {} + FileBlocksProbeResult(FileBlocksProbeResult&&) noexcept = default; + FileBlocksProbeResult& operator=(FileBlocksProbeResult&&) noexcept = delete; + FileBlocksProbeResult(const FileBlocksProbeResult&) = delete; + FileBlocksProbeResult& operator=(const FileBlocksProbeResult&) = delete; + ~FileBlocksProbeResult(); + + /// One entry per cache-block-sized input slot, in offset order. A null entry is a cache miss; + /// a non-null entry covers the whole slot. Its right boundary can exceed the final short slot + /// while a file writer still owns a full-size preallocated tail block. Retaining and releasing + /// a probe result never acquires or completes downloader ownership. + std::vector file_blocks; +}; + // NeedUpdateLRUBlocks keeps FileBlockSPtr entries that require LRU updates in a // deduplicated, sharded container. Entries are keyed by the raw FileBlock // pointer so that multiple shared_ptr copies of the same block are treated as a @@ -180,6 +199,9 @@ class BlockFileCache { BlockFileCache(const std::string& cache_base_path, const FileCacheSettings& cache_settings); virtual ~BlockFileCache() { + if (_async_write_service) { + _async_write_service->shutdown(); + } { std::lock_guard lock(_close_mtx); _close = true; @@ -237,6 +259,29 @@ class BlockFileCache { FileBlocksHolder get_or_set(const UInt128Wrapper& hash, size_t offset, size_t size, CacheContext& context); + /// Probe the block-aligned `[offset, offset + size)` range without creating cache cells or + /// touching LRU state. The result contains one ordered slot per cache block; each slot is null + /// on miss or owns an existing block that starts at and covers the slot. A final short slot can + /// be covered by a full-size block preallocated by a file writer. `context` supplies cache + /// metadata when lazy loading is required. + FileBlocksProbeResult probe(const UInt128Wrapper& hash, size_t offset, size_t size, + const CacheContext& context); + + /// Touch `block` after a successful local read, provided it is still the cached cell. The + /// supplied `context` controls the target LRU queue and query-level accounting. + void touch_probe_block_if_cached(const FileBlockSPtr& block, const CacheContext& context); + + /// Check whether `block` is being deleted while taking cache/block locks in canonical order. + bool is_block_deleting(const FileBlockSPtr& block) const; + + /// Return this cache disk's async-write service. + AsyncCacheWriteService* async_write_service() const { return _async_write_service.get(); } + + /// Return this cache disk's inflight payload index. + InflightWriteBufferIndex* inflight_write_buffer_index() const { + return _inflight_write_buffer_index.get(); + } + /** * Return existing downloaded blocks only if they fully cover [offset, offset + size). * This lookup is read-only: it does not reserve cache space or create EMPTY blocks. @@ -570,6 +615,9 @@ class BlockFileCache { std::unique_ptr _lru_dumper; std::unique_ptr _ttl_mgr; + std::unique_ptr _inflight_write_buffer_index; + std::unique_ptr _async_write_service; + // metrics std::shared_ptr> _cache_capacity_metrics; std::shared_ptr> _cur_cache_size_metrics; @@ -622,6 +670,7 @@ class BlockFileCache { std::shared_ptr _cache_lock_wait_time_us; std::shared_ptr _get_or_set_latency_us; + std::shared_ptr _probe_latency_us; std::shared_ptr _storage_sync_remove_latency_us; std::shared_ptr _storage_retry_sync_remove_latency_us; std::shared_ptr _storage_async_remove_latency_us; diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index e5d1005d24aa53..90554dbe1b1fb0 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -328,6 +328,8 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met size_t task_num = (download_size + one_single_task_size - 1) / one_single_task_size; std::unique_ptr buffer(new char[one_single_task_size]); + IOContext download_ctx = meta.ctx; + download_ctx.cache_write_mode_override = CacheWriteMode::SYNC_WRITE; DBUG_EXECUTE_IF("FileCacheBlockDownloader::download_segment_file_sleep", { if (meta.ctx.is_warmup) { @@ -352,7 +354,9 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met // 1. Directly append buffer data to file cache // 2. Provide `FileReader::async_read()` interface DCHECK(meta.ctx.is_dryrun == config::enable_reader_dryrun_when_download_file_cache); - st = file_reader->read_at(offset, {buffer.get(), size}, &bytes_read, &meta.ctx); + TEST_SYNC_POINT_CALLBACK("FileCacheBlockDownloader::download_segment_file:before_read", + &download_ctx); + st = file_reader->read_at(offset, {buffer.get(), size}, &bytes_read, &download_ctx); if (!st.ok()) { LOG(WARNING) << "failed to download file path=" << meta.path << ", st=" << st; if (meta.download_done) { diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index 0ffc9cea365d6f..8720836d373884 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -70,6 +70,22 @@ size_t FileCacheFactory::try_release(const std::string& base_path) { return 0; } +Status FileCacheFactory::update_async_write_options(const AsyncCacheWriteServiceOptions& options) { + std::lock_guard lock(_mtx); + for (const auto& cache : _caches) { + RETURN_IF_ERROR(cache->async_write_service()->update_options(options)); + } + return Status::OK(); +} + +Status FileCacheFactory::start_async_write_services() { + std::lock_guard lock(_mtx); + for (const auto& cache : _caches) { + RETURN_IF_ERROR(cache->async_write_service()->start()); + } + return Status::OK(); +} + Status FileCacheFactory::create_file_cache(const std::string& cache_base_path, FileCacheSettings file_cache_settings) { if (file_cache_settings.storage == "memory") { @@ -394,3 +410,78 @@ void FileCacheFactory::get_cache_stats_block(Block* block) { } // namespace io } // namespace doris + +namespace doris::config { + +namespace { + +/// Capture all mutable async-write fields after a config update. Returning one complete value +/// keeps every per-disk service on a coherent set of queue, worker, batch, and watchdog settings. +io::AsyncCacheWriteServiceOptions load_async_write_options_from_config() { + return io::AsyncCacheWriteServiceOptions { + .worker_count = static_cast(async_file_cache_write_workers_per_disk), + .max_pending_tasks = + static_cast(async_file_cache_write_max_pending_tasks_per_disk), + .batch_size = static_cast(async_file_cache_write_batch_size), + .watchdog_warn_secs = async_file_cache_write_watchdog_warn_secs, + .watchdog_drop_secs = async_file_cache_write_watchdog_drop_secs, + }; +} + +/// Forward one changed config field through the explicit factory/service update interface. +/// @param config_name Name used only to identify failures in the log. +/// @param old_value Previous config value; equal values require no service update. +/// @param new_value Newly accepted config value. +template +void update_async_write_options(const char* config_name, T old_value, T new_value) { + if (old_value == new_value) { + return; + } + auto* factory = ExecEnv::GetInstance()->file_cache_factory(); + if (factory == nullptr) { + return; + } + Status status = factory->update_async_write_options(load_async_write_options_from_config()); + if (!status.ok()) { + LOG(WARNING) << "Failed to apply async file cache write option " << config_name << " from " + << old_value << " to " << new_value << ": " << status.to_string(); + } +} + +} // namespace + +DEFINE_ON_UPDATE(enable_async_file_cache_write, [](bool old_value, bool new_value) { + if (old_value == new_value || !new_value) { + return; + } + auto* factory = io::FileCacheFactory::instance(); + if (factory == nullptr) { + return; + } + Status status = factory->start_async_write_services(); + if (!status.ok()) { + LOG(WARNING) << "Failed to start async file cache write services: " << status.to_string(); + } +}); + +DEFINE_ON_UPDATE(async_file_cache_write_workers_per_disk, [](int32_t old_value, int32_t new_value) { + update_async_write_options("async_file_cache_write_workers_per_disk", old_value, new_value); +}); +DEFINE_ON_UPDATE(async_file_cache_write_max_pending_tasks_per_disk, + [](int64_t old_value, int64_t new_value) { + update_async_write_options("async_file_cache_write_max_pending_tasks_per_disk", + old_value, new_value); + }); +DEFINE_ON_UPDATE(async_file_cache_write_batch_size, [](int32_t old_value, int32_t new_value) { + update_async_write_options("async_file_cache_write_batch_size", old_value, new_value); +}); +DEFINE_ON_UPDATE(async_file_cache_write_watchdog_warn_secs, [](int64_t old_value, + int64_t new_value) { + update_async_write_options("async_file_cache_write_watchdog_warn_secs", old_value, new_value); +}); +DEFINE_ON_UPDATE(async_file_cache_write_watchdog_drop_secs, [](int64_t old_value, + int64_t new_value) { + update_async_write_options("async_file_cache_write_watchdog_drop_secs", old_value, new_value); +}); + +} // namespace doris::config diff --git a/be/src/io/cache/block_file_cache_factory.h b/be/src/io/cache/block_file_cache_factory.h index f5be334de8b75f..aa696e00175abc 100644 --- a/be/src/io/cache/block_file_cache_factory.h +++ b/be/src/io/cache/block_file_cache_factory.h @@ -95,6 +95,15 @@ class FileCacheFactory { std::vector get_base_paths(); + /// Explicitly replace async-write service settings on every initialized cache disk. + /// @param options Complete settings snapshot produced by the configuration adapter. + /// @return OK after every service is updated; otherwise the first service update error. + Status update_async_write_options(const AsyncCacheWriteServiceOptions& options); + + /// Start async-write workers for every initialized cache disk. Repeated calls are idempotent. + /// @return OK after every service is ready; otherwise the first startup error. + Status start_async_write_services(); + /** * Clears data of all file cache instances * diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 751d422f044c1d..e541890a575e52 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -99,6 +99,17 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(lock_wait_timer); SUBTRACT_FIELD(get_timer); SUBTRACT_FIELD(set_timer); + SUBTRACT_FIELD(async_cache_write_submitted); + SUBTRACT_FIELD(async_cache_write_rejected); + SUBTRACT_FIELD(async_cache_write_buffer_alloc_fail); + SUBTRACT_FIELD(async_cache_write_drop_stale_epoch); + SUBTRACT_FIELD(inflight_write_buffer_index_hit); + SUBTRACT_FIELD(inflight_write_buffer_index_miss); + SUBTRACT_FIELD(probe_downloaded_hit); + SUBTRACT_FIELD(probe_downloading_hit); + SUBTRACT_FIELD(probe_miss); + SUBTRACT_FIELD(block_wait_success); + SUBTRACT_FIELD(block_wait_timeout); SUBTRACT_FIELD(inverted_index_num_local_io_total); SUBTRACT_FIELD(inverted_index_num_remote_io_total); @@ -163,6 +174,27 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _p lock_wait_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LockWaitTimer", cache_profile, 1); get_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "GetTimer", cache_profile, 1); set_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "SetTimer", cache_profile, 1); + async_cache_write_submitted = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "AsyncCacheWriteSubmitted", + TUnit::UNIT, cache_profile, 1); + async_cache_write_rejected = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "AsyncCacheWriteRejected", + TUnit::UNIT, cache_profile, 1); + async_cache_write_buffer_alloc_fail = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AsyncCacheWriteBufferAllocFail", TUnit::UNIT, cache_profile, 1); + async_cache_write_drop_stale_epoch = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AsyncCacheWriteDropStaleEpoch", TUnit::UNIT, cache_profile, 1); + inflight_write_buffer_index_hit = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InflightWriteBufferIndexHit", TUnit::UNIT, cache_profile, 1); + inflight_write_buffer_index_miss = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InflightWriteBufferIndexMiss", TUnit::UNIT, cache_profile, 1); + probe_downloaded_hit = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ProbeDownloadedHit", TUnit::UNIT, + cache_profile, 1); + probe_downloading_hit = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ProbeDownloadingHit", + TUnit::UNIT, cache_profile, 1); + probe_miss = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ProbeMiss", TUnit::UNIT, cache_profile, 1); + block_wait_success = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "BlockWaitSuccess", TUnit::UNIT, + cache_profile, 1); + block_wait_timeout = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "BlockWaitTimeout", TUnit::UNIT, + cache_profile, 1); remote_only_on_miss_triggered = profile->AddHighWaterMarkCounter("RemoteOnlyOnMissTriggered", TUnit::UNIT, cache_profile, 1); remote_only_on_miss_threshold_bytes = profile->AddHighWaterMarkCounter( @@ -257,6 +289,19 @@ void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) con COUNTER_UPDATE(lock_wait_timer, statistics->lock_wait_timer); COUNTER_UPDATE(get_timer, statistics->get_timer); COUNTER_UPDATE(set_timer, statistics->set_timer); + COUNTER_UPDATE(async_cache_write_submitted, statistics->async_cache_write_submitted); + COUNTER_UPDATE(async_cache_write_rejected, statistics->async_cache_write_rejected); + COUNTER_UPDATE(async_cache_write_buffer_alloc_fail, + statistics->async_cache_write_buffer_alloc_fail); + COUNTER_UPDATE(async_cache_write_drop_stale_epoch, + statistics->async_cache_write_drop_stale_epoch); + COUNTER_UPDATE(inflight_write_buffer_index_hit, statistics->inflight_write_buffer_index_hit); + COUNTER_UPDATE(inflight_write_buffer_index_miss, statistics->inflight_write_buffer_index_miss); + COUNTER_UPDATE(probe_downloaded_hit, statistics->probe_downloaded_hit); + COUNTER_UPDATE(probe_downloading_hit, statistics->probe_downloading_hit); + COUNTER_UPDATE(probe_miss, statistics->probe_miss); + COUNTER_UPDATE(block_wait_success, statistics->block_wait_success); + COUNTER_UPDATE(block_wait_timeout, statistics->block_wait_timeout); remote_only_on_miss_triggered->set(statistics->remote_only_on_miss_triggered); remote_only_on_miss_threshold_bytes->set(statistics->remote_only_on_miss_threshold_bytes); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index 2a6e7b33980bb2..c2db1a704c0706 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -89,6 +89,17 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* lock_wait_timer = nullptr; RuntimeProfile::Counter* get_timer = nullptr; RuntimeProfile::Counter* set_timer = nullptr; + RuntimeProfile::Counter* async_cache_write_submitted = nullptr; + RuntimeProfile::Counter* async_cache_write_rejected = nullptr; + RuntimeProfile::Counter* async_cache_write_buffer_alloc_fail = nullptr; + RuntimeProfile::Counter* async_cache_write_drop_stale_epoch = nullptr; + RuntimeProfile::Counter* inflight_write_buffer_index_hit = nullptr; + RuntimeProfile::Counter* inflight_write_buffer_index_miss = nullptr; + RuntimeProfile::Counter* probe_downloaded_hit = nullptr; + RuntimeProfile::Counter* probe_downloading_hit = nullptr; + RuntimeProfile::Counter* probe_miss = nullptr; + RuntimeProfile::Counter* block_wait_success = nullptr; + RuntimeProfile::Counter* block_wait_timeout = nullptr; RuntimeProfile::HighWaterMarkCounter* remote_only_on_miss_triggered = nullptr; RuntimeProfile::HighWaterMarkCounter* remote_only_on_miss_threshold_bytes = nullptr; diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index 47fe02ee2c3e92..91edf328475f8c 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -65,6 +65,7 @@ #include "util/concurrency_stats.h" #include "util/debug_points.h" #include "util/defer_op.h" +#include "util/time.h" namespace doris::io { @@ -118,6 +119,8 @@ static bool use_remote_only_on_cache_miss(const IOContext* io_ctx) { CachedRemoteFileReader::CachedRemoteFileReader(FileReaderSPtr remote_file_reader, const FileReaderOptions& opts) : _is_doris_table(opts.is_doris_table), + _cache_align_mode(opts.align_mode), + _cache_write_mode(opts.cache_write_mode), _tablet_id(opts.tablet_id), _storage_resource_id(opts.storage_resource_id), _remote_file_reader(std::move(remote_file_reader)) { @@ -1258,8 +1261,17 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* return Status::OK(); } - read_st = _read_from_indirect_cache(offset, result, bytes_req, already_read, is_dryrun, - bytes_read, stats, source_read_breakdown, io_ctx); + const CacheWriteMode cache_write_mode = _resolve_cache_write_mode(io_ctx); + DORIS_CHECK(_cache_align_mode == CacheAlignMode::ALIGN_TO_BLOCK); + DORIS_CHECK(cache_write_mode == CacheWriteMode::SYNC_WRITE || + cache_write_mode == CacheWriteMode::ASYNC_WRITE); + if (cache_write_mode == CacheWriteMode::ASYNC_WRITE) { + read_st = _read_async_write_path(offset, result, bytes_req, already_read, bytes_read, stats, + source_read_breakdown, io_ctx); + } else { + read_st = _read_from_indirect_cache(offset, result, bytes_req, already_read, is_dryrun, + bytes_read, stats, source_read_breakdown, io_ctx); + } return read_st; } @@ -1280,6 +1292,7 @@ void CachedRemoteFileReader::prefetch_range(size_t offset, size_t size, const IO dryrun_ctx = *io_ctx; } dryrun_ctx.is_dryrun = true; + dryrun_ctx.cache_write_mode_override = CacheWriteMode::SYNC_WRITE; dryrun_ctx.query_id = nullptr; dryrun_ctx.file_cache_stats = nullptr; dryrun_ctx.file_reader_stats = nullptr; @@ -1360,6 +1373,17 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->lock_wait_timer += read_stats.lock_wait_timer; statis->get_timer += read_stats.get_timer; statis->set_timer += read_stats.set_timer; + statis->async_cache_write_submitted += read_stats.async_cache_write_submitted; + statis->async_cache_write_rejected += read_stats.async_cache_write_rejected; + statis->async_cache_write_buffer_alloc_fail += read_stats.async_cache_write_buffer_alloc_fail; + statis->async_cache_write_drop_stale_epoch += read_stats.async_cache_write_drop_stale_epoch; + statis->inflight_write_buffer_index_hit += read_stats.inflight_write_buffer_index_hit; + statis->inflight_write_buffer_index_miss += read_stats.inflight_write_buffer_index_miss; + statis->probe_downloaded_hit += read_stats.probe_downloaded_hit; + statis->probe_downloading_hit += read_stats.probe_downloading_hit; + statis->probe_miss += read_stats.probe_miss; + statis->block_wait_success += read_stats.block_wait_success; + statis->block_wait_timeout += read_stats.block_wait_timeout; auto update_index_stats = [&](int64_t& num_local_io_total, int64_t& num_remote_io_total, int64_t& num_peer_io_total, int64_t& bytes_read_from_local, diff --git a/be/src/io/cache/cached_remote_file_reader.h b/be/src/io/cache/cached_remote_file_reader.h index 5c562c82513f7b..c74c95ffb7f5fa 100644 --- a/be/src/io/cache/cached_remote_file_reader.h +++ b/be/src/io/cache/cached_remote_file_reader.h @@ -118,6 +118,9 @@ class CachedRemoteFileReader final : public FileReader, const IOContext* io_ctx) override; private: + struct AsyncReadBlock; + struct AsyncReadPlan; + enum class FileCacheReadType { DATA, INVERTED_INDEX, @@ -142,6 +145,107 @@ class CachedRemoteFileReader final : public FileReader, /// @return true if peer read is enabled for this request; otherwise false. bool _should_read_from_peer(const IOContext* io_ctx) const; + /// Resolve the write policy for the current read instead of freezing a global setting in the + /// reader constructor. Explicit cache-population reads always remain synchronous. + /// @param[in] io_ctx Per-read flags and an optional write-mode override. + /// @return The effective synchronous or asynchronous cache-write mode for this read. + CacheWriteMode _resolve_cache_write_mode(const IOContext* io_ctx) const; + + /// Serve a normal read while moving cache-miss writes off the query thread. The method uses + /// the first and last remotely covered blocks as one range, matching the synchronous path's + /// preference for a single remote operation over fine-grained hole processing. + /// @param[in] offset Original user read offset. + /// @param[out] result Destination buffer for the complete user request. + /// @param[in] bytes_req Requested user bytes. + /// @param[in] already_read Prefix bytes already filled by the direct-cache path. + /// @param[out] bytes_read Total completed user bytes. + /// @param[in,out] stats Per-read cache statistics. + /// @param[in,out] source_read_breakdown Local/remote byte attribution for query profiles. + /// @param[in] io_ctx Context passed to cache lookup and remote IO. + /// @return OK on success; otherwise the remote-read error. + Status _read_async_write_path(size_t offset, Slice result, size_t bytes_req, + size_t already_read, size_t* bytes_read, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + const IOContext* io_ctx); + + /// Build block-aligned source coverage for the unread suffix. It first performs one inflight + /// batch lookup; a fully covered request returns without taking the BlockFileCache lock, while + /// incomplete coverage triggers one read-only whole-range probe with one result per aligned + /// block. + /// @param[in] remaining_offset First user byte not filled by the direct-cache path. + /// @param[in] remaining_size Number of unread user bytes. + /// @param[in] write_epoch Epoch captured before any lookup or remote IO. + /// @param[in] io_ctx Context used to build the cache admission/probe context. + /// @param[in,out] stats Lookup and probe counters updated during planning. + /// @return Plan that owns any retained probe blocks and first-to-last remote range. + AsyncReadPlan _build_async_read_plan(size_t remaining_offset, size_t remaining_size, + uint64_t write_epoch, const IOContext* io_ctx, + ReadStatistics& stats); + + /// Copy one block already available from an inflight buffer or downloaded cache file. Cache + /// state is revalidated before IO; a race is reported to the caller as a simple full-range + /// remote fallback. + /// @param[in] plan Plan owning the probed blocks and user boundaries. + /// @param[in] block_index Index of the aligned block and its matching probe result. + /// @param[in] user_offset Original user request offset used to locate the destination slice. + /// @param[out] result Destination buffer for the complete user request. + /// @param[in] cache_context Context used only when a successful local read touches LRU. + /// @param[in,out] stats Local-read timing counters. + /// @param[in,out] materialized_bytes User bytes copied from cache or inflight memory. + /// @param[in,out] need_self_heal Set when a cache file disappears during a local read. + /// @return true when the block was copied; false when the caller should use remote fallback. + bool _materialize_async_block(const AsyncReadPlan& plan, size_t block_index, size_t user_offset, + Slice result, const CacheContext& cache_context, + ReadStatistics& stats, size_t* materialized_bytes, + bool* need_self_heal); + + /// Copy only blocks before the first and after the last REMOTE block. When no REMOTE block + /// exists, copy the entire request. DOWNLOADING blocks outside the remote span keep the + /// existing wait behavior; blocks inside the span are covered by the same remote read. + /// @param[in] plan Classified block list and remote boundaries. + /// @param[in] user_offset Original user request offset. + /// @param[out] result Destination buffer for side data. + /// @param[in] cache_context Context used for successful local-cache touches. + /// @param[in,out] stats Local-read statistics. + /// @param[in,out] source_read_breakdown Local bytes copied from the covered sides. + /// @param[in,out] indirect_read_bytes User bytes copied from the covered sides. + /// @param[in,out] need_self_heal Whether a missing local cache file requires async cleanup. + /// @return true when all selected cache/inflight blocks were copied; false on a race/read error. + bool _materialize_async_cached_sides(const AsyncReadPlan& plan, size_t user_offset, + Slice result, const CacheContext& cache_context, + ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, bool* need_self_heal); + + /// Read the planned middle span once and copy only its overlap with the unread user range. + /// @param[in] plan Source plan containing user boundaries. + /// @param[in] user_offset Original user request offset. + /// @param[out] result Destination buffer for the complete user request. + /// @param[in] need_self_heal Whether stale cache metadata should be removed before remote IO. + /// @param[in] io_ctx Context passed to remote storage. + /// @param[in,out] stats Remote-read timing and source flags. + /// @param[in,out] source_read_breakdown Remote user bytes copied from this span. + /// @param[in,out] indirect_read_bytes User bytes copied by the indirect path. + /// @param[out] remote_buffer Full aligned middle-span payload retained for async tasks. + /// @return OK on success; otherwise the remote-read error. + Status _read_async_remote_range(const AsyncReadPlan& plan, size_t user_offset, Slice result, + bool need_self_heal, const IOContext* io_ctx, + ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, + std::unique_ptr* remote_buffer); + + /// Copy each real cache-miss block from the remote span into tracked memory and enqueue a + /// per-block write task. A final insert-if-absent prevents duplicate ownership after IO. + /// @param[in] plan Classified blocks, remote boundaries, and the epoch captured before IO. + /// @param[in] remote_buffer Full payload for the plan's first-to-last remote span. + /// @param[in] io_ctx Context converted to the worker's admission context. + /// @param[in,out] stats Submission, rejection, allocation, and dedup counters. + /// @return None. + void _submit_async_write_tasks(const AsyncReadPlan& plan, + const std::unique_ptr& remote_buffer, + const IOContext* io_ctx, ReadStatistics& stats); + /// Register a downloaded block in the direct-read map owned by this reader. /// @param[in] file_block Downloaded cache block to insert. /// @return None. @@ -320,6 +424,8 @@ class CachedRemoteFileReader final : public FileReader, FileCacheReadType read_type) const; bool _is_doris_table = false; + CacheAlignMode _cache_align_mode {CacheAlignMode::ALIGN_TO_BLOCK}; + CacheWriteMode _cache_write_mode {CacheWriteMode::DEFAULT}; int64_t _tablet_id = -1; std::string _storage_resource_id; FileReaderSPtr _remote_file_reader; diff --git a/be/src/io/cache/cached_remote_file_reader_async_write.cpp b/be/src/io/cache/cached_remote_file_reader_async_write.cpp new file mode 100644 index 00000000000000..f57535407864d8 --- /dev/null +++ b/be/src/io/cache/cached_remote_file_reader_async_write.cpp @@ -0,0 +1,559 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "cpp/sync_point.h" +#include "io/cache/async_cache_write_service.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/cached_remote_file_reader.h" +#include "io/cache/inflight_write_buffer_index.h" +#include "io/io_common.h" +#include "runtime/runtime_profile.h" +#include "util/defer_op.h" +#include "util/time.h" + +namespace doris::io { + +// These counters are shared by the synchronous and asynchronous indirect-read paths, so their +// definitions remain in cached_remote_file_reader.cpp. +extern bvar::Adder g_read_cache_indirect_num; +extern bvar::Adder g_read_cache_indirect_bytes; +extern bvar::Adder g_read_cache_indirect_total_bytes; +extern bvar::Adder g_read_cache_self_heal_on_not_found; + +namespace { + +bvar::Adder g_cached_remote_reader_probe_total("cached_remote_file_reader_probe_total"); +bvar::Adder g_cached_remote_reader_probe_downloaded( + "cached_remote_file_reader_probe_hit_downloaded_total"); +bvar::Adder g_cached_remote_reader_probe_downloading( + "cached_remote_file_reader_probe_hit_downloading_total"); +bvar::Adder g_cached_remote_reader_probe_miss( + "cached_remote_file_reader_probe_miss_total"); +bvar::Adder g_cached_remote_reader_inflight_hit( + "cached_remote_file_reader_inflight_write_buffer_hit_total"); +bvar::Adder g_cached_remote_reader_async_skip_existing( + "cached_remote_file_reader_async_write_skip_inflight_existing_total"); +bvar::Adder g_cached_remote_reader_block_wait( + "cached_remote_file_reader_block_wait_total"); +bvar::Adder g_cached_remote_reader_block_wait_timeout( + "cached_remote_file_reader_block_wait_timeout_total"); +bvar::Adder g_cached_remote_reader_remote_after_dedup_miss( + "cached_remote_file_reader_remote_read_after_all_dedup_miss_total"); +bvar::Adder g_cached_remote_reader_middle_span_read_bytes( + "cached_remote_file_reader_middle_span_read_bytes"); +bvar::Adder g_cached_remote_reader_middle_span_miss_bytes( + "cached_remote_file_reader_middle_span_miss_bytes"); +bvar::LatencyRecorder g_cached_remote_reader_async_read_plan_latency( + "cached_remote_file_reader_async_read_plan_latency_us"); +bvar::LatencyRecorder g_cached_remote_reader_async_write_submission_latency( + "cached_remote_file_reader_async_write_submission_latency_us"); + +} // namespace + +// One aligned cache block in the simplified read plan. REMOTE blocks delimit the single remote +// range; submit_write distinguishes real cache misses from blocks already being downloaded. +struct CachedRemoteFileReader::AsyncReadBlock { + enum class Source { + INFLIGHT, + CACHE, + DOWNLOADING, + REMOTE, + }; + + explicit AsyncReadBlock(FileBlock::Range range_) : range(range_) {} + + FileBlock::Range range; + Source source {Source::REMOTE}; + bool submit_write {false}; + std::shared_ptr inflight_entry; +}; + +// A read first checks inflight buffers and owns a cache probe result only when that lookup leaves +// uncovered blocks. first_remote_block and remote_block_count identify the inclusive first-to-last +// uncovered span, including any cache hits between those boundaries. +struct CachedRemoteFileReader::AsyncReadPlan { + AsyncReadPlan(uint64_t write_epoch_, size_t user_left_, size_t user_right_) + : write_epoch(write_epoch_), user_left(user_left_), user_right(user_right_) {} + + uint64_t write_epoch {0}; + size_t user_left {0}; + size_t user_right {0}; + std::optional probe_result; + std::vector blocks; + size_t first_remote_block {0}; + size_t remote_block_count {0}; +}; + +// Resolve mode on every read so online configuration changes affect existing readers. +CacheWriteMode CachedRemoteFileReader::_resolve_cache_write_mode(const IOContext* io_ctx) const { + if (io_ctx->is_dryrun || io_ctx->is_warmup || _should_read_from_peer(io_ctx)) { + return CacheWriteMode::SYNC_WRITE; + } + if (io_ctx->cache_write_mode_override.has_value()) { + return *io_ctx->cache_write_mode_override; + } + if (_cache_write_mode != CacheWriteMode::DEFAULT) { + return _cache_write_mode; + } + return config::enable_async_file_cache_write ? CacheWriteMode::ASYNC_WRITE + : CacheWriteMode::SYNC_WRITE; +} + +// Build a deliberately small plan. The inflight index is checked first so a fully covered read +// avoids BlockFileCache::probe and its cache mutex. Only an incomplete inflight lookup performs one +// whole-range probe whose result entries map directly to the logical plan blocks. +CachedRemoteFileReader::AsyncReadPlan CachedRemoteFileReader::_build_async_read_plan( + size_t remaining_offset, size_t remaining_size, uint64_t write_epoch, + const IOContext* io_ctx, ReadStatistics& stats) { + const int64_t plan_start_us = MonotonicMicros(); + Defer record_plan_latency {[&]() { + g_cached_remote_reader_async_read_plan_latency << (MonotonicMicros() - plan_start_us); + }}; + DORIS_CHECK(remaining_size > 0); + const auto [align_left, align_size] = s_align_size(remaining_offset, remaining_size, size()); + const size_t cache_block_size = static_cast(config::file_cache_each_block_size); + DORIS_CHECK(cache_block_size > 0); + + AsyncReadPlan plan(write_epoch, remaining_offset, remaining_offset + remaining_size - 1); + + std::vector block_offsets; + const size_t align_end = align_left + align_size; + for (size_t block_offset = align_left; block_offset < align_end; + block_offset += cache_block_size) { + const size_t block_size = std::min(cache_block_size, size() - block_offset); + DORIS_CHECK(block_size > 0); + const FileBlock::Range block_range(block_offset, block_offset + block_size - 1); + plan.blocks.emplace_back(block_range); + block_offsets.emplace_back(block_offset); + } + DORIS_CHECK(!plan.blocks.empty()); + + const bool inflight_index_enabled = + config::enable_async_file_cache_write_inflight_write_buffer_index; + bool all_blocks_inflight = inflight_index_enabled; + if (inflight_index_enabled) { + auto* inflight_index = _cache->inflight_write_buffer_index(); + DORIS_CHECK(inflight_index != nullptr); + auto inflight_results = inflight_index->lookup_all(_cache_hash, block_offsets, write_epoch); + DORIS_CHECK(inflight_results.size() == plan.blocks.size()); + for (size_t index = 0; index < plan.blocks.size(); ++index) { + auto& entry = inflight_results[index].entry; + if (!entry) { + all_blocks_inflight = false; + ++stats.inflight_write_buffer_index_miss; + continue; + } + + auto& read_block = plan.blocks[index]; + DORIS_CHECK(entry->buffer != nullptr); + DORIS_CHECK(entry->buffer_offset <= read_block.range.left); + DORIS_CHECK(entry->buffer_offset + entry->buffer_size > read_block.range.right); + read_block.source = AsyncReadBlock::Source::INFLIGHT; + read_block.inflight_entry = std::move(entry); + ++stats.inflight_write_buffer_index_hit; + g_cached_remote_reader_inflight_hit << 1; + } + } + if (all_blocks_inflight) { + return plan; + } + + CacheContext cache_context(io_ctx); + cache_context.stats = &stats; + cache_context.tablet_id = _tablet_id; + plan.probe_result.emplace(_cache->probe(_cache_hash, align_left, align_size, cache_context)); + g_cached_remote_reader_probe_total << 1; + const auto& probe_result = *plan.probe_result; + DORIS_CHECK(probe_result.file_blocks.size() == plan.blocks.size()); + + // probe() validates slot coverage while holding the cache mutex. Use the immutable logical + // plan ranges after it returns: a concurrent file writer may shrink a preallocated EOF block + // during finalize(). + for (size_t index = 0; index < plan.blocks.size(); ++index) { + auto& read_block = plan.blocks[index]; + if (read_block.source == AsyncReadBlock::Source::INFLIGHT) { + continue; + } + + const auto& file_block = probe_result.file_blocks[index]; + bool is_miss = file_block == nullptr; + bool is_downloading = false; + if (file_block != nullptr) { + if (_cache->is_block_deleting(file_block)) { + is_miss = true; + } else { + switch (file_block->state()) { + case FileBlock::State::DOWNLOADED: + break; + case FileBlock::State::DOWNLOADING: + is_downloading = true; + break; + case FileBlock::State::EMPTY: + case FileBlock::State::SKIP_CACHE: + is_miss = true; + break; + } + } + } + + if (is_miss) { + read_block.submit_write = true; + ++stats.probe_miss; + g_cached_remote_reader_probe_miss << 1; + } else if (is_downloading) { + read_block.source = AsyncReadBlock::Source::DOWNLOADING; + ++stats.probe_downloading_hit; + g_cached_remote_reader_probe_downloading << 1; + continue; + } else { + read_block.source = AsyncReadBlock::Source::CACHE; + ++stats.probe_downloaded_hit; + g_cached_remote_reader_probe_downloaded << 1; + continue; + } + + if (plan.remote_block_count == 0) { + plan.first_remote_block = index; + } + plan.remote_block_count = index - plan.first_remote_block + 1; + } + return plan; +} + +// Copy one block available from inflight memory or cache. DOWNLOADING blocks outside the remote +// span retain the existing wait semantics. Any race or read failure asks the caller to replace the +// whole planned request with one remote read instead of incrementally repairing the range. +bool CachedRemoteFileReader::_materialize_async_block(const AsyncReadPlan& plan, size_t block_index, + size_t user_offset, Slice result, + const CacheContext& cache_context, + ReadStatistics& stats, + size_t* materialized_bytes, + bool* need_self_heal) { + DORIS_CHECK(block_index < plan.blocks.size()); + const auto& read_block = plan.blocks[block_index]; + DORIS_CHECK(read_block.source != AsyncReadBlock::Source::REMOTE); + DORIS_CHECK(materialized_bytes != nullptr); + DORIS_CHECK(need_self_heal != nullptr); + + const size_t copy_left = std::max(read_block.range.left, plan.user_left); + const size_t copy_right = std::min(read_block.range.right, plan.user_right); + if (copy_left > copy_right) { + return true; + } + const size_t copy_size = copy_right - copy_left + 1; + + if (read_block.source == AsyncReadBlock::Source::INFLIGHT) { + const auto& entry = read_block.inflight_entry; + DORIS_CHECK(entry != nullptr); + const size_t entry_offset = copy_left - entry->buffer_offset; + DORIS_CHECK(entry_offset + copy_size <= entry->buffer_size); + memcpy(result.data + (copy_left - user_offset), entry->buffer->data() + entry_offset, + copy_size); + *materialized_bytes += copy_size; + return true; + } + + DORIS_CHECK(plan.probe_result.has_value()); + DORIS_CHECK(block_index < plan.probe_result->file_blocks.size()); + const auto& file_block = plan.probe_result->file_blocks[block_index]; + DORIS_CHECK(file_block != nullptr); + if (_cache->is_block_deleting(file_block)) { + return false; + } + + FileBlock::State state = file_block->state(); + if (state == FileBlock::State::DOWNLOADING) { + DORIS_CHECK(read_block.source == AsyncReadBlock::Source::DOWNLOADING); + TEST_SYNC_POINT("CachedRemoteFileReader::_materialize_async_block:before_wait"); + { + SCOPED_RAW_TIMER(&stats.remote_wait_timer); + state = file_block->wait(); + } + if (state != FileBlock::State::DOWNLOADED) { + ++stats.block_wait_timeout; + g_cached_remote_reader_block_wait_timeout << 1; + return false; + } + ++stats.block_wait_success; + g_cached_remote_reader_block_wait << 1; + } + if (state != FileBlock::State::DOWNLOADED) { + return false; + } + + Status status; + { + SCOPED_RAW_TIMER(&stats.local_read_timer); + status = file_block->read(Slice(result.data + (copy_left - user_offset), copy_size), + copy_left - read_block.range.left); + } + if (!status.ok()) { + if (status.is()) { + *need_self_heal = true; + g_read_cache_self_heal_on_not_found << 1; + } + LOG_EVERY_N(WARNING, 100) + << "Read probed file cache block failed, falling back to remote. path=" + << path().native() << ", hash=" << _cache_hash.to_string() + << ", offset=" << read_block.range.left << ", status=" << status; + return false; + } + + _cache->touch_probe_block_if_cached(file_block, cache_context); + *materialized_bytes += copy_size; + return true; +} + +// Read cache/inflight blocks outside the first-to-last remote span. A side failure returns false; +// the caller then performs one remote read for the whole aligned request. +bool CachedRemoteFileReader::_materialize_async_cached_sides( + const AsyncReadPlan& plan, size_t user_offset, Slice result, + const CacheContext& cache_context, ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, size_t* indirect_read_bytes, + bool* need_self_heal) { + DORIS_CHECK(indirect_read_bytes != nullptr); + DORIS_CHECK(need_self_heal != nullptr); + + size_t materialized_bytes = 0; + const auto materialize_range = [&](size_t begin, size_t end) { + for (size_t index = begin; index < end; ++index) { + if (!_materialize_async_block(plan, index, user_offset, result, cache_context, stats, + &materialized_bytes, need_self_heal)) { + return false; + } + } + return true; + }; + + const size_t prefix_end = + plan.remote_block_count == 0 ? plan.blocks.size() : plan.first_remote_block; + if (!materialize_range(0, prefix_end)) { + return false; + } + if (plan.remote_block_count != 0) { + const size_t suffix_begin = plan.first_remote_block + plan.remote_block_count; + DORIS_CHECK(suffix_begin <= plan.blocks.size()); + if (!materialize_range(suffix_begin, plan.blocks.size())) { + return false; + } + } + + *indirect_read_bytes += materialized_bytes; + source_read_breakdown.local_bytes += materialized_bytes; + return true; +} + +// Read the single aligned range from the first REMOTE block through the last REMOTE block. Cache +// hits inside that span are intentionally reread to keep one straightforward remote operation. +Status CachedRemoteFileReader::_read_async_remote_range( + const AsyncReadPlan& plan, size_t user_offset, Slice result, bool need_self_heal, + const IOContext* io_ctx, ReadStatistics& stats, SourceReadBreakdown& source_read_breakdown, + size_t* indirect_read_bytes, std::unique_ptr* remote_buffer) { + DORIS_CHECK(indirect_read_bytes != nullptr); + DORIS_CHECK(remote_buffer != nullptr); + DORIS_CHECK(plan.remote_block_count > 0); + const size_t remote_end = plan.first_remote_block + plan.remote_block_count; + DORIS_CHECK(remote_end <= plan.blocks.size()); + const size_t remote_left = plan.blocks[plan.first_remote_block].range.left; + const size_t remote_right = plan.blocks[remote_end - 1].range.right; + const size_t remote_size = remote_right - remote_left + 1; + + stats.hit_cache = false; + stats.from_peer_cache = false; + if (need_self_heal) { + _cache->remove_if_cached_async(_cache_hash); + } + + const std::vector no_peer_blocks; + RETURN_IF_ERROR(_execute_remote_read(no_peer_blocks, remote_left, remote_size, *remote_buffer, + nullptr, stats, io_ctx)); + DORIS_CHECK(*remote_buffer != nullptr); + + const size_t copy_left = std::max(remote_left, plan.user_left); + const size_t copy_right = std::min(remote_right, plan.user_right); + if (copy_left <= copy_right) { + const size_t copy_size = copy_right - copy_left + 1; + memcpy(result.data + (copy_left - user_offset), + remote_buffer->get() + (copy_left - remote_left), copy_size); + *indirect_read_bytes += copy_size; + source_read_breakdown.remote_bytes += copy_size; + } + + size_t miss_bytes = 0; + for (size_t index = plan.first_remote_block; index < remote_end; ++index) { + if (plan.blocks[index].submit_write) { + miss_bytes += plan.blocks[index].range.size(); + } + } + if (miss_bytes > 0) { + g_cached_remote_reader_remote_after_dedup_miss << 1; + } + g_cached_remote_reader_middle_span_read_bytes << remote_size; + g_cached_remote_reader_middle_span_miss_bytes << miss_bytes; + return Status::OK(); +} + +// Submit exactly the real cache misses contained in the remote span. Inflight insertion happens after +// remote IO and immediately before enqueueing, so a concurrent owner wins without duplicate work. +void CachedRemoteFileReader::_submit_async_write_tasks(const AsyncReadPlan& plan, + const std::unique_ptr& remote_buffer, + const IOContext* io_ctx, + ReadStatistics& stats) { + const int64_t submission_start_us = MonotonicMicros(); + Defer record_submission_latency {[&]() { + g_cached_remote_reader_async_write_submission_latency + << (MonotonicMicros() - submission_start_us); + }}; + auto* service = _cache->async_write_service(); + auto* inflight_index = _cache->inflight_write_buffer_index(); + DORIS_CHECK(service != nullptr); + DORIS_CHECK(inflight_index != nullptr); + DORIS_CHECK(remote_buffer != nullptr); + + CacheContext cache_context(io_ctx); + CacheAdmissionContext admission_context { + .query_id = cache_context.query_id, + .cache_type = cache_context.cache_type, + .expiration_time = cache_context.expiration_time, + .tablet_id = _tablet_id, + .is_warmup = cache_context.is_warmup, + }; + DORIS_CHECK(plan.remote_block_count > 0); + const size_t remote_end = plan.first_remote_block + plan.remote_block_count; + DORIS_CHECK(remote_end <= plan.blocks.size()); + const size_t remote_left = plan.blocks[plan.first_remote_block].range.left; + for (size_t index = plan.first_remote_block; index < remote_end; ++index) { + const auto& read_block = plan.blocks[index]; + if (!read_block.submit_write) { + continue; + } + DORIS_CHECK(read_block.source == AsyncReadBlock::Source::REMOTE); + if (!service->is_current_write_epoch(plan.write_epoch)) { + ++stats.async_cache_write_drop_stale_epoch; + continue; + } + + AsyncCacheWriteBufferPtr tracked_buffer; + Status status = service->allocate_tracked_buffer(read_block.range.size(), &tracked_buffer); + if (!status.ok()) { + ++stats.async_cache_write_buffer_alloc_fail; + continue; + } + memcpy(tracked_buffer->data(), remote_buffer.get() + (read_block.range.left - remote_left), + read_block.range.size()); + + AsyncCacheWriteTask task { + .cache_hash = _cache_hash, + .file_offset = read_block.range.left, + .file_size = read_block.range.size(), + .buffer = tracked_buffer, + .admission_ctx = admission_context, + .submit_ts_us = MonotonicMicros(), + .write_epoch = plan.write_epoch, + .on_finalized = nullptr, + }; + std::shared_ptr entry; + if (config::enable_async_file_cache_write_inflight_write_buffer_index) { + entry = std::make_shared( + tracked_buffer, read_block.range.left, read_block.range.size(), + task.submit_ts_us, plan.write_epoch); + TEST_SYNC_POINT_CALLBACK( + "CachedRemoteFileReader::_submit_async_write_tasks:before_inflight_insert", + &task); + auto existing = + inflight_index->insert_if_absent(_cache_hash, read_block.range.left, entry); + if (existing) { + g_cached_remote_reader_async_skip_existing << 1; + continue; + } + task.on_finalized = [cache_hash = _cache_hash, offset = read_block.range.left, + inflight_index, entry](const AsyncCacheWriteTask&) { + inflight_index->remove_if(cache_hash, offset, entry); + }; + } + + if (!service->try_submit(std::move(task))) { + if (entry) { + inflight_index->remove_if(_cache_hash, read_block.range.left, entry); + inflight_index->record_backpressure_rollback(); + } + ++stats.async_cache_write_rejected; + } else { + ++stats.async_cache_write_submitted; + } + } +} + +// Orchestrate one simple plan: read cache/inflight blocks outside the remote span, issue at most +// one remote read, then enqueue only blocks that were actual cache misses. +Status CachedRemoteFileReader::_read_async_write_path(size_t offset, Slice result, size_t bytes_req, + size_t already_read, size_t* bytes_read, + ReadStatistics& stats, + SourceReadBreakdown& source_read_breakdown, + const IOContext* io_ctx) { + DORIS_CHECK(_cache_align_mode == CacheAlignMode::ALIGN_TO_BLOCK); + DORIS_CHECK(!io_ctx->is_dryrun); + DORIS_CHECK(result.data != nullptr); + g_read_cache_indirect_num << 1; + if (already_read == bytes_req) { + *bytes_read = bytes_req; + return Status::OK(); + } + + auto* service = _cache->async_write_service(); + DORIS_CHECK(service != nullptr); + auto plan = _build_async_read_plan(offset + already_read, bytes_req - already_read, + service->current_write_epoch(), io_ctx, stats); + + CacheContext cache_context(io_ctx); + cache_context.stats = &stats; + cache_context.tablet_id = _tablet_id; + size_t indirect_read_bytes = 0; + bool need_self_heal = false; + if (!_materialize_async_cached_sides(plan, offset, result, cache_context, stats, + source_read_breakdown, &indirect_read_bytes, + &need_self_heal)) { + plan.first_remote_block = 0; + plan.remote_block_count = plan.blocks.size(); + } + if (plan.remote_block_count > 0) { + std::unique_ptr remote_buffer; + RETURN_IF_ERROR(_read_async_remote_range(plan, offset, result, need_self_heal, io_ctx, + stats, source_read_breakdown, &indirect_read_bytes, + &remote_buffer)); + _submit_async_write_tasks(plan, remote_buffer, io_ctx, stats); + } + + *bytes_read = bytes_req; + g_read_cache_indirect_bytes << indirect_read_bytes; + g_read_cache_indirect_total_bytes << bytes_req; + return Status::OK(); +} + +} // namespace doris::io diff --git a/be/src/io/cache/file_block.cpp b/be/src/io/cache/file_block.cpp index 944e93baa419bb..d9d57b08bf9c20 100644 --- a/be/src/io/cache/file_block.cpp +++ b/be/src/io/cache/file_block.cpp @@ -331,36 +331,50 @@ std::string FileBlock::get_cache_file() const { return _mgr->_storage->get_local_file(this->_key); } -FileBlocksHolder::~FileBlocksHolder() { - for (auto file_block_it = file_blocks.begin(); file_block_it != file_blocks.end();) { - auto current_file_block_it = file_block_it; - auto& file_block = *current_file_block_it; - BlockFileCache* _mgr = file_block->_mgr; +void FileBlock::release_cache_reference(std::shared_ptr& file_block, + CacheReferenceRole role) { + if (!file_block) { + return; + } + BlockFileCache* mgr = file_block->_mgr; + { + bool should_remove = false; { - bool should_remove = false; - { - std::lock_guard block_lock(file_block->_mutex); + std::lock_guard block_lock(file_block->_mutex); + if (role == CacheReferenceRole::HOLDER) { file_block->complete_unlocked(block_lock); - if (file_block.use_count() == 2 && - (file_block->is_deleting() || - file_block->state_unlock(block_lock) == FileBlock::State::EMPTY)) { - should_remove = true; - } } - if (should_remove) { - SCOPED_CACHE_LOCK(_mgr->_mutex, _mgr); - std::lock_guard block_lock(file_block->_mutex); - if (file_block.use_count() == 2) { - DCHECK(file_block->state_unlock(block_lock) != FileBlock::State::DOWNLOADING); - // one in cache, one in here - if (file_block->is_deleting() || - file_block->state_unlock(block_lock) == FileBlock::State::EMPTY) { - _mgr->remove(file_block, cache_lock, block_lock, false); - } + if (file_block.use_count() == 2 && + (file_block->is_deleting() || + file_block->state_unlock(block_lock) == FileBlock::State::EMPTY)) { + should_remove = true; + } + } + if (should_remove) { + SCOPED_CACHE_LOCK(mgr->_mutex, mgr); + std::lock_guard block_lock(file_block->_mutex); + if (file_block.use_count() == 2) { + DCHECK(file_block->state_unlock(block_lock) != FileBlock::State::DOWNLOADING); + // one in cache, one held by the cache user + if (file_block->is_deleting() || + file_block->state_unlock(block_lock) == FileBlock::State::EMPTY) { + mgr->remove(file_block, cache_lock, block_lock, false); } } } - file_block_it = file_blocks.erase(current_file_block_it); + } + file_block.reset(); +} + +FileBlocksHolder::~FileBlocksHolder() { + for (auto& file_block : file_blocks) { + FileBlock::release_cache_reference(file_block, FileBlock::CacheReferenceRole::HOLDER); + } +} + +FileBlocksProbeResult::~FileBlocksProbeResult() { + for (auto& file_block : file_blocks) { + FileBlock::release_cache_reference(file_block, FileBlock::CacheReferenceRole::PROBE); } } diff --git a/be/src/io/cache/file_block.h b/be/src/io/cache/file_block.h index 6ddd78e717e00c..3e29f8ddb6a035 100644 --- a/be/src/io/cache/file_block.h +++ b/be/src/io/cache/file_block.h @@ -40,11 +40,13 @@ namespace doris { namespace io { struct FileBlocksHolder; +struct FileBlocksProbeResult; class BlockFileCache; struct FileBlockCell; class FileBlock { friend struct FileBlocksHolder; + friend struct FileBlocksProbeResult; friend class BlockFileCache; friend class CachedRemoteFileReader; friend struct FileBlockCell; @@ -154,6 +156,11 @@ class FileBlock { false}; // pocessed by CachedRemoteFileReader::_cache_file_readers private: + enum class CacheReferenceRole { + HOLDER, + PROBE, + }; + std::string get_info_for_log_impl(std::lock_guard& block_lock) const; [[nodiscard]] Status set_downloaded(std::lock_guard& block_lock); @@ -163,6 +170,14 @@ class FileBlock { void reset_downloader_impl(std::lock_guard& block_lock); + /// Release one holder/probe reference and complete deferred EMPTY/deleting block cleanup when + /// it is the last reference outside the cache map. + /// @param[in,out] file_block Reference to release. + /// @param[in] role A holder completes downloader ownership acquired through get_or_set; a + /// read-only probe never changes downloader state. + static void release_cache_reference(std::shared_ptr& file_block, + CacheReferenceRole role); + Range _block_range; State _download_state; diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index 8b52af8d161b9a..868aceee06b9de 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -86,6 +86,17 @@ struct ReadStatistics { int64_t lock_wait_timer = 0; int64_t get_timer = 0; int64_t set_timer = 0; + int64_t async_cache_write_submitted = 0; + int64_t async_cache_write_rejected = 0; + int64_t async_cache_write_buffer_alloc_fail = 0; + int64_t async_cache_write_drop_stale_epoch = 0; + int64_t inflight_write_buffer_index_hit = 0; + int64_t inflight_write_buffer_index_miss = 0; + int64_t probe_downloaded_hit = 0; + int64_t probe_downloading_hit = 0; + int64_t probe_miss = 0; + int64_t block_wait_success = 0; + int64_t block_wait_timeout = 0; }; class BlockFileCache; diff --git a/be/src/io/cache/inflight_write_buffer_index.cpp b/be/src/io/cache/inflight_write_buffer_index.cpp new file mode 100644 index 00000000000000..b3d7af24c4309a --- /dev/null +++ b/be/src/io/cache/inflight_write_buffer_index.cpp @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/inflight_write_buffer_index.h" + +#include + +#include "common/logging.h" +#include "util/time.h" + +namespace doris::io { + +namespace { + +/// Acquire one index shard while recording both contention and critical-section duration. +class TimedShardLock { +public: + TimedShardLock(std::mutex& mutex, bvar::LatencyRecorder* wait_latency, + bvar::LatencyRecorder* hold_latency) + : _lock(mutex, std::defer_lock), + _wait_latency(wait_latency), + _hold_latency(hold_latency) { + DORIS_CHECK(wait_latency != nullptr); + DORIS_CHECK(_hold_latency != nullptr); + const int64_t wait_start_us = MonotonicMicros(); + _lock.lock(); + _acquired_at_us = MonotonicMicros(); + _wait_us = _acquired_at_us - wait_start_us; + } + + ~TimedShardLock() { + const int64_t hold_us = MonotonicMicros() - _acquired_at_us; + _lock.unlock(); + *_wait_latency << _wait_us; + *_hold_latency << hold_us; + } + +private: + std::unique_lock _lock; + bvar::LatencyRecorder* _wait_latency; + bvar::LatencyRecorder* _hold_latency; + int64_t _acquired_at_us {0}; + int64_t _wait_us {0}; +}; + +} // namespace + +InflightWriteBufferIndex::InflightWriteBufferIndex(size_t shard_count, std::string metric_prefix) { + DORIS_CHECK(shard_count > 0); + _shards.reserve(shard_count); + for (size_t index = 0; index < shard_count; ++index) { + _shards.emplace_back(std::make_unique()); + } + + const char* prefix = metric_prefix.c_str(); + _size_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_size", + [](void* index) { return static_cast(index)->size(); }, + this); + _lookup_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_lookup_total"); + _hit_metric = std::make_shared>(prefix, + "inflight_write_buffer_index_hit_total"); + _miss_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_miss_total"); + _insert_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_insert_total"); + _insert_existing_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_insert_existing_total"); + _remove_success_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_remove_if_success_total"); + _remove_failed_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_remove_if_failed_total"); + _stale_epoch_miss_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_stale_epoch_miss_total"); + _stale_epoch_replace_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_stale_epoch_replace_total"); + _rollback_on_backpressure_metric = std::make_shared>( + prefix, "inflight_write_buffer_index_rollback_on_backpressure_total"); + _lock_wait_latency_metric = std::make_shared( + prefix, "inflight_write_buffer_index_lock_wait_latency_us"); + _lock_hold_latency_metric = std::make_shared( + prefix, "inflight_write_buffer_index_lock_hold_latency_us"); +} + +std::shared_ptr InflightWriteBufferIndex::insert_if_absent( + const UInt128Wrapper& cache_hash, size_t block_offset, + std::shared_ptr entry) { + DORIS_CHECK(entry != nullptr); + Key key {.cache_hash = cache_hash, .block_offset = block_offset}; + auto& shard = *_shards[_shard_index(key)]; + bool inserted = false; + { + TimedShardLock lock(shard.mutex, _lock_wait_latency_metric.get(), + _lock_hold_latency_metric.get()); + auto iterator = shard.entries.find(key); + if (iterator == shard.entries.end()) { + shard.entries.emplace(key, std::move(entry)); + _size.fetch_add(1, std::memory_order_relaxed); + inserted = true; + } else if (iterator->second->write_epoch < entry->write_epoch) { + iterator->second = std::move(entry); + *_stale_epoch_replace_metric << 1; + *_insert_metric << 1; + return nullptr; + } else { + *_insert_existing_metric << 1; + return iterator->second; + } + } + + DORIS_CHECK(inserted); + *_insert_metric << 1; + return nullptr; +} + +std::shared_ptr InflightWriteBufferIndex::lookup( + const UInt128Wrapper& cache_hash, size_t block_offset, uint64_t expected_epoch) { + *_lookup_metric << 1; + Key key {.cache_hash = cache_hash, .block_offset = block_offset}; + auto& shard = *_shards[_shard_index(key)]; + { + TimedShardLock lock(shard.mutex, _lock_wait_latency_metric.get(), + _lock_hold_latency_metric.get()); + auto iterator = shard.entries.find(key); + if (iterator == shard.entries.end()) { + *_miss_metric << 1; + return nullptr; + } + if (iterator->second->write_epoch == expected_epoch) { + *_hit_metric << 1; + return iterator->second; + } + *_stale_epoch_miss_metric << 1; + if (iterator->second->write_epoch < expected_epoch) { + shard.entries.erase(iterator); + const size_t old_size = _size.fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_size > 0); + } + } + *_miss_metric << 1; + return nullptr; +} + +std::vector InflightWriteBufferIndex::lookup_all( + const UInt128Wrapper& cache_hash, const std::vector& block_offsets, + uint64_t expected_epoch) { + std::vector results; + results.reserve(block_offsets.size()); + for (size_t block_offset : block_offsets) { + results.emplace_back(LookupResult { + .block_offset = block_offset, + .entry = lookup(cache_hash, block_offset, expected_epoch), + }); + } + return results; +} + +bool InflightWriteBufferIndex::remove_if( + const UInt128Wrapper& cache_hash, size_t block_offset, + const std::shared_ptr& expected) { + DORIS_CHECK(expected != nullptr); + Key key {.cache_hash = cache_hash, .block_offset = block_offset}; + auto& shard = *_shards[_shard_index(key)]; + bool removed = false; + { + TimedShardLock lock(shard.mutex, _lock_wait_latency_metric.get(), + _lock_hold_latency_metric.get()); + auto iterator = shard.entries.find(key); + if (iterator != shard.entries.end() && iterator->second == expected) { + shard.entries.erase(iterator); + const size_t old_size = _size.fetch_sub(1, std::memory_order_relaxed); + DORIS_CHECK(old_size > 0); + removed = true; + } + } + if (removed) { + *_remove_success_metric << 1; + return true; + } + *_remove_failed_metric << 1; + return false; +} + +} // namespace doris::io diff --git a/be/src/io/cache/inflight_write_buffer_index.h b/be/src/io/cache/inflight_write_buffer_index.h new file mode 100644 index 00000000000000..67fe535b47e6e0 --- /dev/null +++ b/be/src/io/cache/inflight_write_buffer_index.h @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io/cache/async_cache_write_service.h" +#include "io/cache/file_cache_common.h" + +namespace doris::io { + +/// Metadata for one block whose remote payload is waiting for asynchronous persistence. +/// `buffer_offset` and `buffer_size` describe the file interval represented by `buffer`. +struct InflightWriteBufferEntry { + AsyncCacheWriteBufferPtr buffer; + size_t buffer_offset {0}; + size_t buffer_size {0}; + int64_t submit_ts_us {0}; + uint64_t write_epoch {0}; + + InflightWriteBufferEntry(AsyncCacheWriteBufferPtr buffer_, size_t offset_, size_t size_, + int64_t submit_ts_us_, uint64_t write_epoch_) + : buffer(std::move(buffer_)), + buffer_offset(offset_), + buffer_size(size_), + submit_ts_us(submit_ts_us_), + write_epoch(write_epoch_) {} +}; + +/// Sharded index from (cache key, aligned block offset) to an accepted async-write payload. +/// Readers use it before probing disk cache so concurrent misses can reuse remote bytes in memory. +class InflightWriteBufferIndex { +public: + /// A batch-lookup result retains the requested offset even when `entry` is null. + struct LookupResult { + size_t block_offset {0}; + std::shared_ptr entry; + }; + + /// @param shard_count Positive number of independently locked hash shards. + /// @param metric_prefix Prefix that makes per-cache-disk bvar names unique. + explicit InflightWriteBufferIndex(size_t shard_count, std::string metric_prefix = {}); + + /// Claim `(cache_hash, block_offset)` for `entry` if no entry from the same/newer epoch exists. + /// An older entry is atomically replaced so invalidation cannot block the current epoch. + /// @return null when insertion/replacement succeeds; otherwise the existing owner. + std::shared_ptr insert_if_absent( + const UInt128Wrapper& cache_hash, size_t block_offset, + std::shared_ptr entry); + + /// Find the payload only when its epoch equals `expected_epoch`. An older stale entry is erased; + /// a newer entry is preserved for readers already operating in that epoch. + std::shared_ptr lookup(const UInt128Wrapper& cache_hash, + size_t block_offset, uint64_t expected_epoch); + + /// Look up aligned `block_offsets` in input order using the same epoch rule as `lookup`. + std::vector lookup_all(const UInt128Wrapper& cache_hash, + const std::vector& block_offsets, + uint64_t expected_epoch); + + /// Remove the key only if it still points to `expected`, preventing an old task callback from + /// deleting a replacement entry. + /// @return true if this exact owner was removed. + bool remove_if(const UInt128Wrapper& cache_hash, size_t block_offset, + const std::shared_ptr& expected); + + /// Return the number of indexed block payloads. + size_t size() const { return _size.load(std::memory_order_relaxed); } + + /// Record removal of an inserted entry because queue submission hit backpressure. + void record_backpressure_rollback() { *_rollback_on_backpressure_metric << 1; } + +private: + struct Key { + UInt128Wrapper cache_hash; + size_t block_offset {0}; + + bool operator==(const Key& other) const { + return cache_hash == other.cache_hash && block_offset == other.block_offset; + } + }; + + struct KeyHash { + size_t operator()(const Key& key) const { + return doris::io::KeyHash()(key.cache_hash) ^ std::hash()(key.block_offset); + } + }; + + struct Shard { + mutable std::mutex mutex; + std::unordered_map, KeyHash> entries; + }; + + size_t _shard_index(const Key& key) const { return KeyHash()(key) % _shards.size(); } + + std::vector> _shards; + std::atomic _size {0}; + + std::shared_ptr> _size_metric; + std::shared_ptr> _lookup_metric; + std::shared_ptr> _hit_metric; + std::shared_ptr> _miss_metric; + std::shared_ptr> _insert_metric; + std::shared_ptr> _insert_existing_metric; + std::shared_ptr> _remove_success_metric; + std::shared_ptr> _remove_failed_metric; + std::shared_ptr> _stale_epoch_miss_metric; + std::shared_ptr> _stale_epoch_replace_metric; + std::shared_ptr> _rollback_on_backpressure_metric; + std::shared_ptr _lock_wait_latency_metric; + std::shared_ptr _lock_hold_latency_metric; +}; + +} // namespace doris::io diff --git a/be/src/io/fs/file_reader.h b/be/src/io/fs/file_reader.h index ab00a9823520e8..6633841dc45c66 100644 --- a/be/src/io/fs/file_reader.h +++ b/be/src/io/fs/file_reader.h @@ -44,6 +44,18 @@ enum class FileCachePolicy : uint8_t { FILE_BLOCK_CACHE, }; +enum class CacheAlignMode : uint8_t { + ALIGN_TO_BLOCK, + UNALIGNED, +}; + +enum class CacheWriteMode : uint8_t { + DEFAULT, + NO_WRITE, + SYNC_WRITE, + ASYNC_WRITE, +}; + inline FileCachePolicy cache_type_from_string(std::string_view type) { if (type == "file_block_cache") { return FileCachePolicy::FILE_BLOCK_CACHE; @@ -55,6 +67,8 @@ inline FileCachePolicy cache_type_from_string(std::string_view type) { // Only affects remote file readers struct FileReaderOptions { FileCachePolicy cache_type {FileCachePolicy::NO_CACHE}; + CacheAlignMode align_mode {CacheAlignMode::ALIGN_TO_BLOCK}; + CacheWriteMode cache_write_mode {CacheWriteMode::DEFAULT}; bool is_doris_table = false; std::string cache_base_path; // Length of the file in bytes, -1 means unset. diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index f42e486eb40aea..a7c7005708a189 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -19,6 +19,8 @@ #include +#include +#include #include #include @@ -41,6 +43,7 @@ enum class ReaderType : uint8_t { namespace io { class RemoteScanCacheWriteLimiter; +enum class CacheWriteMode : uint8_t; enum class FileCacheMissPolicy : uint8_t { READ_THROUGH_AND_WRITE_BACK = 0, @@ -73,6 +76,17 @@ struct FileCacheStatistics { int64_t lock_wait_timer = 0; int64_t get_timer = 0; int64_t set_timer = 0; + int64_t async_cache_write_submitted = 0; + int64_t async_cache_write_rejected = 0; + int64_t async_cache_write_buffer_alloc_fail = 0; + int64_t async_cache_write_drop_stale_epoch = 0; + int64_t inflight_write_buffer_index_hit = 0; + int64_t inflight_write_buffer_index_miss = 0; + int64_t probe_downloaded_hit = 0; + int64_t probe_downloading_hit = 0; + int64_t probe_miss = 0; + int64_t block_wait_success = 0; + int64_t block_wait_timeout = 0; int64_t inverted_index_num_local_io_total = 0; int64_t inverted_index_num_remote_io_total = 0; @@ -134,6 +148,17 @@ struct FileCacheStatistics { lock_wait_timer += other.lock_wait_timer; get_timer += other.get_timer; set_timer += other.set_timer; + async_cache_write_submitted += other.async_cache_write_submitted; + async_cache_write_rejected += other.async_cache_write_rejected; + async_cache_write_buffer_alloc_fail += other.async_cache_write_buffer_alloc_fail; + async_cache_write_drop_stale_epoch += other.async_cache_write_drop_stale_epoch; + inflight_write_buffer_index_hit += other.inflight_write_buffer_index_hit; + inflight_write_buffer_index_miss += other.inflight_write_buffer_index_miss; + probe_downloaded_hit += other.probe_downloaded_hit; + probe_downloading_hit += other.probe_downloading_hit; + probe_miss += other.probe_miss; + block_wait_success += other.block_wait_success; + block_wait_timeout += other.block_wait_timeout; inverted_index_num_local_io_total += other.inverted_index_num_local_io_total; inverted_index_num_remote_io_total += other.inverted_index_num_remote_io_total; @@ -212,6 +237,9 @@ struct IOContext { int64_t predicate_filtered_rows = 0; // if true, bypass peer read / peer-vs-S3 race and read directly from remote storage bool bypass_peer_read {false}; + // Per-call override for cache write completion semantics. An unset value follows the reader + // option and the global async file-cache write switch. + std::optional cache_write_mode_override = std::nullopt; FileCacheMissPolicy file_cache_miss_policy = FileCacheMissPolicy::READ_THROUGH_AND_WRITE_BACK; RemoteScanCacheWriteLimiter* remote_scan_cache_write_limiter = nullptr; // Ref }; diff --git a/be/src/io/tools/CMakeLists.txt b/be/src/io/tools/CMakeLists.txt index 40088892a97280..c956e81211bc2e 100644 --- a/be/src/io/tools/CMakeLists.txt +++ b/be/src/io/tools/CMakeLists.txt @@ -53,6 +53,10 @@ add_executable(file_cache_microbench ${PROTO_SRCS} ) +add_executable(async_file_cache_write_microbench + async_file_cache_write_microbench.cpp +) + # 添加proto生成文件的包含路径 target_include_directories(file_cache_microbench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/build/proto @@ -64,5 +68,10 @@ target_link_libraries(file_cache_microbench protobuf ) +target_link_libraries(async_file_cache_write_microbench + ${DORIS_LINK_LIBS} +) + # 安装规则 install(TARGETS file_cache_microbench DESTINATION ${OUTPUT_DIR}/lib/) +install(TARGETS async_file_cache_write_microbench DESTINATION ${OUTPUT_DIR}/lib/) diff --git a/be/src/io/tools/async_file_cache_write_microbench.cpp b/be/src/io/tools/async_file_cache_write_microbench.cpp new file mode 100644 index 00000000000000..13d7ba51dcedd6 --- /dev/null +++ b/be/src/io/tools/async_file_cache_write_microbench.cpp @@ -0,0 +1,1072 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/file_cache_common.h" + +#if defined(BE_TEST) && defined(BUILD_FILE_CACHE_MICROBENCH_TOOL) + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cloud/config.h" +#include "common/config.h" +#include "common/status.h" +#include "io/cache/async_cache_write_service.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_factory.h" +#include "io/cache/cached_remote_file_reader.h" +#include "io/cache/fs_file_cache_storage.h" +#include "io/cache/inflight_write_buffer_index.h" +#include "io/fs/file_reader.h" +#include "io/fs/path.h" +#include "runtime/exec_env.h" +#include "runtime/thread_context.h" +#include "util/cpu_info.h" +#include "util/disk_info.h" +#include "util/mem_info.h" +#include "util/slice.h" +#include "util/time.h" + +DEFINE_string(benchmark_mode, "all", + "Comma-separated benchmark groups: reader, service, index, or all"); +DEFINE_string(cache_path, "./output/async_file_cache_write_microbench", + "Directory used by the real filesystem-backed BlockFileCache"); +DEFINE_uint64(block_size, 1024 * 1024, + "File-cache block size and stride used by cold reader misses"); +DEFINE_uint64(request_size, 64 * 1024, "Bytes returned to the caller by each reader operation"); +DEFINE_uint64(reader_operations, 128, "Cold read operations in each sync/async reader case"); +DEFINE_uint64(service_task_size, 1024 * 1024, "Payload bytes in each direct service task"); +DEFINE_uint64(service_operations, 256, "Attempted tasks in each direct service case"); +DEFINE_uint64(service_key_count, 64, "Logical remote files spread across direct service tasks"); +DEFINE_uint64(index_operations_per_thread, 100000, + "Inflight-index lookups performed by each producer thread"); +DEFINE_uint64(index_key_count, 4096, "Keys used by the representative sharded index case"); +DEFINE_int32(producer_threads, 16, "Concurrent foreground readers or task producers"); +DEFINE_int32(reader_workers, 16, "Async write workers used by the reader comparison"); +DEFINE_string(worker_counts, "1,4,16", + "Comma-separated async write worker counts used by service scaling cases"); +DEFINE_uint64(repetitions, 5, "Measured repetitions of every selected benchmark case"); +DEFINE_uint64(backpressure_pending_tasks, 64, + "Pending-task limit used by the saturated service case"); +DEFINE_uint64(queue_sample_interval_us, 50, + "Sampling interval for pending, queued, and inflight peak values"); +DEFINE_uint64(timeout_seconds, 120, "Maximum time to wait for one benchmark case to drain"); +DEFINE_bool(keep_cache, false, "Keep benchmark cache files after the process exits"); + +namespace doris::io { +namespace { + +using Clock = std::chrono::steady_clock; +using Nanoseconds = std::chrono::nanoseconds; + +constexpr size_t kMiB = 1024 * 1024; + +/// Compact percentile summary for foreground operation latency. +struct LatencySummary { + double average_us {0}; + double p50_us {0}; + double p95_us {0}; + double p99_us {0}; + double maximum_us {0}; +}; + +/// Return one nearest-rank percentile from an already sorted nanosecond sample set. +/// @param sorted_ns Ascending latency samples in nanoseconds. +/// @param percentile Requested percentile in the inclusive range [0, 1]. +double percentile_us(const std::vector& sorted_ns, double percentile) { + DORIS_CHECK(!sorted_ns.empty()); + DORIS_CHECK(percentile > 0 && percentile <= 1); + const size_t index = + static_cast(std::ceil(percentile * static_cast(sorted_ns.size()))) - 1; + return static_cast(sorted_ns[std::min(index, sorted_ns.size() - 1)]) / 1000.0; +} + +/// Merge per-thread samples and calculate stable latency percentiles. +/// @param per_thread_ns Independently owned samples, one vector per producer. +LatencySummary summarize_latencies(const std::vector>& per_thread_ns) { + size_t sample_count = 0; + for (const auto& samples : per_thread_ns) { + sample_count += samples.size(); + } + DORIS_CHECK(sample_count > 0); + + std::vector sorted_ns; + sorted_ns.reserve(sample_count); + for (const auto& samples : per_thread_ns) { + sorted_ns.insert(sorted_ns.end(), samples.begin(), samples.end()); + } + std::sort(sorted_ns.begin(), sorted_ns.end()); + const int64_t total_ns = std::accumulate(sorted_ns.begin(), sorted_ns.end(), int64_t {0}); + return LatencySummary { + .average_us = + static_cast(total_ns) / static_cast(sample_count) / 1000.0, + .p50_us = percentile_us(sorted_ns, 0.50), + .p95_us = percentile_us(sorted_ns, 0.95), + .p99_us = percentile_us(sorted_ns, 0.99), + .maximum_us = static_cast(sorted_ns.back()) / 1000.0, + }; +} + +/// Parse a comma-separated positive integer list used for worker scaling. +/// @param text Raw gflag value such as "1,4,16". +/// @param values Parsed worker counts in input order. +Status parse_positive_integer_list(std::string_view text, std::vector* values) { + DORIS_CHECK(values != nullptr); + values->clear(); + std::stringstream stream {std::string(text)}; + std::string token; + while (std::getline(stream, token, ',')) { + try { + const long long value = std::stoll(token); + if (value <= 0) { + return Status::InvalidArgument("worker count must be positive: {}", token); + } + values->push_back(static_cast(value)); + } catch (const std::exception& error) { + return Status::InvalidArgument("invalid worker count '{}': {}", token, error.what()); + } + } + if (values->empty()) { + return Status::InvalidArgument("worker_counts cannot be empty"); + } + return Status::OK(); +} + +/// Split the selected benchmark groups while preserving a small command-line surface. +/// @param text Raw mode string. +std::vector parse_modes(std::string_view text) { + std::stringstream stream {std::string(text)}; + std::vector modes; + std::string mode; + while (std::getline(stream, mode, ',')) { + if (!mode.empty()) { + modes.emplace_back(std::move(mode)); + } + } + return modes; +} + +/// Return whether a benchmark group was requested explicitly or through "all". +/// @param modes Parsed benchmark groups. +/// @param target Group to test. +bool mode_enabled(const std::vector& modes, std::string_view target) { + return std::find(modes.begin(), modes.end(), "all") != modes.end() || + std::find(modes.begin(), modes.end(), target) != modes.end(); +} + +/// Validate sizes and concurrency before allocating cache capacity or starting threads. +Status validate_flags(const std::vector& modes, + const std::vector& worker_counts) { + if (modes.empty()) { + return Status::InvalidArgument("benchmark_mode cannot be empty"); + } + for (const auto& mode : modes) { + if (mode != "all" && mode != "reader" && mode != "service" && mode != "index") { + return Status::InvalidArgument("unsupported benchmark mode: {}", mode); + } + } + DORIS_CHECK(!worker_counts.empty()); + if (FLAGS_producer_threads <= 0 || FLAGS_reader_workers <= 0) { + return Status::InvalidArgument("producer_threads and reader_workers must be positive"); + } + if (FLAGS_block_size == 0 || FLAGS_request_size == 0 || FLAGS_request_size > FLAGS_block_size) { + return Status::InvalidArgument("sizes must satisfy 0 < request_size <= block_size"); + } + if (FLAGS_reader_operations < static_cast(FLAGS_producer_threads) || + FLAGS_service_operations < static_cast(FLAGS_producer_threads)) { + return Status::InvalidArgument( + "reader_operations and service_operations must be at least producer_threads"); + } + if (FLAGS_service_task_size == 0 || FLAGS_service_task_size > FLAGS_block_size || + FLAGS_service_key_count == 0 || FLAGS_index_operations_per_thread == 0 || + FLAGS_index_key_count == 0 || FLAGS_backpressure_pending_tasks == 0 || + FLAGS_queue_sample_interval_us == 0 || FLAGS_timeout_seconds == 0 || + FLAGS_repetitions == 0) { + return Status::InvalidArgument( + "operation counts, timeouts, and 0 < service_task_size <= block_size are required"); + } + return Status::OK(); +} + +/// Produce deterministic data without adding network or object-store latency to reader results. +class SyntheticRemoteFileReader final : public FileReader { +public: + /// @param path Stable logical path used to derive the file-cache hash. + /// @param file_size Virtual file length; no payload is allocated for it. + /// @param block_size Pattern granularity used to validate copied bytes. + SyntheticRemoteFileReader(Path path, size_t file_size, size_t block_size) + : _path(std::move(path)), _file_size(file_size), _block_size(block_size) {} + + Status close() override { + _closed.store(true, std::memory_order_release); + return Status::OK(); + } + + const Path& path() const override { return _path; } + size_t size() const override { return _file_size; } + bool closed() const override { return _closed.load(std::memory_order_acquire); } + int64_t mtime() const override { return 0; } + + /// Return the byte value expected at an aligned benchmark block. + /// @param offset File offset inside the virtual source. + char expected_byte(size_t offset) const { + return static_cast((offset / _block_size) % 251); + } + +protected: + /// Fill the requested span from a deterministic virtual file. + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override { + DORIS_CHECK(bytes_read != nullptr); + if (offset > _file_size || result.size > _file_size - offset) { + return Status::InvalidArgument("synthetic read [{}, {}) exceeds file size {}", offset, + offset + result.size, _file_size); + } + size_t copied = 0; + while (copied < result.size) { + const size_t current_offset = offset + copied; + const size_t block_end = + std::min(_file_size, (current_offset / _block_size + 1) * _block_size); + const size_t bytes = std::min(result.size - copied, block_end - current_offset); + std::memset(result.data + copied, expected_byte(current_offset), bytes); + copied += bytes; + } + *bytes_read = result.size; + return Status::OK(); + } + +private: + Path _path; + size_t _file_size; + size_t _block_size; + std::atomic _closed {false}; +}; + +/// Record the first worker-thread error without obscuring the performance hot path. +class ConcurrentError { +public: + /// Store the first non-OK status observed by any producer. + void set(Status status) { + if (status.ok()) { + return; + } + bool expected = false; + if (_failed.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + std::lock_guard lock(_mutex); + _status = std::move(status); + } + } + + bool failed() const { return _failed.load(std::memory_order_acquire); } + + /// Return the stored error after producer threads have joined. + Status status() const { + std::lock_guard lock(_mutex); + return _status; + } + +private: + std::atomic _failed {false}; + mutable std::mutex _mutex; + Status _status; +}; + +/// Sample public queue gauges so short benchmark cases still report a meaningful high-water mark. +class QueuePeakSampler { +public: + /// @param service Service whose pending and queued gauges are sampled. + /// @param index Inflight index paired with the service. + QueuePeakSampler(AsyncCacheWriteService* service, InflightWriteBufferIndex* index) + : _service(service), _index(index) { + DORIS_CHECK(_service != nullptr); + DORIS_CHECK(_index != nullptr); + } + + /// Start the sampling thread. The sampler does not mutate benchmark state. + void start() { + _running.store(true, std::memory_order_release); + _thread = std::thread([this]() { + while (_running.load(std::memory_order_acquire)) { + update_max(&_peak_pending, _service->pending_count()); + update_max(&_peak_queued, _service->queued_count()); + update_max(&_peak_inflight, _index->size()); + std::this_thread::sleep_for( + std::chrono::microseconds(FLAGS_queue_sample_interval_us)); + } + update_max(&_peak_pending, _service->pending_count()); + update_max(&_peak_queued, _service->queued_count()); + update_max(&_peak_inflight, _index->size()); + }); + } + + /// Stop sampling and join before reading peak values. + void stop() { + _running.store(false, std::memory_order_release); + if (_thread.joinable()) { + _thread.join(); + } + } + + ~QueuePeakSampler() { stop(); } + + size_t peak_pending() const { return _peak_pending.load(std::memory_order_relaxed); } + size_t peak_queued() const { return _peak_queued.load(std::memory_order_relaxed); } + size_t peak_inflight() const { return _peak_inflight.load(std::memory_order_relaxed); } + +private: + /// Atomically preserve the largest sampled gauge value. + static void update_max(std::atomic* maximum, size_t value) { + size_t current = maximum->load(std::memory_order_relaxed); + while (current < value && + !maximum->compare_exchange_weak(current, value, std::memory_order_relaxed)) { + } + } + + AsyncCacheWriteService* _service; + InflightWriteBufferIndex* _index; + std::atomic _running {false}; + std::thread _thread; + std::atomic _peak_pending {0}; + std::atomic _peak_queued {0}; + std::atomic _peak_inflight {0}; +}; + +/// Poll one asynchronous completion predicate with a bounded failure mode. +/// @param predicate Returns true after the expected state is reached. +/// @param description Included in timeout diagnostics. +template +Status wait_until(Predicate&& predicate, std::string_view description) { + const auto deadline = Clock::now() + std::chrono::seconds(FLAGS_timeout_seconds); + while (!predicate()) { + if (Clock::now() >= deadline) { + return Status::TimedOut("timed out waiting for {}", description); + } + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + return Status::OK(); +} + +/// Own one real filesystem cache and install its factory into the otherwise minimal ExecEnv. +class BenchmarkEnvironment { +public: + BenchmarkEnvironment() = default; + + ~BenchmarkEnvironment() { + if (_factory) { + ExecEnv::GetInstance()->set_file_cache_factory(nullptr); + _factory.reset(); + } + if (_owns_cache_path && !FLAGS_keep_cache) { + std::error_code error; + std::filesystem::remove_all(FLAGS_cache_path, error); + } + } + + /// Configure globals, create the cache, and wait for its asynchronous metadata open. + /// @param cache_capacity Bytes reserved for the benchmark's normal queue. + Status initialize(size_t cache_capacity) { + std::error_code error; + const auto cache_path = + std::filesystem::absolute(FLAGS_cache_path, error).lexically_normal(); + if (error) { + return Status::IOError("failed to resolve cache path {}: {}", FLAGS_cache_path, + error.message()); + } + const auto current_path = std::filesystem::current_path(error); + if (error) { + return Status::IOError("failed to resolve current directory: {}", error.message()); + } + if (cache_path == cache_path.root_path() || cache_path == current_path) { + return Status::InvalidArgument("cache path must be a dedicated subdirectory: {}", + cache_path.string()); + } + const bool cache_path_exists = std::filesystem::exists(cache_path, error); + if (error) { + return Status::IOError("failed to inspect cache path {}: {}", cache_path.string(), + error.message()); + } + if (cache_path_exists) { + const bool cache_path_is_directory = std::filesystem::is_directory(cache_path, error); + if (error) { + return Status::IOError("failed to inspect cache path {}: {}", cache_path.string(), + error.message()); + } + if (!cache_path_is_directory) { + return Status::InvalidArgument("cache path is not a directory: {}", + cache_path.string()); + } + const bool cache_path_is_empty = std::filesystem::is_empty(cache_path, error); + if (error) { + return Status::IOError("failed to inspect cache path {}: {}", cache_path.string(), + error.message()); + } + if (!cache_path_is_empty) { + return Status::InvalidArgument( + "cache path must not exist or must be an empty directory: {}", + cache_path.string()); + } + } else { + std::filesystem::create_directories(cache_path, error); + if (error) { + return Status::IOError("failed to create cache path {}: {}", cache_path.string(), + error.message()); + } + } + _owns_cache_path = true; + + config::enable_async_file_cache_write = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::clear_file_cache = true; + config::enable_evict_file_cache_in_advance = false; + config::file_cache_enter_disk_resource_limit_mode_percent = 99; + config::file_cache_each_block_size = static_cast(FLAGS_block_size); + // Benchmark reads are external-table style with tablet_id=0, so they do not register + // tablet-scoped TTL work. Short intervals only bound cache teardown latency; production + // defaults let the otherwise idle TTL threads sleep for three minutes before join(). + config::file_cache_background_ttl_gc_interval_ms = 100; + config::file_cache_background_ttl_info_update_interval_ms = 100; + config::file_cache_background_tablet_id_flush_interval_ms = 100; + config::async_file_cache_write_workers_per_disk = FLAGS_reader_workers; + config::async_file_cache_write_max_pending_tasks_per_disk = + static_cast(std::max(FLAGS_reader_operations, FLAGS_service_operations)); + config::async_file_cache_write_batch_size = 16; + + DORIS_CHECK(ExecEnv::GetInstance()->file_cache_factory() == nullptr); + ExecEnv::GetInstance()->set_file_cache_open_fd_cache(std::make_unique()); + _factory = std::make_unique(); + ExecEnv::GetInstance()->set_file_cache_factory(_factory.get()); + + FileCacheSettings settings; + settings.capacity = cache_capacity; + settings.max_file_block_size = FLAGS_block_size; + settings.max_query_cache_size = 0; + const size_t auxiliary_queue_size = FLAGS_block_size; + settings.disposable_queue_size = auxiliary_queue_size; + settings.disposable_queue_elements = 128; + settings.index_queue_size = auxiliary_queue_size; + settings.index_queue_elements = 128; + settings.ttl_queue_size = auxiliary_queue_size; + settings.ttl_queue_elements = 128; + settings.query_queue_size = cache_capacity - 3 * auxiliary_queue_size; + settings.query_queue_elements = std::max( + 1024, 2 * std::max(FLAGS_reader_operations, FLAGS_service_operations)); + + RETURN_IF_ERROR(_factory->create_file_cache(FLAGS_cache_path, settings)); + _cache = _factory->get_by_path(FLAGS_cache_path); + DORIS_CHECK(_cache != nullptr); + RETURN_IF_ERROR(wait_until([&]() { return _cache->get_async_open_success(); }, + "file cache initialization")); + return Status::OK(); + } + + /// Drain accepted writes, then synchronously invalidate and clear all cached blocks. + Status clear_cache() { + RETURN_IF_ERROR(wait_for_idle()); + std::string clear_result; + RETURN_IF_ERROR(_factory->clear_file_caches(true, &clear_result)); + return wait_for_idle(); + } + + /// Apply one explicit worker/queue snapshot to the production service. + /// @param workers Active background writer count. + /// @param max_pending Maximum accepted tasks, including active workers. + Status configure_service(size_t workers, size_t max_pending) { + auto options = service()->options(); + options.worker_count = workers; + options.max_pending_tasks = max_pending; + options.batch_size = 16; + options.watchdog_warn_secs = static_cast(FLAGS_timeout_seconds); + options.watchdog_drop_secs = static_cast(FLAGS_timeout_seconds * 2); + return service()->update_options(options); + } + + /// Wait until both queue ownership and reader-visible inflight payloads are gone. + Status wait_for_idle() { + return wait_until( + [&]() { + return service()->pending_count() == 0 && service()->queued_count() == 0 && + index()->size() == 0; + }, + "async write queue and inflight index to drain"); + } + + BlockFileCache* cache() const { return _cache; } + AsyncCacheWriteService* service() const { return _cache->async_write_service(); } + InflightWriteBufferIndex* index() const { return _cache->inflight_write_buffer_index(); } + +private: + std::unique_ptr _factory; + BlockFileCache* _cache {nullptr}; + bool _owns_cache_path {false}; +}; + +/// Verify that a complete range can be resolved from the final cache state. +/// @param cache Target cache. +/// @param hash Logical file hash. +/// @param offset Range offset to verify. +/// @param size Expected persisted bytes. +Status verify_cached_range(BlockFileCache* cache, const UInt128Wrapper& hash, size_t offset, + size_t size) { + DORIS_CHECK(cache != nullptr); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + FileBlocks blocks; + bool fully_covered = false; + RETURN_IF_ERROR(cache->get_downloaded_blocks_if_fully_covered(hash, offset, size, context, + &blocks, &fully_covered)); + if (!fully_covered) { + return Status::InternalError("cache range [{}, {}) was not persisted", offset, + offset + size); + } + return Status::OK(); +} + +/// Shared result fields printed for reader and direct-service cases. +struct AsyncWriteResult { + std::string benchmark; + std::string variant; + size_t producers {0}; + size_t workers {0}; + size_t operations {0}; + size_t accepted {0}; + size_t rejected {0}; + size_t persisted {0}; + size_t bytes_per_operation {0}; + double foreground_seconds {0}; + double drain_seconds {0}; + double total_seconds {0}; + size_t peak_pending {0}; + size_t peak_queued {0}; + size_t peak_inflight {0}; + LatencySummary latency; +}; + +/// Print one machine-readable line without hiding the foreground/drain distinction. +/// @param result Completed benchmark result. +/// @param repetition One-based repetition index. +void print_async_write_result(const AsyncWriteResult& result, size_t repetition) { + const double foreground_ops_per_sec = + static_cast(result.operations) / result.foreground_seconds; + const double persisted_mib_per_sec = + result.total_seconds > 0 + ? static_cast(result.persisted * result.bytes_per_operation) / + static_cast(kMiB) / result.total_seconds + : 0; + std::cout << std::fixed << std::setprecision(3) << "RESULT" + << " benchmark=" << result.benchmark << " variant=" << result.variant + << " repetition=" << repetition << " producers=" << result.producers + << " workers=" << result.workers << " operations=" << result.operations + << " accepted=" << result.accepted << " rejected=" << result.rejected + << " persisted=" << result.persisted + << " bytes_per_operation=" << result.bytes_per_operation + << " foreground_seconds=" << result.foreground_seconds + << " drain_seconds=" << result.drain_seconds + << " total_seconds=" << result.total_seconds + << " foreground_ops_per_sec=" << foreground_ops_per_sec + << " persisted_mib_per_sec=" << persisted_mib_per_sec + << " avg_us=" << result.latency.average_us << " p50_us=" << result.latency.p50_us + << " p95_us=" << result.latency.p95_us << " p99_us=" << result.latency.p99_us + << " max_us=" << result.latency.maximum_us << " peak_pending=" << result.peak_pending + << " peak_queued=" << result.peak_queued << " peak_inflight=" << result.peak_inflight + << '\n'; +} + +/// Compare cold-miss caller latency with synchronous and asynchronous cache persistence. +/// @param environment Shared real cache, cleared before the case. +/// @param mode Explicit write policy applied to every CachedRemoteFileReader. +/// @param variant Stable output label. +/// @param repetition One-based repetition index included in output. +Status run_reader_case(BenchmarkEnvironment* environment, CacheWriteMode mode, std::string variant, + size_t repetition) { + DORIS_CHECK(environment != nullptr); + RETURN_IF_ERROR(environment->clear_cache()); + RETURN_IF_ERROR(environment->configure_service( + static_cast(FLAGS_reader_workers), + static_cast(FLAGS_reader_operations + FLAGS_reader_workers))); + + const size_t producer_count = static_cast(FLAGS_producer_threads); + const size_t operation_count = static_cast(FLAGS_reader_operations); + const size_t file_size = operation_count * FLAGS_block_size; + const std::string file_name = "async_write_reader_" + variant + ".bin"; + const Path path("/synthetic/" + file_name); + + std::vector producers; + producers.reserve(producer_count); + std::vector> latencies(producer_count); + std::vector statistics(producer_count); + std::barrier start_barrier(static_cast(producer_count + 1)); + ConcurrentError error; + QueuePeakSampler sampler(environment->service(), environment->index()); + sampler.start(); + + for (size_t producer = 0; producer < producer_count; ++producer) { + producers.emplace_back([&, producer]() { + SCOPED_INIT_THREAD_CONTEXT(); + auto remote = + std::make_shared(path, file_size, FLAGS_block_size); + FileReaderOptions options; + options.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + options.cache_write_mode = mode; + options.cache_base_path = FLAGS_cache_path; + options.tablet_id = 0; + auto reader = std::make_shared(remote, options); + IOContext io_context; + io_context.file_cache_stats = &statistics[producer]; + io_context.bypass_peer_read = true; + std::string buffer(FLAGS_request_size, '\0'); + latencies[producer].reserve((operation_count + producer_count - 1) / producer_count); + + start_barrier.arrive_and_wait(); + for (size_t operation = producer; operation < operation_count; + operation += producer_count) { + if (error.failed()) { + break; + } + const size_t offset = operation * FLAGS_block_size; + size_t bytes_read = 0; + const auto start = Clock::now(); + Status status = reader->read_at(offset, Slice(buffer.data(), buffer.size()), + &bytes_read, &io_context); + const auto elapsed = std::chrono::duration_cast(Clock::now() - start); + latencies[producer].push_back(elapsed.count()); + if (!status.ok()) { + error.set(std::move(status)); + break; + } + if (bytes_read != buffer.size() || + buffer.front() != remote->expected_byte(offset) || + buffer.back() != remote->expected_byte(offset + buffer.size() - 1)) { + error.set(Status::InternalError( + "reader result mismatch at operation {}, bytes_read={}", operation, + bytes_read)); + break; + } + } + }); + } + + const auto foreground_start = Clock::now(); + start_barrier.arrive_and_wait(); + for (auto& producer : producers) { + producer.join(); + } + const auto foreground_end = Clock::now(); + if (error.failed()) { + sampler.stop(); + return error.status(); + } + + RETURN_IF_ERROR(environment->wait_for_idle()); + const auto drain_end = Clock::now(); + sampler.stop(); + + FileCacheStatistics total_stats; + for (const auto& local_stats : statistics) { + total_stats.merge_from(local_stats); + } + const auto hash = BlockFileCache::hash(path.native() + ":0"); + RETURN_IF_ERROR(verify_cached_range(environment->cache(), hash, 0, file_size)); + + const double foreground_seconds = + std::chrono::duration(foreground_end - foreground_start).count(); + const double drain_seconds = std::chrono::duration(drain_end - foreground_end).count(); + AsyncWriteResult result { + .benchmark = "reader", + .variant = std::move(variant), + .producers = producer_count, + .workers = mode == CacheWriteMode::ASYNC_WRITE + ? static_cast(FLAGS_reader_workers) + : 0, + .operations = operation_count, + .accepted = mode == CacheWriteMode::ASYNC_WRITE + ? static_cast(total_stats.async_cache_write_submitted) + : 0, + .rejected = static_cast(total_stats.async_cache_write_rejected), + .persisted = operation_count, + .bytes_per_operation = FLAGS_block_size, + .foreground_seconds = foreground_seconds, + .drain_seconds = drain_seconds, + .total_seconds = foreground_seconds + drain_seconds, + .peak_pending = sampler.peak_pending(), + .peak_queued = sampler.peak_queued(), + .peak_inflight = sampler.peak_inflight(), + .latency = summarize_latencies(latencies), + }; + print_async_write_result(result, repetition); + return Status::OK(); +} + +/// One accepted direct-service task retained for post-timing persistence verification. +struct ServiceTaskRecord { + UInt128Wrapper hash; + size_t offset {0}; + size_t size {0}; +}; + +/// Exercise producer admission, inflight publication, MPMC consumption, and real persistence. +/// @param environment Shared real cache, cleared before the case. +/// @param variant Stable output label. +/// @param workers Active service consumers. +/// @param max_pending Bounded pending-task limit. +/// @param repetition One-based repetition index included in output. +Status run_service_case(BenchmarkEnvironment* environment, std::string variant, size_t workers, + size_t max_pending, size_t repetition) { + DORIS_CHECK(environment != nullptr); + RETURN_IF_ERROR(environment->clear_cache()); + RETURN_IF_ERROR(environment->configure_service(workers, max_pending)); + + const size_t producer_count = static_cast(FLAGS_producer_threads); + const size_t operation_count = static_cast(FLAGS_service_operations); + std::vector producers; + producers.reserve(producer_count); + std::vector> latencies(producer_count); + std::vector> accepted_records(producer_count); + std::barrier start_barrier(static_cast(producer_count + 1)); + std::atomic accepted {0}; + std::atomic rejected {0}; + std::atomic completed {0}; + ConcurrentError error; + QueuePeakSampler sampler(environment->service(), environment->index()); + sampler.start(); + + for (size_t producer = 0; producer < producer_count; ++producer) { + producers.emplace_back([&, producer]() { + SCOPED_INIT_THREAD_CONTEXT(); + latencies[producer].reserve((operation_count + producer_count - 1) / producer_count); + accepted_records[producer].reserve((operation_count + producer_count - 1) / + producer_count); + start_barrier.arrive_and_wait(); + + for (size_t operation = producer; operation < operation_count; + operation += producer_count) { + if (error.failed()) { + break; + } + const size_t key_index = operation % FLAGS_service_key_count; + const size_t block_index = operation / FLAGS_service_key_count; + const auto hash = BlockFileCache::hash("async_write_service_" + variant + "_" + + std::to_string(key_index)); + const size_t offset = block_index * FLAGS_block_size; + const auto start = Clock::now(); + + AsyncCacheWriteBufferPtr buffer; + Status status = environment->service()->allocate_tracked_buffer( + FLAGS_service_task_size, &buffer); + if (!status.ok()) { + error.set(std::move(status)); + break; + } + std::memset(buffer->data(), static_cast(operation % 251), buffer->size()); + const uint64_t epoch = environment->service()->current_write_epoch(); + auto entry = std::make_shared( + buffer, offset, buffer->size(), MonotonicMicros(), epoch); + auto existing = environment->index()->insert_if_absent(hash, offset, entry); + if (existing != nullptr) { + error.set(Status::InternalError( + "unexpected duplicate inflight owner at operation {}", operation)); + break; + } + + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = offset, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = epoch, + .on_finalized = + [index = environment->index(), hash, offset, entry, + &completed](const AsyncCacheWriteTask&) { + index->remove_if(hash, offset, entry); + completed.fetch_add(1, std::memory_order_release); + }, + }; + const bool submitted = environment->service()->try_submit(std::move(task)); + const auto elapsed = std::chrono::duration_cast(Clock::now() - start); + latencies[producer].push_back(elapsed.count()); + if (submitted) { + accepted.fetch_add(1, std::memory_order_relaxed); + accepted_records[producer].push_back(ServiceTaskRecord { + .hash = hash, .offset = offset, .size = FLAGS_service_task_size}); + } else { + environment->index()->remove_if(hash, offset, entry); + environment->index()->record_backpressure_rollback(); + rejected.fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + + const auto foreground_start = Clock::now(); + start_barrier.arrive_and_wait(); + for (auto& producer : producers) { + producer.join(); + } + const auto foreground_end = Clock::now(); + if (error.failed()) { + sampler.stop(); + return error.status(); + } + + RETURN_IF_ERROR(wait_until( + [&]() { + return environment->service()->pending_count() == 0 && + completed.load(std::memory_order_acquire) == + accepted.load(std::memory_order_acquire) && + environment->index()->size() == 0; + }, + "direct service tasks to complete")); + const auto drain_end = Clock::now(); + sampler.stop(); + + size_t persisted = 0; + for (const auto& records : accepted_records) { + for (const auto& record : records) { + Status status = verify_cached_range(environment->cache(), record.hash, record.offset, + record.size); + if (status.ok()) { + ++persisted; + } + } + } + if (persisted != accepted.load(std::memory_order_relaxed)) { + return Status::InternalError("only {} of {} accepted service tasks persisted", persisted, + accepted.load(std::memory_order_relaxed)); + } + + const double foreground_seconds = + std::chrono::duration(foreground_end - foreground_start).count(); + const double drain_seconds = std::chrono::duration(drain_end - foreground_end).count(); + AsyncWriteResult result { + .benchmark = "service", + .variant = std::move(variant), + .producers = producer_count, + .workers = workers, + .operations = operation_count, + .accepted = accepted.load(std::memory_order_relaxed), + .rejected = rejected.load(std::memory_order_relaxed), + .persisted = persisted, + .bytes_per_operation = FLAGS_service_task_size, + .foreground_seconds = foreground_seconds, + .drain_seconds = drain_seconds, + .total_seconds = foreground_seconds + drain_seconds, + .peak_pending = sampler.peak_pending(), + .peak_queued = sampler.peak_queued(), + .peak_inflight = sampler.peak_inflight(), + .latency = summarize_latencies(latencies), + }; + print_async_write_result(result, repetition); + return Status::OK(); +} + +/// Print one inflight-index result using the same latency field names as write cases. +/// @param repetition One-based repetition index. +void print_index_result(std::string_view variant, size_t repetition, size_t operations, + double elapsed_seconds, const LatencySummary& latency) { + std::cout << std::fixed << std::setprecision(3) << "RESULT benchmark=index" + << " variant=" << variant << " repetition=" << repetition + << " producers=" << FLAGS_producer_threads << " operations=" << operations + << " elapsed_seconds=" << elapsed_seconds + << " operations_per_sec=" << static_cast(operations) / elapsed_seconds + << " avg_us=" << latency.average_us << " p50_us=" << latency.p50_us + << " p95_us=" << latency.p95_us << " p99_us=" << latency.p99_us + << " max_us=" << latency.maximum_us << '\n'; +} + +/// Measure either representative sharded misses or worst-case single-key hit contention. +/// @param variant Stable output label. +/// @param key_count Number of distinct cache hashes used by the workload. +/// @param populate Whether every lookup should hit a pre-populated entry. +/// @param repetition One-based repetition index included in output. +Status run_index_case(std::string variant, size_t key_count, bool populate, size_t repetition) { + const size_t producer_count = static_cast(FLAGS_producer_threads); + const size_t operations_per_thread = static_cast(FLAGS_index_operations_per_thread); + InflightWriteBufferIndex index( + 64, FLAGS_cache_path + "_index_" + variant + "_" + std::to_string(repetition)); + std::vector hashes; + hashes.reserve(key_count); + const uint64_t epoch = 1; + auto entry = + std::make_shared(nullptr, 0, 1, MonotonicMicros(), epoch); + for (size_t key = 0; key < key_count; ++key) { + hashes.emplace_back( + BlockFileCache::hash("inflight_index_" + variant + "_" + std::to_string(key))); + if (populate) { + DORIS_CHECK(index.insert_if_absent(hashes.back(), 0, entry) == nullptr); + } + } + + std::vector producers; + producers.reserve(producer_count); + std::vector> latencies(producer_count); + std::barrier start_barrier(static_cast(producer_count + 1)); + ConcurrentError error; + for (size_t producer = 0; producer < producer_count; ++producer) { + producers.emplace_back([&, producer]() { + latencies[producer].reserve(operations_per_thread); + start_barrier.arrive_and_wait(); + for (size_t operation = 0; operation < operations_per_thread; ++operation) { + const size_t key = (operation * 2654435761ULL + producer) % key_count; + const auto start = Clock::now(); + auto result = index.lookup(hashes[key], 0, epoch); + const auto elapsed = std::chrono::duration_cast(Clock::now() - start); + latencies[producer].push_back(elapsed.count()); + if ((result != nullptr) != populate) { + error.set(Status::InternalError("unexpected index lookup result")); + break; + } + } + }); + } + + const auto start = Clock::now(); + start_barrier.arrive_and_wait(); + for (auto& producer : producers) { + producer.join(); + } + const auto end = Clock::now(); + if (error.failed()) { + return error.status(); + } + + const size_t total_operations = producer_count * operations_per_thread; + print_index_result(variant, repetition, total_operations, + std::chrono::duration(end - start).count(), + summarize_latencies(latencies)); + return Status::OK(); +} + +/// Run the selected benchmark groups in a fixed, directly comparable order. +Status run_benchmarks(const std::vector& modes, + const std::vector& worker_counts) { + const size_t reader_bytes = FLAGS_reader_operations * FLAGS_block_size; + const size_t service_bytes = FLAGS_service_operations * FLAGS_service_task_size; + const size_t workload_bytes = std::max(reader_bytes, service_bytes); + const size_t cache_capacity = std::max(256 * kMiB, workload_bytes * 4); + if (cache_capacity <= 3 * FLAGS_block_size) { + return Status::InvalidArgument("calculated cache capacity is too small"); + } + + BenchmarkEnvironment environment; + RETURN_IF_ERROR(environment.initialize(cache_capacity)); + std::cout << "CONFIG cache_path=" << FLAGS_cache_path << " cache_capacity=" << cache_capacity + << " block_size=" << FLAGS_block_size << " request_size=" << FLAGS_request_size + << " producer_threads=" << FLAGS_producer_threads + << " repetitions=" << FLAGS_repetitions << '\n'; + + for (size_t repetition = 1; repetition <= FLAGS_repetitions; ++repetition) { + if (mode_enabled(modes, "reader")) { + RETURN_IF_ERROR(run_reader_case(&environment, CacheWriteMode::SYNC_WRITE, "sync_write", + repetition)); + RETURN_IF_ERROR(run_reader_case(&environment, CacheWriteMode::ASYNC_WRITE, + "async_write", repetition)); + } + if (mode_enabled(modes, "service")) { + for (size_t workers : worker_counts) { + RETURN_IF_ERROR(run_service_case( + &environment, "drain_workers_" + std::to_string(workers), workers, + static_cast(FLAGS_service_operations + workers), repetition)); + } + RETURN_IF_ERROR(run_service_case( + &environment, "backpressure", worker_counts.back(), + std::min(FLAGS_backpressure_pending_tasks, FLAGS_service_operations), + repetition)); + } + if (mode_enabled(modes, "index")) { + RETURN_IF_ERROR( + run_index_case("sharded_miss", FLAGS_index_key_count, false, repetition)); + RETURN_IF_ERROR(run_index_case("sharded_hit", FLAGS_index_key_count, true, repetition)); + RETURN_IF_ERROR(run_index_case("hot_key_hit", 1, true, repetition)); + } + } + return Status::OK(); +} + +} // namespace +} // namespace doris::io + +int main(int argc, char** argv) { + google::ParseCommandLineFlags(&argc, &argv, true); + FLAGS_logtostderr = true; + // Keep RESULT lines parseable when the documented runner merges stdout and stderr. + FLAGS_minloglevel = google::GLOG_ERROR; + google::InitGoogleLogging(argv[0]); + + // Doris configuration defaults contain ${DORIS_HOME}; a standalone benchmark has no launcher + // to provide it, and only needs a valid expansion rather than a configuration file. + if (std::getenv("DORIS_HOME") == nullptr) { + const std::string current_directory = std::filesystem::current_path().string(); + if (::setenv("DORIS_HOME", current_directory.c_str(), 0) != 0) { + std::cerr << "Benchmark failed: cannot initialize DORIS_HOME\n"; + return 1; + } + } + if (!doris::config::init(nullptr, true)) { + std::cerr << "Benchmark failed: cannot initialize Doris configuration defaults\n"; + return 1; + } + doris::CpuInfo::init(); + doris::DiskInfo::init(); + doris::MemInfo::init(); + SCOPED_INIT_THREAD_CONTEXT(); + doris::ExecEnv::GetInstance()->init_mem_tracker(); + DORIS_CHECK(doris::thread_context()->thread_mem_tracker_mgr->init()); + + std::vector worker_counts; + doris::Status status = + doris::io::parse_positive_integer_list(FLAGS_worker_counts, &worker_counts); + const auto modes = doris::io::parse_modes(FLAGS_benchmark_mode); + if (status.ok()) { + status = doris::io::validate_flags(modes, worker_counts); + } + if (status.ok()) { + status = doris::io::run_benchmarks(modes, worker_counts); + } + if (!status.ok()) { + std::cerr << "Benchmark failed: " << status.to_string() << '\n'; + return 1; + } + return 0; +} + +#endif diff --git a/be/src/io/tools/readme.md b/be/src/io/tools/readme.md index a5804958da07b9..803e02a4c2b161 100644 --- a/be/src/io/tools/readme.md +++ b/be/src/io/tools/readme.md @@ -5,10 +5,13 @@ To compile the project, run the following command: ```bash -./build.sh --clean --file-cache-microbench --be +./build.sh --be --file-cache-microbench -j100 ``` This will generate the `file_cache_microbench` executable in the `apache_doris/output/be/lib` directory. +It also generates `async_file_cache_write_microbench`, a standalone benchmark for the +asynchronous file-cache write path, and installs its runner as +`output/be/bin/run-async-file-cache-write-microbench.sh`. ## Usage @@ -131,3 +134,181 @@ curl "http://localhost:{port}/MicrobenchService/get_help" ### Version Information: you can see it in get_help return msg +## Asynchronous write benchmark + +### Scope + +`async_file_cache_write_microbench` measures the asynchronous cache-write components added below +`CachedRemoteFileReader`. It uses a real filesystem-backed `BlockFileCache` and production +`InflightWriteBufferIndex`, admission control, bounded MPMC queue, worker pool, `get_or_set`, +`append`, and `finalize` implementations. Its deterministic in-memory remote reader removes S3 and +network latency from the comparison. + +This complements the existing `file_cache_microbench`: + +- The existing S3 read mode is useful for a complete remote-read environment, but its results also + contain object-store and network latency. +- The existing direct `get_or_set` mode measures the lower-level cache lookup path, but does not + exercise asynchronous buffer publication, queue admission, background workers, or persistence. +- The asynchronous benchmark connects those production components and verifies the final cache + state, while keeping the source read deterministic. + +The measured reader flow is: + +```text +SyntheticRemoteFileReader + -> CachedRemoteFileReader + -> inflight-buffer lookup + -> BlockFileCache probe + -> foreground remote fill + -> InflightWriteBufferIndex publication + -> AsyncCacheWriteService admission and MPMC queue + -> worker get_or_set, append, and finalize + -> inflight entry cleanup +``` + +### Benchmark groups + +The benchmark groups are: + +- `reader` + - Starts every case with an empty cache. + - Issues concurrent, non-overlapping cold reads through `CachedRemoteFileReader`. + - Compares forced synchronous writes with asynchronous writes using the same source data and + requested ranges. + - Validates returned bytes and verifies that the complete aligned block range is readable from + the final cache state. + - Separates caller-visible foreground time from the time needed to drain accepted asynchronous + writes. +- `service` + - Submits unique real buffers directly to `AsyncCacheWriteService` from concurrent producers. + - Measures 1, 4, and 16 workers by default, including allocation, inflight publication, queue + admission, worker consumption, `get_or_set`, `append`, `finalize`, and completion cleanup. + - Verifies every accepted task in the final `BlockFileCache`. + - Includes a deliberately bounded backpressure case. `accepted` and `rejected` show admission + behavior, while peak gauges show where work accumulated. +- `index` + - Measures `InflightWriteBufferIndex::lookup` independently from disk writes. + - `sharded_miss` spreads absent keys across shards. + - `sharded_hit` spreads pre-populated keys across shards. + - `hot_key_hit` sends all producers to one key to expose the upper bound of lock contention. + +The integrated reader and service groups cover index insertion and conditional removal. The index +group intentionally measures only lookup contention so it does not duplicate those flows. + +### Disk baseline + +Run the installed wrapper instead of invoking the binary directly when comparing machines. With +the default `RUN_FIO=auto`, the wrapper checks whether `fio` is installed and, if available, runs +these baselines on the same filesystem before starting the cache benchmark: + +- `seqwrite_qd1`: direct 1 MiB sequential writes at queue depth 1, showing single-stream device + throughput and latency. +- `randwrite_qd16`: direct 1 MiB random writes at queue depth 16, showing concurrent device + throughput and latency for a workload closer to writes spread across cache files. + +The fio files are created in a unique directory next to `--cache_path`, use `--unlink=1`, and are +not placed in `/tmp`. Direct I/O avoids filling the page cache immediately before the cache +benchmark. These fio results describe the storage environment; they are not numerically equivalent +to cache `persisted_mib_per_sec`, because production cache files use buffered writes without an +`fsync` or `fdatasync` in the measured completion path. + +Environment controls for the wrapper are: + +| Variable | Default | Meaning | +| --- | --- | --- | +| `RUN_FIO` | `auto` | Run when fio exists. Use `1` to require fio or `0` to skip it. | +| `FIO_SIZE` | `1G` | Address space used by each direct-I/O fio case. | +| `FIO_RUNTIME` | `5` | Measured seconds for each fio case, after a one-second ramp. | + +### Defaults and repetitions + +The defaults represent small reads that cause full 1 MiB cache-block writes: + +| Setting | Default | Purpose | +| --- | ---: | --- | +| `block_size` | 1 MiB | Cache alignment and persisted bytes per reader miss | +| `request_size` | 64 KiB | Bytes returned by each caller read | +| `reader_operations` | 128 | Cold blocks in each synchronous or asynchronous reader case | +| `service_task_size` | 1 MiB | Payload in each direct service task | +| `service_operations` | 256 | Attempts in each service case | +| `producer_threads` | 16 | Concurrent readers or submitters | +| `reader_workers` | 16 | Workers in the asynchronous reader comparison | +| `worker_counts` | 1, 4, 16 | Worker scaling points in the service group | +| `backpressure_pending_tasks` | 64 | Pending-task limit in the rejection case | +| `index_operations_per_thread` | 100,000 | Lookups performed by each index producer | +| `index_key_count` | 4,096 | Keys in each sharded index case | +| `repetitions` | 5 | Repeated executions of every selected case | + +Each repetition clears and drains the real cache before every reader or service case. The process +prints the one-based `repetition` on every `RESULT` line. Use the median across repetitions as the +primary comparison and retain the minimum-to-maximum range to expose scheduler, filesystem, page +cache, and background writeback noise. The first repetition is intentionally retained instead of +being silently treated as warm-up. + +For formal comparisons, use the same Release build, host, cache filesystem, arguments, and idle +machine state. Five repetitions are the default for a quick comparison; increase +`--repetitions` when the min-to-max spread is large. + +### Build and run + +Performance numbers should come from a Release build. Build both microbenchmarks and the wrapper +with: + +```bash +./build.sh --be --file-cache-microbench -j100 +``` + +Run all groups, the fio baseline, and five repetitions with: + +```bash +./output/be/bin/run-async-file-cache-write-microbench.sh \ + --benchmark_mode=all \ + --cache_path=./output/async_file_cache_write_microbench \ + --producer_threads=16 \ + --worker_counts=1,4,16 \ + --repetitions=5 2>&1 | tee ./output/async_file_cache_write_microbench.log +``` + +> `--cache_path` is benchmark-owned and must not exist or must be an empty directory. The benchmark +> rejects a non-empty path instead of clearing it, and removes only the directory it accepted on +> exit unless `--keep_cache` is set. Always use a dedicated path under `output/`; never point it at +> an existing cache or data directory. + +Invoke `output/be/lib/async_file_cache_write_microbench` directly only when the fio baseline is not +wanted. Use `--help` to adjust operation counts, request and task sizes, queue limit, worker counts, +repetitions, and cache retention. + +### Result fields + +Each measured case emits one machine-readable `RESULT` line: + +- Identity and shape: `benchmark`, `variant`, `repetition`, `producers`, `workers`, and + `operations`. +- Admission and correctness: `accepted`, `rejected`, and `persisted`. A case fails instead of + printing a successful result if returned data or final cache coverage is incorrect. +- Timing: `foreground_seconds` ends when producer or reader calls return; `drain_seconds` is the + remaining background completion time; `total_seconds` includes both. +- Rates: `foreground_ops_per_sec` measures caller or submitter completion. + `persisted_mib_per_sec` divides verified bytes by total time and does not claim durable-media + completion. +- Foreground latency: `avg_us`, `p50_us`, `p95_us`, `p99_us`, and `max_us`. +- Queue shape: `peak_pending`, `peak_queued`, and `peak_inflight` are sampled high-water marks. + `pending` includes queued and active accepted tasks. `inflight` can briefly exceed `pending` + because a producer publishes its buffer before admission and conditionally removes it after a + rejection. +- Index-only rates: `elapsed_seconds` and `operations_per_sec` replace write-specific rate fields. + +Use reader `foreground_ops_per_sec` and latency to quantify caller benefit from asynchronous +writes. Use service `persisted_mib_per_sec`, `drain_seconds`, and peak gauges together to determine +whether workers or admission are limiting progress. Use the backpressure acceptance ratio to +validate bounded overload behavior, and compare sharded versus hot-key index results to quantify +lock-contention sensitivity. + +### Out of scope + +This benchmark does not measure S3 or network latency, complete scanner/query throughput, +cache-hit read throughput, restart recovery, multiple-cache-disk balancing, or durable `fsync` +throughput. It is also not a replacement for BE unit and regression tests: it validates data and +final cache coverage to reject invalid performance samples, but its primary purpose is controlled +performance comparison. diff --git a/be/src/storage/segment/segment_index_file_cache_loader.cpp b/be/src/storage/segment/segment_index_file_cache_loader.cpp index 13a93460cbc2a3..46a1ffa9525be3 100644 --- a/be/src/storage/segment/segment_index_file_cache_loader.cpp +++ b/be/src/storage/segment/segment_index_file_cache_loader.cpp @@ -150,6 +150,7 @@ Status SegmentIndexFileCacheLoader::load_segment_index_to_file_cache( .is_index_data = true, .is_dryrun = true, .is_warmup = false, + .cache_write_mode_override = io::CacheWriteMode::SYNC_WRITE, }; TEST_SYNC_POINT_RETURN_WITH_VALUE( "SegmentIndexFileCacheLoader::load_segment_index_to_file_cache", Status::OK(), &ctx, diff --git a/be/test/io/cache/async_cache_write_service_test.cpp b/be/test/io/cache/async_cache_write_service_test.cpp new file mode 100644 index 00000000000000..8eaad8c41aa980 --- /dev/null +++ b/be/test/io/cache/async_cache_write_service_test.cpp @@ -0,0 +1,1189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/async_cache_write_service.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "cpp/sync_point.h" +#include "io/cache/block_file_cache_test_common.h" +#include "util/defer_op.h" +#include "util/time.h" + +namespace doris::io { +namespace { + +FileCacheSettings async_write_cache_settings() { + FileCacheSettings settings; + settings.query_queue_size = 4_mb; + settings.query_queue_elements = 1024; + settings.index_queue_size = 1_mb; + settings.index_queue_elements = 256; + settings.disposable_queue_size = 1_mb; + settings.disposable_queue_elements = 256; + settings.capacity = 8_mb; + settings.max_file_block_size = 4096; + settings.max_query_cache_size = 0; + return settings; +} + +class AsyncCacheWriteServiceTest : public BlockFileCacheTest { +protected: + std::unique_ptr create_cache(const std::string& name) { + auto path = caches_dir / name; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + _paths.emplace_back(path); + auto cache = std::make_unique(path.string(), async_write_cache_settings()); + EXPECT_TRUE(cache->initialize().ok()); + wait_until_cache_ready(*cache); + EXPECT_TRUE(cache->async_write_service()->start().ok()); + return cache; + } + + void TearDown() override { + for (const auto& path : _paths) { + std::error_code error; + fs::remove_all(path, error); + } + } + +private: + std::vector _paths; +}; + +TEST_F(AsyncCacheWriteServiceTest, TaskWritesDownloadedBlockAndCleansInflightEntry) { + auto cache = create_cache("async_write_service_single_task"); + auto* service = cache->async_write_service(); + auto* index = cache->inflight_write_buffer_index(); + ASSERT_NE(service, nullptr); + ASSERT_NE(index, nullptr); + const uint64_t baseline_submitted = service->_submitted_metric->get_value(); + const uint64_t baseline_submitted_bytes = service->_submitted_bytes_metric->get_value(); + const uint64_t baseline_finished = service->_finished_metric->get_value(); + const uint64_t baseline_persisted_blocks = service->_persisted_blocks_metric->get_value(); + const uint64_t baseline_persisted_bytes = service->_persisted_bytes_metric->get_value(); + const int64_t baseline_submit_latency_count = service->_submit_latency_metric->count(); + const int64_t baseline_buffer_alloc_latency_count = + service->_buffer_alloc_latency_metric->count(); + const int64_t baseline_queue_wait_latency_count = service->_queue_wait_latency_metric->count(); + const int64_t baseline_worker_task_latency_count = + service->_worker_task_latency_metric->count(); + const int64_t baseline_get_or_set_latency_count = service->_get_or_set_latency_metric->count(); + const int64_t baseline_append_latency_count = service->_append_latency_metric->count(); + const int64_t baseline_finalize_latency_count = service->_finalize_latency_metric->count(); + + constexpr size_t block_size = 4096; + const auto hash = BlockFileCache::hash("async_single_task"); + const std::string payload(block_size, 'a'); + const int64_t baseline_memory = service->buffer_memory_bytes(); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(block_size, &buffer).ok()); + memcpy(buffer->data(), payload.data(), payload.size()); + EXPECT_GE(service->buffer_memory_bytes(), baseline_memory + static_cast(block_size)); + + const uint64_t epoch = service->current_write_epoch(); + auto entry = std::make_shared(buffer, 0, block_size, + MonotonicMicros(), epoch); + ASSERT_EQ(index->insert_if_absent(hash, 0, entry), nullptr); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = block_size, + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = epoch, + .on_finalized = + [index, hash, entry, &finished](const AsyncCacheWriteTask&) { + index->remove_if(hash, 0, entry); + finished.set_value(); + }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(service->queued_count(), 0); + EXPECT_EQ(service->active_task_count(), 0); + EXPECT_EQ(index->lookup(hash, 0, epoch), nullptr); + EXPECT_EQ(service->_submitted_metric->get_value() - baseline_submitted, 1); + EXPECT_EQ(service->_submitted_bytes_metric->get_value() - baseline_submitted_bytes, block_size); + EXPECT_EQ(service->_finished_metric->get_value() - baseline_finished, 1); + EXPECT_EQ(service->_persisted_blocks_metric->get_value() - baseline_persisted_blocks, 1); + EXPECT_EQ(service->_persisted_bytes_metric->get_value() - baseline_persisted_bytes, block_size); + EXPECT_EQ(service->_submit_latency_metric->count() - baseline_submit_latency_count, 1); + EXPECT_EQ(service->_buffer_alloc_latency_metric->count() - baseline_buffer_alloc_latency_count, + 1); + EXPECT_EQ(service->_queue_wait_latency_metric->count() - baseline_queue_wait_latency_count, 1); + EXPECT_EQ(service->_worker_task_latency_metric->count() - baseline_worker_task_latency_count, + 1); + EXPECT_EQ(service->_get_or_set_latency_metric->count() - baseline_get_or_set_latency_count, 1); + EXPECT_EQ(service->_append_latency_metric->count() - baseline_append_latency_count, 1); + EXPECT_EQ(service->_finalize_latency_metric->count() - baseline_finalize_latency_count, 1); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + FileBlocks blocks; + bool fully_covered = false; + ASSERT_TRUE(cache->get_downloaded_blocks_if_fully_covered(hash, 0, block_size, context, &blocks, + &fully_covered) + .ok()); + ASSERT_TRUE(fully_covered); + ASSERT_EQ(blocks.size(), 1); + std::string actual(block_size, '\0'); + ASSERT_TRUE(blocks.front()->read(Slice(actual.data(), actual.size()), 0).ok()); + EXPECT_EQ(actual, payload); + + blocks.clear(); + entry.reset(); + buffer.reset(); + for (int attempt = 0; attempt < 100 && service->buffer_memory_bytes() != baseline_memory; + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(service->buffer_memory_bytes(), baseline_memory); +} + +TEST_F(AsyncCacheWriteServiceTest, PendingLimitRejectsWhileWorkerIsActive) { + auto cache = create_cache("async_write_service_backpressure"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + auto options = service->options(); + options.max_pending_tasks = 1; + ASSERT_TRUE(service->update_options(options).ok()); + + std::mutex mutex; + std::condition_variable cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + AsyncCacheWriteBufferPtr first_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &first_buffer).ok()); + memset(first_buffer->data(), 'b', first_buffer->size()); + std::promise first_finished; + auto first_future = first_finished.get_future(); + AsyncCacheWriteTask first_task { + .cache_hash = BlockFileCache::hash("backpressure_first"), + .file_offset = 0, + .file_size = first_buffer->size(), + .buffer = first_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&first_finished](const AsyncCacheWriteTask&) { first_finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(first_task))); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); + } + + AsyncCacheWriteBufferPtr rejected_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &rejected_buffer).ok()); + AsyncCacheWriteTask rejected_task { + .cache_hash = BlockFileCache::hash("backpressure_rejected"), + .file_offset = 0, + .file_size = rejected_buffer->size(), + .buffer = rejected_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = nullptr, + }; + EXPECT_FALSE(service->try_submit(std::move(rejected_task))); + EXPECT_EQ(service->pending_count(), 1); + EXPECT_EQ(service->queued_count(), 0); + EXPECT_EQ(service->active_task_count(), 1); + EXPECT_EQ(service->_active_get_or_set_count.load(std::memory_order_relaxed), 1); + EXPECT_GE(service->_reject_backpressure_metric->get_value(), 1); + + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + ASSERT_EQ(first_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); +} + +TEST_F(AsyncCacheWriteServiceTest, WatchdogDropsExpiredTaskAndCleansInflightEntry) { + auto cache = create_cache("async_write_service_watchdog"); + auto* service = cache->async_write_service(); + auto* index = cache->inflight_write_buffer_index(); + ASSERT_NE(service, nullptr); + ASSERT_NE(index, nullptr); + auto options = service->options(); + options.watchdog_warn_secs = 0; + options.watchdog_drop_secs = 1; + ASSERT_TRUE(service->update_options(options).ok()); + + const auto hash = BlockFileCache::hash("watchdog_drop"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), 'w', buffer->size()); + const uint64_t epoch = service->current_write_epoch(); + auto entry = std::make_shared(buffer, 0, buffer->size(), + MonotonicMicros(), epoch); + ASSERT_EQ(index->insert_if_absent(hash, 0, entry), nullptr); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros() - 2 * 1000 * 1000, + .write_epoch = epoch, + .on_finalized = + [index, hash, entry, &finished](const AsyncCacheWriteTask&) { + index->remove_if(hash, 0, entry); + finished.set_value(); + }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(index->lookup(hash, 0, epoch), nullptr); + EXPECT_GE(service->_watchdog_warn_metric->get_value(), 1); + EXPECT_GE(service->_watchdog_drop_metric->get_value(), 1); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + auto probe_result = cache->probe(hash, 0, 4096, context); + ASSERT_EQ(probe_result.file_blocks.size(), 1); + EXPECT_EQ(probe_result.file_blocks[0], nullptr); +} + +TEST_F(AsyncCacheWriteServiceTest, ShutdownDrainsAcceptedTask) { + auto cache = create_cache("async_write_service_shutdown_drain"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + const auto hash = BlockFileCache::hash("shutdown_drain"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), 's', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + service->shutdown(); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(0)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + FileBlocks blocks; + bool fully_covered = false; + ASSERT_TRUE(cache->get_downloaded_blocks_if_fully_covered(hash, 0, 4096, context, &blocks, + &fully_covered) + .ok()); + EXPECT_TRUE(fully_covered); + ASSERT_EQ(blocks.size(), 1); + std::string actual(4096, '\0'); + ASSERT_TRUE(blocks.front()->read(Slice(actual.data(), actual.size()), 0).ok()); + EXPECT_EQ(actual, std::string(4096, 's')); +} + +TEST_F(AsyncCacheWriteServiceTest, OneTaskWritesMultipleContainedCells) { + auto cache = create_cache("async_write_service_multiple_cells"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t cell_size = 4096; + constexpr size_t task_size = cell_size * 2; + const auto hash = BlockFileCache::hash("multiple_cells"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(task_size, &buffer).ok()); + memset(buffer->data(), 'm', cell_size); + memset(buffer->data() + cell_size, 'n', cell_size); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = task_size, + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + FileBlocks blocks; + bool fully_covered = false; + ASSERT_TRUE(cache->get_downloaded_blocks_if_fully_covered(hash, 0, task_size, context, &blocks, + &fully_covered) + .ok()); + EXPECT_TRUE(fully_covered); + ASSERT_EQ(blocks.size(), 2); + auto iterator = blocks.begin(); + std::string first(cell_size, '\0'); + ASSERT_TRUE((*iterator)->read(Slice(first.data(), first.size()), 0).ok()); + EXPECT_EQ(first, std::string(cell_size, 'm')); + ++iterator; + std::string second(cell_size, '\0'); + ASSERT_TRUE((*iterator)->read(Slice(second.data(), second.size()), 0).ok()); + EXPECT_EQ(second, std::string(cell_size, 'n')); +} + +TEST_F(AsyncCacheWriteServiceTest, ExistingAndDeletingCellsKeepTheirOwners) { + auto cache = create_cache("async_write_service_existing_states"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t cell_size = 4096; + constexpr size_t task_size = cell_size * 3; + const auto hash = BlockFileCache::hash("existing_states"); + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + auto holder = cache->get_or_set(hash, 0, task_size, context); + ASSERT_EQ(holder.file_blocks.size(), 3); + auto iterator = holder.file_blocks.begin(); + const auto downloaded_block = *iterator; + ++iterator; + const auto downloading_block = *iterator; + ++iterator; + const auto deleting_block = *iterator; + + ASSERT_EQ(downloaded_block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string original(cell_size, 'x'); + ASSERT_TRUE(downloaded_block->append(Slice(original.data(), original.size())).ok()); + ASSERT_TRUE(downloaded_block->finalize().ok()); + ASSERT_EQ(downloading_block->get_or_set_downloader(), FileBlock::get_caller_id()); + deleting_block->set_deleting(); + + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(task_size, &buffer).ok()); + memset(buffer->data(), 'y', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = task_size, + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_GE(service->_skip_downloaded_metric->get_value(), 1); + EXPECT_GE(service->_skip_downloading_metric->get_value(), 1); + EXPECT_GE(service->_skip_deleting_metric->get_value(), 1); + + std::string actual(cell_size, '\0'); + ASSERT_TRUE(downloaded_block->read(Slice(actual.data(), actual.size()), 0).ok()); + EXPECT_EQ(actual, original); + EXPECT_EQ(downloading_block->state(), FileBlock::State::DOWNLOADING); + EXPECT_EQ(downloading_block->get_downloader(), FileBlock::get_caller_id()); + EXPECT_TRUE(cache->is_block_deleting(deleting_block)); +} + +TEST_F(AsyncCacheWriteServiceTest, RemoveDuringAppendDoesNotLeaveResurrectedCacheData) { + auto cache = create_cache("async_write_service_remove_during_append"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + std::mutex mutex; + std::condition_variable cv; + bool before_append = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_append", + [&](auto&&) { + std::unique_lock lock(mutex); + before_append = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + const auto hash = BlockFileCache::hash("remove_during_append"); + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), 'r', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return before_append; })); + } + EXPECT_EQ(service->_active_append_count.load(std::memory_order_relaxed), 1); + + fs::path cache_file; + { + ReadStatistics probe_stats; + CacheContext probe_context; + probe_context.stats = &probe_stats; + auto probe_result = cache->probe(hash, 0, 4096, probe_context); + ASSERT_EQ(probe_result.file_blocks.size(), 1); + ASSERT_NE(probe_result.file_blocks[0], nullptr); + EXPECT_EQ(probe_result.file_blocks[0]->state(), FileBlock::State::DOWNLOADING); + cache_file = probe_result.file_blocks[0]->get_cache_file(); + } + const uint64_t old_epoch = service->current_write_epoch(); + cache->remove_if_cached_async(hash); + EXPECT_EQ(service->current_write_epoch(), old_epoch + 1); + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(service->_active_append_count.load(std::memory_order_relaxed), 0); + + bool removed = false; + for (int attempt = 0; attempt < 5000; ++attempt) { + ReadStatistics probe_stats; + CacheContext probe_context; + probe_context.stats = &probe_stats; + bool metadata_removed = false; + { + auto probe_result = cache->probe(hash, 0, 4096, probe_context); + metadata_removed = + probe_result.file_blocks.size() == 1 && probe_result.file_blocks[0] == nullptr; + } + if (metadata_removed && !fs::exists(cache_file)) { + removed = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(removed); +} + +TEST_F(AsyncCacheWriteServiceTest, PartialOverlapIsSkippedWithoutOutOfBoundsWrite) { + auto cache = create_cache("async_write_service_partial_overlap"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t existing_size = 4096; + const auto hash = BlockFileCache::hash("partial_overlap"); + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + { + auto holder = cache->get_or_set(hash, 0, existing_size, context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + ASSERT_EQ(block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(existing_size, 'x'); + ASSERT_TRUE(block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(block->finalize().ok()); + } + + constexpr size_t task_offset = 1024; + constexpr size_t task_size = 4096; + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(task_size, &buffer).ok()); + memset(buffer->data(), 'y', buffer->size()); + std::promise finished; + auto finished_future = finished.get_future(); + AsyncCacheWriteTask task { + .cache_hash = hash, + .file_offset = task_offset, + .file_size = task_size, + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = [&finished](const AsyncCacheWriteTask&) { finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + ASSERT_EQ(finished_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_GE(service->_skip_partial_overlap_metric->get_value(), 1); + + FileBlocks blocks; + bool fully_covered = false; + ASSERT_TRUE(cache->get_downloaded_blocks_if_fully_covered(hash, 0, task_offset + task_size, + context, &blocks, &fully_covered) + .ok()); + EXPECT_TRUE(fully_covered); + ASSERT_EQ(blocks.size(), 2); + auto iterator = blocks.begin(); + std::string existing(existing_size, '\0'); + ASSERT_TRUE((*iterator)->read(Slice(existing.data(), existing.size()), 0).ok()); + EXPECT_EQ(existing, std::string(existing_size, 'x')); + ++iterator; + std::string tail(task_offset, '\0'); + ASSERT_TRUE((*iterator)->read(Slice(tail.data(), tail.size()), 0).ok()); + EXPECT_EQ(tail, std::string(task_offset, 'y')); +} + +TEST_F(AsyncCacheWriteServiceTest, RemoveInvalidatesActiveAndQueuedTasksAndCleansEmptyCells) { + auto cache = create_cache("async_write_service_epoch"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + auto options = service->options(); + options.worker_count = 1; + ASSERT_TRUE(service->update_options(options).ok()); + + std::mutex mutex; + std::condition_variable cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + const auto active_hash = BlockFileCache::hash("epoch_drop_active"); + AsyncCacheWriteBufferPtr active_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &active_buffer).ok()); + memset(active_buffer->data(), 'a', active_buffer->size()); + std::promise active_finished; + auto active_future = active_finished.get_future(); + AsyncCacheWriteTask active_task { + .cache_hash = active_hash, + .file_offset = 0, + .file_size = active_buffer->size(), + .buffer = active_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&active_finished](const AsyncCacheWriteTask&) { active_finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(active_task))); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); + } + + const auto queued_hash = BlockFileCache::hash("epoch_drop_queued"); + AsyncCacheWriteBufferPtr queued_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &queued_buffer).ok()); + memset(queued_buffer->data(), 'q', queued_buffer->size()); + std::promise queued_finished; + auto queued_future = queued_finished.get_future(); + AsyncCacheWriteTask queued_task { + .cache_hash = queued_hash, + .file_offset = 0, + .file_size = queued_buffer->size(), + .buffer = queued_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&queued_finished](const AsyncCacheWriteTask&) { queued_finished.set_value(); }, + }; + ASSERT_TRUE(service->try_submit(std::move(queued_task))); + ASSERT_EQ(service->pending_count(), 2); + + const uint64_t old_epoch = service->current_write_epoch(); + cache->remove_if_cached_async(active_hash); + EXPECT_EQ(service->current_write_epoch(), old_epoch + 1); + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + ASSERT_EQ(active_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + ASSERT_EQ(queued_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_GE(service->_drop_stale_epoch_metric->get_value(), 2); + + ReadStatistics read_stats; + CacheContext context; + context.stats = &read_stats; + const auto expect_cache_gap = [&](const UInt128Wrapper& hash) { + auto probe_result = cache->probe(hash, 0, 4096, context); + ASSERT_EQ(probe_result.file_blocks.size(), 1); + EXPECT_EQ(probe_result.file_blocks[0], nullptr); + }; + expect_cache_gap(active_hash); + expect_cache_gap(queued_hash); +} + +TEST_F(AsyncCacheWriteServiceTest, UpdateOptionsValidatesAndAppliesAtRuntime) { + auto cache = create_cache("async_write_service_resize"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + auto options = service->options(); + auto invalid_options = options; + invalid_options.worker_count = 0; + EXPECT_TRUE(service->update_options(invalid_options).is()); + invalid_options = options; + invalid_options.max_pending_tasks = 0; + EXPECT_TRUE(service->update_options(invalid_options).is()); + invalid_options = options; + invalid_options.batch_size = 0; + EXPECT_TRUE(service->update_options(invalid_options).is()); + invalid_options = options; + invalid_options.watchdog_drop_secs = invalid_options.watchdog_warn_secs; + EXPECT_TRUE(service->update_options(invalid_options).is()); + + options.worker_count = 3; + options.max_pending_tasks = 7; + options.batch_size = 2; + options.watchdog_warn_secs = 1; + options.watchdog_drop_secs = 2; + ASSERT_TRUE(service->update_options(options).ok()); + const auto updated = service->options(); + EXPECT_EQ(updated.worker_count, 3); + EXPECT_EQ(updated.max_pending_tasks, 7); + EXPECT_EQ(updated.batch_size, 2); + EXPECT_EQ(updated.watchdog_warn_secs, 1); + EXPECT_EQ(updated.watchdog_drop_secs, 2); + EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 3); + + options.worker_count = 1; + ASSERT_TRUE(service->update_options(options).ok()); + EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 1); +} + +TEST_F(AsyncCacheWriteServiceTest, RuntimeGrowAndShrinkPreserveConcurrentTasks) { + auto cache = create_cache("async_write_service_mpmc_consumers"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t worker_count = 8; + auto options = service->options(); + options.worker_count = 1; + options.max_pending_tasks = worker_count; + options.batch_size = 1; + ASSERT_TRUE(service->update_options(options).ok()); + options.worker_count = worker_count; + ASSERT_TRUE(service->update_options(options).ok()); + + std::mutex mutex; + std::condition_variable cv; + size_t entered_workers = 0; + size_t finished_tasks = 0; + bool release_workers = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + ++entered_workers; + cv.notify_all(); + cv.wait(lock, [&]() { return release_workers; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_workers = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + for (size_t task_id = 0; task_id < worker_count; ++task_id) { + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + memset(buffer->data(), static_cast('a' + task_id), buffer->size()); + AsyncCacheWriteTask task { + .cache_hash = BlockFileCache::hash("mpmc_consumer_" + std::to_string(task_id)), + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&](const AsyncCacheWriteTask&) { + std::lock_guard lock(mutex); + ++finished_tasks; + cv.notify_all(); + }, + }; + ASSERT_TRUE(service->try_submit(std::move(task))); + } + + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return entered_workers == worker_count; })); + EXPECT_EQ(service->pending_count(), worker_count); + } + + auto shrink_options = service->options(); + shrink_options.worker_count = 1; + auto shrink_future = std::async(std::launch::async, [service, shrink_options]() { + return service->update_options(shrink_options); + }); + for (int attempt = 0; + attempt < 5000 && service->_desired_worker_count.load(std::memory_order_acquire) != 1; + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(service->_desired_worker_count.load(std::memory_order_acquire), 1); + EXPECT_EQ(shrink_future.wait_for(std::chrono::milliseconds(0)), std::future_status::timeout); + { + std::lock_guard lock(mutex); + release_workers = true; + } + cv.notify_all(); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return finished_tasks == worker_count; })); + } + ASSERT_TRUE(shrink_future.get().ok()); + EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(service->options().worker_count, 1); +} + +TEST_F(AsyncCacheWriteServiceTest, DynamicMpmcQueueGrowthRejectionAndDrain) { + auto cache = create_cache("async_write_service_dynamic_backpressure"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + constexpr size_t producer_count = 4; + constexpr size_t fill_wave_count = 4; + constexpr size_t fill_tasks_per_producer = 4; + constexpr size_t accepted_producer_tasks = + producer_count * fill_wave_count * fill_tasks_per_producer; + constexpr size_t rejected_tasks_per_producer = 32; + constexpr size_t rejected_producer_tasks = producer_count * rejected_tasks_per_producer; + constexpr size_t total_producer_tasks = accepted_producer_tasks + rejected_producer_tasks; + constexpr size_t max_pending_tasks = accepted_producer_tasks + 1; + constexpr size_t fast_worker_count = 4; + constexpr size_t block_size = 4096; + + auto options = service->options(); + options.worker_count = 1; + options.max_pending_tasks = max_pending_tasks; + options.batch_size = 1; + ASSERT_TRUE(service->update_options(options).ok()); + + std::mutex mutex; + std::condition_variable cv; + size_t consumer_entries = 0; + size_t finished_tasks = 0; + bool slow_consumers = true; + bool record_drain_samples = false; + std::vector drain_queue_sizes; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + ++consumer_entries; + cv.notify_all(); + cv.wait(lock, [&]() { return !slow_consumers; }); + if (record_drain_samples) { + // This excludes active tasks, unlike pending_count(), and therefore observes + // the actual MPMC backlog after each dequeue. + drain_queue_sizes.emplace_back(service->queued_count()); + } + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + slow_consumers = false; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + service->shutdown(); + }}; + + std::vector tasks; + tasks.reserve(total_producer_tasks + 1); + for (size_t task_id = 0; task_id <= total_producer_tasks; ++task_id) { + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(block_size, &buffer).ok()); + memset(buffer->data(), static_cast('a' + task_id % 26), buffer->size()); + tasks.emplace_back(AsyncCacheWriteTask { + .cache_hash = + BlockFileCache::hash("dynamic_backpressure_" + std::to_string(task_id)), + .file_offset = 0, + .file_size = buffer->size(), + .buffer = std::move(buffer), + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = + [&](const AsyncCacheWriteTask&) { + std::lock_guard lock(mutex); + ++finished_tasks; + cv.notify_all(); + }, + }); + } + + const uint64_t baseline_submitted = service->_submitted_metric->get_value(); + const uint64_t baseline_rejected = service->_rejected_metric->get_value(); + const uint64_t baseline_backpressure = service->_reject_backpressure_metric->get_value(); + ASSERT_TRUE(service->try_submit(std::move(tasks[0]))); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return consumer_entries == 1; })); + } + + std::atomic producer_accepts {0}; + std::atomic producer_rejects {0}; + // Each producer owns a contiguous task slice beginning at first_task; tasks_per_producer + // controls whether this invocation is one fill wave or the final rejection burst. + const auto submit_wave = [&](size_t first_task, size_t tasks_per_producer) { + std::vector producers; + producers.reserve(producer_count); + std::barrier producer_start(producer_count); + for (size_t producer_id = 0; producer_id < producer_count; ++producer_id) { + producers.emplace_back([&, producer_id]() { + producer_start.arrive_and_wait(); + const size_t producer_first = first_task + producer_id * tasks_per_producer; + for (size_t task_offset = 0; task_offset < tasks_per_producer; ++task_offset) { + if (service->try_submit(std::move(tasks[producer_first + task_offset]))) { + producer_accepts.fetch_add(1, std::memory_order_relaxed); + } else { + producer_rejects.fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + for (auto& producer : producers) { + producer.join(); + } + }; + + // With the only consumer blocked, every producer continuously submits several tasks per wave. + // The stable snapshots make backlog growth deterministic instead of depending on polling + // timing. + for (size_t wave = 0; wave < fill_wave_count; ++wave) { + submit_wave(1 + wave * producer_count * fill_tasks_per_producer, fill_tasks_per_producer); + EXPECT_EQ(service->queued_count(), (wave + 1) * producer_count * fill_tasks_per_producer); + EXPECT_EQ(service->pending_count(), + 1 + (wave + 1) * producer_count * fill_tasks_per_producer); + } + + // The queue is now bounded by max_pending_tasks. A larger concurrent burst must be dropped by + // backpressure without changing either the queued or active task count. + submit_wave(1 + accepted_producer_tasks, rejected_tasks_per_producer); + EXPECT_EQ(producer_accepts.load(std::memory_order_relaxed), accepted_producer_tasks); + EXPECT_EQ(producer_rejects.load(std::memory_order_relaxed), rejected_producer_tasks); + EXPECT_GT(producer_rejects.load(std::memory_order_relaxed), + producer_accepts.load(std::memory_order_relaxed)); + EXPECT_EQ(service->queued_count(), accepted_producer_tasks); + EXPECT_EQ(service->pending_count(), max_pending_tasks); + EXPECT_EQ(service->_submitted_metric->get_value() - baseline_submitted, max_pending_tasks); + EXPECT_EQ(service->_rejected_metric->get_value() - baseline_rejected, rejected_producer_tasks); + EXPECT_EQ(service->_reject_backpressure_metric->get_value() - baseline_backpressure, + rejected_producer_tasks); + + // Producers have stopped. Grow from one to four consumers while writes remain blocked, then + // remove the artificial delay and increase the batch size to model faster disk consumption. + options.worker_count = fast_worker_count; + options.batch_size = 4; + ASSERT_TRUE(service->update_options(options).ok()); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return consumer_entries == fast_worker_count; })); + EXPECT_EQ(service->queued_count(), accepted_producer_tasks - (fast_worker_count - 1)); + record_drain_samples = true; + slow_consumers = false; + } + cv.notify_all(); + + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(10), + [&]() { return finished_tasks == max_pending_tasks; })); + } + EXPECT_EQ(service->pending_count(), 0); + EXPECT_EQ(service->queued_count(), 0); + EXPECT_TRUE(std::any_of(drain_queue_sizes.begin(), drain_queue_sizes.end(), [&](size_t size) { + return size > 0 && size <= accepted_producer_tasks / 2; + })); + EXPECT_TRUE(std::any_of(drain_queue_sizes.begin(), drain_queue_sizes.end(), + [](size_t size) { return size == 0; })); +} + +TEST_F(AsyncCacheWriteServiceTest, MutableConfigUpdatesServicesExplicitly) { + auto* factory = FileCacheFactory::instance(); + factory->_caches.clear(); + factory->_path_to_cache.clear(); + factory->_capacity = 0; + + const bool old_enable = config::enable_async_file_cache_write; + const int32_t old_workers = config::async_file_cache_write_workers_per_disk; + const int64_t old_max_pending = config::async_file_cache_write_max_pending_tasks_per_disk; + const int32_t old_batch_size = config::async_file_cache_write_batch_size; + const int64_t old_warn_secs = config::async_file_cache_write_watchdog_warn_secs; + const int64_t old_drop_secs = config::async_file_cache_write_watchdog_drop_secs; + const auto path = caches_dir / "async_write_service_config_update"; + std::error_code error; + Defer restore {[&]() { + EXPECT_TRUE(config::set_config("async_file_cache_write_workers_per_disk", + std::to_string(old_workers)) + .ok()); + EXPECT_TRUE(config::set_config("async_file_cache_write_max_pending_tasks_per_disk", + std::to_string(old_max_pending)) + .ok()); + EXPECT_TRUE(config::set_config("async_file_cache_write_batch_size", + std::to_string(old_batch_size)) + .ok()); + EXPECT_TRUE(config::set_config("async_file_cache_write_watchdog_warn_secs", + std::to_string(old_warn_secs)) + .ok()); + EXPECT_TRUE(config::set_config("async_file_cache_write_watchdog_drop_secs", + std::to_string(old_drop_secs)) + .ok()); + EXPECT_TRUE( + config::set_config("enable_async_file_cache_write", old_enable ? "true" : "false") + .ok()); + factory->_caches.clear(); + factory->_path_to_cache.clear(); + factory->_capacity = 0; + fs::remove_all(path, error); + }}; + + ASSERT_TRUE(config::set_config("enable_async_file_cache_write", "false").ok()); + fs::remove_all(path, error); + fs::create_directories(path); + ASSERT_TRUE(factory->create_file_cache(path.string(), async_write_cache_settings()).ok()); + auto* cache = factory->get_by_path(path.string()); + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + ASSERT_FALSE(cache->async_write_service()->_started.load(std::memory_order_acquire)); + AsyncCacheWriteBufferPtr disabled_buffer; + ASSERT_TRUE(cache->async_write_service()->allocate_tracked_buffer(4096, &disabled_buffer).ok()); + AsyncCacheWriteTask disabled_task { + .cache_hash = BlockFileCache::hash("disabled_async_write_service"), + .file_offset = 0, + .file_size = disabled_buffer->size(), + .buffer = disabled_buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = cache->async_write_service()->current_write_epoch(), + .on_finalized = nullptr, + }; + EXPECT_FALSE(cache->async_write_service()->try_submit(std::move(disabled_task))); + EXPECT_EQ(cache->async_write_service()->pending_count(), 0); + + const int32_t new_workers = old_workers == 1 ? 2 : 1; + const int64_t new_max_pending = old_max_pending + 1; + const int32_t new_batch_size = old_batch_size + 1; + const int64_t new_warn_secs = old_warn_secs + 1; + const int64_t new_drop_secs = std::max(old_drop_secs + 1, new_warn_secs + 1); + ASSERT_TRUE(config::set_config("async_file_cache_write_workers_per_disk", + std::to_string(new_workers)) + .ok()); + ASSERT_TRUE(config::set_config("async_file_cache_write_max_pending_tasks_per_disk", + std::to_string(new_max_pending)) + .ok()); + ASSERT_TRUE( + config::set_config("async_file_cache_write_batch_size", std::to_string(new_batch_size)) + .ok()); + ASSERT_TRUE(config::set_config("async_file_cache_write_watchdog_drop_secs", + std::to_string(new_drop_secs)) + .ok()); + ASSERT_TRUE(config::set_config("async_file_cache_write_watchdog_warn_secs", + std::to_string(new_warn_secs)) + .ok()); + ASSERT_TRUE(config::set_config("enable_async_file_cache_write", "true").ok()); + + const auto updated = cache->async_write_service()->options(); + EXPECT_TRUE(cache->async_write_service()->_started.load(std::memory_order_acquire)); + EXPECT_EQ(updated.worker_count, new_workers); + EXPECT_EQ(updated.max_pending_tasks, new_max_pending); + EXPECT_EQ(updated.batch_size, new_batch_size); + EXPECT_EQ(updated.watchdog_warn_secs, new_warn_secs); + EXPECT_EQ(updated.watchdog_drop_secs, new_drop_secs); +} + +TEST_F(AsyncCacheWriteServiceTest, ShutdownWaitsForRegisteredSubmitterAndRejectsItsTask) { + auto cache = create_cache("async_write_service_shutdown"); + auto* service = cache->async_write_service(); + ASSERT_NE(service, nullptr); + + std::mutex mutex; + std::condition_variable cv; + bool submitter_registered = false; + bool release_submitter = false; + std::promise shutdown_stopped_accepting; + auto shutdown_stopped_accepting_future = shutdown_stopped_accepting.get_future(); + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard submit_guard; + SyncPoint::CallbackGuard shutdown_guard; + sync_point->set_call_back( + "AsyncCacheWriteService::try_submit:after_register", + [&](auto&&) { + std::unique_lock lock(mutex); + submitter_registered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_submitter; }); + }, + &submit_guard); + sync_point->set_call_back( + "AsyncCacheWriteService::shutdown:after_stop_accepting", + [&](auto&&) { shutdown_stopped_accepting.set_value(); }, &shutdown_guard); + sync_point->enable_processing(); + std::future submit_future; + std::future shutdown_future; + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_submitter = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + AsyncCacheWriteBufferPtr buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(4096, &buffer).ok()); + AsyncCacheWriteTask task { + .cache_hash = BlockFileCache::hash("shutdown_submitter"), + .file_offset = 0, + .file_size = buffer->size(), + .buffer = buffer, + .admission_ctx = {}, + .submit_ts_us = MonotonicMicros(), + .write_epoch = service->current_write_epoch(), + .on_finalized = nullptr, + }; + submit_future = std::async(std::launch::async, [service, task = std::move(task)]() mutable { + return service->try_submit(std::move(task)); + }); + { + std::unique_lock lock(mutex); + ASSERT_TRUE( + cv.wait_for(lock, std::chrono::seconds(5), [&]() { return submitter_registered; })); + } + + shutdown_future = std::async(std::launch::async, [service]() { service->shutdown(); }); + ASSERT_EQ(shutdown_stopped_accepting_future.wait_for(std::chrono::seconds(5)), + std::future_status::ready); + EXPECT_EQ(shutdown_future.wait_for(std::chrono::milliseconds(0)), std::future_status::timeout); + { + std::lock_guard lock(mutex); + release_submitter = true; + } + cv.notify_all(); + + EXPECT_FALSE(submit_future.get()); + ASSERT_EQ(shutdown_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + EXPECT_EQ(service->pending_count(), 0); +} + +} // namespace +} // namespace doris::io diff --git a/be/test/io/cache/block_file_cache_downloader_test.cpp b/be/test/io/cache/block_file_cache_downloader_test.cpp index 79974ef4860ad1..c74e9f0e9b9505 100644 --- a/be/test/io/cache/block_file_cache_downloader_test.cpp +++ b/be/test/io/cache/block_file_cache_downloader_test.cpp @@ -26,6 +26,10 @@ #include "cloud/cloud_storage_engine.h" #include "common/config.h" +#include "cpp/sync_point.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "util/defer_op.h" namespace doris::io { @@ -237,4 +241,70 @@ TEST_F(FileCacheBlockDownloaderTest, TestCheckDownloadTaskInflightStatus) { EXPECT_TRUE(done) << "Task should complete with decremented inflight count"; } +TEST_F(FileCacheBlockDownloaderTest, DownloadSegmentForcesSynchronousCacheWriteForBothDryRunModes) { + const Path directory = "ut_dir/block_file_cache_downloader_sync_override"; + const Path file = directory / "segment.dat"; + static_cast(global_local_filesystem()->delete_file(file)); + static_cast(global_local_filesystem()->delete_directory(directory)); + ASSERT_TRUE(global_local_filesystem()->create_directory(directory).ok()); + Defer cleanup {[&]() { + static_cast(global_local_filesystem()->delete_file(file)); + static_cast(global_local_filesystem()->delete_directory(directory)); + }}; + FileWriterPtr writer; + ASSERT_TRUE(global_local_filesystem()->create_file(file, &writer).ok()); + ASSERT_TRUE(writer->append(Slice("x", 1)).ok()); + ASSERT_TRUE(writer->close().ok()); + + const bool old_dryrun_config = config::enable_reader_dryrun_when_download_file_cache; + const int64_t old_buffer_size = config::s3_write_buffer_size; + Defer restore_config {[&]() { + config::enable_reader_dryrun_when_download_file_cache = old_dryrun_config; + config::s3_write_buffer_size = old_buffer_size; + }}; + config::s3_write_buffer_size = 1; + + std::vector> observed_contexts; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "FileCacheBlockDownloader::download_segment_file:before_read", + [&](auto&& values) { + auto* context = try_any_cast(values.back()); + ASSERT_TRUE(context->cache_write_mode_override.has_value()); + observed_contexts.emplace_back(context->is_dryrun, + *context->cache_write_mode_override); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + FileCacheBlockDownloader downloader(_engine); + for (bool is_dryrun : {false, true}) { + config::enable_reader_dryrun_when_download_file_cache = is_dryrun; + Status completion_status = Status::InternalError("download completion was not called"); + DownloadFileMeta meta { + .path = file, + .file_size = 1, + .offset = 0, + .download_size = 1, + .file_system = global_local_filesystem(), + .ctx = IOContext {.is_dryrun = is_dryrun}, + .download_done = [&](Status status) { completion_status = std::move(status); }, + .tablet_id = 10086, + }; + DownloadTask task(std::move(meta)); + downloader.download_blocks(task); + EXPECT_TRUE(completion_status.ok()); + } + + EXPECT_EQ(observed_contexts, (std::vector> { + {false, CacheWriteMode::SYNC_WRITE}, + {true, CacheWriteMode::SYNC_WRITE}, + })); +} + } // namespace doris::io diff --git a/be/test/io/cache/block_file_cache_probe_test.cpp b/be/test/io/cache/block_file_cache_probe_test.cpp new file mode 100644 index 00000000000000..11c88f9d5dd9d0 --- /dev/null +++ b/be/test/io/cache/block_file_cache_probe_test.cpp @@ -0,0 +1,329 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "io/cache/block_file_cache_test_common.h" +#include "util/defer_op.h" + +namespace doris::io { +namespace { + +FileCacheSettings probe_cache_settings() { + FileCacheSettings settings; + settings.query_queue_size = 4_mb; + settings.query_queue_elements = 1024; + settings.index_queue_size = 1_mb; + settings.index_queue_elements = 256; + settings.disposable_queue_size = 1_mb; + settings.disposable_queue_elements = 256; + settings.capacity = 8_mb; + settings.max_file_block_size = 4096; + settings.max_query_cache_size = 0; + return settings; +} + +TEST_F(BlockFileCacheTest, ProbeMissDoesNotCreateCacheCell) { + const auto path = caches_dir / "block_file_cache_probe_miss"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_miss"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + const size_t cache_size_before = cache._cur_cache_size; + const int64_t probe_latency_count = cache._probe_latency_us->count(); + const int64_t cache_lock_wait_count = cache._cache_lock_wait_time_us->count(); + + auto result = cache.probe(hash, 0, 4096, context); + ASSERT_EQ(result.file_blocks.size(), 1); + EXPECT_EQ(result.file_blocks[0], nullptr); + EXPECT_EQ(cache._cur_cache_size, cache_size_before); + EXPECT_EQ(cache._probe_latency_us->count() - probe_latency_count, 1); + EXPECT_EQ(cache._cache_lock_wait_time_us->count() - cache_lock_wait_count, 1); + std::lock_guard cache_lock(cache._mutex); + EXPECT_EQ(cache._files.find(hash), cache._files.end()); + } + fs::remove_all(path, error); +} + +TEST_F(BlockFileCacheTest, ProbeReturnsDownloadedBlockAndNullMissSlot) { + const auto path = caches_dir / "block_file_cache_probe_mixed"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_mixed"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + { + auto holder = cache.get_or_set(hash, 0, 4096, context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + ASSERT_EQ(block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(4096, 'p'); + ASSERT_TRUE(block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(block->finalize().ok()); + } + const size_t cache_size_before = cache._cur_cache_size; + { + std::lock_guard cache_lock(cache._mutex); + cache._files.at(hash).begin()->second.atime = 123; + } + + auto result = cache.probe(hash, 0, 8192, context); + ASSERT_EQ(result.file_blocks.size(), 2); + ASSERT_NE(result.file_blocks[0], nullptr); + EXPECT_EQ(result.file_blocks[0]->state(), FileBlock::State::DOWNLOADED); + EXPECT_EQ(result.file_blocks[0]->range().left, 0); + EXPECT_EQ(result.file_blocks[0]->range().right, 4095); + EXPECT_EQ(result.file_blocks[1], nullptr); + EXPECT_EQ(cache._cur_cache_size, cache_size_before); + { + std::lock_guard cache_lock(cache._mutex); + EXPECT_EQ(cache._files.at(hash).begin()->second.atime, 123); + } + } + fs::remove_all(path, error); +} + +TEST_F(BlockFileCacheTest, ProbeAcceptsPreallocatedBlockCoveringFileTail) { + const auto path = caches_dir / "block_file_cache_probe_preallocated_file_tail"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_preallocated_file_tail"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + + // File writers allocate a full block before the final buffer size is known. During that + // window, the last cache block can extend beyond the actual file tail passed to probe(). + auto holder = cache.get_or_set(hash, 0, 8192, context); + ASSERT_EQ(holder.file_blocks.size(), 2); + const auto& preallocated_tail = *std::next(holder.file_blocks.begin()); + ASSERT_EQ(preallocated_tail->range().left, 4096); + ASSERT_EQ(preallocated_tail->range().right, 8191); + + auto result = cache.probe(hash, 0, 6144, context); + ASSERT_EQ(result.file_blocks.size(), 2); + ASSERT_EQ(result.file_blocks[1], preallocated_tail); + EXPECT_EQ(result.file_blocks[1]->range().left, 4096); + EXPECT_EQ(result.file_blocks[1]->range().right, 8191); + } + fs::remove_all(path, error); +} + +TEST_F(BlockFileCacheTest, ProbeTouchAndRemovalRevalidateTheRetainedBlock) { + const auto path = caches_dir / "block_file_cache_probe_touch_remove"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + const bool old_async_touch = config::enable_file_cache_async_touch_on_get_or_set; + config::enable_file_cache_async_touch_on_get_or_set = false; + Defer restore_config { + [&]() { config::enable_file_cache_async_touch_on_get_or_set = old_async_touch; }}; + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_touch_remove"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + { + auto holder = cache.get_or_set(hash, 0, 4096, context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + ASSERT_EQ(block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(4096, 't'); + ASSERT_TRUE(block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(block->finalize().ok()); + } + { + std::lock_guard cache_lock(cache._mutex); + cache._files.at(hash).begin()->second.atime = 123; + } + + fs::path cache_file; + const uint64_t old_epoch = cache.async_write_service()->current_write_epoch(); + { + auto probe_result = cache.probe(hash, 0, 4096, context); + ASSERT_EQ(probe_result.file_blocks.size(), 1); + const auto& block = probe_result.file_blocks[0]; + ASSERT_NE(block, nullptr); + cache_file = block->get_cache_file(); + { + std::lock_guard cache_lock(cache._mutex); + EXPECT_EQ(cache._files.at(hash).begin()->second.atime, 123); + } + + EXPECT_FALSE(cache.is_block_deleting(block)); + cache.touch_probe_block_if_cached(block, context); + { + std::lock_guard cache_lock(cache._mutex); + EXPECT_NE(cache._files.at(hash).begin()->second.atime, 123); + } + + cache.remove_if_cached(hash); + EXPECT_EQ(cache.async_write_service()->current_write_epoch(), old_epoch + 1); + EXPECT_TRUE(cache.is_block_deleting(block)); + cache.touch_probe_block_if_cached(block, context); + } + + bool removed = false; + for (int attempt = 0; attempt < 5000; ++attempt) { + bool metadata_removed = false; + { + auto probe_result = cache.probe(hash, 0, 4096, context); + metadata_removed = probe_result.file_blocks.size() == 1 && + probe_result.file_blocks[0] == nullptr; + } + if (metadata_removed && !fs::exists(cache_file)) { + removed = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(removed); + } + fs::remove_all(path, error); +} + +TEST_F(BlockFileCacheTest, ProbeObservesDownloadingAndEmptyBlocksWithoutTakingDownloader) { + const auto path = caches_dir / "block_file_cache_probe_states"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_states"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + std::mutex mutex; + std::condition_variable cv; + bool downloader_ready = false; + bool release_downloader = false; + + std::thread downloader([&]() { + ReadStatistics downloader_stats; + CacheContext downloader_context; + downloader_context.stats = &downloader_stats; + auto holder = cache.get_or_set(hash, 0, 6144, downloader_context); + DORIS_CHECK(holder.file_blocks.size() == 2); + DORIS_CHECK(holder.file_blocks.front()->get_or_set_downloader() == + FileBlock::get_caller_id()); + { + std::lock_guard lock(mutex); + downloader_ready = true; + } + cv.notify_all(); + std::unique_lock lock(mutex); + cv.wait(lock, [&]() { return release_downloader; }); + }); + Defer stop_downloader {[&]() { + { + std::lock_guard lock(mutex); + release_downloader = true; + } + cv.notify_all(); + if (downloader.joinable()) { + downloader.join(); + } + }}; + bool ready = false; + { + std::unique_lock lock(mutex); + ready = cv.wait_for(lock, std::chrono::seconds(5), [&]() { return downloader_ready; }); + } + if (!ready) { + FAIL() << "downloader thread did not become ready"; + } + + { + auto result = cache.probe(hash, 0, 6144, context); + ASSERT_EQ(result.file_blocks.size(), 2); + ASSERT_NE(result.file_blocks[0], nullptr); + ASSERT_NE(result.file_blocks[1], nullptr); + EXPECT_EQ(result.file_blocks[0]->state(), FileBlock::State::DOWNLOADING); + EXPECT_EQ(result.file_blocks[0]->range().left, 0); + EXPECT_EQ(result.file_blocks[0]->range().right, 4095); + EXPECT_EQ(result.file_blocks[1]->state(), FileBlock::State::EMPTY); + EXPECT_EQ(result.file_blocks[1]->range().left, 4096); + EXPECT_EQ(result.file_blocks[1]->range().right, 6143); + } + } + fs::remove_all(path, error); +} + +TEST_F(BlockFileCacheTest, ProbeResultDoesNotCompleteDownloaderOwnedByCaller) { + const auto path = caches_dir / "block_file_cache_probe_preserves_downloader"; + std::error_code error; + fs::remove_all(path, error); + fs::create_directories(path); + { + BlockFileCache cache(path.string(), probe_cache_settings()); + ASSERT_TRUE(cache.initialize().ok()); + wait_until_cache_ready(cache); + const auto hash = BlockFileCache::hash("probe_preserves_downloader"); + ReadStatistics stats; + CacheContext context; + context.stats = &stats; + auto holder = cache.get_or_set(hash, 0, 4096, context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + const uint64_t downloader = FileBlock::get_caller_id(); + ASSERT_EQ(block->get_or_set_downloader(), downloader); + + { + auto result = cache.probe(hash, 0, 4096, context); + ASSERT_EQ(result.file_blocks.size(), 1); + ASSERT_EQ(result.file_blocks[0], block); + EXPECT_EQ(result.file_blocks[0]->state(), FileBlock::State::DOWNLOADING); + } + + // A probe only retains the block against deletion. It must not complete downloader + // ownership established independently by a FileBlocksHolder, even on the same thread. + EXPECT_EQ(block->state(), FileBlock::State::DOWNLOADING); + EXPECT_EQ(block->get_downloader(), downloader); + } + fs::remove_all(path, error); +} + +} // namespace +} // namespace doris::io diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index c2aea1cd825253..107da8fc5abd6c 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -54,6 +54,17 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.inverted_index_io_timer = multiplier * 28; stats.remote_only_on_miss_triggered = multiplier * 29; stats.remote_only_on_miss_threshold_bytes = multiplier * 30; + stats.async_cache_write_submitted = multiplier * 31; + stats.async_cache_write_rejected = multiplier * 32; + stats.async_cache_write_buffer_alloc_fail = multiplier * 33; + stats.async_cache_write_drop_stale_epoch = multiplier * 34; + stats.inflight_write_buffer_index_hit = multiplier * 35; + stats.inflight_write_buffer_index_miss = multiplier * 36; + stats.probe_downloaded_hit = multiplier * 37; + stats.probe_downloading_hit = multiplier * 38; + stats.probe_miss = multiplier * 39; + stats.block_wait_success = multiplier * 40; + stats.block_wait_timeout = multiplier * 41; return stats; } @@ -94,6 +105,19 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.remote_only_on_miss_triggered, expected.remote_only_on_miss_triggered); EXPECT_EQ(actual.remote_only_on_miss_threshold_bytes, expected.remote_only_on_miss_threshold_bytes); + EXPECT_EQ(actual.async_cache_write_submitted, expected.async_cache_write_submitted); + EXPECT_EQ(actual.async_cache_write_rejected, expected.async_cache_write_rejected); + EXPECT_EQ(actual.async_cache_write_buffer_alloc_fail, + expected.async_cache_write_buffer_alloc_fail); + EXPECT_EQ(actual.async_cache_write_drop_stale_epoch, + expected.async_cache_write_drop_stale_epoch); + EXPECT_EQ(actual.inflight_write_buffer_index_hit, expected.inflight_write_buffer_index_hit); + EXPECT_EQ(actual.inflight_write_buffer_index_miss, expected.inflight_write_buffer_index_miss); + EXPECT_EQ(actual.probe_downloaded_hit, expected.probe_downloaded_hit); + EXPECT_EQ(actual.probe_downloading_hit, expected.probe_downloading_hit); + EXPECT_EQ(actual.probe_miss, expected.probe_miss); + EXPECT_EQ(actual.block_wait_success, expected.block_wait_success); + EXPECT_EQ(actual.block_wait_timeout, expected.block_wait_timeout); } } // namespace @@ -117,6 +141,17 @@ TEST(FileCacheProfileReporterTest, DiffReturnsZeroWithoutNewData) { make_file_cache_stats(0)); } +TEST(FileCacheProfileReporterTest, MergeIncludesEveryAsyncReadAndWriteField) { + auto aggregate = make_file_cache_stats(2); + aggregate.merge_from(make_file_cache_stats(3)); + + auto expected = make_file_cache_stats(5); + // These two pre-existing fields merge by OR/max instead of addition. + expected.remote_only_on_miss_triggered = true; + expected.remote_only_on_miss_threshold_bytes = 90; + expect_file_cache_stats_eq(aggregate, expected); +} + TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTotals) { auto profile = std::make_unique("test_profile"); io::FileCacheProfileReporter reporter(profile.get()); @@ -139,6 +174,27 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot EXPECT_EQ(profile->get_counter("CacheGetOrSetTimer")->value(), after_second_report.cache_get_or_set_timer); EXPECT_EQ(profile->get_counter("LockWaitTimer")->value(), after_second_report.lock_wait_timer); + EXPECT_EQ(profile->get_counter("AsyncCacheWriteSubmitted")->value(), + after_second_report.async_cache_write_submitted); + EXPECT_EQ(profile->get_counter("AsyncCacheWriteRejected")->value(), + after_second_report.async_cache_write_rejected); + EXPECT_EQ(profile->get_counter("AsyncCacheWriteBufferAllocFail")->value(), + after_second_report.async_cache_write_buffer_alloc_fail); + EXPECT_EQ(profile->get_counter("AsyncCacheWriteDropStaleEpoch")->value(), + after_second_report.async_cache_write_drop_stale_epoch); + EXPECT_EQ(profile->get_counter("InflightWriteBufferIndexHit")->value(), + after_second_report.inflight_write_buffer_index_hit); + EXPECT_EQ(profile->get_counter("InflightWriteBufferIndexMiss")->value(), + after_second_report.inflight_write_buffer_index_miss); + EXPECT_EQ(profile->get_counter("ProbeDownloadedHit")->value(), + after_second_report.probe_downloaded_hit); + EXPECT_EQ(profile->get_counter("ProbeDownloadingHit")->value(), + after_second_report.probe_downloading_hit); + EXPECT_EQ(profile->get_counter("ProbeMiss")->value(), after_second_report.probe_miss); + EXPECT_EQ(profile->get_counter("BlockWaitSuccess")->value(), + after_second_report.block_wait_success); + EXPECT_EQ(profile->get_counter("BlockWaitTimeout")->value(), + after_second_report.block_wait_timeout); } } // namespace doris diff --git a/be/test/io/cache/cached_remote_file_reader_test.cpp b/be/test/io/cache/cached_remote_file_reader_test.cpp index 84cce46c9980aa..911cb1869ba8c6 100644 --- a/be/test/io/cache/cached_remote_file_reader_test.cpp +++ b/be/test/io/cache/cached_remote_file_reader_test.cpp @@ -15,9 +15,693 @@ // specific language governing permissions and limitations // under the License. +#include +#include + #include "block_file_cache_test_common.h" +#include "cloud/config.h" +#include "util/time.h" namespace doris::io { +namespace { + +FileCacheSettings async_reader_cache_settings() { + FileCacheSettings settings; + settings.query_queue_size = 8_mb; + settings.query_queue_elements = 16; + settings.index_queue_size = 2_mb; + settings.index_queue_elements = 4; + settings.disposable_queue_size = 2_mb; + settings.disposable_queue_elements = 4; + settings.capacity = 16_mb; + settings.max_file_block_size = 1_mb; + settings.max_query_cache_size = 0; + return settings; +} + +void reset_async_reader_cache_factory() { + FileCacheFactory::instance()->_caches.clear(); + FileCacheFactory::instance()->_path_to_cache.clear(); + FileCacheFactory::instance()->_capacity = 0; +} + +class CountingFileReader final : public FileReader { +public: + explicit CountingFileReader(FileReaderSPtr delegate) : _delegate(std::move(delegate)) {} + + Status close() override { return _delegate->close(); } + const Path& path() const override { return _delegate->path(); } + size_t size() const override { return _delegate->size(); } + bool closed() const override { return _delegate->closed(); } + int64_t mtime() const override { return _delegate->mtime(); } + + size_t read_count() const { return _read_count; } + size_t last_offset() const { return _last_offset; } + size_t last_size() const { return _last_size; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override { + ++_read_count; + _last_offset = offset; + _last_size = result.size; + return _delegate->read_at(offset, result, bytes_read, io_ctx); + } + +private: + FileReaderSPtr _delegate; + size_t _read_count {0}; + size_t _last_offset {0}; + size_t _last_size {0}; +}; + +// Provides one isolated cache disk with phase-one async-read settings for concise end-to-end +// reader tests. Every test gets a fresh FileCacheFactory entry and restores global knobs afterward. +class AsyncCachedRemoteFileReaderTest : public BlockFileCacheTest { +protected: + void SetUp() override { + _old_enable_async = config::enable_async_file_cache_write; + _old_enable_inflight = config::enable_async_file_cache_write_inflight_write_buffer_index; + _old_enable_direct = config::enable_read_cache_file_directly; + _old_enable_peer = config::enable_cache_read_from_peer; + _old_block_size = config::file_cache_each_block_size; + _old_wait_timeout_ms = config::block_cache_wait_timeout_ms; + + config::enable_async_file_cache_write = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + reset_async_reader_cache_factory(); + // FDCache is process-wide and keyed by cache hash plus offset, while these tests recreate + // the same logical file on a different cache path for every case. Reset it together with + // FileCacheFactory so one case cannot read an open descriptor retained by another case. + ExecEnv::GetInstance()->set_file_cache_open_fd_cache(std::make_unique()); + } + + void TearDown() override { + reset_async_reader_cache_factory(); + if (!_cache_path.empty()) { + std::error_code error; + fs::remove_all(_cache_path, error); + } + config::enable_async_file_cache_write = _old_enable_async; + config::enable_async_file_cache_write_inflight_write_buffer_index = _old_enable_inflight; + config::enable_read_cache_file_directly = _old_enable_direct; + config::enable_cache_read_from_peer = _old_enable_peer; + config::file_cache_each_block_size = _old_block_size; + config::block_cache_wait_timeout_ms = _old_wait_timeout_ms; + } + + // Create and initialize the single cache disk used by the current test. + void create_cache(std::string_view name) { + _cache_path = caches_dir / name; + std::error_code error; + fs::remove_all(_cache_path, error); + fs::create_directories(_cache_path); + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(_cache_path.string(), async_reader_cache_settings()) + .ok()); + _cache = FileCacheFactory::instance()->_path_to_cache[_cache_path.string()]; + ASSERT_NE(_cache, nullptr); + wait_until_cache_ready(*_cache); + } + + // Open the deterministic 10MB fixture file used as the remote source in cache tests. + FileReaderSPtr open_remote_file() { + FileReaderSPtr reader; + EXPECT_TRUE(global_local_filesystem()->open_file(tmp_file, &reader).ok()); + return reader; + } + + // Wrap a supplied remote source with the normal file-block-cache reader options. + std::shared_ptr create_reader(FileReaderSPtr remote_reader) { + FileReaderOptions options; + options.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + options.is_doris_table = true; + options.tablet_id = 10086; + return std::make_shared(std::move(remote_reader), options); + } + + // Wait until accepted writes and their inflight payload entries have both been finalized. + void wait_for_async_writes() { + ASSERT_NE(_cache, nullptr); + for (int attempt = 0; + attempt < 5000 && (_cache->async_write_service()->pending_count() != 0 || + _cache->inflight_write_buffer_index()->size() != 0); + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(_cache->async_write_service()->pending_count(), 0); + EXPECT_EQ(_cache->inflight_write_buffer_index()->size(), 0); + } + + BlockFileCache* cache() const { return _cache; } + +private: + fs::path _cache_path; + BlockFileCache* _cache = nullptr; + bool _old_enable_async = false; + bool _old_enable_inflight = false; + bool _old_enable_direct = false; + bool _old_enable_peer = false; + int64_t _old_block_size = 0; + int64_t _old_wait_timeout_ms = 0; +}; + +} // namespace + +TEST_F(AsyncCachedRemoteFileReaderTest, preallocated_cache_block_can_cover_the_short_file_tail) { + create_cache("cached_remote_reader_async_preallocated_file_tail"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + ASSERT_EQ(reader->size(), 10_mb + 1); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + const size_t file_tail_offset = reader->size() - 1; + auto holder = cache()->get_or_set(reader->_cache_hash, file_tail_offset, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& preallocated_tail = holder.file_blocks.front(); + ASSERT_EQ(preallocated_tail->range().left, file_tail_offset); + ASSERT_EQ(preallocated_tail->range().right, file_tail_offset + 1_mb - 1); + + std::string result(1, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(file_tail_offset, Slice(result.data(), result.size()), &bytes_read, + &context) + .ok()); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result, "0"); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 9_mb); + EXPECT_EQ(counting_reader->last_size(), 1_mb + 1); + EXPECT_EQ(stats.probe_miss, 2); + wait_for_async_writes(); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + downloading_preallocated_tail_can_finalize_short_while_reader_waits) { + create_cache("cached_remote_reader_async_downloading_preallocated_tail"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + ASSERT_EQ(reader->size(), 10_mb + 1); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + const size_t file_tail_offset = reader->size() - 1; + auto holder = cache()->get_or_set(reader->_cache_hash, file_tail_offset, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& downloading_tail = holder.file_blocks.front(); + ASSERT_EQ(downloading_tail->range().right, file_tail_offset + 1_mb - 1); + ASSERT_EQ(downloading_tail->get_or_set_downloader(), FileBlock::get_caller_id()); + + std::mutex wait_mutex; + std::condition_variable wait_cv; + bool reader_reached_wait = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "CachedRemoteFileReader::_materialize_async_block:before_wait", + [&](auto&&) { + { + std::lock_guard lock(wait_mutex); + reader_reached_wait = true; + } + wait_cv.notify_all(); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + std::string result(1, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + auto read_future = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return reader->read_at(file_tail_offset, Slice(result.data(), result.size()), &bytes_read, + &context); + }); + { + std::unique_lock lock(wait_mutex); + ASSERT_TRUE(wait_cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return reader_reached_wait; })); + } + EXPECT_EQ(read_future.wait_for(std::chrono::milliseconds(100)), std::future_status::timeout); + + const std::string tail_payload(1, '0'); + ASSERT_TRUE(downloading_tail->append(Slice(tail_payload.data(), tail_payload.size())).ok()); + ASSERT_TRUE(downloading_tail->finalize().ok()); + ASSERT_EQ(read_future.wait_for(std::chrono::seconds(5)), std::future_status::ready); + ASSERT_TRUE(read_future.get().ok()); + + EXPECT_EQ(downloading_tail->range().left, file_tail_offset); + EXPECT_EQ(downloading_tail->range().right, file_tail_offset); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result, tail_payload); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 9_mb); + EXPECT_EQ(counting_reader->last_size(), 1_mb); + EXPECT_EQ(stats.bytes_read_from_local, 1); + EXPECT_EQ(stats.bytes_read_from_remote, 0); + EXPECT_EQ(stats.probe_miss, 1); + EXPECT_EQ(stats.probe_downloading_hit, 1); + EXPECT_EQ(stats.block_wait_success, 1); + EXPECT_EQ(stats.async_cache_write_submitted, 1); + wait_for_async_writes(); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + partial_inflight_coverage_combines_with_existing_cache_block) { + create_cache("cached_remote_reader_async_partial_inflight_and_cache"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache()->get_or_set(reader->_cache_hash, 0, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& cached_block = holder.file_blocks.front(); + ASSERT_EQ(cached_block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string cached_payload(1_mb, '0'); + ASSERT_TRUE(cached_block->append(Slice(cached_payload.data(), cached_payload.size())).ok()); + ASSERT_TRUE(cached_block->finalize().ok()); + + auto* service = cache()->async_write_service(); + auto* inflight_index = cache()->inflight_write_buffer_index(); + AsyncCacheWriteBufferPtr inflight_buffer; + ASSERT_TRUE(service->allocate_tracked_buffer(1_mb, &inflight_buffer).ok()); + const std::string inflight_payload(1_mb, '1'); + memcpy(inflight_buffer->data(), inflight_payload.data(), inflight_payload.size()); + auto inflight_entry = std::make_shared( + inflight_buffer, 1_mb, 1_mb, MonotonicMicros(), service->current_write_epoch()); + ASSERT_EQ(inflight_index->insert_if_absent(reader->_cache_hash, 1_mb, inflight_entry), nullptr); + Defer remove_inflight {[&]() { + static_cast(inflight_index->remove_if(reader->_cache_hash, 1_mb, inflight_entry)); + }}; + + std::string result(2_mb, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result.substr(0, 1_mb), cached_payload); + EXPECT_EQ(result.substr(1_mb), inflight_payload); + EXPECT_EQ(counting_reader->read_count(), 0); + EXPECT_EQ(stats.bytes_read_from_local, 2_mb); + EXPECT_EQ(stats.bytes_read_from_remote, 0); + EXPECT_EQ(stats.inflight_write_buffer_index_hit, 1); + EXPECT_EQ(stats.inflight_write_buffer_index_miss, 1); + EXPECT_EQ(stats.probe_downloaded_hit, 1); + EXPECT_EQ(stats.probe_miss, 0); + EXPECT_EQ(stats.async_cache_write_submitted, 0); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + missing_cache_file_falls_back_and_removes_the_complete_cache_hash) { + create_cache("cached_remote_reader_async_self_heal"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + std::string initial_result(2_mb, '\0'); + FileCacheStatistics initial_stats; + IOContext initial_context; + initial_context.file_cache_stats = &initial_stats; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, Slice(initial_result.data(), initial_result.size()), &bytes_read, + &initial_context) + .ok()); + ASSERT_EQ(bytes_read, initial_result.size()); + wait_for_async_writes(); + + std::vector cache_files; + { + ReadStatistics probe_stats; + CacheContext probe_context; + probe_context.stats = &probe_stats; + auto probe_result = cache()->probe(reader->_cache_hash, 0, 2_mb, probe_context); + ASSERT_EQ(probe_result.file_blocks.size(), 2); + for (const auto& block : probe_result.file_blocks) { + ASSERT_NE(block, nullptr); + ASSERT_EQ(block->state(), FileBlock::State::DOWNLOADED); + cache_files.emplace_back(block->get_cache_file()); + } + } + ASSERT_EQ(cache_files.size(), 2); + ASSERT_NE(cache_files[0], cache_files[1]); + std::error_code remove_error; + ASSERT_TRUE(fs::remove(cache_files[0], remove_error)); + ASSERT_FALSE(remove_error); + + const uint64_t epoch_before_self_heal = cache()->async_write_service()->current_write_epoch(); + std::string fallback_result(4096, '\0'); + FileCacheStatistics fallback_stats; + IOContext fallback_context; + fallback_context.file_cache_stats = &fallback_stats; + bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, Slice(fallback_result.data(), fallback_result.size()), + &bytes_read, &fallback_context) + .ok()); + EXPECT_EQ(bytes_read, fallback_result.size()); + EXPECT_EQ(fallback_result, std::string(fallback_result.size(), '0')); + EXPECT_EQ(counting_reader->read_count(), 2); + EXPECT_EQ(fallback_stats.bytes_read_from_remote, fallback_result.size()); + EXPECT_EQ(fallback_stats.async_cache_write_submitted, 0); + EXPECT_EQ(cache()->async_write_service()->current_write_epoch(), epoch_before_self_heal + 1); + + bool hash_removed = false; + for (int attempt = 0; attempt < 5000; ++attempt) { + ReadStatistics probe_stats; + CacheContext probe_context; + probe_context.stats = &probe_stats; + bool metadata_removed = false; + { + auto probe_result = cache()->probe(reader->_cache_hash, 0, 2_mb, probe_context); + metadata_removed = + probe_result.file_blocks.size() == 2 && + std::all_of(probe_result.file_blocks.begin(), probe_result.file_blocks.end(), + [](const auto& block) { return block == nullptr; }); + } + const bool files_removed = + std::none_of(cache_files.begin(), cache_files.end(), + [](const fs::path& path) { return fs::exists(path); }); + if (metadata_removed && files_removed) { + hash_removed = true; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(hash_removed); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + downloading_cache_block_timeout_falls_back_without_duplicate_submission) { + create_cache("cached_remote_reader_async_wait_timeout"); + config::block_cache_wait_timeout_ms = 10; + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + std::mutex mutex; + std::condition_variable cv; + bool downloader_ready = false; + bool release_downloader = false; + std::thread downloader([&]() { + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache()->get_or_set(reader->_cache_hash, 0, 1_mb, cache_context); + DORIS_CHECK(holder.file_blocks.size() == 1); + DORIS_CHECK(holder.file_blocks.front()->get_or_set_downloader() == + FileBlock::get_caller_id()); + { + std::lock_guard lock(mutex); + downloader_ready = true; + } + cv.notify_all(); + std::unique_lock lock(mutex); + cv.wait(lock, [&]() { return release_downloader; }); + }); + Defer stop_downloader {[&]() { + { + std::lock_guard lock(mutex); + release_downloader = true; + } + cv.notify_all(); + if (downloader.joinable()) { + downloader.join(); + } + }}; + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return downloader_ready; })); + } + + std::string result(4096, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(result, std::string(result.size(), '0')); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 0); + EXPECT_EQ(counting_reader->last_size(), 1_mb); + EXPECT_EQ(stats.probe_downloading_hit, 1); + EXPECT_EQ(stats.block_wait_success, 0); + EXPECT_EQ(stats.block_wait_timeout, 1); + EXPECT_EQ(stats.bytes_read_from_remote, result.size()); + EXPECT_EQ(stats.async_cache_write_submitted, 0); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + direct_cache_prefix_is_preserved_while_async_path_fills_the_unread_suffix) { + create_cache("cached_remote_reader_async_direct_prefix"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache()->get_or_set(reader->_cache_hash, 0, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& prefix_block = holder.file_blocks.front(); + ASSERT_EQ(prefix_block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string cached_prefix(1_mb, 'x'); + ASSERT_TRUE(prefix_block->append(Slice(cached_prefix.data(), cached_prefix.size())).ok()); + ASSERT_TRUE(prefix_block->finalize().ok()); + config::enable_read_cache_file_directly = true; + reader->_insert_file_reader(prefix_block); + + std::string result(1_mb + 4096, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result.substr(0, 1_mb), cached_prefix); + EXPECT_EQ(result.substr(1_mb), std::string(4096, '1')); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 1_mb); + EXPECT_EQ(counting_reader->last_size(), 1_mb); + EXPECT_EQ(stats.bytes_read_from_local, 1_mb); + EXPECT_EQ(stats.bytes_read_from_remote, 4096); + EXPECT_EQ(stats.async_cache_write_submitted, 1); + wait_for_async_writes(); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, concurrent_cold_reads_publish_only_one_async_write_task) { + create_cache("cached_remote_reader_async_concurrent_dedup"); + auto first_counting_reader = std::make_shared(open_remote_file()); + auto second_counting_reader = std::make_shared(open_remote_file()); + auto first_reader = create_reader(first_counting_reader); + auto second_reader = create_reader(second_counting_reader); + + std::mutex insert_mutex; + std::condition_variable insert_cv; + size_t insert_arrivals = 0; + bool release_inserters = false; + std::mutex worker_mutex; + std::condition_variable worker_cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard insert_guard; + SyncPoint::CallbackGuard worker_guard; + sync_point->set_call_back( + "CachedRemoteFileReader::_submit_async_write_tasks:before_inflight_insert", + [&](auto&&) { + std::unique_lock lock(insert_mutex); + ++insert_arrivals; + if (insert_arrivals == 2) { + release_inserters = true; + insert_cv.notify_all(); + } + insert_cv.wait(lock, [&]() { return release_inserters; }); + }, + &insert_guard); + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(worker_mutex); + worker_entered = true; + worker_cv.notify_all(); + worker_cv.wait(lock, [&]() { return release_worker; }); + }, + &worker_guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(insert_mutex); + release_inserters = true; + } + insert_cv.notify_all(); + { + std::lock_guard lock(worker_mutex); + release_worker = true; + } + worker_cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + std::string first_result(4096, '\0'); + std::string second_result(4096, '\0'); + FileCacheStatistics first_stats; + FileCacheStatistics second_stats; + IOContext first_context; + IOContext second_context; + first_context.file_cache_stats = &first_stats; + second_context.file_cache_stats = &second_stats; + size_t first_bytes_read = 0; + size_t second_bytes_read = 0; + auto first_future = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return first_reader->read_at(0, Slice(first_result.data(), first_result.size()), + &first_bytes_read, &first_context); + }); + auto second_future = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return second_reader->read_at(0, Slice(second_result.data(), second_result.size()), + &second_bytes_read, &second_context); + }); + const auto first_status = first_future.wait_for(std::chrono::seconds(5)); + const auto second_status = second_future.wait_for(std::chrono::seconds(5)); + if (first_status != std::future_status::ready || second_status != std::future_status::ready) { + { + std::lock_guard lock(insert_mutex); + release_inserters = true; + } + insert_cv.notify_all(); + { + std::lock_guard lock(worker_mutex); + release_worker = true; + } + worker_cv.notify_all(); + } + ASSERT_EQ(first_status, std::future_status::ready); + ASSERT_EQ(second_status, std::future_status::ready); + ASSERT_TRUE(first_future.get().ok()); + ASSERT_TRUE(second_future.get().ok()); + EXPECT_EQ(first_result, std::string(first_result.size(), '0')); + EXPECT_EQ(second_result, std::string(second_result.size(), '0')); + EXPECT_EQ(first_counting_reader->read_count(), 1); + EXPECT_EQ(second_counting_reader->read_count(), 1); + EXPECT_EQ(first_stats.async_cache_write_submitted + second_stats.async_cache_write_submitted, + 1); + EXPECT_EQ(cache()->async_write_service()->pending_count(), 1); + EXPECT_EQ(cache()->inflight_write_buffer_index()->size(), 1); + { + std::unique_lock lock(worker_mutex); + ASSERT_TRUE(worker_cv.wait_for(lock, std::chrono::seconds(5), + [&]() { return worker_entered; })); + release_worker = true; + } + worker_cv.notify_all(); + wait_for_async_writes(); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + async_buffer_allocation_failure_returns_remote_data_without_publishing_state) { + create_cache("cached_remote_reader_async_allocation_failure"); + auto counting_reader = std::make_shared(open_remote_file()); + auto reader = create_reader(counting_reader); + + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::allocate_tracked_buffer:inject_failure", + [&](auto&& values) { + auto* status = try_any_cast(values.back()); + *status = Status::MemoryAllocFailed("injected async write buffer failure"); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + std::string result(4096, '\0'); + FileCacheStatistics stats; + IOContext context; + context.file_cache_stats = &stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(result, std::string(result.size(), '0')); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(stats.async_cache_write_buffer_alloc_fail, 1); + EXPECT_EQ(stats.async_cache_write_submitted, 0); + EXPECT_EQ(cache()->async_write_service()->pending_count(), 0); + EXPECT_EQ(cache()->inflight_write_buffer_index()->size(), 0); + EXPECT_GE(cache()->async_write_service()->_buffer_alloc_fail_metric->get_value(), 1); +} + +TEST_F(AsyncCachedRemoteFileReaderTest, + external_table_reader_uses_async_write_and_reuses_downloaded_cache) { + create_cache("cached_remote_reader_async_external_table"); + // Disabling the optional inflight index must only remove memory reuse/deduplication; accepted + // tasks still persist through AsyncCacheWriteService and become readable cache blocks. + config::enable_async_file_cache_write_inflight_write_buffer_index = false; + auto first_remote = std::make_shared(open_remote_file()); + FileReaderOptions options; + options.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + options.cache_base_path = cache()->get_base_path(); + options.mtime = 123; + auto first_reader = std::make_shared(first_remote, options); + + std::string first_result(4096, '\0'); + FileCacheStatistics first_stats; + IOContext first_context; + first_context.file_cache_stats = &first_stats; + size_t bytes_read = 0; + ASSERT_TRUE(first_reader + ->read_at(0, Slice(first_result.data(), first_result.size()), &bytes_read, + &first_context) + .ok()); + EXPECT_EQ(first_result, std::string(first_result.size(), '0')); + EXPECT_EQ(first_remote->read_count(), 1); + EXPECT_EQ(first_stats.async_cache_write_submitted, 1); + EXPECT_EQ(first_stats.inflight_write_buffer_index_hit, 0); + EXPECT_EQ(first_stats.inflight_write_buffer_index_miss, 0); + wait_for_async_writes(); + + auto second_remote = std::make_shared(open_remote_file()); + auto second_reader = std::make_shared(second_remote, options); + std::string second_result(4096, '\0'); + FileCacheStatistics second_stats; + IOContext second_context; + second_context.file_cache_stats = &second_stats; + bytes_read = 0; + ASSERT_TRUE(second_reader + ->read_at(4096, Slice(second_result.data(), second_result.size()), + &bytes_read, &second_context) + .ok()); + EXPECT_EQ(second_result, std::string(second_result.size(), '0')); + EXPECT_EQ(second_remote->read_count(), 0); + EXPECT_EQ(second_stats.probe_downloaded_hit, 1); + EXPECT_EQ(second_stats.async_cache_write_submitted, 0); +} TEST_F(BlockFileCacheTest, direct_partial_hit_with_downloaded_remainder_should_not_read_remote_again) { @@ -122,4 +806,588 @@ TEST_F(BlockFileCacheTest, config::enable_read_cache_file_directly = false; } +TEST_F(BlockFileCacheTest, async_write_reuses_inflight_buffer_then_reads_downloaded_block) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + config::enable_async_file_cache_write = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_async_write"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + std::mutex mutex; + std::condition_variable cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(local_reader, opts); + + std::string first_page(4096, '\0'); + FileCacheStatistics first_stats; + IOContext first_ctx; + first_ctx.file_cache_stats = &first_stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(first_page.data(), first_page.size()), &bytes_read, &first_ctx) + .ok()); + EXPECT_EQ(bytes_read, first_page.size()); + EXPECT_EQ(first_page, std::string(first_page.size(), '0')); + EXPECT_EQ(first_stats.async_cache_write_submitted, 1); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); + } + + std::string second_page(4096, '\0'); + FileCacheStatistics second_stats; + IOContext second_ctx; + second_ctx.file_cache_stats = &second_stats; + bytes_read = 0; + std::future second_read; + { + // Full inflight coverage must complete while the BlockFileCache mutex is unavailable, + // proving that this fast path does not call BlockFileCache::probe. + std::lock_guard cache_lock(cache->_mutex); + second_read = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return reader->read_at(4096, Slice(second_page.data(), second_page.size()), &bytes_read, + &second_ctx); + }); + EXPECT_EQ(second_read.wait_for(std::chrono::seconds(5)), std::future_status::ready); + } + ASSERT_TRUE(second_read.get().ok()); + EXPECT_EQ(bytes_read, second_page.size()); + EXPECT_EQ(second_page, std::string(second_page.size(), '0')); + EXPECT_EQ(second_stats.inflight_write_buffer_index_hit, 1); + EXPECT_EQ(second_stats.bytes_read_from_remote, 0); + EXPECT_EQ(second_stats.async_cache_write_submitted, 0); + + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + for (int attempt = 0; attempt < 500 && cache->async_write_service()->pending_count() != 0; + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_EQ(cache->async_write_service()->pending_count(), 0); + for (int attempt = 0; attempt < 500 && cache->inflight_write_buffer_index()->size() != 0; + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_EQ(cache->inflight_write_buffer_index()->size(), 0); + + std::string third_page(4096, '\0'); + FileCacheStatistics third_stats; + IOContext third_ctx; + third_ctx.file_cache_stats = &third_stats; + bytes_read = 0; + ASSERT_TRUE(reader->read_at(8192, Slice(third_page.data(), third_page.size()), &bytes_read, + &third_ctx) + .ok()); + EXPECT_EQ(bytes_read, third_page.size()); + EXPECT_EQ(third_page, std::string(third_page.size(), '0')); + EXPECT_EQ(third_stats.probe_downloaded_hit, 1); + EXPECT_EQ(third_stats.bytes_read_from_remote, 0); +} + +TEST_F(BlockFileCacheTest, async_write_waits_for_downloading_block_outside_remote_span) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const int64_t old_block_size = config::file_cache_each_block_size; + config::enable_async_file_cache_write = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + config::file_cache_each_block_size = old_block_size; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_wait_downloading"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + auto counting_reader = std::make_shared(local_reader); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(counting_reader, opts); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache->get_or_set(reader->_cache_hash, 0, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& downloading_block = holder.file_blocks.front(); + ASSERT_EQ(downloading_block->get_or_set_downloader(), FileBlock::get_caller_id()); + + std::string result(4096, '\0'); + FileCacheStatistics read_stats; + IOContext context; + context.file_cache_stats = &read_stats; + size_t bytes_read = 0; + auto read_future = std::async(std::launch::async, [&]() { + SCOPED_ATTACH_TASK(ExecEnv::GetInstance()->orphan_mem_tracker()); + return reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context); + }); + EXPECT_EQ(read_future.wait_for(std::chrono::milliseconds(100)), std::future_status::timeout); + + const std::string payload(1_mb, '0'); + ASSERT_TRUE(downloading_block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(downloading_block->finalize().ok()); + ASSERT_TRUE(read_future.get().ok()); + + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result, std::string(result.size(), '0')); + EXPECT_EQ(counting_reader->read_count(), 0); + EXPECT_EQ(read_stats.bytes_read_from_remote, 0); + EXPECT_EQ(read_stats.probe_downloading_hit, 1); + EXPECT_EQ(read_stats.block_wait_success, 1); + EXPECT_EQ(read_stats.block_wait_timeout, 0); + EXPECT_EQ(read_stats.async_cache_write_submitted, 0); +} + +TEST_F(BlockFileCacheTest, async_write_reads_only_middle_span_between_cached_sides) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const int64_t old_block_size = config::file_cache_each_block_size; + config::enable_async_file_cache_write = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + config::file_cache_each_block_size = old_block_size; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_middle_span"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(local_reader, opts); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto populate_block = [&](size_t offset, char value) { + auto holder = cache->get_or_set(reader->_cache_hash, offset, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& block = holder.file_blocks.front(); + ASSERT_EQ(block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(1_mb, value); + ASSERT_TRUE(block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(block->finalize().ok()); + }; + populate_block(0, '0'); + populate_block(2_mb, '2'); + + std::string result(3_mb, '\0'); + FileCacheStatistics read_stats; + IOContext context; + context.file_cache_stats = &read_stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result.substr(0, 1_mb), std::string(1_mb, '0')); + EXPECT_EQ(result.substr(1_mb, 1_mb), std::string(1_mb, '1')); + EXPECT_EQ(result.substr(2_mb, 1_mb), std::string(1_mb, '2')); + EXPECT_EQ(read_stats.bytes_read_from_local, 2_mb); + EXPECT_EQ(read_stats.bytes_read_from_remote, 1_mb); + EXPECT_EQ(read_stats.probe_downloaded_hit, 2); + EXPECT_EQ(read_stats.probe_miss, 1); + EXPECT_EQ(read_stats.async_cache_write_submitted, 1); + + for (int attempt = 0; attempt < 5000 && (cache->async_write_service()->pending_count() != 0 || + cache->inflight_write_buffer_index()->size() != 0); + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(cache->async_write_service()->pending_count(), 0); + EXPECT_EQ(cache->inflight_write_buffer_index()->size(), 0); +} + +TEST_F(BlockFileCacheTest, async_write_middle_hit_uses_one_remote_span_and_submits_only_misses) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const int64_t old_block_size = config::file_cache_each_block_size; + config::enable_async_file_cache_write = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + config::file_cache_each_block_size = old_block_size; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_middle_hit"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + auto counting_reader = std::make_shared(local_reader); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(counting_reader, opts); + + ReadStatistics cache_stats; + CacheContext cache_context; + cache_context.stats = &cache_stats; + auto holder = cache->get_or_set(reader->_cache_hash, 1_mb, 1_mb, cache_context); + ASSERT_EQ(holder.file_blocks.size(), 1); + const auto& middle_block = holder.file_blocks.front(); + ASSERT_EQ(middle_block->get_or_set_downloader(), FileBlock::get_caller_id()); + const std::string payload(1_mb, '1'); + ASSERT_TRUE(middle_block->append(Slice(payload.data(), payload.size())).ok()); + ASSERT_TRUE(middle_block->finalize().ok()); + { + std::lock_guard cache_lock(cache->_mutex); + cache->_files.at(reader->_cache_hash).at(1_mb).atime = 123; + } + + std::string result(3_mb, '\0'); + FileCacheStatistics read_stats; + IOContext context; + context.file_cache_stats = &read_stats; + size_t bytes_read = 0; + ASSERT_TRUE( + reader->read_at(0, Slice(result.data(), result.size()), &bytes_read, &context).ok()); + EXPECT_EQ(bytes_read, result.size()); + EXPECT_EQ(result.substr(0, 1_mb), std::string(1_mb, '0')); + EXPECT_EQ(result.substr(1_mb, 1_mb), std::string(1_mb, '1')); + EXPECT_EQ(result.substr(2_mb, 1_mb), std::string(1_mb, '2')); + EXPECT_EQ(counting_reader->read_count(), 1); + EXPECT_EQ(counting_reader->last_offset(), 0); + EXPECT_EQ(counting_reader->last_size(), 3_mb); + EXPECT_EQ(read_stats.bytes_read_from_local, 0); + EXPECT_EQ(read_stats.bytes_read_from_remote, 3_mb); + EXPECT_EQ(read_stats.probe_downloaded_hit, 1); + EXPECT_EQ(read_stats.probe_miss, 2); + EXPECT_EQ(read_stats.async_cache_write_submitted, 2); + { + std::lock_guard cache_lock(cache->_mutex); + EXPECT_EQ(cache->_files.at(reader->_cache_hash).at(1_mb).atime, 123); + } + + for (int attempt = 0; attempt < 5000 && (cache->async_write_service()->pending_count() != 0 || + cache->inflight_write_buffer_index()->size() != 0); + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(cache->async_write_service()->pending_count(), 0); + EXPECT_EQ(cache->inflight_write_buffer_index()->size(), 0); +} + +TEST_F(BlockFileCacheTest, async_write_backpressure_rolls_back_inflight_entry) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_inflight = + config::enable_async_file_cache_write_inflight_write_buffer_index; + const bool old_enable_direct = config::enable_read_cache_file_directly; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const int64_t old_block_size = config::file_cache_each_block_size; + config::enable_async_file_cache_write = true; + config::enable_async_file_cache_write_inflight_write_buffer_index = true; + config::enable_read_cache_file_directly = false; + config::enable_cache_read_from_peer = false; + config::file_cache_each_block_size = 1_mb; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_async_file_cache_write_inflight_write_buffer_index = old_enable_inflight; + config::enable_read_cache_file_directly = old_enable_direct; + config::enable_cache_read_from_peer = old_enable_peer; + config::file_cache_each_block_size = old_block_size; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_backpressure"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + auto* cache = FileCacheFactory::instance()->_path_to_cache[cache_path.string()]; + ASSERT_NE(cache, nullptr); + wait_until_cache_ready(*cache); + auto async_write_options = cache->async_write_service()->options(); + async_write_options.max_pending_tasks = 1; + ASSERT_TRUE(cache->async_write_service()->update_options(async_write_options).ok()); + + std::mutex mutex; + std::condition_variable cv; + bool worker_entered = false; + bool release_worker = false; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sync_point->set_call_back( + "AsyncCacheWriteService::_write_one:before_get_or_set", + [&](auto&&) { + std::unique_lock lock(mutex); + worker_entered = true; + cv.notify_all(); + cv.wait(lock, [&]() { return release_worker; }); + }, + &guard); + sync_point->enable_processing(); + Defer clear_sync_point {[&]() { + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + sync_point->disable_processing(); + sync_point->clear_all_call_backs(); + }}; + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(local_reader, opts); + + std::string first_page(4096, '\0'); + FileCacheStatistics first_stats; + IOContext first_context; + first_context.file_cache_stats = &first_stats; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, Slice(first_page.data(), first_page.size()), &bytes_read, + &first_context) + .ok()); + EXPECT_EQ(first_page, std::string(first_page.size(), '0')); + EXPECT_EQ(first_stats.async_cache_write_submitted, 1); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(cv.wait_for(lock, std::chrono::seconds(5), [&]() { return worker_entered; })); + } + + std::string second_page(4096, '\0'); + FileCacheStatistics second_stats; + IOContext second_context; + second_context.file_cache_stats = &second_stats; + bytes_read = 0; + ASSERT_TRUE(reader->read_at(1_mb, Slice(second_page.data(), second_page.size()), &bytes_read, + &second_context) + .ok()); + EXPECT_EQ(second_page, std::string(second_page.size(), '1')); + EXPECT_EQ(second_stats.async_cache_write_rejected, 1); + EXPECT_EQ(second_stats.bytes_read_from_remote, 4096); + EXPECT_EQ(cache->async_write_service()->pending_count(), 1); + EXPECT_EQ( + cache->inflight_write_buffer_index()->lookup( + reader->_cache_hash, 1_mb, cache->async_write_service()->current_write_epoch()), + nullptr); + EXPECT_EQ(cache->inflight_write_buffer_index()->size(), 1); + + { + std::lock_guard lock(mutex); + release_worker = true; + } + cv.notify_all(); + for (int attempt = 0; attempt < 5000 && (cache->async_write_service()->pending_count() != 0 || + cache->inflight_write_buffer_index()->size() != 0); + ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_EQ(cache->async_write_service()->pending_count(), 0); + EXPECT_EQ(cache->inflight_write_buffer_index()->size(), 0); +} + +TEST_F(BlockFileCacheTest, cache_write_mode_is_resolved_for_each_read_context) { + const bool old_enable_async = config::enable_async_file_cache_write; + const bool old_enable_peer = config::enable_cache_read_from_peer; + const std::string old_deploy_mode = config::deploy_mode; + config::enable_cache_read_from_peer = false; + Defer restore_config {[&]() { + config::enable_async_file_cache_write = old_enable_async; + config::enable_cache_read_from_peer = old_enable_peer; + config::deploy_mode = old_deploy_mode; + }}; + + reset_async_reader_cache_factory(); + const auto cache_path = caches_dir / "cached_remote_reader_mode_resolution"; + std::error_code error; + fs::remove_all(cache_path, error); + fs::create_directories(cache_path); + Defer cleanup_cache {[&]() { + reset_async_reader_cache_factory(); + std::error_code cleanup_error; + fs::remove_all(cache_path, cleanup_error); + }}; + ASSERT_TRUE(FileCacheFactory::instance() + ->create_file_cache(cache_path.string(), async_reader_cache_settings()) + .ok()); + + FileReaderSPtr local_reader; + ASSERT_TRUE(global_local_filesystem()->open_file(tmp_file, &local_reader).ok()); + FileReaderOptions opts; + opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE; + opts.is_doris_table = true; + opts.tablet_id = 10086; + auto reader = std::make_shared(local_reader, opts); + IOContext context; + + config::enable_async_file_cache_write = false; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + config::enable_async_file_cache_write = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::ASYNC_WRITE); + + context.cache_write_mode_override = CacheWriteMode::SYNC_WRITE; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + context.cache_write_mode_override = CacheWriteMode::ASYNC_WRITE; + context.is_dryrun = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + context.is_dryrun = false; + context.is_warmup = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + + context.is_warmup = false; + config::deploy_mode = "cloud"; + config::enable_cache_read_from_peer = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + context.bypass_peer_read = true; + EXPECT_EQ(reader->_resolve_cache_write_mode(&context), CacheWriteMode::ASYNC_WRITE); + + context.bypass_peer_read = false; + config::enable_cache_read_from_peer = false; + context.cache_write_mode_override.reset(); + opts.cache_write_mode = CacheWriteMode::SYNC_WRITE; + auto sync_reader = std::make_shared(local_reader, opts); + EXPECT_EQ(sync_reader->_resolve_cache_write_mode(&context), CacheWriteMode::SYNC_WRITE); + context.cache_write_mode_override = CacheWriteMode::ASYNC_WRITE; + EXPECT_EQ(sync_reader->_resolve_cache_write_mode(&context), CacheWriteMode::ASYNC_WRITE); +} + } // namespace doris::io diff --git a/be/test/io/cache/inflight_write_buffer_index_test.cpp b/be/test/io/cache/inflight_write_buffer_index_test.cpp new file mode 100644 index 00000000000000..bd5bfdf45c1da2 --- /dev/null +++ b/be/test/io/cache/inflight_write_buffer_index_test.cpp @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/cache/inflight_write_buffer_index.h" + +#include + +#include +#include +#include +#include + +#include "io/cache/block_file_cache.h" +#include "util/time.h" + +namespace doris::io { +namespace { + +std::shared_ptr make_entry(uint64_t epoch, size_t offset = 0, + size_t size = 4096) { + return std::make_shared(nullptr, offset, size, MonotonicMicros(), + epoch); +} + +TEST(InflightWriteBufferIndexTest, InsertLookupAndConditionalRemove) { + InflightWriteBufferIndex index(8, "inflight_index_basic_test"); + const auto hash = BlockFileCache::hash("basic"); + auto entry = make_entry(7); + const int64_t lock_wait_count = index._lock_wait_latency_metric->count(); + const int64_t lock_hold_count = index._lock_hold_latency_metric->count(); + + EXPECT_EQ(index.insert_if_absent(hash, 0, entry), nullptr); + EXPECT_EQ(index.size(), 1); + EXPECT_EQ(index.lookup(hash, 0, 7), entry); + + auto unexpected = make_entry(7); + EXPECT_FALSE(index.remove_if(hash, 0, unexpected)); + EXPECT_EQ(index.lookup(hash, 0, 7), entry); + EXPECT_TRUE(index.remove_if(hash, 0, entry)); + EXPECT_EQ(index.size(), 0); + EXPECT_EQ(index.lookup(hash, 0, 7), nullptr); + EXPECT_EQ(index._lock_wait_latency_metric->count() - lock_wait_count, 6); + EXPECT_EQ(index._lock_hold_latency_metric->count() - lock_hold_count, 6); +} + +TEST(InflightWriteBufferIndexTest, LookupAllSupportsPartialHit) { + InflightWriteBufferIndex index(8, "inflight_index_lookup_all_test"); + const auto hash = BlockFileCache::hash("lookup_all"); + auto first = make_entry(3, 0); + auto third = make_entry(3, 8192); + ASSERT_EQ(index.insert_if_absent(hash, 0, first), nullptr); + ASSERT_EQ(index.insert_if_absent(hash, 8192, third), nullptr); + + auto results = index.lookup_all(hash, {0, 4096, 8192}, 3); + ASSERT_EQ(results.size(), 3); + EXPECT_EQ(results[0].block_offset, 0); + EXPECT_EQ(results[0].entry, first); + EXPECT_EQ(results[1].block_offset, 4096); + EXPECT_EQ(results[1].entry, nullptr); + EXPECT_EQ(results[2].block_offset, 8192); + EXPECT_EQ(results[2].entry, third); +} + +TEST(InflightWriteBufferIndexTest, NewEpochReplacesOldWithoutOldCallbackDeletingNew) { + InflightWriteBufferIndex index(8, "inflight_index_epoch_test"); + const auto hash = BlockFileCache::hash("epoch"); + auto old_entry = make_entry(10); + auto new_entry = make_entry(11); + ASSERT_EQ(index.insert_if_absent(hash, 0, old_entry), nullptr); + + EXPECT_EQ(index.lookup(hash, 0, 11), nullptr); + EXPECT_EQ(index.size(), 0); + ASSERT_EQ(index.insert_if_absent(hash, 0, old_entry), nullptr); + EXPECT_EQ(index.insert_if_absent(hash, 0, new_entry), nullptr); + EXPECT_EQ(index.size(), 1); + EXPECT_FALSE(index.remove_if(hash, 0, old_entry)); + EXPECT_EQ(index.lookup(hash, 0, 11), new_entry); + + auto stale_candidate = make_entry(10); + EXPECT_EQ(index.insert_if_absent(hash, 0, stale_candidate), new_entry); + EXPECT_EQ(index.lookup(hash, 0, 11), new_entry); +} + +TEST(InflightWriteBufferIndexTest, ConcurrentInsertPublishesExactlyOneEntry) { + InflightWriteBufferIndex index(64, "inflight_index_concurrent_test"); + const auto hash = BlockFileCache::hash("concurrent"); + constexpr size_t thread_count = 32; + std::vector> candidates; + candidates.reserve(thread_count); + for (size_t i = 0; i < thread_count; ++i) { + candidates.emplace_back(make_entry(20)); + } + + std::atomic inserted_count {0}; + std::vector threads; + threads.reserve(thread_count); + for (size_t i = 0; i < thread_count; ++i) { + threads.emplace_back([&, i]() { + if (index.insert_if_absent(hash, 0, candidates[i]) == nullptr) { + inserted_count.fetch_add(1, std::memory_order_relaxed); + } + }); + } + for (auto& thread : threads) { + thread.join(); + } + + EXPECT_EQ(inserted_count.load(std::memory_order_relaxed), 1); + EXPECT_EQ(index.size(), 1); + auto published = index.lookup(hash, 0, 20); + ASSERT_NE(published, nullptr); + EXPECT_TRUE(std::find(candidates.begin(), candidates.end(), published) != candidates.end()); +} + +TEST(InflightWriteBufferIndexTest, ConcurrentNewEpochReplacementSurvivesOldRemoval) { + InflightWriteBufferIndex index(16, "inflight_index_concurrent_replace_test"); + const auto hash = BlockFileCache::hash("concurrent_replace"); + auto old_entry = make_entry(30); + auto new_entry = make_entry(31); + ASSERT_EQ(index.insert_if_absent(hash, 0, old_entry), nullptr); + + std::atomic start {false}; + std::vector threads; + threads.emplace_back([&]() { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + EXPECT_EQ(index.insert_if_absent(hash, 0, new_entry), nullptr); + }); + for (size_t thread_index = 0; thread_index < 16; ++thread_index) { + threads.emplace_back([&]() { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + index.remove_if(hash, 0, old_entry); + }); + } + start.store(true, std::memory_order_release); + for (auto& thread : threads) { + thread.join(); + } + + EXPECT_EQ(index.size(), 1); + EXPECT_EQ(index.lookup(hash, 0, 31), new_entry); + EXPECT_FALSE(index.remove_if(hash, 0, old_entry)); + EXPECT_EQ(index.lookup(hash, 0, 31), new_entry); +} + +} // namespace +} // namespace doris::io diff --git a/be/test/storage/cloud_file_cache_write_index_only_test.cpp b/be/test/storage/cloud_file_cache_write_index_only_test.cpp index c330b93fee489b..49f43e80cf16b0 100644 --- a/be/test/storage/cloud_file_cache_write_index_only_test.cpp +++ b/be/test/storage/cloud_file_cache_write_index_only_test.cpp @@ -404,6 +404,8 @@ class CloudFileCacheWriteIndexOnlyTest : public testing::Test { EXPECT_TRUE(io_ctx->is_index_data); EXPECT_TRUE(io_ctx->is_dryrun); EXPECT_FALSE(io_ctx->is_warmup); + ASSERT_TRUE(io_ctx->cache_write_mode_override.has_value()); + EXPECT_EQ(*io_ctx->cache_write_mode_override, io::CacheWriteMode::SYNC_WRITE); observed->push_back(ObservedIndexPreload { .reason = ctx->reason, .segment_id = ctx->segment_id, diff --git a/bin/run-async-file-cache-write-microbench.sh b/bin/run-async-file-cache-write-microbench.sh new file mode 100755 index 00000000000000..a4d06c3410e1e0 --- /dev/null +++ b/bin/run-async-file-cache-write-microbench.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +doris_home="$(cd "${curdir}/.." && pwd)" +benchmark="${doris_home}/lib/async_file_cache_write_microbench" +if [[ ! -x "${benchmark}" ]]; then + benchmark="${doris_home}/output/be/lib/async_file_cache_write_microbench" +fi +if [[ ! -x "${benchmark}" ]]; then + echo "Cannot find async_file_cache_write_microbench under ${doris_home}." >&2 + exit 1 +fi + +cache_path="./output/async_file_cache_write_microbench" +benchmark_args=("$@") +for ((argument_index = 0; argument_index < ${#benchmark_args[@]}; ++argument_index)); do + argument="${benchmark_args[argument_index]}" + case "${argument}" in + --cache_path=*) + cache_path="${argument#*=}" + ;; + --cache_path) + argument_index=$((argument_index + 1)) + if [[ "${argument_index}" -ge "${#benchmark_args[@]}" ]]; then + echo "--cache_path requires a value." >&2 + exit 1 + fi + cache_path="${benchmark_args[argument_index]}" + ;; + esac +done +cache_path="${cache_path%/}" +if [[ -z "${cache_path}" ]]; then + echo "--cache_path cannot be empty or root." >&2 + exit 1 +fi + +run_fio="${RUN_FIO:-auto}" +case "${run_fio}" in +auto) + if command -v fio &>/dev/null; then + run_fio=1 + else + run_fio=0 + echo "DISK_BASELINE skipped: fio is not installed." + fi + ;; +1 | true | on) + if ! command -v fio &>/dev/null; then + echo "RUN_FIO=${run_fio}, but fio is not installed." >&2 + exit 1 + fi + run_fio=1 + ;; +0 | false | off) + run_fio=0 + ;; +*) + echo "RUN_FIO must be auto, 1, or 0." >&2 + exit 1 + ;; +esac + +if [[ "${run_fio}" -eq 1 ]]; then + fio_size="${FIO_SIZE:-1G}" + fio_runtime="${FIO_RUNTIME:-5}" + if [[ ! "${fio_runtime}" =~ ^[1-9][0-9]*$ ]]; then + echo "FIO_RUNTIME must be a positive integer number of seconds." >&2 + exit 1 + fi + + cache_parent="$(dirname "${cache_path}")" + mkdir -p "${cache_parent}" + fio_dir="$(mktemp -d "${cache_path}_fio.XXXXXX")" + echo "DISK_BASELINE path=${fio_dir} size=${fio_size} runtime_seconds=${fio_runtime}" + df -hT "${fio_dir}" + + fio --name=seqwrite_qd1 \ + --filename="${fio_dir}/seqwrite_qd1.data" \ + --size="${fio_size}" \ + --runtime="${fio_runtime}" \ + --ramp_time=1 \ + --time_based=1 \ + --rw=write \ + --bs=1m \ + --ioengine=libaio \ + --iodepth=1 \ + --numjobs=1 \ + --direct=1 \ + --invalidate=1 \ + --refill_buffers=1 \ + --group_reporting=1 \ + --eta=never \ + --unlink=1 + + fio --name=randwrite_qd16 \ + --filename="${fio_dir}/randwrite_qd16.data" \ + --size="${fio_size}" \ + --runtime="${fio_runtime}" \ + --ramp_time=1 \ + --time_based=1 \ + --rw=randwrite \ + --bs=1m \ + --ioengine=libaio \ + --iodepth=16 \ + --numjobs=1 \ + --direct=1 \ + --invalidate=1 \ + --refill_buffers=1 \ + --randrepeat=1 \ + --group_reporting=1 \ + --eta=never \ + --unlink=1 + + rmdir "${fio_dir}" +fi + +if [[ -n "${JAVA_HOME:-}" && -d "${JAVA_HOME}/lib/server" ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH:-}" +fi +exec "${benchmark}" "$@" diff --git a/build.sh b/build.sh index 7bcd0dc7439000..133d2ce24bd0e8 100755 --- a/build.sh +++ b/build.sh @@ -1138,6 +1138,8 @@ EOF if [[ "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" = "ON" ]]; then cp -r -p "${DORIS_HOME}/be/output/lib/file_cache_microbench" "${DORIS_OUTPUT}/be/lib"/ + cp -r -p "${DORIS_HOME}/be/output/lib/async_file_cache_write_microbench" "${DORIS_OUTPUT}/be/lib"/ + cp -r -p "${DORIS_HOME}/bin/run-async-file-cache-write-microbench.sh" "${DORIS_OUTPUT}/be/bin"/ fi if [[ "${BUILD_INDEX_TOOL}" = "ON" ]]; then diff --git a/regression-test/data/cloud_p0/cache/test_async_file_cache_write.out b/regression-test/data/cloud_p0/cache/test_async_file_cache_write.out new file mode 100644 index 00000000000000..d53f5b895d051e --- /dev/null +++ b/regression-test/data/cloud_p0/cache/test_async_file_cache_write.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !async_file_cache_first -- +536854528 16777216 + +-- !async_file_cache_second -- +536854528 16777216 + +-- !async_file_cache_sync_fallback -- +536854528 16777216 diff --git a/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy b/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy new file mode 100644 index 00000000000000..e78c339883125f --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/test_async_file_cache_write.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_async_file_cache_write", "docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + options.msNum = 1 + options.beConfigs += [ + "enable_file_cache=true", + "enable_async_file_cache_write=true", + "enable_async_file_cache_write_inflight_write_buffer_index=true", + "enable_read_cache_file_directly=false", + "enable_cache_read_from_peer=false", + "disable_storage_page_cache=true", + "disable_segment_cache=true", + "enable_evict_file_cache_in_advance=false", + "file_cache_each_block_size=262144", + "file_cache_enter_disk_resource_limit_mode_percent=99", + "file_cache_path=[{\"path\":\"/opt/apache-doris/be/storage/file_cache\",\"total_size\":134217728,\"query_limit\":134217728}]" + ] + + docker(options) { + def clusters = sql "SHOW CLUSTERS" + assertFalse(clusters.isEmpty()) + sql "use @${clusters[0][0]}" + + def backends = sql "SHOW BACKENDS" + assertEquals(backends.size(), 1) + def beHost = backends[0][1] + def beHttpPort = backends[0][4] + def beBrpcPort = backends[0][5] + + def clearFileCache = { + def (code, out, err) = curl( + "GET", "http://${beHost}:${beHttpPort}/api/file_cache?op=clear&sync=true") + assertTrue(code == 0 && out.contains("OK"), + "clear file cache failed, out=${out}, err=${err}") + } + + def readMetric = { String metricSuffix -> + def metrics = new URL("http://${beHost}:${beBrpcPort}/brpc_metrics").text + long total = 0 + metrics.eachLine { String line -> + if (!line.startsWith("#") && line.contains(metricSuffix)) { + def matcher = line =~ ~".*${metricSuffix}\\s+([0-9]+)\\s*" + if (matcher.matches()) { + total += matcher[0][1] as long + } + } + } + return total + } + + def setBeConfig = { String name, String value -> + def (code, out, err) = curl( + "POST", "http://${beHost}:${beHttpPort}/api/update_config?${name}=${value}") + assertTrue(code == 0 && out.contains("OK"), + "update_config ${name}=${value} failed, out=${out}, err=${err}") + } + + def waitForPendingWrites = { + long deadline = System.currentTimeMillis() + 30000L + while (System.currentTimeMillis() < deadline && + readMetric("async_cache_write_pending_count") != 0L) { + sleep(100) + } + assertEquals(readMetric("async_cache_write_pending_count"), 0L) + } + + sql "DROP TABLE IF EXISTS test_async_file_cache_write_phase1" + sql """ + CREATE TABLE test_async_file_cache_write_phase1 ( + k BIGINT NOT NULL, + v STRING NOT NULL + ) + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ) + """ + sql """ + INSERT INTO test_async_file_cache_write_phase1 + SELECT number, repeat(md5(cast(number AS STRING)), 16) + FROM numbers("number" = "32768") + """ + sql "SYNC" + + clearFileCache() + long submittedBefore = readMetric("async_cache_write_submitted_total") + order_qt_async_file_cache_first """ + SELECT sum(k), sum(length(v)) + FROM test_async_file_cache_write_phase1 + """ + long submittedAfter = readMetric("async_cache_write_submitted_total") + assertTrue(submittedAfter > submittedBefore, + "cold query should submit async cache writes: before=${submittedBefore}, " + + "after=${submittedAfter}") + + waitForPendingWrites() + long probeHitBefore = readMetric("cached_remote_file_reader_probe_hit_downloaded_total") + order_qt_async_file_cache_second """ + SELECT sum(k), sum(length(v)) + FROM test_async_file_cache_write_phase1 + """ + long probeHitAfter = readMetric("cached_remote_file_reader_probe_hit_downloaded_total") + assertTrue(probeHitAfter > probeHitBefore, + "second query should read downloaded cache blocks: before=${probeHitBefore}, " + + "after=${probeHitAfter}") + + setBeConfig("enable_async_file_cache_write", "false") + clearFileCache() + submittedBefore = readMetric("async_cache_write_submitted_total") + order_qt_async_file_cache_sync_fallback """ + SELECT sum(k), sum(length(v)) + FROM test_async_file_cache_write_phase1 + """ + submittedAfter = readMetric("async_cache_write_submitted_total") + assertEquals(submittedAfter, submittedBefore) + } +}