From 975b02943bce5a95b813a3c4639ec96605c52ac2 Mon Sep 17 00:00:00 2001 From: Ahmad Date: Fri, 5 Jun 2026 11:58:07 +0300 Subject: [PATCH] MDEV-39858: Reloading COSINE metric index from disk degrades search recall due to abs2 quantization noise When a vector is created in-memory using FVector::create() during normal inserts, its squared magnitude (abs2) under the COSINE metric is hardcoded to 0.5f. However, when the index is reloaded from disk (after a server restart, FLUSH TABLES, or ALTER TABLE), the index uses FVectorNode::load_from_record(). This method reads the stored scale and quantized int16 coordinates from the database record, and runs postprocess(). Inside postprocess(), abs2 is dynamically recomputed using floating-point math: abs2 = subabs2 + scale * scale * dot_product(d, d, vec_len) / 2; Because the coordinates stored on disk are quantized int16 values, this recalculation introduces rounding noise. This affects high dimensions datasets, and it is increasing as M increases. Added hardcoded abs2=0.5 to FVectorNode::load_from_record and removed postprocess() --- sql/vector_mhnsw.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/vector_mhnsw.cc b/sql/vector_mhnsw.cc index 14942735e887f..d018ffd71f1fa 100644 --- a/sql/vector_mhnsw.cc +++ b/sql/vector_mhnsw.cc @@ -103,7 +103,8 @@ struct FVector vec->scale= std::nextafter(vec->scale, FLT_MAX); for (size_t i= 0; i < vec_len; i++) vec->dims[i] = static_cast(std::round(get_float(v + i) / vec->scale)); - vec->postprocess(vec_len); + vec->fix_tail(vec_len); + vec->abs2= vec->scale * vec->scale * dot_product(vec->dims, vec->dims, vec_len) / 2; if (metric == COSINE) { if (vec->abs2 > 0.0f) @@ -113,12 +114,6 @@ struct FVector return vec; } - void postprocess(size_t vec_len) - { - fix_tail(vec_len); - abs2= scale * scale * dot_product(dims, dims, vec_len) / 2; - } - #ifdef AVX2_IMPLEMENTATION /************* AVX2 *****************************************************/ static constexpr size_t AVX2_bytes= 256/8; @@ -882,7 +877,12 @@ int FVectorNode::load_from_record(TABLE *graph) return my_errno= HA_ERR_CRASHED; FVector *vec_ptr= FVector::align_ptr(tref() + tref_len()); memcpy(vec_ptr->data(), v->ptr(), v->length()); - vec_ptr->postprocess(ctx->vec_len); + vec_ptr->fix_tail(ctx->vec_len); + if (ctx->metric == COSINE) + vec_ptr->abs2= 0.5f; + else + vec_ptr->abs2= vec_ptr->scale * vec_ptr->scale * + vec_ptr->dot_product(vec_ptr->dims, vec_ptr->dims, ctx->vec_len) / 2; longlong layer= graph->field[FIELD_LAYER]->val_int(); if (layer > 100) // 10e30 nodes at M=2, more at larger M's