From 88541c7b208fa6c81fa55d15d06a8f2d0756955f Mon Sep 17 00:00:00 2001 From: EthanPeng-2048 Date: Sun, 25 Oct 2026 21:07:16 +0800 Subject: [PATCH] =?UTF-8?q?perf(kvrec):=20key=20index=20for=20O(1)=20keyed?= =?UTF-8?q?=20lookup,=20fixing=20O(n=C2=B2)=20per-key=20reads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KeyValueRecord::get/has linearly scanned fields_ on every call. Real callers (e.g. go.neuralnet.cpp parse_nngames over a .nngames container) do per-key reads over the whole record, yielding O(n²) string compares: 80k keys took 33.7s, and full-dataset parse was 51% of verify time. Add a private key_index_ (key -> first-occurrence offset): - set() overloads emplace into the index (first occurrence wins) - parse()/parse_at() rebuild the index after populating fields_ - has()/get() are read-only lookups; concurrent const reads stay safe - duplicate keys with type mismatch keep the old "first key+type match" semantics via forward fallback scan from the first occurrence - nested Record get() rebuilds the sub-record's index The container byte layout is unchanged (downstream data identity contract); this only touches the in-memory read path. Benchmark (issue #24, clang -std=c++23 -O2): 80k per-key gets 33.7s -> 0.014s; scaling now linear. Fixes #24 --- .../neuralnet.cpp/model_keyvalue_record.hpp | 61 +++++++++++++------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/include/neuralnet.cpp/model_keyvalue_record.hpp b/include/neuralnet.cpp/model_keyvalue_record.hpp index c7ea7fa8..b5e36c1f 100644 --- a/include/neuralnet.cpp/model_keyvalue_record.hpp +++ b/include/neuralnet.cpp/model_keyvalue_record.hpp @@ -49,6 +49,7 @@ #include #include #include +#include #include #include "core_errors.hpp" @@ -215,6 +216,31 @@ class KeyValueRecord }; std::vector fields_; + // 键索引：key → fields_ 首次出现下标（issue #24：逐键 get 的 O(n²)）。 + // set()/parse() 构建与维护，get()/has() 只读 —— 并发 const 读取安全。 + // 重复键：emplace 保留首个下标；类型不符时 find_field_ 从首个出现处 + // 继续前向扫描，与旧版"返回首个 key+type 匹配"语义逐例一致。 + std::unordered_map key_index_; + + void rebuild_index_() + { + key_index_.clear(); + for (std::size_t i = 0; i < fields_.size(); ++i) + key_index_.emplace(fields_[i].key, i); + } + + // 返回首个 key+type 匹配的字段；缺失或类型不符返回 nullptr + [[nodiscard]] const Field *find_field_(const std::string &key, Type type) const + { + const auto it = key_index_.find(key); + if (it == key_index_.end()) return nullptr; + // 首个出现处类型即匹配时 O(1)；否则（重复键 + 类型错位）前向兜底扫描 + for (std::size_t i = it->second; i < fields_.size(); ++i) + if (fields_[i].key == key && fields_[i].type == type) + return &fields_[i]; + return nullptr; + } + // v2 内部构件 static constexpr std::uint64_t SENTINEL_INLINE_SIZE = 4 + 13 + 1 + 4 + 8; [[nodiscard]] static std::uint64_t value_size_(const Field &f); @@ -232,24 +258,28 @@ class KeyValueRecord inline KeyValueRecord &KeyValueRecord::set(const std::string &key, uint64_t v) { + key_index_.emplace(key, fields_.size()); // emplace 保留首个出现下标 fields_.push_back(Field{key, Type::UInt, v, {}, {}, {}}); return *this; } inline KeyValueRecord &KeyValueRecord::set(const std::string &key, const std::string &v) { + key_index_.emplace(key, fields_.size()); // emplace 保留首个出现下标 fields_.push_back(Field{key, Type::Str, 0, v, {}, {}}); return *this; } inline KeyValueRecord &KeyValueRecord::set(const std::string &key, const std::vector &v) { + key_index_.emplace(key, fields_.size()); // emplace 保留首个出现下标 fields_.push_back(Field{key, Type::UIntArray, 0, {}, v, {}}); return *this; } inline KeyValueRecord &KeyValueRecord::set(const std::string &key, const KeyValueRecord &v) { + key_index_.emplace(key, fields_.size()); // emplace 保留首个出现下标 fields_.push_back(Field{key, Type::Record, 0, {}, {}, v.fields_}); return *this; } @@ -509,6 +539,7 @@ inline Result KeyValueRecord::parse_at(const ReadAt &read, + std::to_string(version) + "（当前支持 v2）"); NN_TRY(fields, parse_fields_v2_(read, c.pos, *count)); rec.fields_ = std::move(*fields); + rec.rebuild_index_(); return rec; } @@ -550,6 +581,7 @@ inline Result KeyValueRecord::parse_at(const ReadAt &read, NN_FAIL("KeyValueRecord::parse: 字段 '" + f.key + "' 值长度错误"); rec.fields_.push_back(std::move(f)); } + rec.rebuild_index_(); return rec; } @@ -560,42 +592,37 @@ inline Result KeyValueRecord::parse(std::string_view bytes) inline bool KeyValueRecord::has(const std::string &key) const { - for (const auto &f : fields_) - if (f.key == key) return true; - return false; + return key_index_.contains(key); } inline bool KeyValueRecord::get(const std::string &key, uint64_t &out) const { - for (const auto &f : fields_) - if (f.key == key && f.type == Type::UInt) { out = f.u; return true; } + if (const Field *f = find_field_(key, Type::UInt)) { out = f->u; return true; } return false; } inline bool KeyValueRecord::get(const std::string &key, std::string &out) const { - for (const auto &f : fields_) - if (f.key == key && f.type == Type::Str) { out = f.s; return true; } + if (const Field *f = find_field_(key, Type::Str)) { out = f->s; return true; } return false; } inline bool KeyValueRecord::get(const std::string &key, std::vector &out) const { - for (const auto &f : fields_) - if (f.key == key && f.type == Type::UIntArray) { out = f.arr; return true; } + if (const Field *f = find_field_(key, Type::UIntArray)) { out = f->arr; return true; } return false; } inline bool KeyValueRecord::get(const std::string &key, KeyValueRecord &out) const { - for (const auto &f : fields_) - if (f.key == key && f.type == Type::Record) - { - KeyValueRecord sub; - sub.fields_ = f.rec; - out = std::move(sub); - return true; - } + if (const Field *f = find_field_(key, Type::Record)) + { + KeyValueRecord sub; + sub.fields_ = f->rec; + sub.rebuild_index_(); + out = std::move(sub); + return true; + } return false; }