From ea9d3ec5bc0bd436764a1c799fd926b214df7301 Mon Sep 17 00:00:00 2001 From: Mohamed Arbi Date: Tue, 24 Mar 2026 08:14:26 +0100 Subject: [PATCH 01/49] fix: pass timeout parameter to query_points in Qdrant clients (#725) --- vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py | 1 + vectordb_bench/backend/clients/qdrant_local/qdrant_local.py | 1 + 2 files changed, 2 insertions(+) diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index a2c8f9020..e7d7054d9 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -205,6 +205,7 @@ def search_embedding( query_filter=self.query_filter, search_params=self.db_case_config.search_param(), with_payload=self.db_case_config.with_payload, + timeout=timeout, ) res = points_res.points diff --git a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py index d1cf736d9..15c790c61 100644 --- a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py +++ b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py @@ -227,6 +227,7 @@ def search_embedding( limit=k, query_filter=f, search_params=SearchParams(**self.search_parameter), + timeout=timeout, ).points return [result.id for result in res] From 99c311510648dff36f57960c2beeaae3ecb9ad61 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Tue, 31 Mar 2026 18:02:43 +0800 Subject: [PATCH 02/49] enhance: Migrate PyMilvus orm to MilvusClient (#738) * enhance: Migrate PyMilvus orm to MilvusClient * chore: add .worktrees/ to .gitignore Signed-off-by: yangxuan --- .gitignore | 3 + README.md | 13 +- pyproject.toml | 33 +-- tests/pytest.ini | 3 +- tests/test_bench_runner.py | 3 +- tests/test_milvus.py | 39 ++++ .../backend/clients/milvus/milvus.py | 204 +++++++++--------- 7 files changed, 147 insertions(+), 151 deletions(-) create mode 100644 tests/test_milvus.py diff --git a/.gitignore b/.gitignore index 3ddb942c4..8985eeb4d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ venv/ results/ logs/ +# Worktrees +.worktrees/ + # AI rules CLAUDE.md AGENTS.md diff --git a/README.md b/README.md index ef498c263..fe215e7a7 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,6 @@ python >= 3.11 pip install vectordb-bench ``` -**Install all database clients** - -``` shell -pip install 'vectordb-bench[all]' -``` **Install the specific database client** ```shell @@ -42,7 +37,6 @@ All the database client supported | Optional database client | install command | |--------------------------|---------------------------------------------| | pymilvus, zilliz_cloud (*default*) | `pip install vectordb-bench` | -| all (*clients requirements might be conflict with each other*) | `pip install vectordb-bench[all]` | | qdrant | `pip install vectordb-bench[qdrant]` | | pinecone | `pip install vectordb-bench[pinecone]` | | weaviate | `pip install vectordb-bench[weaviate]` | @@ -225,7 +219,6 @@ Options: --ondisk Ondisk mode with binary quantization(32x compression) --oversample-factor Controls the degree of oversampling applied to minority classes in imbalanced datasets to improve model performance by balancing class distributions.(default 1.0) - # Quantization Type --quantization-type TEXT which type of quantization to use valid values [fp32, fp16, bq] @@ -294,13 +287,13 @@ Options: # Connection --cloud-id TEXT Elastic Cloud ID [required] --password TEXT Elastic Cloud password [required] - + # HNSW Index Parameters --m INTEGER HNSW M parameter [default: 16] --ef-construction INTEGER HNSW efConstruction parameter [default: 100] --num-candidates INTEGER Number of candidates for search [default: 100] --element-type [float|byte] Element type for vectors (float: 4 bytes, byte: 1 byte) [default: float] - + # Index Configuration --number-of-shards INTEGER Number of shards [default: 1] --number-of-replicas INTEGER Number of replicas [default: 0] @@ -311,7 +304,7 @@ Options: --use-routing BOOLEAN Whether to use routing [default: False] --use-rescore BOOLEAN Whether to use rescore [default: False] --oversample-ratio FLOAT Oversample ratio for rescore [default: 2.0] - + # Common Options --case-type [CapacityDim128|CapacityDim960|Performance768D100M|...] Case type diff --git a/pyproject.toml b/pyproject.toml index c5e30aa6b..905996f5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ "pydantic=0.10.1", ] dynamic = ["version"] @@ -51,37 +50,7 @@ test = [ "ruff", "pytest", ] -restful = [ "flask" ] - -all = [ - "grpcio==1.53.0", # for qdrant-client and pymilvus - "grpcio-tools==1.53.0", # for qdrant-client and pymilvus - "qdrant-client", - "pinecone", - "weaviate-client", - "elasticsearch", - "sqlalchemy", - "redis", - "chromadb", - "pgvector", - "psycopg", - "psycopg-binary", - "pgvecto_rs[psycopg3]>=0.2.2", - "opensearch-dsl", - "opensearch-py", - "memorydb", - "alibabacloud_ha3engine_vector", - "mariadb", - "PyMySQL", - "clickhouse-connect", - "pyvespa", - "lancedb", - "mysql-connector-python", - "turbopuffer[fast]", - 'zvec', - "endee==0.1.10", # compatible with pydantic<2 -] - +restful = [ "flask" ] qdrant = [ "qdrant-client" ] pinecone = [ "pinecone" ] weaviate = [ "weaviate-client" ] diff --git a/tests/pytest.ini b/tests/pytest.ini index ad082137c..e5915e89e 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,4 +1,5 @@ [pytest] -filterwarnings = +filterwarnings = ignore::UserWarning + ignore::DeprecationWarning diff --git a/tests/test_bench_runner.py b/tests/test_bench_runner.py index 5fab91067..7aff0c27e 100644 --- a/tests/test_bench_runner.py +++ b/tests/test_bench_runner.py @@ -1,5 +1,7 @@ import time import logging + +import ujson from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import ( DB, IndexType, CaseType, TaskConfig, CaseConfig, @@ -55,6 +57,5 @@ def test_performance_case_no_error(self): d = t.json(exclude={'db_config': {'password', 'api_key'}}) log.info(f"{d}") - import ujson loads = ujson.loads(d) log.info(f"{loads}") diff --git a/tests/test_milvus.py b/tests/test_milvus.py new file mode 100644 index 000000000..8cc391acc --- /dev/null +++ b/tests/test_milvus.py @@ -0,0 +1,39 @@ +"""E2E test for Milvus client using MilvusClient API. + +Requires a running Milvus instance at localhost:19530. +""" + +import logging + +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import IndexType +from vectordb_bench.backend.clients.milvus.config import MilvusConfig +from vectordb_bench.backend.cases import CaseType +from vectordb_bench.interface import BenchMarkRunner +from vectordb_bench.models import CaseConfig, TaskConfig + + +log = logging.getLogger(__name__) + + +class TestMilvus: + """E2E test for Milvus using Performance1536D50K (OpenAI 50K dataset).""" + + def test_performance_1536d_50k(self): + """Full benchmark: download dataset, insert, optimize (force merge), search.""" + runner = BenchMarkRunner() + + task_config = TaskConfig( + db=DB.Milvus, + db_config=MilvusConfig(uri=SecretStr("http://localhost:19530")), + db_case_config=DB.Milvus.case_config_cls(index_type=IndexType.Flat)(), + case_config=CaseConfig(case_id=CaseType.Performance1536D50K), + ) + + runner.run([task_config]) + runner._sync_running_task() + result = runner.get_results() + log.info(f"test result: {result}") + assert len(result) > 0 diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index b177af332..ead2979ff 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -5,7 +5,7 @@ from collections.abc import Iterable from contextlib import contextmanager -from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, MilvusException, utility +from pymilvus import DataType, MilvusClient, MilvusException from vectordb_bench.backend.filter import Filter, FilterOp @@ -51,76 +51,72 @@ def __init__( self._scalar_id_index_name = "id_sort_idx" self._scalar_labels_index_name = "labels_idx" - from pymilvus import connections - - connections.connect( + client = MilvusClient( uri=self.db_config.get("uri"), user=self.db_config.get("user"), password=self.db_config.get("password"), timeout=30, ) - if drop_old and utility.has_collection(self.collection_name): + + if drop_old and client.has_collection(self.collection_name): log.info(f"{self.name} client drop_old collection: {self.collection_name}") - utility.drop_collection(self.collection_name) + client.drop_collection(self.collection_name) + + if not client.has_collection(self.collection_name): + schema = MilvusClient.create_schema() + schema.add_field(self._primary_field, DataType.INT64, is_primary=True) + schema.add_field(self._scalar_id_field, DataType.INT64) + schema.add_field(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) - if not utility.has_collection(self.collection_name): - fields = [ - FieldSchema(self._primary_field, DataType.INT64, is_primary=True), - FieldSchema(self._scalar_id_field, DataType.INT64), - FieldSchema(self._vector_field, DataType.FLOAT_VECTOR, dim=dim), - ] if self.with_scalar_labels: is_partition_key = db_case_config.use_partition_key log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") - fields.append( - FieldSchema( - self._scalar_label_field, - DataType.VARCHAR, - max_length=256, - is_partition_key=is_partition_key, - ) + schema.add_field( + self._scalar_label_field, + DataType.VARCHAR, + max_length=256, + is_partition_key=is_partition_key, ) log.info(f"{self.name} create collection: {self.collection_name}") - # Create the collection - col = Collection( - name=self.collection_name, - schema=CollectionSchema(fields), - consistency_level="Session", + index_params = self._build_index_params() + client.create_collection( + collection_name=self.collection_name, + schema=schema, num_shards=self.db_config.get("num_shards", 1), + consistency_level="Session", + ) + client.create_index(self.collection_name, index_params) + client.load_collection( + self.collection_name, + replica_number=self.db_config.get("replica_number", 1), ) - self.create_index() - col.load(replica_number=self.db_config.get("replica_number", 1)) - - connections.disconnect("default") + client.close() - def create_index(self): - col = Collection(self.collection_name) - # vector index - col.create_index( - self._vector_field, - self.case_config.index_param(), + def _build_index_params(self): + index_params = MilvusClient.prepare_index_params() + vec_idx = self.case_config.index_param() + index_params.add_index( + field_name=self._vector_field, index_name=self._vector_index_name, + index_type=vec_idx.get("index_type", ""), + metric_type=vec_idx.get("metric_type", ""), + params=vec_idx.get("params", {}), ) - # scalar index for range-expr (int-filter) - col.create_index( - self._scalar_id_field, - index_params={ - "index_type": "STL_SORT", - }, + index_params.add_index( + field_name=self._scalar_id_field, index_name=self._scalar_id_index_name, + index_type="STL_SORT", ) - # scalar index for varchar (label-filter) if self.with_scalar_labels: - col.create_index( - self._scalar_label_field, - index_params={ - "index_type": "BITMAP", - }, + index_params.add_index( + field_name=self._scalar_label_field, index_name=self._scalar_labels_index_name, + index_type="BITMAP", ) + return index_params @contextmanager def init(self): @@ -130,65 +126,58 @@ def init(self): >>> self.insert_embeddings() >>> self.search_embedding() """ - from pymilvus import connections - - self.col: Collection | None = None - - connections.connect(**self.db_config, timeout=60) - # Grab the existing colection with connections - self.col = Collection(self.collection_name) - + self.client: MilvusClient | None = None + self.client = MilvusClient( + uri=self.db_config.get("uri"), + user=self.db_config.get("user"), + password=self.db_config.get("password"), + timeout=60, + ) yield - connections.disconnect("default") + self.client.close() + self.client = None + + def _wait_for_index(self): + while True: + info = self.client.describe_index(self.collection_name, self._vector_index_name) + if info.get("pending_index_rows", -1) == 0: + break + time.sleep(5) + + def _wait_for_compaction(self, compaction_id: int): + while True: + state = self.client.get_compaction_state(compaction_id) + if state == "Completed": + break + time.sleep(0.5) def _optimize(self): log.info(f"{self.name} optimizing before search") - self._post_insert() try: - self.col.load(refresh=True) - except Exception as e: - log.warning(f"{self.name} optimize error: {e}") - raise e from None - - def _post_insert(self): - try: - self.col.flush() - # wait for index done and load refresh - self.create_index() - - utility.wait_for_index_building_complete(self.collection_name, index_name=self._vector_index_name) - - def wait_index(): - while True: - progress = utility.index_building_progress(self.collection_name, index_name=self._vector_index_name) - if progress.get("pending_index_rows", -1) == 0: - break - time.sleep(5) - - wait_index() - - # Skip compaction if use GPU indexType + self.client.flush(self.collection_name) + self._wait_for_index() if self.case_config.is_gpu_index: - log.debug("skip compaction for gpu index type.") + log.debug("skip force merge compaction for gpu index type.") else: try: - self.col.compact() - self.col.wait_for_compaction_completed() - log.info("compactation completed. waiting for the rest of index buliding.") + compaction_id = self.client.compact(self.collection_name, target_size=(2**63 - 1)) + if compaction_id > 0: + self._wait_for_compaction(compaction_id) + log.info(f"{self.name} force merge compaction completed.") + self._wait_for_index() except Exception as e: log.warning(f"{self.name} compact error: {e}") - if hasattr(e, "code"): - if e.code().name == "PERMISSION_DENIED": - log.warning("Skip compact due to permission denied.") + if hasattr(e, "code") and e.code().name == "PERMISSION_DENIED": + log.warning("Skip compact due to permission denied.") else: - raise e from e - wait_index() + raise e from None + self.client.refresh_load(self.collection_name) except Exception as e: log.warning(f"{self.name} optimize error: {e}") raise e from None def optimize(self, data_size: int | None = None): - assert self.col, "Please call self.init() before" + assert self.client, "Please call self.init() before" self._optimize() def need_normalize_cosine(self) -> bool: @@ -207,22 +196,24 @@ def insert_embeddings( **kwargs, ) -> tuple[int, Exception]: """Insert embeddings into Milvus. should call self.init() first""" - # use the first insert_embeddings to init collection - assert self.col is not None + assert self.client is not None assert len(embeddings) == len(metadata) insert_count = 0 try: for batch_start_offset in range(0, len(embeddings), self.batch_size): batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) - insert_data = [ - metadata[batch_start_offset:batch_end_offset], - metadata[batch_start_offset:batch_end_offset], - embeddings[batch_start_offset:batch_end_offset], - ] - if self.with_scalar_labels: - insert_data.append(labels_data[batch_start_offset:batch_end_offset]) - res = self.col.insert(insert_data) - insert_count += len(res.primary_keys) + batch_data = [] + for i in range(batch_start_offset, batch_end_offset): + row = { + self._primary_field: metadata[i], + self._scalar_id_field: metadata[i], + self._vector_field: embeddings[i], + } + if self.with_scalar_labels: + row[self._scalar_label_field] = labels_data[i] + batch_data.append(row) + res = self.client.insert(self.collection_name, batch_data) + insert_count += res["insert_count"] except MilvusException as e: log.info(f"Failed to insert data: {e}") return insert_count, e @@ -246,16 +237,15 @@ def search_embedding( timeout: int | None = None, ) -> list[int]: """Perform a search on a query embedding and return results.""" - assert self.col is not None + assert self.client is not None - # Perform the search. - res = self.col.search( + res = self.client.search( + collection_name=self.collection_name, data=[query], anns_field=self._vector_field, - param=self.case_config.search_param(), + search_params=self.case_config.search_param(), limit=k, - expr=self.expr, + filter=self.expr, ) - # Organize results. - return [result.id for result in res[0]] + return [result[self._primary_field] for result in res[0]] From 6f7a1534a3cdf753285296dc6c8a9d87587e7a83 Mon Sep 17 00:00:00 2001 From: nanlongyu Date: Wed, 1 Apr 2026 11:49:16 +0800 Subject: [PATCH 03/49] feat: add support for PolarDB (#737) - add PolarDB vector search client with FAISS_HNSW_FLAT, FAISS_HNSW_PQ, and FAISS_HNSW_SQ index types - add CLI integration with hnswflat, hnswpq, and hnswsq benchmark commands - add frontend (Streamlit) UI support with index type selection, HNSW/PQ/SQ parameter configuration --- README.md | 42 +++ install/requirements_py3.11.txt | 3 +- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 16 + .../backend/clients/polardb/__init__.py | 0 vectordb_bench/backend/clients/polardb/cli.py | 248 +++++++++++++++ .../backend/clients/polardb/config.py | 148 +++++++++ .../backend/clients/polardb/polardb.py | 286 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 8 + .../frontend/config/dbCaseConfigs.py | 127 ++++++++ vectordb_bench/frontend/config/styles.py | 1 + vectordb_bench/models.py | 5 + 12 files changed, 884 insertions(+), 1 deletion(-) create mode 100644 vectordb_bench/backend/clients/polardb/__init__.py create mode 100644 vectordb_bench/backend/clients/polardb/cli.py create mode 100644 vectordb_bench/backend/clients/polardb/config.py create mode 100644 vectordb_bench/backend/clients/polardb/polardb.py diff --git a/README.md b/README.md index fe215e7a7..1b0f46309 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ All the database client supported | hologres | `pip install vectordb-bench[hologres]` | | tencent_es | `pip install vectordb-bench[tencent_es]` | | alisql | `pip install 'vectordb-bench[alisql]'` | +| polardb | `pip install vectordb-bench[polardb]` | | doris | `pip install vectordb-bench[doris]` | | zvec | `pip install vectordb-bench[zvec]` | | endee | `pip install vectordb-bench[endee]` | @@ -520,6 +521,47 @@ To list the options for Lindorm, execute `vectordbbench lindormhnsw --help`, The --ef-search INTEGER hnsw ef-search [required] ``` +### Run PolarDB from command line + +PolarDB supports index types: faiss_hnsw_flat, faiss_hnsw_pq, and faiss_hnsw_sq. + +**Example: Run faiss_hnsw_flat benchmark** + +```shell +vectordbbench polardbhnswflat \ + --case-type Performance768D1M \ + --username \ + --password '' \ + --host \ + --port 3306 \ + --m 16 \ + --ef-construction 256 \ + --ef-search 256 \ + --insert-workers 64 \ + --num-concurrency '10,20,40,60,80' \ + --concurrency-duration 60 \ + --task-label \ + --db-label \ + --skip-search-serial \ + --post-load-index +``` + +To list the options for PolarDB, execute `vectordbbench polardbhnswflat --help`. The following are some PolarDB-specific command-line options. + +```text + --username TEXT Username [required] + --password TEXT Password + --host TEXT Db host [default: 127.0.0.1] + --port INTEGER Db Port [default: 3306] + --database TEXT Database name [default: vectordbbench] + --m INTEGER M parameter (max_degree) in HNSW + --ef-construction INTEGER ef_construction parameter in HNSW + --ef-search INTEGER polar_vector_index_hnsw_ef_search session variable + --insert-workers INTEGER Number of concurrent threads for data insertion + --post-load-index / --inline-index + Create index after load or inline at table creation +``` + #### Using a configuration file. The vectordbbench command can optionally read some or all the options from a yaml formatted configuration file. diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 5d5702492..4214267a3 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -26,5 +26,6 @@ pymilvus clickhouse_connect pyvespa mysql-connector-python +PyMySQL packaging -hdrhistogram>=0.10.1 \ No newline at end of file +hdrhistogram>=0.10.1 diff --git a/pyproject.toml b/pyproject.toml index 905996f5e..d7bf42633 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ vespa = [ "pyvespa" ] lancedb = [ "lancedb" ] oceanbase = [ "mysql-connector-python" ] alisql = [ "mysql-connector-python" ] +polardb = [ "PyMySQL" ] doris = [ "doris-vector-search" ] turbopuffer = [ "turbopuffer" ] zvec = [ "zvec" ] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index b5f3a4d6c..214d85e96 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -59,6 +59,7 @@ class DB(Enum): Zvec = "Zvec" Endee = "Endee" Lindorm = "Lindorm" + PolarDB = "PolarDB" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -246,6 +247,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormVector + if self == DB.PolarDB: + from .polardb.polardb import PolarDB + + return PolarDB + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -435,6 +441,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormConfig + if self == DB.PolarDB: + from .polardb.config import PolarDBConfig + + return PolarDBConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -581,6 +592,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return AliSQLIndexConfig + if self == DB.PolarDB: + from .polardb.config import _polardb_case_config + + return _polardb_case_config.get(index_type) + if self == DB.Doris: from .doris.config import DorisCaseConfig diff --git a/vectordb_bench/backend/clients/polardb/__init__.py b/vectordb_bench/backend/clients/polardb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/polardb/cli.py b/vectordb_bench/backend/clients/polardb/cli.py new file mode 100644 index 000000000..6f7a8f79c --- /dev/null +++ b/vectordb_bench/backend/clients/polardb/cli.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, Unpack + +if TYPE_CHECKING: + from .config import PolarDBConfig + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class PolarDBTypedDict(CommonTypedDict): + user_name: Annotated[ + str, + click.option( + "--username", + type=str, + help="Username", + required=True, + ), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="Password", + default="", + ), + ] + + host: Annotated[ + str, + click.option( + "--host", + type=str, + help="Db host", + default="127.0.0.1", + ), + ] + + port: Annotated[ + int, + click.option( + "--port", + type=int, + default=3306, + help="Db Port", + ), + ] + + database: Annotated[ + str, + click.option( + "--database", + type=str, + help="Database name", + default="vectordbbench", + ), + ] + + unix_socket: Annotated[ + str, + click.option( + "--unix-socket", + type=str, + help="Unix socket path (overrides host/port if set)", + default="", + ), + ] + + +class PolarDBHNSWTypedDict(PolarDBTypedDict): + m: Annotated[ + int, + click.option( + "--m", + type=int, + help="M parameter (max_degree) in HNSW", + default=16, + ), + ] + + ef_construction: Annotated[ + int, + click.option( + "--ef-construction", + type=int, + help="ef_construction parameter in HNSW", + default=200, + ), + ] + + ef_search: Annotated[ + int, + click.option( + "--ef-search", + type=int, + help="polar_vector_index_hnsw_ef_search session variable", + default=64, + ), + ] + + insert_workers: Annotated[ + int, + click.option( + "--insert-workers", + type=int, + help="Number of concurrent threads for data insertion", + default=10, + ), + ] + + post_load_index: Annotated[ + bool, + click.option( + "--post-load-index/--inline-index", + type=bool, + help="Create vector index via ALTER TABLE after data load; " + "otherwise create index inline during table creation", + default=False, + ), + ] + + +class PolarDBHNSWPQTypedDict(PolarDBHNSWTypedDict): + pq_m: Annotated[ + int, + click.option( + "--pq-m", + type=int, + help="PQ subquantizer count (must divide dimension)", + default=1, + ), + ] + + pq_nbits: Annotated[ + int, + click.option( + "--pq-nbits", + type=int, + help="PQ bits per subquantizer (max 24)", + default=8, + ), + ] + + +class PolarDBHNSWSQTypedDict(PolarDBHNSWTypedDict): + sq_type: Annotated[ + str, + click.option( + "--sq-type", + type=str, + help="SQ quantizer type (8bit, 4bit, fp16, bf16, 6bit, etc.)", + default="8bit", + ), + ] + + +def _build_db_config(parameters: dict) -> PolarDBConfig: + from .config import PolarDBConfig + + pwd = parameters["password"] + sock = parameters["unix_socket"] + return PolarDBConfig( + db_label=parameters["db_label"], + user_name=parameters["username"], + password=SecretStr(pwd) if pwd else None, + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + unix_socket=sock if sock else None, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PolarDBHNSWTypedDict) +def PolarDBHNSWFlat( + **parameters: Unpack[PolarDBHNSWTypedDict], +): + from .config import PolarDBHNSWFlatConfig + + run( + db=DB.PolarDB, + db_config=_build_db_config(parameters), + db_case_config=PolarDBHNSWFlatConfig( + M=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + insert_workers=parameters["insert_workers"], + post_load_index=parameters["post_load_index"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PolarDBHNSWPQTypedDict) +def PolarDBHNSWPQ( + **parameters: Unpack[PolarDBHNSWPQTypedDict], +): + from .config import PolarDBHNSWPQConfig + + run( + db=DB.PolarDB, + db_config=_build_db_config(parameters), + db_case_config=PolarDBHNSWPQConfig( + M=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + insert_workers=parameters["insert_workers"], + post_load_index=parameters["post_load_index"], + pq_m=parameters["pq_m"], + pq_nbits=parameters["pq_nbits"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(PolarDBHNSWSQTypedDict) +def PolarDBHNSWSQ( + **parameters: Unpack[PolarDBHNSWSQTypedDict], +): + from .config import PolarDBHNSWSQConfig + + run( + db=DB.PolarDB, + db_config=_build_db_config(parameters), + db_case_config=PolarDBHNSWSQConfig( + M=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + insert_workers=parameters["insert_workers"], + post_load_index=parameters["post_load_index"], + sq_type=parameters["sq_type"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/polardb/config.py b/vectordb_bench/backend/clients/polardb/config.py new file mode 100644 index 000000000..c75448c49 --- /dev/null +++ b/vectordb_bench/backend/clients/polardb/config.py @@ -0,0 +1,148 @@ +from typing import TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class PolarDBConfigDict(TypedDict): + user: str + password: str + host: str + port: int + database: str + unix_socket: str | None + + +class PolarDBConfig(DBConfig): + user_name: str = "root" + password: SecretStr | None = None + host: str = "127.0.0.1" + port: int = 3306 + database: str = "vectordbbench" + unix_socket: str | None = None + + @staticmethod + def common_long_configs() -> list[str]: + return ["note", "unix_socket"] + + def to_dict(self) -> PolarDBConfigDict: + pwd_str = self.password.get_secret_value() if self.password else "" + return { + "host": self.host, + "port": self.port, + "user": self.user_name, + "password": pwd_str, + "database": self.database, + "unix_socket": self.unix_socket or None, + } + + +class PolarDBIndexConfig(BaseModel): + """Base config for PolarDB vector index""" + + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "EUCLIDEAN" + if self.metric_type == MetricType.COSINE: + return "COSINE" + if self.metric_type == MetricType.IP: + return "INNER_PRODUCT" + msg = f"Metric type {self.metric_type} is not supported!" + raise ValueError(msg) + + def parse_metric_for_distance(self) -> str: + """Return the metric name used in DISTANCE() function""" + if self.metric_type == MetricType.L2: + return "EUCLIDEAN" + if self.metric_type == MetricType.COSINE: + return "COSINE" + if self.metric_type == MetricType.IP: + return "DOT" + msg = f"Metric type {self.metric_type} is not supported!" + raise ValueError(msg) + + +class PolarDBHNSWBaseConfig(PolarDBIndexConfig, DBCaseConfig): + """Shared HNSW config fields for all PolarDB HNSW variants.""" + + M: int = 16 + ef_construction: int = 200 + ef_search: int = 64 + insert_workers: int = 10 + post_load_index: bool = False # If True, create index after data load via ALTER TABLE + index: IndexType = IndexType.HNSW + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric_for_distance(), + "ef_search": self.ef_search, + } + + +class PolarDBHNSWFlatConfig(PolarDBHNSWBaseConfig): + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "metric_type_distance": self.parse_metric_for_distance(), + "index_type": "FAISS_HNSW_FLAT", + "M": self.M, + "ef_construction": self.ef_construction, + "vector_index_comment": ( + f"imci_vector_index=FAISS_HNSW_FLAT(" + f"metric={self.parse_metric()}," + f"max_degree={self.M}," + f"ef_construction={self.ef_construction})" + ), + } + + +class PolarDBHNSWPQConfig(PolarDBHNSWBaseConfig): + pq_m: int = 1 + pq_nbits: int = 8 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "metric_type_distance": self.parse_metric_for_distance(), + "index_type": "FAISS_HNSW_PQ", + "M": self.M, + "ef_construction": self.ef_construction, + "vector_index_comment": ( + f"imci_vector_index=FAISS_HNSW_PQ(" + f"metric={self.parse_metric()}," + f"max_degree={self.M}," + f"ef_construction={self.ef_construction}," + f"pq_m={self.pq_m}," + f"pq_nbits={self.pq_nbits})" + ), + } + + +class PolarDBHNSWSQConfig(PolarDBHNSWBaseConfig): + sq_type: str = "8bit" + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "metric_type_distance": self.parse_metric_for_distance(), + "index_type": "FAISS_HNSW_SQ", + "M": self.M, + "ef_construction": self.ef_construction, + "vector_index_comment": ( + f"imci_vector_index=FAISS_HNSW_SQ(" + f"metric={self.parse_metric()}," + f"max_degree={self.M}," + f"ef_construction={self.ef_construction}," + f"sq_type={self.sq_type})" + ), + } + + +_polardb_case_config = { + IndexType.HNSW: PolarDBHNSWFlatConfig, + IndexType.HNSW_PQ: PolarDBHNSWPQConfig, + IndexType.HNSW_SQ: PolarDBHNSWSQConfig, +} diff --git a/vectordb_bench/backend/clients/polardb/polardb.py b/vectordb_bench/backend/clients/polardb/polardb.py new file mode 100644 index 000000000..f42b6fca5 --- /dev/null +++ b/vectordb_bench/backend/clients/polardb/polardb.py @@ -0,0 +1,286 @@ +import concurrent.futures +import logging +import time +from contextlib import contextmanager + +import numpy as np +import pymysql + +from ..api import VectorDB +from .config import PolarDBConfigDict, PolarDBIndexConfig + +log = logging.getLogger(__name__) + + +class PolarDB(VectorDB): + def __init__( + self, + dim: int, + db_config: PolarDBConfigDict, + db_case_config: PolarDBIndexConfig, + collection_name: str = "vec_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "PolarDB" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + conn, cursor = self._create_connection() + + if drop_old: + log.info(f"PolarDB dropping old table: {self.table_name}") + self._create_db_table(cursor, dim) + + cursor.close() + conn.close() + + def _create_connection(self): + connect_kwargs = { + "user": self.db_config["user"], + "password": self.db_config["password"], + "autocommit": True, + } + if self.db_config.get("unix_socket"): + connect_kwargs["unix_socket"] = self.db_config["unix_socket"] + else: + connect_kwargs["host"] = self.db_config["host"] + connect_kwargs["port"] = self.db_config["port"] + + conn = pymysql.connect(**connect_kwargs) + cursor = conn.cursor() + # Disable query cache to ensure accurate benchmarking + cursor.execute("SET query_cache_type = OFF") + return conn, cursor + + def _create_db_table(self, cursor: pymysql.cursors.Cursor, dim: int) -> None: + index_param = self.case_config.index_param() + vector_index_comment = index_param["vector_index_comment"] + post_load_index = getattr(self.case_config, "post_load_index", False) + + try: + log.info(f"PolarDB creating database: {self.db_config['database']}") + cursor.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_config['database']}") + cursor.execute(f"USE {self.db_config['database']}") + cursor.execute(f"DROP TABLE IF EXISTS {self.table_name}") + + if post_load_index: + # Post-load mode: create table without vector index, will add via ALTER TABLE later + create_sql = ( + f"CREATE TABLE {self.table_name} (" + f"id INT PRIMARY KEY, " + f"v VECTOR({dim}) NOT NULL" + f") ENGINE=InnoDB COMMENT 'COLUMNAR=1'" + ) + log.info(f"PolarDB creating table (post-load index mode): {create_sql}") + else: + # Inline mode: create table with vector index comment + create_sql = ( + f"CREATE TABLE {self.table_name} (" + f"id INT PRIMARY KEY, " + f'v VECTOR({dim}) NOT NULL COMMENT "{vector_index_comment}"' + f") ENGINE=InnoDB COMMENT 'COLUMNAR=1'" + ) + log.info(f"PolarDB creating table: {create_sql}") + cursor.execute(create_sql) + except Exception as e: + log.warning(f"Failed to create table: {self.table_name} error: {e}") + raise + + @contextmanager + def init(self): + self.conn, self.cursor = self._create_connection() + + search_param = self.case_config.search_param() + + # Force PolarDB vector search to use the IMCI engine. + self.cursor.execute("SET use_imci_engine = FORCED") + self.cursor.execute("SET imci_enable_vector_search = ON") + self.cursor.execute("SET imci_max_dop = 1") + self.cursor.execute("SET cost_threshold_for_imci = 0") + + # Set ef_search + if search_param.get("ef_search") is not None: + self.cursor.execute(f"SET polar_vector_index_hnsw_ef_search = {search_param['ef_search']}") + + metric_type = search_param["metric_type"] + db_name = self.db_config["database"] + hint = "/*+ SET_VAR(imci_enable_fast_vector_search=on) */" + + self.insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" # noqa: S608 + self.select_sql = ( + f"SELECT {hint} id FROM {db_name}.{self.table_name} " # noqa: S608 + f"ORDER BY DISTANCE(v, _binary %s, '{metric_type}') " + f"LIMIT %s" + ) + self.select_sql_with_filter = ( + f"SELECT id FROM {db_name}.{self.table_name} " # noqa: S608 + f"WHERE id >= %s " + f"ORDER BY DISTANCE(v, _binary %s, '{metric_type}') " + f"LIMIT %s" + ) + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def ready_to_load(self) -> bool: + pass + + def optimize(self, data_size: int | None = None) -> None: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + db_name = self.db_config["database"] + index_param = self.case_config.index_param() + post_load_index = getattr(self.case_config, "post_load_index", False) + + # Ensure vector index builds even for small datasets + try: + self.cursor.execute("SET GLOBAL imci_vector_index_dump_rows_threshold = 1") + except Exception as e: + log.warning(f"Cannot SET GLOBAL imci_vector_index_dump_rows_threshold (need SUPER): {e}") + + start_time = time.time() + + if post_load_index: + # Post-load mode: issue ALTER TABLE to add vector index after data load + vector_index_comment = index_param["vector_index_comment"] + alter_sql = ( + f"ALTER TABLE {db_name}.{self.table_name} " + f'MODIFY COLUMN v VECTOR({self.dim}) NOT NULL COMMENT "{vector_index_comment}"' + ) + log.info(f"PolarDB creating vector index via ALTER TABLE: {alter_sql}") + self.cursor.execute(alter_sql) + log.info("ALTER TABLE completed.") + + analyze_sql = f"/* FORCE_IMCI_NODES */ ANALYZE TABLE {db_name}.{self.table_name}" + log.info(f"PolarDB analyzing table: {analyze_sql}") + analyze_start = time.time() + self.cursor.execute(analyze_sql) + analyze_elapsed = time.time() - analyze_start + + log.info(f"ANALYZE TABLE completed in {analyze_elapsed:.1f}s, waiting for vector index to be built...") + + last_vectors = 0 + while True: + self.cursor.execute( + "SELECT VECTORS FROM information_schema.imci_vector_index_stats " + "WHERE SCHEMA_NAME=%s AND TABLE_NAME=%s", + (db_name, self.table_name), + ) + result = self.cursor.fetchone() + + if result is None: + log.info("Vector index stats not yet available, waiting...") + time.sleep(2) + continue + + vectors = int(result[0]) + + if vectors != last_vectors: + elapsed = max(0.0, time.time() - start_time - analyze_elapsed) + log.info( + f"Vector index building: {vectors} vectors indexed " + f"(target: {data_size}), elapsed: {elapsed:.1f}s" + ) + last_vectors = vectors + + if data_size is not None and vectors >= data_size: + break + + # Also check imci_index_stats for vector_rows as secondary indicator + self.cursor.execute( + "SELECT VECTOR_ROWS FROM information_schema.imci_index_stats WHERE SCHEMA_NAME=%s AND TABLE_NAME=%s", + (db_name, self.table_name), + ) + idx_result = self.cursor.fetchone() + if idx_result and data_size is not None and int(idx_result[0]) >= data_size: + break + + time.sleep(2) + + total_time = max(0.0, time.time() - start_time - analyze_elapsed) + log.info(f"PolarDB vector index build completed in {total_time:.1f}s") + + @staticmethod + def vector_to_hex(v: list[float]) -> bytes: + return np.array(v, "float32").tobytes() + + def _insert_batch(self, embeddings: list[list[float]], metadata: list[int], offset: int, size: int) -> None: + """Insert a batch of embeddings using a dedicated connection.""" + conn, cursor = self._create_connection() + try: + db_name = self.db_config["database"] + insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" # noqa: S608 + batch_data = [] + for i in range(offset, offset + size): + batch_data.append((int(metadata[i]), self.vector_to_hex(embeddings[i]))) + + cursor.executemany(insert_sql, batch_data) + finally: + cursor.close() + conn.close() + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + workers = self.case_config.insert_workers + total = len(embeddings) + batch_size = max(1, total // workers) + + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = [] + for i in range(0, total, batch_size): + offset = i + size = min(batch_size, total - i) + future = executor.submit(self._insert_batch, embeddings, metadata, offset, size) + futures.append(future) + + done, pending = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_EXCEPTION) + for future in done: + future.result() + for future in pending: + future.cancel() + + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into table ({self.table_name}), error: {e}") + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + if filters: + self.cursor.execute( + self.select_sql_with_filter, + (filters.get("id"), self.vector_to_hex(query), k), + ) + else: + self.cursor.execute(self.select_sql, (self.vector_to_hex(query), k)) + return [row[0] for row in self.cursor.fetchall()] + except Exception: + log.exception("Failed to execute search query") + raise diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index dd704ae91..b48d5900c 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -25,6 +25,11 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn from ..backend.clients.pinecone.cli import Pinecone +from ..backend.clients.polardb.cli import ( + PolarDBHNSWFlat, + PolarDBHNSWPQ, + PolarDBHNSWSQ, +) from ..backend.clients.qdrant_cloud.cli import QdrantCloud from ..backend.clients.qdrant_local.cli import QdrantLocal from ..backend.clients.redis.cli import Redis @@ -82,6 +87,9 @@ cli.add_command(LindormHNSW) cli.add_command(LindormIVFBQ) cli.add_command(Pinecone) +cli.add_command(PolarDBHNSWFlat) +cli.add_command(PolarDBHNSWPQ) +cli.add_command(PolarDBHNSWSQ) if __name__ == "__main__": diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index e8c81c1d1..387f7fb4a 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -2847,6 +2847,129 @@ class FilterType(Enum): CaseConfigParamInput_NumberOfRegions_Lindorm, ] +# PolarDB configs +CaseConfigParamInput_IndexType_PolarDB = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_SQ.value, + ], + }, +) + +CaseConfigParamInput_M_PolarDB = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={ + "min": 2, + "max": 1024, + "value": 16, + }, + inputHelp="HNSW M parameter", +) + +CaseConfigParamInput_EFConstruction_PolarDB = CaseConfigInput( + label=CaseConfigParamType.ef_construction, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 8192, + "value": 200, + }, + inputHelp="ef_construction", +) + +CaseConfigParamInput_EFSearch_PolarDB = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 8192, + "value": 200, + }, + inputHelp="ef_search", +) + +CaseConfigParamInput_InsertWorkers_PolarDB = CaseConfigInput( + label=CaseConfigParamType.insert_workers, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 1024, + "value": 10, + }, + inputHelp="Number of insert workers", +) + +CaseConfigParamInput_PostLoadIndex_PolarDB = CaseConfigInput( + label=CaseConfigParamType.post_load_index, + inputType=InputType.Bool, + inputConfig={ + "value": True, + }, + inputHelp="Create index after data load via ALTER TABLE", +) + +CaseConfigParamInput_PQM_PolarDB = CaseConfigInput( + label=CaseConfigParamType.pq_m, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 16383, + "value": 1, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW_PQ.value, + inputHelp="PQ M parameter", +) + +CaseConfigParamInput_PQNbits_PolarDB = CaseConfigInput( + label=CaseConfigParamType.pq_nbits, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 24, + "value": 8, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW_PQ.value, + inputHelp="PQ nbits parameter", +) + +CaseConfigParamInput_SQType_PolarDB = CaseConfigInput( + label=CaseConfigParamType.sq_type, + inputType=InputType.Option, + inputConfig={ + "options": [ + "8bit", + "4bit", + "8bit_uniform", + "4bit_uniform", + "8bit_direct", + "8bit_direct_signed", + "fp16", + "bf16", + "6bit", + ], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW_SQ.value, + inputHelp="Scalar quantizer type", +) + +PolarDBConfig = [ + CaseConfigParamInput_IndexType_PolarDB, + CaseConfigParamInput_M_PolarDB, + CaseConfigParamInput_EFConstruction_PolarDB, + CaseConfigParamInput_EFSearch_PolarDB, + CaseConfigParamInput_InsertWorkers_PolarDB, + CaseConfigParamInput_PostLoadIndex_PolarDB, + CaseConfigParamInput_PQM_PolarDB, + CaseConfigParamInput_PQNbits_PolarDB, + CaseConfigParamInput_SQType_PolarDB, +] + # Map DB to config CASE_CONFIG_MAP = { DB.Milvus: { @@ -2937,6 +3060,10 @@ class FilterType(Enum): CaseLabel.Load: LindormLoadConfig, CaseLabel.Performance: LindormPerformanceConfig, }, + DB.PolarDB: { + CaseLabel.Load: PolarDBConfig, + CaseLabel.Performance: PolarDBConfig, + }, } diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index fabe9bdcf..268a2cd7d 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -74,6 +74,7 @@ def getPatternShape(i): DB.Zvec: "https://zvec.org/img/zvec-logo-light.svg", DB.Endee: "data:image/svg+xml,%3c?xml%20version=%271.0%27%20encoding=%27UTF-8%27?%3e%3csvg%20id=%27Layer_1%27%20xmlns=%27http://www.w3.org/2000/svg%27%20version=%271.1%27%20viewBox=%270%200%20600%20600%27%3e%3c!--%20Generator:%20Adobe%20Illustrator%2030.0.0,%20SVG%20Export%20Plug-In%20.%20SVG%20Version:%202.1.1%20Build%20123)%20--%3e%3cdefs%3e%3cstyle%3e%20.st0%20{%20fill:%20%233266a4;%20}%20%3c/style%3e%3c/defs%3e%3cpath%20class=%27st0%27%20d=%27M106.22,490.02H10.36l-.04-184.85c11.19-163.31,232.1-211.74,306.42-61.94,23.96,48.3,15.59,99.83,17,152.01h61.62c15.25,0,42.7-13.75,54.75-23.2,66.11-51.86,57.28-158.2-16.28-198.49-9.65-5.28-33.16-14.19-43.74-14.19h-154.31v-91.09c0-.87,2.55-1.86,3.63-1.63,104.05,4.23,201.15-21.64,284.48,55.84,119.18,110.8,69.12,325.47-91.33,362.6-28.65,6.63-85.47,7.76-115.02,4.8-19.71-1.97-43.29-16.57-55.97-31.45-37.98-44.56-20.77-98.07-24.7-151.16-5.18-69.99-100-85.31-125.9-20.47-1.16,2.92-4.76,13.88-4.76,16.3v186.91Z%27/%3e%3c/svg%3e", DB.Lindorm: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAJgAAACKCAYAAABW3IOxAAAMT2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSIQQIREBK6E0QqQGkhNACSC+CqIQkQCgxJgQVO7K4gmsXESwrugqi2FZAFhvqqiuLgr0uFlSUdXFd7MqbEECXfeV7831z57//nPnnnHPn3rkDAL2LL5XmopoA5EnyZbEhAazJySksUg8gAmNABjZAiy+QSznR0REAluH27+X1NYAo28sOSq1/9v/XoiUUyQUAINEQpwvlgjyIfwQAbxFIZfkAEKWQN5+VL1XidRDryKCDENcocaYKtyhxugpfGrSJj+VC/AgAsjqfL8sEQKMP8qwCQSbUocNogZNEKJZA7A+xb17eDCHEiyC2gTZwTrpSn53+lU7m3zTTRzT5/MwRrIplsJADxXJpLn/O/5mO/13ychXDc1jDqp4lC41Vxgzz9ihnRrgSq0P8VpIeGQWxNgAoLhYO2isxM0sRmqCyR20Eci7MGWBCPFGeG8cb4mOF/MBwiA0hzpDkRkYM2RRliIOVNjB/aIU4nxcPsR7ENSJ5UNyQzQnZjNjhea9lyLicIf4pXzbog1L/syIngaPSx7SzRLwhfcyxMCs+CWIqxIEF4sRIiDUgjpTnxIUP2aQWZnEjh21kilhlLBYQy0SSkACVPlaeIQuOHbLfnScfjh07kSXmRQ7hzvys+FBVrrBHAv6g/zAWrE8k4SQM64jkkyOGYxGKAoNUseNkkSQhTsXjetL8gFjVWNxOmhs9ZI8HiHJDlLwZxPHygrjhsQX5cHGq9PESaX50vMpPvDKbHxat8gffDyIAFwQCFlDAmg5mgGwgbu9t7IV3qp5gwAcykAlEwGGIGR6RNNgjgdc4UAh+h0gE5CPjAgZ7RaAA8p9GsUpOPMKprg4gY6hPqZIDHkOcB8JBLrxXDCpJRjxIBI8gI/6HR3xYBTCGXFiV/f+eH2a/MBzIRAwxiuEZWfRhS2IQMZAYSgwm2uIGuC/ujUfAqz+szjgb9xyO44s94TGhg/CAcJXQRbg5XVwkG+XlJNAF9YOH8pP+dX5wK6jphgfgPlAdKuNM3AA44K5wHg7uB2d2gyx3yG9lVlijtP8WwVdPaMiO4kRBKWMo/hSb0SM17DTcRlSUuf46Pypf00fyzR3pGT0/96vsC2EbPtoS+xY7hJ3FTmLnsRasEbCw41gT1oYdVeKRFfdocMUNzxY76E8O1Bm9Zr48WWUm5U51Tj1OH1V9+aLZ+cqXkTtDOkcmzszKZ3HgjiFi8SQCx3EsZydnNwCU+4/q8/YqZnBfQZhtX7glvwHgc3xgYOCnL1zYcQAOeMBPwpEvnA0bbi1qAJw7IlDIClQcrrwQ4JeDDt8+fbi/mcP9zQE4A3fgDfxBEAgDUSAeJINp0PssuM5lYBaYBxaDElAGVoH1oBJsBdtBDdgLDoJG0AJOgp/BBXAJXAW34erpBs9BH3gNPiAIQkJoCAPRR0wQS8QecUbYiC8ShEQgsUgykoZkIhJEgcxDliBlyBqkEtmG1CIHkCPISeQ80oHcRO4jPcifyHsUQ9VRHdQItULHo2yUg4aj8ehUNBOdiRaixegKtAKtRvegDehJ9AJ6Fe1Cn6P9GMDUMCZmijlgbIyLRWEpWAYmwxZgpVg5Vo3VY83wOV/GurBe7B1OxBk4C3eAKzgUT8AF+Ex8Ab4cr8Rr8Ab8NH4Zv4/34Z8JNIIhwZ7gReARJhMyCbMIJYRywk7CYcIZ+C51E14TiUQm0ZroAd/FZGI2cS5xOXEzcR/xBLGD+JDYTyKR9En2JB9SFIlPyieVkDaS9pCOkzpJ3aS3ZDWyCdmZHExOIUvIReRy8m7yMXIn+Qn5A0WTYknxokRRhJQ5lJWUHZRmykVKN+UDVYtqTfWhxlOzqYupFdR66hnqHeorNTU1MzVPtRg1sdoitQq1/Wrn1O6rvVPXVrdT56qnqivUV6jvUj+hflP9FY1Gs6L501Jo+bQVtFraKdo92lsNhoajBk9DqLFQo0qjQaNT4wWdQrekc+jT6IX0cvoh+kV6ryZF00qTq8nXXKBZpXlE87pmvxZDa4JWlFae1nKt3VrntZ5qk7SttIO0hdrF2tu1T2k/ZGAMcwaXIWAsYexgnGF06xB1rHV4Otk6ZTp7ddp1+nS1dV11E3Vn61bpHtXtYmJMKyaPmctcyTzIvMZ8P8ZoDGeMaMyyMfVjOse80Rur568n0ivV26d3Ve+9Pks/SD9Hf7V+o/5dA9zAziDGYJbBFoMzBr1jdcZ6jxWMLR17cOwtQ9TQzjDWcK7hdsM2w34jY6MQI6nRRqNTRr3GTGN/42zjdcbHjHtMGCa+JmKTdSbHTZ6xdFkcVi6rgnWa1WdqaBpqqjDdZtpu+sHM2izBrMhsn9ldc6o52zzDfJ15q3mfhYnFJIt5FnUWtywplmzLLMsNlmct31hZWyVZLbVqtHpqrWfNsy60rrO+Y0Oz8bOZaVNtc8WWaMu2zbHdbHvJDrVzs8uyq7K7aI/au9uL7Tfbd4wjjPMcJxlXPe66g7oDx6HAoc7hviPTMcKxyLHR8cV4i/Ep41ePPzv+s5ObU67TDqfbE7QnhE0omtA84U9nO2eBc5XzFReaS7DLQpcml5eu9q4i1y2uN9wYbpPclrq1un1y93CXude793hYeKR5bPK4ztZhR7OXs895EjwDPBd6tni+83L3yvc66PWHt4N3jvdu76cTrSeKJu6Y+NDHzIfvs82ny5flm+b7vW+Xn6kf36/a74G/ub/Qf6f/E44tJ5uzh/MiwClAFnA44A3XizufeyIQCwwJLA1sD9IOSgiqDLoXbBacGVwX3BfiFjI35EQoITQ8dHXodZ4RT8Cr5fWFeYTNDzsdrh4eF14Z/iDCLkIW0TwJnRQ2ae2kO5GWkZLIxigQxYtaG3U32jp6ZvRPMcSY6JiqmMexE2LnxZ6NY8RNj9sd9zo+IH5l/O0EmwRFQmsiPTE1sTbxTVJg0pqkrsnjJ8+ffCHZIFmc3JRCSklM2ZnSPyVoyvop3aluqSWp16ZaT5099fw0g2m5045Op0/nTz+URkhLStud9pEfxa/m96fz0jel9wm4gg2C50J/4Tphj8hHtEb0JMMnY03G00yfzLWZPVl+WeVZvWKuuFL8Mjs0e2v2m5yonF05A7lJufvyyHlpeUck2pIcyekZxjNmz+iQ2ktLpF0zvWaun9knC5ftlCPyqfKmfB34o9+msFF8o7hf4FtQVfB2VuKsQ7O1Zktmt82xm7NszpPC4MIf5uJzBXNb55nOWzzv/nzO/G0LkAXpC1oXmi8sXti9KGRRzWLq4pzFvxY5Fa0p+mtJ0pLmYqPiRcUPvwn5pq5Eo0RWcn2p99Kt3+Lfir9tX+aybOOyz6XC0l/KnMrKyz4uFyz/5bsJ31V8N7AiY0X7SveVW1YRV0lWXVvtt7pmjdaawjUP105a27COta503V/rp68/X+5avnUDdYNiQ1dFREXTRouNqzZ+rMyqvFoVULVvk+GmZZvebBZu7tziv6V+q9HWsq3vvxd/f2NbyLaGaqvq8u3E7QXbH+9I3HH2B/YPtTsNdpbt/LRLsqurJrbmdK1Hbe1uw90r69A6RV3PntQ9l/YG7m2qd6jfto+5r2w/2K/Y/+xA2oFrB8MPth5iH6r/0fLHTYcZh0sbkIY5DX2NWY1dTclNHUfCjrQ2ezcf/snxp10tpi1VR3WPrjxGPVZ8bOB44fH+E9ITvSczTz5snd56+9TkU1dOx5xuPxN+5tzPwT+fOss5e/ycz7mW817nj/zC/qXxgvuFhja3tsO/uv16uN29veGix8WmS56Xmjsmdhzr9Os8eTnw8s9XeFcuXI282nEt4dqN66nXu24Ibzy9mXvz5a2CWx9uL7pDuFN6V/Nu+T3De9W/2f62r8u96+j9wPttD+Ie3H4oePj8kfzRx+7ix7TH5U9MntQ+dX7a0hPcc+nZlGfdz6XPP/SW/K71+6YXNi9+/MP/j7a+yX3dL2UvB/5c/kr/1a6/XP9q7Y/uv/c67/WHN6Vv9d/WvGO/O/s+6f2TD7M+kj5WfLL91Pw5/POdgbyBASlfxh/8FcCA8miTAcCfuwCgJQPAgOdG6hTV+XCwIKoz7SAC/wmrzpCDxR2AevhPH9ML/26uA7B/BwBWUJ+eCkA0DYB4T4C6uIzU4bPc4LlTWYjwbPD9tE/peeng3xTVmfQrv0e3QKnqCka3/wLmpoMnuLWGFQAAAIplWElmTU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAACQAAAAAQAAAJAAAAABAAOShgAHAAAAEgAAAHigAgAEAAAAAQAAAJigAwAEAAAAAQAAAIoAAAAAQVNDSUkAAABTY3JlZW5zaG90rCu3yAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAdZpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDYuMC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iPgogICAgICAgICA8ZXhpZjpQaXhlbFlEaW1lbnNpb24+MTM4PC9leGlmOlBpeGVsWURpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6UGl4ZWxYRGltZW5zaW9uPjE1MjwvZXhpZjpQaXhlbFhEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PlNjcmVlbnNob3Q8L2V4aWY6VXNlckNvbW1lbnQ+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpf6GgjAAAAHGlET1QAAAACAAAAAAAAAEUAAAAoAAAARQAAAEUAACFD9bmq1QAAIQ9JREFUeAGcncu2JFdxhk+LBQJzsw1mgO0ZSIwl8RDm5ocw6pY9smH5LUD2DHXDU3DxQ4AaewYC7JFnXJbBmJuE2vFF5rfPX3Eyz2nYUtXeEfHHH7FzR+3MyqpTfe/tt99+clXt3r17V0+ePLno0duw0cCVT/fasseeTT90jJ955pk03znWP3kzfuqNISm29Gc88WKzT5/UOyb+XfM4iiWvPGe5pK8+ic35n3HJkf6Oz3zUZ6/PWXz1E4eM7V4lu1VOsaLAwMMDKJB+NvH22pFp00e9uLPemNjnYibnEV8e/OQ5wh7Fx4cH8zdW+h7pkgc7eHoeHseJUU5udNPvrvkbTz5leeS3N4b41KuTA/mu+OnjGM4VvwZdObkwAukzgR26dMriJT7zm3r95El/bfZpMyf9znj1fRrc2fzlmPHh5GEu5pCxjnTixck7uYxrL05O+vTJsT639TM+2Iwxfc1bPf48jl5AYujvFagLbO/axlhCe+30Sao+SZv4KXcx/TMOOuXJexb/NnxyPA3uaeObe/LfNTb+bb5PGz9jySd/2o7G4rXph/4oftqPxvLMfp0icZLYfoLV2087sjsBfI7RTxnd5EH21OQkwM02/bCjs+lL/NteDMbTXznznpxgxGnLXt85X2Xzkcdc6dGJS7382ulnEy8G+5z/lBPLmAf5OQc45GVsA2fTfqRrnnpaaME6Z58JpH6OM9C03SbjR/y7cjiz628M85h49eK0e1CR5Uqs+umnbJ8+6rI33hEubcZLHT5ZoPBOnLHkT3/xYrI/m7/+YOFUPuLXJpb+3u9///vCXieOkgYYvUQ5MW3g0u5YPTgSp+mfSbRhPMlhjMRjU04cFMg8jCPW+PoZbsrypV1O+ne84x1Xr//gN1eP3/jd1YPPfmDNO3mSgzE2F8685NeesmN6fM8w2sTpl/GnTYz95FZvj51H5i3/UfyJd95dYDjYACKrqwLsIMrgkswFNBGTkEc5OWcsZfqV2J7TEY84YyJnHGViwocNLHLi5BZvn7ky/+/+6M2rr37zf6vAfntVR+bq/mfeT8CrB5/7IC4XTX65nU/migN2HuqnHzmIO8o7fRu4P8mDKEfaHYM74kBnvNnLeeSLTju9814X+Sgld0x/V3MS6Tt9ZnDt0yfl6XOXvCa0L0zGkNcDhi3zVoZjLvhXv/XLq4ff/EUfGznFP/jMB7ZiK8WMb75g/5j4+NGSZ9McPxtDa85P2x8yf3nsZx7JD8YY4rXfWmBHThlIEklv6/VzktnrJyblXPCMN3ObcnI4tgebvOjhzgLjdHj/Sz++urffGG7+2r3qdd80nQvD2mgotJc//b5VtADAn7Xb4jtH+skxdcqzJy462uRQd9f8xSWPcbAd6Y/s/S5SQzpBQlO3SduzSU8/rL6SwWA/wugv520YbROLjM14jGnKM34b60lc5mAMek6Dng7bZy8i/amvLjIKiJh7jy+F9vlPvXfFyFiO4TG3HKdOPT1+t9mcr/zK+qiHKxt2mnb8LLrEMRbL2HwcGw/ZJob+4jYFgCRLh7SZlHZtEiPnGE6TP0pIPL3NPPSl59FJs7B7E4eYMXOcfo7TD9/v/uh3V4++sV1nIa/WC1HxCEkO1XEdxsgebGdUed3/9PuvXnz+XVcvPfdu1J0zseb8yU87vXIr68n80lddYtXJoez8sweDXZ0+6lNmTLPw8NP3KP6RDf8+RWqchA2IxUSmgbfPYOhMXk7ldgj79MMub/qc4RIjN736s/hgtIl/+I1fXD2qi/iukp7bXlBiMWz1UM6lXOMS5vHZ7fc/9b6r+/Vu03yIZSO+BafO3gVVtj86NnMeR9ipQ57xk2fG10ZPy/VQd6TXr99FAnDC6ZSOOqATc9vBEwPehs44qXNM7wT0T/lsnP7G0P/MBpfXWR13v9ZKfI/ruG671dXVS8+/++rF557ti35tq9h03At02+u267N+11n2zC3nguuZLK07PzjnNsdgZ4Hoj89RfPA01yW5py/xzDNtjNUbf/G89dZbTzAaQMeckDp69YwlZSwhY1rakLWjZzztYGzYxaSf9uTgNgKy+U+8XNrh+M4bv67rrF9utx3K96LtO9ATDnzZXICv/fNHrl78+LMN5Z3la9/4edXW9c4247rTPan72G8/efvq7z/3p/2OM3PPuOmfY+ObP7bkYP407diygdcHvb5iJn/aGWPHX365jDP95bW/2MF01jhl9OiyGUgsAdGpF6vf1GunnxjkqdNfvb0HYMbPA/D4h7+9esh11hu/WWHlawVzY4H2KVb0q5c51dV9L/nFE5fT6qNv1am1GnIXHM6xyMTHhj9vAl58/tm6PtsKtR33JzDNMY5d5j/xyPjQyIuHHK2sp4wvjl6/5NfXOYKb/uhoYtIfvRyMaes2hQYDb+ZLIieBbeLE2x9NVpuxlOmnTn4nknZt+KU9ZcbpQzH0rsPisyjd19Bqgqv+U/5knQ7vf7Yu2GvXgkeuo3hyE9PWxdahLLwtFv4P6o3Ay1W0cuEjPz0tbcgsJDoeYtHb9FMWhyyXGHpfMOrEKeuDHh0PdKnHZtNPWdx6F4kBpcBJqF5HZQnF26tPvGNt9MmT9jygZ3j0+ohXJ6/vDr/z/V8D3vCs4X4m6VPh5tQF9uJz71qFBbc8QNgBv/uDN3tHU298TrvYHtapc8UpH3E1uNbXkELOG7VbCpfx5Ma2eBCqpQ3Z+YObNuypTzt+NHQ8Ms6UxbXDeJrxjXFxHwxyK1t/A85gEpzhEp9j8fo7cXrHYNKunHZ50GlP3eMfctvhF+vjHRY0W/LXoa2AdUFeOwvv/Gjy6rOuu2oRLIzMR/zC7ZyNqfGMLy/9K5/9YN+oTb7MDwxy2tHRjLtJlzht+oqhl18dWB5z/bUf9fJjSz70xryxg02iJJFIgsQm7sgudgWuA0ZLP23oTTjtqWdM0y6ed4bcdnj8g99tgPFMMbHY4vGnYPgAOxt6MP1O88s/WSaLcfqZB0DGX/u3/zv+iOmk2Lw+e/Hj7+pYmd8KXgP16jJu6iZOG3p8pv1Ip89RP+PKhz65btwHk0wCE0KPbla4OP0MpHzWZxJzjI882HgQl96WdnXzohu9uN7A9tNi44uL2w5cC7GoC7eTcTpcN17JAX31+dERKvwoDj4qUqY3V3Lis8xZ2HCVcwEbXU81rv+5UettDSw0uIjjcVDXxv3J/I2rnBh50HlKA8f4bF2Ni89R/BnH+OBp6yIfIRMQKMGUwdNMVF/xm/X6+cz/GrHFV5Zn+hlHHPG5znLXyh0GjDw1uc2lDij3sh7sF/Dy0MPtNRsfGWW7yKOoehekIqiQ4swbq/iJZ0yREb8LjfttpLK7XvZ7IZfZ0zD+2ZyPx8HjD2ba0g8czULS3x7bHKNLzilj08ceDGPbKrALZTnO+0s6SopscMnEpE3eI+wRXq55QNSnTxfW17frrLQzvhFvLwqud+YOIWfuNr3wbYCsHrv/2omw1XGyOU8Lw/jqkeXXJ/vEEYt7Z+j+4W//bH2QjizvXB/95VS2oNCnP8cXOe36GkM5/dTN+2/q9XX91ikSAEYM2esggf2RnkRMBjtjW+LFYNPHiWpbCe6LqL+cFNd9ro32he84xiufxiPv/p56zEs+/LjO4nTIdduTWlh8ln3nvJ5JRwp7yVVnnVfE98J9Q2/H1jFvPh6yq1mf5Qd/776MKv4qgBo//OKH12eb5uXxyePmsQGjnS9K0tLmmB6svX70PGjyGKeVQz/55GneIug7+YIkmL0BTeYIL2b66pP69Mcv5cQdjcHfuPjmgGyrdFF03M96+TPvW3fhzZF4XGf1Hf03fluulyUkbsb3Dr96cT1HKoZ1KSr4uKHa13nxVZ7G7fNlR+sbtTsezuRTdldEZsHFaKe3JT86sOrE3NaDpWWMxBt/2o2RffPUVtcFJomOEFG1ZwHV2+tHbxA57RPreGLlmfHl0L4KzAPCtY2tdFxnccsh75rnweGabbv4Lqc9Z9y3XYQRB3ovGMS9mbcy9UTblmVfmIq/eW/+Fgi+PPK48tUg4nMTmGZ8dtJ792pORaE/9hkfHc3jYgxkdRvi2jf14FNO7Jl+xtPHXk781ykS4Sj5qTeoJJDmOIMw1l8/dBnnTA/OBt4tWt8+RX5pu32Quw98uSBy2Hdh8TmiBbnXUdsrTiUs9LovvQWksvNWWS7mpd2egsHbvHi3mfMBh9zfnN1v0uprn/fn1M1+xs/jeoQ9ssOhXj5kx/Ck3TF6MalrvDuYBoFtjIOtHhxjHvpMLLJ4x2ATr789OFr6GSv9wLATUWAPXv3pwudCPn7tL4GtBqenw+/U55Bg+X+rmpOiwpsC2nGWWMfp3QUOjHsrrBhVq/dY7ccu32jkrvrg1Z/0jeFrvxpViPmC8RjN46Ie/7SpT50xsD2NPjkY65e+6uTuPGqCpb8Okg4AsNmmTb198ug3faYemUfuUOkjnj713Km//+UfG3rrybUWMQuMQlz3sy7RXUBdFHDvO1rHsQALb/wupb1AoFlFunM2Lnh29Wm3fTFx++DbGBZYcmOjwPxKdh4DbMqMeeRxPA1eBv3AJI8+6GZLn2lDPuK5uJMPaJIYaOrB3tZu8zMRMfDIr03uxKijZ0d68OXawbZtqE29u5ScBQbvxe0BjpsbD4vSniP+UYEVD818Ot/g6rzDb6fdusB1wKLi5i5vPj75/Hsag/9tBeatlduOkzYIzZMxemXH2YOZTbz65Fb3NP26BgMsKWSTMG1s675SDKKdHl8wNHDKcopF5qFP6vVPDHzi120KlLWAq1Qq/uOHf9Xx4TM+MD8nrID7Ou/zjIIDV0Eu8rco2kYs7lG1sOVDYVu0vsvsuezx4SMmvYX1wse2Tw/Iz3nff/XH9YF5fcQ14h+dIjkONHwdT1lej5k9+lw/ORLf5Af86F0bxnDKi0xDpjWv12CtiSdBAsO0CFJncupIAg4noh2dttRlPDjAYHcBtNuzg/V9sD2gXIjsYB4E/PXBxm722tf/p7hLqOPwDAdjPyDYbenf4B1DHAqKG6E0ue3Nwzn2/PdYj+p+Fl//0SZWX3YwTv20jH/bRf7kaOc7nuB2XczFHKYr/DzEMwZrftPvhlwOvhgXtyQophmCqQMncdrUyZO8acOeTY7EpI7xLLD09xSZ/tjlYNz3oKrY3PkoGsb2YHKMTENXTEy4ZTmPYunvDgRWnH369ymy7sl1CNgrDDnpjwq/LJD0x24zVtrViaFPu/rE/TF2eejXR0UqIeRhxTqZGQiZydrr7ysCOXkSl2P97LHpR08zjmN67oN5DeZCoi/nPkVuwydX//6fb/af+3sNo56eltdni6fnVqeuXmHmuY8Lb2G0814AXXRbvXX8AvUc+G4Z39Lwq9b6Mi9O8fwMQX5Ifp8djM9Ae97srDWs3eaV+rq1uI4becAlLzZlesfoE4NsA0PTrt9cf/GzzxjYUu5xPXUECyMDJRn6HdrJMOZhIomdY/3QT355wTA+sssnD5jeweoPY7fDsyEsEHcwtF535a0B9JPr9e/V13z2rz9jt5lP11o9tbwXVpGsd5+NJ5kqCm7u+unB8peweosa3MMvfLhzIZ9X/uWnl7cpdh92MH8PQz7wPDz+jLHRZxOv7sx+tv7pb4zkSrv67NcOZuCjJNMB3JxU2udYXvUmNPVpT9vEG59vO3BKSawcWWAupjyebvRTjyy2SHklNF3aUezatrHDaW9FPXmdNfmxL/4dTIG99k8f2qW63+V9sIiP0ZwFGpMYOdaevXZ15qWsXT2yYzDTnjo5busvblNATPHQT2IDZw+xyST+aCw2bcYzQeN6WtaHfvpx6uMi3/hyUAC8i6TB03fI/T4WF+Z74bBonHZ8sYA3fn8YHX/Mgc3TYMfj+NRxohnfHTJ3AvnYbfuPTepr2/rhyyn00Rf+gmE33kVy2ixWNsxuzNsCMxa9eavb4cuHQR5HZPNxnDJxlKcf+Gy3xZQDfI+LrOeiwYWUMMkYOzHti2hfuNTfNpbXeMaffIlzDGbdpqjsXQ6LgB1M7Nw1MicW7qVPPNvXSBlfDL7rM0LmV/PPhQfXn3nW/az8S270zIvC8iav+ThfMJ4iGWPvU6QX+fu8wPtiyByT54j7SEccm3ZlejkzTtpTrz8+qU98c5bRY9Y2RR11eJrgM5A+cMgrn6+SxCQOPA8LeuL4Iwt2sAt94dkhLDBsFthRfAry3jPbAvqVaXHG/4//eqt3FT7D3M+PVPTVCx9/Z/9xiDdKM3fGD7/+876mMz/75OebFo/qGoyGnVMkH2XROD51AK6eqa/bUGDzGqzthfP44AN3xmFsPPXgsk17ysmXPjmeeG3kR8x1DXbDUEaahaCdPknVq1Omz0mZrDh6Hn5fST8Ts0efB1Hct7//q6tXXv3Z9aLvBmKeFRgF5R7UebAg3Cejcup/FtIfL8Fu/vQWKsXFYv/d3/xJ5yXGvPxu2be/9ysOwMq9cfuuBBZ+TpFf/eJHVpwusP2vnzo+SVXAfBdpPL7wx3FRNr69+dPTjnBHNo+7eHnktVc/OZTJD451DYagkyTZ65iBsacsx9TLK/aMVxz2u+J5H2zhtqDtR4EZy3ebpWCtupCAXox3mTv0L33iPX3qytsajb/jicLyr8WbLhe2xlvorZCxH31t+/oarADUVjXmR+Gbj/ParNfHCflpjv86Xvt6y4NvbiaJA3Mm648dDvNb+CLtEs/kcFJOB8fYJWScbRGzoNWUxRzxYjvS4+vuNeNthVMX+fWfOxCriJwFZty1A6Eo3l7BfREbg28VmBfhXvfknBt38HR9/2qjBeK89U/57M58F1j+NVTlSaYWmFzw2+ZxyThglMWr47imDW53L7HYz44/Nnxmn76MewdLIEqdGJ81MSbl5JXxU8cYPC11yPLYo6Mlftqwcw3G13W6RvZibr869/MuUp85Ny/czaNxxbK+4AdJFJ6Li9pmboff4WKe5LP68mpx+4br/BKknBy3V/71Z3W9d/ntWl487F5eI4p3fh7vi/nEwou/q4fnrJiOfI1vL0Z55VWDXvlMEHDKR+MkdAyOADR9GKfOBNDT0rZptmf19EcT907+KoZ9QcG7gx1xMHHegfZfYZ/8NGbm0YVSQVhg76bzeWH/SJ3v+HDY4y9fDsNeqP3X4lUkvNMkpzw24tGv+2Aq6UvPNRhFduTrHIEe8W4U1zGPOMTQ2+RKfI7F2R/ZWldPtE6Onia5zvRHNv2m/QwrX/oRSzn9HOMjxjE9BZY/cYmuW03h9dc+ujgpBm4HyCcXfZ82fXdYftvpFpaqjL04OrcSsLGbwcePp+BfpCS3xfWZQ4hqt7kDHsXnNO9vX8DH99s27uuPpqD1Hhtjmlyd2x6/89nMCzN1+k6/lHMnC7obQ3zSL2Olbf26jgD7ybi2vHlAdyCkNHp3nN209NroiUPPg2Zc+9SLw6b9xg7WLBWfHerRXy9eTmN8vvcC378aN1ZxgZvd6LXazWjyt8BTpUdxFbCLiVMWcuNIfdRX40r5Uv2KDh8D2ZwDx4bC4sYrrl/5xz9vCHxHO5inSHYwc5vHBoK0OW7ieMIPW/ZyuWbKk0Mf6ZQn3gJd+lJwmFYzARQ5XoBdPwtuEdYEaMg8SJw+E9amHXza9c/42uXyIt8FRm/zFInsxT12dhNOdeRuk5f+ckfbCkqc/D27fY5ddzxFY7fkV3m88YpJX8YP6vPGx9zrKo6jj4r6Jw8Iwnz2OOTt6RkOc2Yst7opg6GlnvkrW1gbauPGNtdXOz2xpl0+ccrrNgUGlDibrDqd6LW5SCaor72T0K6fXPqrp9dXDD06mrgW6qkLLH4zon2r2rhY5yJffBZY8xSG3cAFE2f87cJ9280sXvzO8lC/FdYHrl742Ds7trzazYMdSd5ZYP2Fwx++WcGIWG17rfYLY+a7AW7mNRf+CEdO5mWe4NAheyyU5aAX7/q5vmKNb39RYBInEeOZjGRpY0wzAcbTTx16E0NHS85Nc/k87X0NFnfyMxY7mG3tSihqwRrHgaydNXeGzBuoBdGLjR+7Hgcfkn3hwdG8ztqky2dPh7w7XH4scMHyTj559e42frIAoG8wZo5Ect5GnRjsqUPm4fFPf3DKRz7oxKQ980g9XBfXYDpnsjPgmdxklQBt8ky+mUTaHSefuuxvFNgqgOsPu8FfF0qt1H4As0j4TQna/Fc7jK//nDc+ng7zQl0/CsvPIcHWynX8bVxi/Yd/XqddFFjgbytg8zIu/Lcd38SBdSfCZ67blMHbMgY6dyzG6bfug2HI4CaOnpZOKRtIvHI71ZOc2tVPHHqxjsXSG1+/s6/rUDyPH97cwbY945pRno5ZPp421V8jt9OxtzXQUxjzV3mcH/58bklhErP5qraMLz94eB59cXybYr/RmnxPU2DmKz9ycjgWZy8eOw/laVeWB1ziU5/YdYqcxDrrqJPEKTMWp10+9YmXW91RL+bIH7zXYEf217/y0XWgJo4CXAtfPOmPjULjQXMujikaGnb9cp7udmXEubFpR0EM/sefwskbqOsTgcJhF/uHFpi5dQLjKeeEKfM7sw2KNffpr5zx+8NutjfOyReG/QDh1JPdKxb5qJmotuRCN+1uqeiNr+/sj+Kvr+tMcMl5DYYvC++OIryXnzmyu/AR0f5n+uadi4puHp/FUxw3TodllGfhqlzcxbYdcPuKT84/v02x/Iqfj5bYMc1D2x/Tmxc9sTO+usmr3h4742zw0NQrr1MkhiQQkCQSYBNroWBLHwNNfzH62yf39ElZfO9M/dMB+27BhJlkdXmKNA967nfxtWh3hi6uerqWLYEtIkVGY8c6ypubro/q/lnfWoj4RbgO9MZUz9irPaifj2LXch722F588N+kdNGISx63vYsEkzwXBLuA3ZZ4/eh5ePEv9qzX72j9teF7eorEkZYB0zHH4JCzGfjM/4h/+iO7c7jA6IhFgfFZ5GzmMe9+6wfPOpWFs34dp6bin6U9UzsbBeFpUx6LVQr9lTNfdPOLieLwm/msggdUuRA7i7LV5UeMGVfeGV/97Ke/fkfrA9aYfh0n1ze55e0CQxAoiQEMqPOU1eO3SGvi+vt9r7Q5pp8FhI4Y+DsmRuaHvE6R1PW2O/cu0dzl33kWFzvGvGaSdy0suTcNRDWK/PvvJkuLjp3khefqi4Zxe2R33OZePN0iPrciuPHKH9rSyMv4vEge8gN6+41XCovTNbE8ztwe6dN1Fbn+PYinnnPJ9vhmnID20PgIjJXTx/UzjyMOdK4f44y//EvZR0UjQNptxCaSOPzTRz6x2iaug9VT4tFNWRyJw9WnyLjRir0XqFacnv9p8HANw30vC82csHtjdSux63nP+MpFCAjXi9bzKxM85pF/AOL8cepTa73T9Gc6xSeh+M4/3gw4/7Trhw68NvWpy7F2evS06dvKeDrzD0gPF18lfPEDdAa4i0iCSewk0cslBh8e7kbojSOfPhzIxCWH496B9t/Vulh0DnIVFLvLKpxa9gd1qvn8/nmk8eAi5vXvhW3Fkxf+YBKPvJoLQ7xq4Py+1zwWzIlT66P6fNS8yuGiYPvFsTHVM78nu/1uvx89mUceJ+DKjMGcHbtpQ555ypGc6I7azAdM6tZXplHyOEtsks/gSTqx2tQfxUGX+vQ505vDOtWNyXU8Xpix4cBFAeSNVWPRz58f7xjktidvTLCrGHZ+3x1y41UcbmDN8QZfscA+9f5LI5xa8Z/r0vH3nZTxbBnfHMSc8aU+ORnLZ5+6xGo31sW7SJQTIFAS7HObTsz0xy99Zwxs+ExM6icn8eXRxgLyrQn/aEJ9A12APU4XRhUF1zZ+B79x9YQfp9/Xv1934ouzi7PWzxLbimqfU2HBe+OVfs6DTxzYHbePijY81dpFRXGNay5+HCU/LPc4mB89uqknD2ODyflP/LSlrzb5s5d38qVef3S03sEk2VTbM7rZZiLpN4MaaPKol/vML+2MMxayucgnT19T7V+9EUdvHomnWLgu48I9/zpIvLsOchEQtIc8daGV+PALH1o/DZA58iYkPyqa8ZvIwi3u/GzUHMHol9zoEyMOjDudfuKO5LQdjeWlp4nZpMs1gT/ji1mnSBX2JqQseerVgUGPrF3bmV4f+e31U5ZPOfvEzjgWR2NcyHK2MJq3crZo5m2NjCtXxhafOGIhg/dfYWufg/ibnn1s++jpwee237DAf84r4zpODLozP3HmmTI6C1Leu3huw8kthn7tYAiZhKfBmQA4bGKxQ6xsguqU7adeecaXD3tiGBufsfmJhwe9fP6Vdmt2fSW7FRarW21bZgZ1fVanTf+l2rbFglM4vPN7+dPvXbsWsYzXhZi/s1q+nRdx95iN3cfbvbHtXxoxFn1yItOcHzbGxtys13Zt4pTt1ae/NnrHaZ/xPf7o5/qjw1dMf5tikgEiEA0CgyI75kYbzQVuIZ7kFC8fesf2YsO9E0Se8dGBZwI0459xob84bfa0eKpFZ93hKb7ZcoeCI+NkfMZcs+WfrCUXvuyaFjFj/k3v+1XIXLMxj6P5p865wZv6GSdlcR0/jjl6deCNn3j0yowzvjaPf+K00WPvOPW0v46vi6cNBwcdR5pJMTaANMrYZjviTT/tqZscyOIcH2HMI7nYzfxqND5z4SkCdfizmx19RGN8CiuvszKfjl9H1sLqHOtQe+P3KL/G1JP8N/h2QB5/fY56YuT8HYs1B2Xj2quf/ZFdbjnFrGswFfQ85ivWINhskiEf6dElRr/spx8yD3cusLdxGEMeZXzUTQ7veWUejXGn6byf6e/Vf75Oh/MNQPptp8XrnxVwR+ycmcsOZrfyT9bMEZM5OkdkHh5/islx4hk7R3vtyeXYXgw9Tb05zfjqN/T5MzgbnPKsa7C7iHSCJJOSNPWpY5zBtcmh7KtS/fSZ8Z1AHny56PXXT97EWByUQZ8k68DQtmLY/rXbxE9OZHjhodFfl9Tm2R8VxY+jbNqbz85/Wo7yFmN8ZXpzVKe/emXt+sg17eknJn08/mnTTr8+7EawTbB6emySKqf9LMHEMD7DoSfGWcv4d+WZHDOeNncz7Z4WlcWZ04yvnb4LNm6R+FERtsmHLpv86HJe0y9t6e84edDpr175NnzG0E/85JRP3A25Xjm9mhgktk/SHEuWOol9JSIzzmLUZpKTBxl8xhdjfviKc0yfLXOZ8ZXhSB4Kjest7JmnvHIim5O27LFxD8w/stXm8ZCHHqwyYx63zR+uxCkbQxnOyZ82x8bTH1k/deanj30eR7H2+nAc/x8AAP//4/JLOgAAIrJJREFUnZwJ1GVVdedPzSNUUQwWQ0xWEzEYjVFBMYDGqEA3Dq0SVBBMbBEihIA4QZYDmABL44AE0RZjWu2lcaVNFCFZ6IqmUUHRdEgMajRGoxQWWEzFUHPl/M59v/vtt+ve7/voE9/bZ+/9///3Pvedd+999yuyYNeuXbvLZCxYsMDpoN29e3cRw5yBPxRXIOPH4mqIF2cdfDHMI07MbLGMyX49Dk0TDeuIsZ5xfHPiI8Z5xDGPI/ZKPNaPuKE65CN/CBPzUc955BDDz5wYYx79iCXOGIzVZL/BAOlieS1cuJBwI5tDaOfOnS1mHox55kN54g6biRxyxjnguT55YmDIM8Rjoxb16c3+zIlv5PpmXD/mYy7qj2HAm8sbxrh1tLEGsYiLOeL6ERN1yMf1Rpxc8TFHzHyME4u+uBwzHvF+/gtqcGqDeWAEY2laGOLMEVi0aNFgAxRUJzYjl/zY3JwbCN+DxpxhfeLqa+1zaIOJ6VSmD6o8chFHH0PrFxd56mLz+qOmOLnknJOL2LnqD2kRQy/q6FvHmmKMwzU2pGNsCEfM0a+/TtoGywUFxsLELJ7jMScXC46Xm0Se9cTqz6YPdiyPrjk1xY/lIo55xNmnmCFtc1gPqLGIVyvGxEXrZiKWOfPxx/Qz15rEI2eovlhtxBNTmzk5fXELZttggiEzJDG3OS2xoaGGXHxe8awALzanTtZWg7y5ISsfGznG5ehrY5x59Id05GHFYhkZH+NiGzC85Q8YXPxiRs0xjSA3NR2qnzWG6seaca541oh1wPSXyCguOdpMBG9BrPkmOvHNEzMvNubIz1ZfLjj5cR5jOY4PP9bLehkzlAfjMI9mPA7m7WeoprGoAQ8/5pybi9rMrStPvHEwxsQQY1i78zrcbMc/crKm2mpGH2y/wQQoYPFIYD72jXJhmZ+bG8ON1Xfh5O1lqMZsurFvca4va9mHeawYc/rkjGGJxxx5hhhy4mKsQ3XvOZ7Xn4+/eNjqO8eS5yUvrx9MHLl/uOpGnchhnnXl9BsMUG6GmIMcwwYydigfsepgwbpgfbHxgJIzzjyOXD/mmLtg+GDVmQ9vrDdrqIfNw3rErZkxkZcx/7/rV1O9uP5YHxwvcVoxMWcsWzHYPNTr69fJnqjKEqhAT6gfGMMi5mOxyI1x5nx4kYsuww814omrFTnEs0+MYXxIJ+aiLrwxX82MyXjyDGq4li4y8259I7HHMb0Ydw5/Li21hzhyxaAXcfh5yCGe5xmLltpTN/kC+2QAxhwCczUkHqsec4vPxfc5Sv6w8kZH85GOoQMU+yFvHeu7BrlaeeajT0x+7FFujuHDJ58fs4i1Lyza1jOvRQMMj5JyPeLEGPaXdTz+xrXWV9O4etFvmEpolUhkEA0YYx6HQjHGvIlWLUfmj/EyFx4vD0DWG9PJ9eWNWfvL68/1xanjgbY/62rJM8zLG7PWow97IZb5xBi5vrqxfuTH40VcHePW1CdPDevHOLXkm8ePGq6/vweLAJvFSnIec8wtZAMxP5az1hBfjjpD9TNGLDb2Ic6YPpaY8cgfm8sxr5b+fLTkYOMHMx8NN5R11Brz0QRj3hpj8YzN+rm+euL0c73+EgmApIRcMBMVFB/5UUec+egzd6fnAz5WP9aLWhE/VH+ufF5fxMc6zof6sG7WguMHxDzmXT+xGM/1rScm+1k/57MefTDEMVebuXrGxOmDYRiP+hEzdQYD5Afd0WcEIClGTpEYM24xc5krDhuxzB9pfTQYsVYX6d6jfozbf4yJzTH8ITzxsbrk4hBnTD1rYp2LiVZ8jMW5+uDU0UbcbPOIj3pyjOHbT4wZjzpTG0ySgti5BHJezpAWuXgAxGIZQ5ysn/kdc6bPnJ8vX51s5Q/1BnYoTyzjIy7mjaNl78wZ0c96HWLmPepE/bl4MwrTs6hBRn17Utd4ZBPzRNE2GKdDA7MRyCmsoHjj8/XVyg0P6aqJtc+M07cPfbnWMY4Vay9acvKcy5dDnKEf8V2mezdvTJy1yHv8jYnFEjOOnWv9cq0Lh5H9GHsk9dVpouEtx/u6VXy3SYOB10/F9IHJhObMadGJ88whFzHmY8xeso6+HOvLzXl1tOQzBi3yvPwAxVsnWvjko7UPY+KtFfXEZBvry488YuqZt679RE0wmU8sasjD8nL94BhjfHnWB5t1W6wC2xZXKIIAMBTrvJmiLsZ4tOrlhiOGuTjjQ/XN8UDl4a27y6rlMx+wufnoxHVwYBhz9QdG3oNbdpcHt3TPkJYs2lXWrl68B98+PPBRXx0085Dn+mfrL2PRMqau/lh99a2nhedcLW3M+ZxMfeuJVaNdIk0SjCKAbSQLKQBGvpacPB70MV79J3eV7TvatLz0WSvLf3vaqubAseZQfTVv+NaWcun/vqc88PDu8rRfWVauPHe/sqj7t5BTOjjoZC11jIMjxrqsbwyMw3WAO/nijeUHG7a31NG/urRccc5+UxsMHbWcu358hvVjjaGcdcVpI7YJTt6sF2sQG6sPjrz9+PlSl1iuNynTx+1PXq6vbn+TrwBARi4QfedysLFZxYkz51nuEb93O24bZz1/7/Ka5+2t26x1p4LV6bRKeeb5t7fN1T76qnnpq9eV449Y0cOt3wfCJObiPEBGp7Gvky/ZWP7t9rrBav1jn7C8vO/sfaeOk+vGOuDnmuLARKwcbKyLL84NQCwO87mW9eNGEAtffK4XtZnLsb6+vOg7b7xK6O/BomguHEkZF33mcuWwZ4846/Ya7y5LZ71gTb/BxOZGWcjMQSnl6b9/e9m2feb0fcFvrymnPHv14Aeh5lBf5NSNvQ5h6d++yLcNtqE7DbvBMg8/1pCPFmtiUN84vseJucP1ixOjL07fNeHzij5Y1xJzxKmjtjjjsU8x1tOPa4KX60+dwUhKBMzIBAuQExt5MS+GM9iRr92A2/TOPHF1OfMFa5tvDK3MFUD8musfKB/6/OaGWb1iQbn+sgPL6hXdNTLWjxzm9hjj4P0AiMvHOiLPeNxgz/i1Fe0MJn+IZyxi8jr1tXJyfX170Y/azI1HHHPj4rM/FFdPLX1sHFmfnJz2JD8Wi0SBOe+ujXHnCkcdNthT2WCTTRQvkX6DhvjEYvM/u3tn+cldO8qTfnlJWVivQnKoFef2bdyesh75sfpDemywH27YWXvaVY594opyxdn7zRzI1CvaDuq7DnX151tfLWzkjK3NOHhrWZuYQy39aCNePWJj88jtMXXCmPUDgjiEiYJijE03Vy+R4R7szOft1V8i5UW8MbWwP7pjR7n1h9taH0uXLCgnHrWqX+h3frytfO8n25u/csWidm9224+2lxu/vaXc+oNtTebQgxaXo+t909MftzzKtjlrc8C75V+3ln/6t23l/od2lafWHxTPeuLycujBi8tL33Fn+eEdOxv0GO7BXruuP25otFdZWG7854fLv/xoW/nuj7eXLdt2N+7hj15ajn788rLfmu5Hj8cTu/GeHeXm27o+ET/+yJX1Xm9b+cQXHygbNu0qh+y/qJz6nJXlcb+4rPBj5+H6S7bUlp902LJ2Fv/8zQ+Wb353a9lZL3dPPHRZeXKNH/nYmXXe/J2t5evf2VK+XY/fsnrsDv+lpeVFx6wqB+073YvHYMjmjTj0ebEWrwzMGVOXyCjcA+qOfSRjiEetfoPV+ZnP7zYY2NyotdTRv/pzm8tH/qa7RPLr8RsfOLhxwb3rL+4rn/rSA6X+9inLl5Xy8t9aXT76tw9IbR9G/fibf2y9tL3rzHVl6eLps+POepZ996fvK3/x5Qcbvkq1YR8XnrK2fPrLD8xssMcva78iLQLuxxt3lvOu2tTOsvLI0xf1+WJc8sp15bgjV3SbcbL+r9Yvwh9cdbdS5U0vW1su/+Q9zff4nP3CvcornrO63ove0ePY5N+oG4t7U+pZB8BLnrG6vPGla8r5tZ+bbtva6hMHw/84hhe+fE150bGrCc854noA25fEVj/sFfFTjyn6YAVmAYWw4DJWHwtXTKezoBzJGWwSz/dgURM8L74xaPCNwP/AZ+9vG4x9smjh7nLzVQe1lsizwT75d5ub3/bFhF8Fum9U9Zlb/7Uv2Ku86r/u1XS7/ko5+3131Q9iS4uBmxrsTUNtn+6uZ6Nl5b31DGZ///D9re1RjHWaLjUZVW83N/gT3dOfs6qc99v7dLn6zgb7/St/3npsvIpj/Yx2RqgyZ//3yQY7hw1WA7HH6voFsj7HbsXSUrZur42DDetnvqAeN2Q++ZYDymGHLGm14DA8JsyJ4cecvZFv/TGZDPG4bV4fmA3+ihwqogjWIhEX87Gh2vLUGew1J67qb/JtaBo/rU/ug9c+0G2wWmRRPV43X3VgK7fHBpsc+KcfvrScUr/x1P7c1x4qX/jWw1179RguXLCr8g+pB647eF//7rby2rrBGKyHb/cFJ68tj//FJWXrjt3lw9fdX88UM5cwPpijH7+0f0yxY+fucuJFG8vP7+sun+gcf8TK8vzfWFmWL11Q/l+9TF/11/cRrt3wYe0qn3rr+vbBsrav/cvWboORn/TP8eWMd8A+i8vGTTvK+fVX80uOXdWdwdgHkw2/ZtXC8saXrSn77b2wfPEfHq5n2XoGrgM+WhyfY+ql+bTjVrczHWfoG299qObqIqvGr9f72Y+8/oDGGXqjP172BYZ5ftA6xG3YSu627QgipxHPBaEO4YxTof2KZFL53IOdcWJ3BpE3l65nMD4gzmBfr5dIh5dI/aN/dXl7EGt97IUfvrvc8M2HWn38a/94fTl4v8Wt7xe/bWP5j3p544CzeT9zyaPqfc9iYC1P23/wp3eXr9UzHJuLDfKMJ67sf0V+9G83lyv/arKB6vpOf+7qcu6L9576UG79wdbyu++8sztzVN3DDl7Szh6sv10iqz5nIdbH/06rZ7lzX7y2bpDWRnvbXjf7UWdvaPU5jnwRrr9sfbuvcwO85c/uKdd/46HWN1qH1y/JJy7qNpCf2yl/dGf53k/r87y6lqX1C3DTld3VYKbSnrP8OYmwrn62g/dgNgKYbwLfglggi+Cbl+uG6XJusI7JPRgbLA+bVYO89TkDcA8GhgP79XqJtMaffPr+/hJZGyl/efH68l8OXNL3hA6XsNe8exPT9kF+4Lz9y1GHL6t/etpVjj53Q7cZ6gF/4TEry1tesbbzK9b6P62/Xl/4lo29ZnxMcfrld5Vv//vWps1N9P9930Flcb1/Zh1eQphf9JF7yg231DNp3UCMr77/oHrT3V0iz/3TTTXcba4jH7u0fPD8/ftaHhc22NPOnnlg/cKjV5W3njbTK5pf/seHywUfrJu11qtv5ZJX7Vued9RKUv245vrN5erP3d/7N15xUFm5bNJUjcbj77zp1Zy9GO9FJpMYb/P6xpgSBRuF9MHFOH48gOSIiW+T+kboKWf9VLedwc58/preZ+IpfSoYHG7yr7m+OyiL6zOKb1x9cF/LMxgf0MJ6dvMHgHR64hHHiRf9rO/fvwR8vz6Zf1n9deg90mX1LwTHhb8QqMHanvP6DeXuzfXSU//v2F9b3u7BiD/z/A3trwycEZ77lOXlsjPWSWv1PG6cqdhIjk9cuH89wyxtZ7BzuQerWoz3/N6+5Zn1lysjHs9t23eVo86pX4b6f5ztzj9p73Lac7u/iFjjth9tLa+4rLvcw/+fr9u/POWwpf26iXHLcPHHuh8R9Px371lf9qrPFlkLL4Z1m1PfYnxoLm4PXv1gux0holpAceOEVJsOFdhDeNIohPlssMhnbg1re4lEbHE9hbHBOu2ZX5H4K+o38Sv1GylfXe6Pjn/Tnhvs72/dUl539ab+gF7z+v3Lrx+6pOe3IpO3V07OVNy/HFM32BX1T0U8huAM6Di93uuc+6LpP4ORo59/r49aTqp/z3Rc/Dv7tLNL3nh/dfEB5dGPmr7xhtNdIm+vWt118031V+DJv9n9CnSd3/2P7eXUS++0RGE9T/rl6Q32+ZsfKm/7826DwfvSew4se6/sNOlTrV6kTjyentFjLs4zt/8n0wprJSmsj0VkKJ5z4iq8/amoMvnC9Gcw8/AY+tou2r33G6y6i+rNeb4Ha48Xam75kt3lK+/vHmHEs+Km+3eV497QPexF8bIz9i3H1bMNN+BnvPvn3UGtjV513gH10jn9gYBnvKw+aP0+fyqqOB538LfI+hup3l/OXLZe/qzV5Q31MQMjfxicXU67vJ6p6mCN73zNPuXZT17ZncE4s9UYm+faPz6gHLhvdw/YwPWN4902WD2DgeM4vvnla/fYYDwT5AzGWa4CyzVvOKBtMHQ8Hmywt/+ve5tOFW4bjDMYPXk7NPb5ojP0+RCLOfgNV4t2mZrNolko+01xHm/U9jkYGtzk50vkXNptg3GJrI17D0Zpen7np+7tn1/xHIwzWB6ewYyzwfhj+V337iwnvPln/SXyD0/dpz5D6v6lh1gs/R1z7u1lCz/764h/izzhzXeUO++p/++s6v8d8dhl5UOvm37CDx7+Z+ul6R0frx/sxP/r+mPiFw5YPHWJROPzlz6qbTA4DtbJJdLnYOTyBgPDBuMM5mfpGSxq9RtsIs4ZbHW9IsORZ91o0RjLm7OOuP4mPyfY7bz85x4QMiYWZ24R5v6M7XgzG4xc/BWJz7ChWCPqscG4BwPnYwryfOO4B+sekNZnP8sXtg1mfS/1d927Y+oS+UevWltOqE/M0XtqPQPt7B47lcc9ekn5WL03QpuX6+dh5dlX1DNDxXP64B6MMxjjnCt+Xm6qT8sroX0Brrt0fdl/cotpfXgvefvGepmsv97qwL+lXub5k1e8RFITPmcw5q1eY5SZDVbL7Kq/ZNlgL33WzI8l8N/7yY7uEjnpZWyDvfWjd7cvxILaQLxEomF/uf6kjSkjnmDEe7bcY4MBZFECIHnaNKdoXDy5OPiAGR2322B8Ozm5/4/6kPN3T+j+1MMHGHWsa2xpvVIw739FVg3+idlNV65vcXL9Bqv1vAcb2mAnvKne/3QnoMIG4xJJ/Us+fk/57FfrI4w6uNk/76S15dRnr2y9E9t4z65y+uV3tudcrIH/HfOE+iS//i2Sccv3tpaz3ttd+vB/oT7iuOaCdWWfvbr/B3h8Zpd/6r7yf/7+wbZ+MDxpv+iUNW0NX/lnnuRPfuHW+tfVP+R7ifRDw3KJfDoPWmv9XfX4Xnjquv4SiSajvwejaD02cYNxrNDhDMYG4wuxsK4/brB8/K3fqc+8G8cOjV6nThrC4tg4xgQihrkF5cvDp0T3HKwCla9V2Wzi6xSVGcAE960Pdjfz/QarevkS2X5F8iS/5txgqDHog9fdm3eX499YL4VdoXLZq+s92BHLW/3u/mzyhHzS4GPq3x6fcOjScm/91cjfNPlzDMN+j6lP8uO/B7vg6rvKl/5xa59fUr8Y/MPINasW1Q24pdx5b/f3Q+rT4xfe1T0aoDfOYDxnc/3X1Wdbj9qnu+luRSd1+3uwSZAz2EnP6B5BoENvbjD7/PAF+/U/Woy1m/yP1pv8yTF2g7EpPOOi90iH+pE3eJMvIBdBIMb0o83cLjd9iawiFdadzci30dZT3/QJVty3PnRIS7PB/uxv6t8XK9wNJtfHFOydFfWfU8d7MPtlE/ErsvlV97Iz9us3GAW+8M2Hyx9+ZFOpf9do9cTFfjx7AWCD8S9aHfc+sLM+jb+73FbvgRjWtUdxy+ufb95/zr710cHMY4i4wTgMXCLXr5v+T/7R2WOD1R8TJz2zu8xTD0y+B2OD8SvSAebamx4sb//zei84xwZTM3JdFzG0MkasduoSCWFoZBF8Y1q5+lr0+jMYTj2CnkVwG4+jSmKyYvNspFuu7jbYh6/bXP9cVG/yKyw+6+Jb9/7P3F8+/sXuTyT+irS+B8ENZn2fg1FZzA/rYwQuVXds4p/ktKZqR/WY1P+98vi9yuYHd5XP3Nhd5rgH4xLZXwrawS7lYzfcX66+dnPZzhmP4zlZFnr8q4y3/c66/pkTtYnfUv8MddZ7uTHvzlo3vOvAsm/9008c4OIlkkv5m0/Zp53B6L8dx0r4wU+3lpPfUZ+D0X+NX1P/DPTkx8xsMDTbBuNXZB3o8qub4+bZyzjWY6M+MQY8Yq6/i87EzfcbTEC2Q6fNWBQhR26COHleNp8bypyol3XNRY4xsPaV57PVzzXw+Y87vvPjreWO+nD20AMXl8ccsrQsmfzrC7XljdXnksg/IeI/EjmsXm5/af2Ser8384G4hsjP2vjkXVdcR4zbizh1IsZ6Hn99MHHk+FDNiJdvLXP20jaYScEmAVtQIlacPGIRZ96YPjhG5umLkxexMedcXhOdvMElbk4tOeblmLcWdjaMvIiXYy7WMqa1N79wxO2VefxC4882Ii/WjHP4Eaevrmv1OIhVA5yYPBdrHMuQC29B/bXVtrAFtBEkaSiXcfkAmVeDA0sMLax5tbXgHeLIySMX5xFrDTFYNfIHK09r/dhXnIvT5lxevzgsWHu2DniGfWW9lqxvcvWzlYdVC4zxXNec+cgxh5UHjrl+zDnH5vVP3eQDYFg0i5FDwEL4ecglzpzGYwxubiJryAXrUAOrRuxDrDj9zJfrAR3Cx5hzdbBR27wxfXDMXb954kPrlyfOxyz6WnDM/RzQGxriyanNXK4xj0PEyyEmjphDDfzME4NtvdZG+zOYYpGUY9mPgsxdOJbhAppT32w61jDXGgqbyjjWusb01dFaX38IH+tEnTjPPP3ZdMFYP68/8mJ9dYnx8njpy9OKRz/iiYOxvngwcRBXW46WuDw5Q3xyc9Xv+6iTvoMsHotYPGNswHj04xwtNXLcOtFmTN/wHBvQPmK9qJvjuY6+HPWIO1cDa0yevpghfwgLnjGUyzF98FEff7YhD+tGFk9sNq18/NWSny1aTTPeg8UCFsxCEYNozJOzkVxQPfmRB9a4PHWMi9cXpxWPDybjrS9eC4/hAZcnfqyeOPNz1TcvXr59DMWJiRvKw41x8cbUxlqfecyrPxQfwhmLPLj45LTEGP1/tkYiJhXqYN17zBPBz2OoSMSqKzf6Ud941JdDzHyMGVfHHFjn6kW+eXnqYOWJkaeOdgynpnnxQzoZIxY7hI/5obl69h4xWQ8srxyXMxa3xhCu6dWd3e+S2EgWVMi4vsLRkvOMYFx85kefecYN8cFEnhhtzBvTWmOIbw7sUD7G1dPaNz5z1x/j5KyhPrE4Il4s+TF85DqPGsa0amY9OOYyFj/jxeRae2hUAKMXYB6HBK05Cw7hxWSb60QN53LUta7WfMYb10a+sWhjL1w+8oZQHxwv8mqiY17NmDOWcWDkecnCt37Mq0feOTb3GWvNNrdu1CKmfrRZJ3PyujI++lNP8hWaAtQm4gBjs8Q9UMZjLvLAMTxAMedcDXx7yXoRI27o4GR+9merSc66mTdW3+NgL+qrk9evTrTMPT7M59OHddXJ9e0jWjH2Rg5+rB/x5o1l3pBej62ibSUT0+IQfA7jgiVowStMLBYVg1XXvL5WfX04YpkzyM0Vi3w4+QPNfDCMzBvC5fpyItZYpzqzBuNi9XN/maePhSPfOJ8Pw+NnXNyQ/pCO/chTJ/LHeHDky9Ma7/9URAICwtFaWKtAtDbQi07OevpqY4lFPAdI3FDduNBYM8/VJa4OMf/BYOzf+mDlkecVc+RzffKMqIevjjm18BnixZmPNs471p71Yxwt1mdPWDR40bd5OGJyDeK83KjMI585w3xz0pvaWms0rfo2ddOV3KkDg24jTYri24DxaM1HTfHkhoZYcfpi1c/56DOPPDlqiMU3F/HisGJj3hj5HFePnCPG4tx8tOrFGuSNMycXfWKMzOmi01xx8ufi5LwnILW14tB13mpVwuB/2Y1Q3LU2pOBs1iKxEHibi3GxWc/4WF010GTYK3i5YOQzz1h49sScEXXVJB518B1qyiMONnLFEs85/MiN2BgHxzCmrzWOHdO0V3qTFzVbgckb2LE1RJw6YvF56feXSBuTbMP4EKIvRhvzzBkZH+MRD9ZcjqtjXl9OrCHGWPSzbvbVI+6BIeYYw1sLHJg4xnLEI1YcMV65vnFx2lgrztWOOGL6s+llnLrEh/pSExyYOMy1f02BY0CgfiQ5j5g4J599Y8RjHbXyGcQ4NvYQdZkP6RmDK975UO2YEx9rkh8aYK0V8WrAMS7WWIwzV0eueesO5cewcHIOX41cL/rW08rRGsdmTXPEHa6jP4PlhGCBeSMYlxeLGos242MO7mx5sNaPuMgbqy+ePCP6cd6SKW8MG2vhw1UTn5ExxKzBfGxEXpyDjzVizSHdjI38Ibz5WHM2XK4feWph4+ifg9mcBcZ8yeL0LSbPODZjzcnRn82qq1bm6otTK+KJRR+sp355Ma/GmAUrT+3oGxvjg7WemBxTT1z25WlzPuuJw4qNMevEWJxHzhA25uFNncFMQuSMweADwI855tGPheKZJmKYi4vzVmTyFvHWh5Prk3NjRD7zXN9aWjDMebk2Ygzj1uyi3Ts5BrnZ6oMDw4h6xl2LmAZMWGuQG1rnbPVdv7Vdo3E0yTGoYz/4OU7MEXNo4cPN/VlH3f4eTCEtQAYCiuFDZJjX18ZGGnDgLerFeYS6iFwfjLUi3nmur36MexDMRb2Iy5r6EW9szFoj6hqD4zzmiXt88/rFgxka6mjdAPr0rrZaQ+uJMbnWixrqmxMrv79EArBgBpnDKp6FyTGGuF1mJodvA+aw1s9zfaz1h/jk4wAT+3EORr4xfXIxFufkGHxAY+vvEDNriXznYob6yzGxcsn7BTGHjf3HeOQxF+c85uWZ08dGXMwbj1jm1uk3mCQsLw6gZMDOFVJAX+vi5Qzh0MrxGHNuTbWsoTWuJS43z2O9jAFrnhwv1+8cDAOfIb454Y28/TjPeDUCrU0jz/pyhzjiYy+xZuREDJoxZw3j5PwCqSd+SMc+5OMb22ODAZrvsCh4CxvTn48WHF6RE+dqRIycsQMhRx3wQyPmmYuLcXiz5cxj5TNnRJ5+S6Q3v5gp3LtRxzm1xtYPxgEu+sTt03jGmFcj48jzsr64PXg1sMeRJ6SgRG3OSReffXljVnzME5ut8VgrzqOG8/nmxVM71s/9xRwc83PVUV+cvlYdfOaz4Tw2ERf5aMg3rk+OMRaPuQacvMmPtwexvlh19afOYCRpfogIwXgUsbAxMcRjM/Dzt1QsOfD6ahnHxjGUH4rNVj/WQxs+L9dvPdcnRp75bNEAwyuuN/vquVn0xcX4WA05WHj6Q/Oc02+k+gbH3rURM5aXr43ctv4aaGcwD4bNSdAC4+XCJ7Q9FmZcnno5bj5ascbgxFj01TMfc/CH8upi5RkbWn/UZM5raP2xlnN157L2AW9oHvkRM594xIzNY7/Wj9iYZ+76I2ZoLm/wv4uUIAif4vrMh/49Umwg520+alhnzIKVJ8ZY1icf64uTpyXOUFffvJYNx/Cf+xifTTfnrCF3yOb6kZP1om9/8QOfSyvWVyvWI68Glpf64uRFLXkxx+cDt/+PPiREEDEWApA4g0LMe4Hqx2Ej5Jnra7OOcTVy/eyDm0/9IZ5cLHXB5Lk8rfkGnLxFnnFj+ti8tphjLif2Qjzy7CNiwYwdf3Fz6bhB8xeIOPW16LjJmDOskfvsst27/KkzGMExMcmKWsQ41lyMgeNFLuejBjlxmR/92eZRX2114cX8kM5QfXFRj9iQH/XNiyU3Vy+PpH7uS649zKe+nKg1xDePHctnLXGzbrDYZBRnrqCW2NAY0pATczRkXJ2YJybGvL6LMY6Vay76uY68HI9+nKuvtvxowY8NcvmLDHa2GjmntnFtjDvPFqxXpbgG4nFkzYzFFxMtGmL/E7AKXaYE1pgUAAAAAElFTkSuQmCC", + DB.PolarDB: "data:image/svg+xml;base64,PHN2ZyB0PSIxNzczNTU4MDAyMzQ5IiBjbGFzcz0iaWNvbiIgdmlld0JveD0iMCAwIDMwNTEgMTAyNCIgdmVyc2lvbj0iMS4xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHAtaWQ9IjE3MzgiIHdpZHRoPSIyMDAiIGhlaWdodD0iMjAwIj48cGF0aCBkPSJNMCAwaDQ3Ny40NzA3MmMyNjMuMjgwNjQgMCA0NzYuNzIzMiAyMTMuNDMyMzIgNDc2LjcyMzIgNDc2LjcyMzJ2NjguMDg1NzZjMCAyNjMuMjgwNjQtMjEzLjQ0MjU2IDQ3Ni43MjMyLTQ3Ni43MjMyIDQ3Ni43MjMySDBWMHoiIGZpbGw9IiNGRTY5MDIiIHAtaWQ9IjE3MzkiPjwvcGF0aD48cGF0aCBkPSJNMzY0LjkwMjQgMTAyNGMyNC43ODA4LTEzMC4yMzIzMiA2Ni44MzY0OC0yMzIuNzk2MTYgMTI2LjE3NzI4LTMwNy42NzEwNEM1OTkuODQ4OTYgNTc5LjA1MTUyIDcwNi41NiA1MTAuMDQ0MTYgNzE1LjAwOCA0OTkuMTU5MDRjMTguNjQ3MDQtMjQuMDQzNTIgOS4wNDE5Mi01MC4wODM4NCAxNC4yMDI4OC02MC45NDg0OHMzNS4zMjgtMzEuMTkxMDQgNDEuMDAwOTYtNDkuMDQ5NmM1LjY3Mjk2LTE3LjgzODA4IDUuNjcyOTYtOTEuMjA3NjgtMjIuODk2NjQtMTA1LjQwMDMyLTE5LjA0NjQtOS40NzItNjcuNzY4MzItMi4zNTUyLTE0Ni4xODYyNCAyMS4zNTA0bC0xMjkuMDU0NzItMzEuNDA2MDhjLTk1Ljg3NzEyLTEwLjQ4NTc2LTE1Ni44MTUzNi0xMC40ODU3Ni0xODIuNzg0IDAtMjUuOTg5MTIgMTAuNDc1NTItMTIyLjQxOTIgNzMuOTYzNTItMjg5LjI5MDI0IDE5MC40NjRWMTAyNGgzNjQuOTAyNHoiIGZpbGw9IiNGRUZGRkEiIHAtaWQ9IjE3NDAiPjwvcGF0aD48cGF0aCBkPSJNMzM5LjY3MTA0IDM2Mi4xODg4YzM1LjE3NDQtMjkuNzQ3MiA1Ni4wMTI4LTQ0LjYxNTY4IDYyLjUyNTQ0LTQ0LjYxNTY4IDExLjQ3OTA0IDAgMTkuMjIwNDggNS43MjQxNiAyNS40NjY4OCAxMy41ODg0OGE0MjkuMDU2IDQyOS4wNTYgMCAwIDEgMTQuMjMzNiAxOS41ODkxMiA1LjE4MTQ0IDUuMTgxNDQgMCAwIDEtNC4wNDQ4IDguMTUxMDRjLTI2LjM1Nzc2IDAuOTYyNTYtNDUuOTY3MzYgMi4zMjQ0OC01OC44MTg1NiA0LjA3NTUyLTkuMjg3NjggMS4yNjk3Ni0yMC44MDc2OCAzLjk3MzEyLTM0LjU3MDI0IDguMTEwMDh2MC4wMTAyNGE1LjE4MTQ0IDUuMTgxNDQgMCAwIDEtNC44MTI4LTguOTI5Mjh6TTYyMS41MTY4IDMzMS4yMjMwNGMzLjUwMjA4LTE4LjczOTIgOTUuOTQ4OC01NC4zNDM2OCAxMTUuNTg5MTItNDAuNDI3NTIgMTkuNjQwMzIgMTMuOTI2NCAxMC40MTQwOCA0MC40Mjc1Mi0xMi45NjM4NCA2OS40Mzc0NC0zMC41NjY0IDMzLjI4LTEwNi4xMTcxMi0xMC4yODA5Ni0xMDIuNjI1MjgtMjkuMDA5OTJ6IiBmaWxsPSIjRkU2OTAyIiBwLWlkPSIxNzQxIj48L3BhdGg+PHBhdGggZD0iTTExOS42MzM5MiA0NjMuOTg0NjRsLTQ4LjQ4NjQtNDYuNTkyYy0yNi43ODc4NC0yNS43MjI4OC0yNS45MTc0NC02OS44NDcwNCAxLjkzNTM2LTk4LjUzOTUyIDI3Ljg1MjgtMjguNzAyNzIgNzIuMTQwOC0zMS4wOTg4OCA5OC45MTg0LTUuMzc2bDQ4LjUwNjg4IDQ2LjU5MiIgZmlsbD0iI0ZFRkZGQSIgcC1pZD0iMTc0MiI+PC9wYXRoPjxwYXRoIGQ9Ik02OS4zOTY0OCAzMTUuMjM4NGMyOS43ODgxNi0zMC43MDk3NiA3Ny4zMjIyNC0zMy4zMTA3MiAxMDYuMjA5MjgtNS41Mjk2bDQ4LjQ1NTY4IDQ2LjU5Mi03LjE4ODQ4IDcuNDc1Mi00OC40NTU2OC00Ni41OTJjLTI0LjY0NzY4LTIzLjY5NTM2LTY1LjY1ODg4LTIxLjQ1MjgtOTEuNTc2MzIgNS4yNzM2LTI1LjkwNzIgMjYuNzE2MTYtMjYuNzI2NCA2Ny41NjM1Mi0yLjA5OTIgOTEuMjM4NGw0OC40NTU2OCA0Ni41OTItNy4xODg0OCA3LjQ3NTItNDguNDU1NjgtNDYuNTkyYy0yOC44OTcyOC0yNy43ODExMi0yNy45NTUyLTc1LjIxMjggMS44NDMyLTEwNS45MzI4eiIgZmlsbD0iI0ZCNkQwMSIgcC1pZD0iMTc0MyI+PC9wYXRoPjxwYXRoIGQ9Ik00NDQuNzMzNDQgMjk0LjA3MjMyYTYyLjIyODQ4IDU2Ljc1MDA4IDAgMSAwIDEyNC40NTY5NiAwIDYyLjIyODQ4IDU2Ljc1MDA4IDAgMSAwLTEyNC40NTY5NiAwWiIgZmlsbD0iI0ZFRkZGQSIgcC1pZD0iMTc0NCI+PC9wYXRoPjxwYXRoIGQ9Ik0xOTEuMzY1MTIgNjEwLjg1Njk2YzAgOS44OTE4NCA2MC44MjU2IDU1LjI5NiAxMDAuNzIwNjQgNjIuOTU1NTIgMzkuODg0OCA3LjY2OTc2IDg2LjI3Mi0yNi4yNzU4NCA4Ni4yNzItMzIuODI5NDQgMC02LjU1MzYtNDcuODIwOCAxMC4zNDI0LTg2LjI3MiA0LjE0NzItMzguNDUxMi02LjE5NTItMTAwLjcyMDY0LTQ0LjE3NTM2LTEwMC43MjA2NC0zNC4yNzMyOHpNNzI3LjM3NzkyIDQ1NC4yODczNmMtMC4wNjE0NCAxMi42MjU5Mi0wLjE5NDU2IDI5LjE3Mzc2LTEyLjM2OTkyIDQ0Ljg3MTY4LTguNDM3NzYgMTAuODg1MTItMTE1LjE1OTA0IDc5Ljg5MjQ4LTIyMy45MjgzMiAyMTcuMTY5OTItNTkuMzQwOCA3NC44NzQ4OC0xMDEuMzk2NDggMTc3LjQzODcyLTEyNi4xNzcyOCAzMDcuNjcxMDRINzIuMzA0NjRjMTExLjYxNi0xNzQuMjc0NTYgMjIxLjMzNzYtMzA3LjA5NzYgMzI5LjE0NDMyLTM5OC40OTk4NEM1MjUuMjA5NiA1MjAuNjAxNiA2MzUuMjU4ODggNDYzLjM3MDI0IDczMS42Mjc1MiA0NTMuODQ3MDR6IiBmaWxsPSIjRkVEQkJCIiBwLWlkPSIxNzQ1Ij48L3BhdGg+PHBhdGggZD0iTTExNTMuODYzNjggMzMwLjM0MjR2MzYwLjk5MDcyaDU1LjM5ODRWNTUwLjc3ODg4aDk0LjAxMzQ0Yzg2LjkwNjg4IDAgMTMwLjYxMTItMzYuOTA0OTYgMTMwLjYxMTItMTEwLjcyNTEyIDAtNzMuMzE4NC00My4xOTIzMi0xMDkuNzIxNi0xMjkuNTg3Mi0xMDkuNzIxNmgtMTUwLjQyNTZ6IG01NS4zOTg0IDQ3LjAxMTg0aDkwLjQ2MDE2YzI2LjkzMTIgMCA0Ni43NTU4NCA1LjA1ODU2IDU5LjQ2MzY4IDE1LjE3NTY4IDEyLjY5NzYgOS4wOTMxMiAxOS4zMDI0IDI1LjI3MjMyIDE5LjMwMjQgNDcuNTEzNiAwIDIyLjI1MTUyLTYuNjA0OCAzOC40MzA3Mi0xOC44MDA2NCA0OC41Mzc2LTEyLjY5NzYgMTAuMTE3MTItMzIuNTIyMjQgMTUuMTc1NjgtNTkuOTY1NDQgMTUuMTc1NjhoLTkwLjQ2MDE2VjM3Ny4zNTQyNHpNMTYwMS4wODU0NCA0MjIuODYwOGMtMzkuNjI4OCAwLTcxLjY0OTI4IDEzLjE0ODE2LTk1LjUzOTIgMzkuNDM0MjQtMjMuODg5OTIgMjUuNzg0MzItMzUuNTczNzYgNTguNjU0NzItMzUuNTczNzYgOTguNjAwOTYgMCAzOS40MjQgMTEuNjgzODQgNzIuMjk0NCAzNS4wNjE3NiA5Ny41NzY5NiAyNC40MDE5MiAyNi4yOTYzMiA1Ni40MjI0IDM5LjkzNiA5Ni4wNTEyIDM5LjkzNiAzOS42MzkwNCAwIDcxLjY1OTUyLTEzLjYzOTY4IDk2LjA1MTItMzkuOTM2IDIzLjM3NzkyLTI1LjI4MjU2IDM1LjA3Mi01OC4xNDI3MiAzNS4wNzItOTcuNTg3MiAwLTM5LjkzNi0xMi4xOTU4NC03Mi44MDY0LTM1LjU3Mzc2LTk4LjU5MDcyLTIzLjg4OTkyLTI2LjI4NjA4LTU1LjkxMDQtMzkuNDM0MjQtOTUuNTM5Mi0zOS40MzQyNHogbTAgNDMuOTkxMDRjMjQuOTAzNjggMCA0NC4yMTYzMiA5LjYwNTEyIDU4LjQ0OTkyIDI5LjMxNzEyIDEyLjE4NTYgMTYuNjkxMiAxOC4yODg2NCAzOC40MzA3MiAxOC4yODg2NCA2NC43MTY4IDAgMjUuNzk0NTYtNi4wOTI4IDQ3LjAyMjA4LTE4LjI4ODY0IDY0LjIxNTA0LTE0LjIzMzYgMTkuMjIwNDgtMzMuNTQ2MjQgMjkuMzI3MzYtNTguNDQ5OTIgMjkuMzI3MzYtMjQuOTAzNjggMC00NC4yMDYwOC0xMC4xMDY4OC01Ny45Mjc2OC0yOS4zMjczNi0xMi4yMDYwOC0xNi42OTEyLTE3Ljc4Njg4LTM3LjkxODcyLTE3Ljc4Njg4LTY0LjIwNDggMC0yNi4yOTYzMiA1LjU4MDgtNDguMDM1ODQgMTcuNzg2ODgtNjQuNzE2OCAxMy43MjE2LTE5LjcyMjI0IDMzLjAyNC0yOS4zMjczNiA1Ny45Mjc2OC0yOS4zMjczNnpNMTc4OS42MzQ1NiAzMjMuMjU2MzJ2MzY4LjA3NjhoNTMuODYyNFYzMjMuMjU2MzJ6TTIwMjkuMDA0OCA0MjIuODYwOGMtMzIuNTMyNDggMC01OC45NTE2OCA1LjU2MDMyLTc4LjI2NDMyIDE3LjY5NDcyLTIyLjM2NDE2IDEzLjE0ODE2LTM2LjU5Nzc2IDM0LjM4NTkyLTQyLjE4ODggNjIuNjk5NTJsNTMuMzcwODggNC41NDY1NmMzLjA0MTI4LTE0LjY2MzY4IDEwLjY3MDA4LTI1LjI4MjU2IDIyLjg2NTkyLTMyLjM1ODQgMTAuMTY4MzItNi4wNjIwOCAyMy44ODk5Mi05LjEwMzM2IDQwLjY1MjgtOS4xMDMzNiAzOS42MzkwNCAwIDU5LjQ2MzY4IDE4LjIwNjcyIDU5LjQ2MzY4IDU0LjYwOTkydjEwLjYxODg4bC01OC45NTE2OCAxLjUxNTUyYy0zOC42MjUyOCAxLjAxMzc2LTY5LjEyIDguNjAxNi05MC40NjAxNiAyMy43NTY4LTIzLjM3NzkyIDE1LjY3NzQ0LTM1LjA3MiAzOC40MzA3Mi0zNS4wNzIgNjcuNzU4MDggMCAyMS43Mzk1MiA4LjEzMDU2IDM5LjQzNDI0IDI0LjkwMzY4IDUzLjA4NDE2IDE1LjI1NzYgMTMuNjQ5OTIgMzYuNTk3NzYgMjAuNzM2IDY0LjA0MDk2IDIwLjczNiAyMy4zNzc5MiAwIDQzLjcwNDMyLTQuNTU2OCA2MC45NzkyLTEyLjY0NjRhMTA5LjMxMiAxMDkuMzEyIDAgMCAwIDM4LjExMzI4LTMxLjM0NDY0djM2LjkwNDk2aDQ5LjgwNzM2VjUyNC40OTI4YzAtMzEuODU2NjQtOC4xMzA1Ni01Ni4xMjU0NC0yMy44Nzk2OC03Mi44MDY0LTE4LjI5ODg4LTE5LjIyMDQ4LTQ2Ljc1NTg0LTI4LjgyNTYtODUuMzgxMTItMjguODI1NnogbTU1LjkwMDE2IDE0Ny42NDAzMnYxNS4xNjU0NGMwIDIwLjIyNC04LjY0MjU2IDM3LjQxNjk2LTI0LjkwMzY4IDUxLjA2Njg4LTE2LjI2MTEyIDEzLjY0OTkyLTM1LjU3Mzc2IDIwLjcyNTc2LTU4LjQzOTY4IDIwLjcyNTc2LTEzLjcyMTYgMC0yNC45MDM2OC0zLjUzMjgtMzMuMDM0MjQtMTAuMTA2ODgtOC42NDI1Ni02LjU3NDA4LTEyLjcwNzg0LTE0LjY2MzY4LTEyLjcwNzg0LTI0Ljc4MDggMC0zMi4zNTg0IDI0LjM5MTY4LTQ5LjU0MTEyIDczLjY4NzA0LTUwLjU1NDg4bDU1LjM5ODQtMS41MTU1MnpNMjMyMS43MzU2OCA0MjIuODYwOGMtMTYuMjcxMzYgMC0zMC40OTQ3MiA0LjU0NjU2LTQyLjcwMDggMTQuNjYzNjgtMTAuMTU4MDggNy4wNzU4NC0xOC44MDA2NCAxNy42OTQ3Mi0yNS4zOTUyIDMxLjg0NjR2LTM5LjQyNGgtNTMuODgyODh2MjYxLjM4NjI0aDUzLjg3MjY0VjU1Mi44MDY0YzAtMjIuNzUzMjggNi42MDQ4LTQxLjQ3MiAyMC4zMjY0LTU1LjYyMzY4IDEyLjcwNzg0LTEzLjE0ODE2IDI3LjQ0MzItMTkuNzEyIDQzLjcwNDMyLTE5LjcxMiAxMi4yMDYwOCAwIDI0LjkwMzY4IDEuNTE1NTIgMzguMTIzNTIgNS41NjAzMnYtNTMuNTk2MTZjLTkuMTU0NTYtNC41NDY1Ni0yMC44Mzg0LTYuNTc0MDgtMzQuMDQ4LTYuNTc0MDh6TTIzOTMuODk2OTYgMzMwLjM0MjR2MzYwLjk5MDcyaDEzMS4xMTI5NmM1OC40NDk5MiAwIDEwMi42NjYyNC0xNi4xNzkyIDEzMy4xNTA3Mi00OC41Mzc2IDI4Ljk3OTItMzEuMzM0NCA0My43MTQ1Ni03NS4zMzU2OCA0My43MTQ1Ni0xMzEuOTYyODggMC01Ny4xMjg5Ni0xNC4yMzM2LTEwMS4xMi00Mi43MDA4LTEzMS40NTA4OC0zMC40ODQ0OC0zMi44NzA0LTc0LjcwMDgtNDkuMDQ5Ni0xMzMuMTQwNDgtNDkuMDQ5NmgtMTMyLjEzNjk2eiBtNTUuMzk4NCA0Ny4wMTE4NGg2Ni41NzAyNGM0NS43NDIwOCAwIDc5LjI3ODA4IDEwLjYxODg4IDEwMC42Mjg0OCAzMi4zNTg0IDIwLjMyNjQgMjEuMjM3NzYgMzAuOTk2NDggNTQuNjA5OTIgMzAuOTk2NDggMTAxLjEyIDAgNDUuNTA2NTYtMTAuNjcwMDggNzguODc4NzItMzEuNTA4NDggMTAwLjYxODI0LTIxLjM0MDE2IDIxLjczOTUyLTU1LjM5ODQgMzIuODcwNC0xMDEuMTMwMjQgMzIuODcwNGgtNjUuNTU2NDh2LTI2Ni45NTY4ek0yNzU4LjI4NzM2IDMzMC4zNDI0djM2MC45OTA3MmgxNjUuNjcyOTZjMzguNjI1MjggMCA2OC42MDgtNy4wNjU2IDg5Ljk0ODE2LTIxLjIyNzUyIDI0LjkwMzY4LTE3LjIwMzIgMzcuNjExNTItNDMuNDg5MjggMzcuNjExNTItNzkuODkyNDggMC0yNC4yNjg4LTYuMTAzMDQtNDMuOTgwOC0xOC4yOTg4OC01OC42NDQ0OC0xMi4xODU2LTE0LjY2MzY4LTMwLjQ4NDQ4LTI0LjI2ODgtNTQuMzc0NC0yOC44MjU2IDE4LjI5ODg4LTYuNTc0MDggMzIuMDIwNDgtMTYuNjgwOTYgNDIuMTg4OC0yOS44MjkxMiAxMC4xNTgwOC0xNC4xNTE2OCAxNS4yMzcxMi0zMS4zNDQ2NCAxNS4yMzcxMi01MS41Njg2NCAwLTI3LjgxMTg0LTkuNjU2MzItNDkuNTUxMzYtMjguOTY4OTYtNjUuNzMwNTYtMjAuMzI2NC0xNy4xOTI5Ni00Ny43Njk2LTI1LjI4MjU2LTgzLjM1MzYtMjUuMjgyNTZoLTE2NS42NjI3MnogbTU1LjM5ODQgNDUuNDk2MzJoOTYuNTUyOTZjMjQuMzkxNjggMCA0Mi42OTA1NiA0LjA0NDggNTMuODYyNCAxMi42NDY0IDExLjE5MjMyIDguMDg5NiAxNi43NzMxMiAyMS4yMjc1MiAxNi43NzMxMiAzOS40MjQgMCAxOS4yMjA0OC01LjU4MDggMzMuMzgyNC0xNi43NjI4OCA0Mi40NzU1Mi0xMS4xODIwOCA4LjYwMTYtMjkuNDgwOTYgMTMuMTQ4MTYtNTQuODg2NCAxMy4xNDgxNmgtOTUuNTM5MlYzNzUuODM4NzJ6IG0wIDE1Mi42OTg4OGgxMDQuMTcxNTJjMjYuNDI5NDQgMCA0Ni4yNTQwOCA0LjU0NjU2IDU4Ljk1MTY4IDE0LjE1MTY4IDEyLjcwNzg0IDkuNjA1MTIgMTkuMzEyNjQgMjUuMjgyNTYgMTkuMzEyNjQgNDYuNTIwMzIgMCAyMC43MjU3Ni04LjYzMjMyIDM1Ljg5MTItMjQuOTAzNjggNDUuNDk2MzItMTMuMjA5NiA3LjA4NjA4LTMxLjUwODQ4IDExLjEzMDg4LTU0Ljg4NjQgMTEuMTMwODhoLTEwMi42NTZWNTI4LjUzNzZ6IiBmaWxsPSIjMTExMTExIiBwLWlkPSIxNzQ2Ij48L3BhdGg+PC9zdmc+", } # RedisCloud color: #0D6EFD diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 27be85b6e..0369b2e33 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -155,6 +155,11 @@ class CaseConfigParamType(Enum): optimize_after_write = "optimize_after_write" read_dur_after_write = "read_dur_after_write" + # PolarDB parameters + insert_workers = "insert_workers" + post_load_index = "post_load_index" + pq_nbits = "pq_nbits" + # Lindorm parameters efSearch = "efSearch" pq_m = "pq_m" From 7e251b6a96654116ed13d98c1a8f8cf16d1061ec Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Wed, 1 Apr 2026 13:39:04 +0800 Subject: [PATCH 04/49] Add concurrent insert in performence case (#741) 1. Fix concurrent insert memory and process cleanup 2. Add configurable load concurrency for performance cases 3. Make CLI Ctrl+C work by polling has_running() instead of blocking on concurrent.futures.wait(), which swallows SIGINT. 4. Remove perf-case insert from SerialInsertRunner 5. Ignore S608 lint rule and fix formatting Signed-off-by: yangxuan --- pyproject.toml | 2 +- tests/test_concurrent_runner.py | 159 ++++++++++ vectordb_bench/__init__.py | 1 + .../backend/clients/alisql/alisql.py | 8 +- vectordb_bench/backend/clients/api.py | 5 + vectordb_bench/backend/clients/doris/doris.py | 2 + .../backend/clients/mariadb/mariadb.py | 6 +- .../backend/clients/milvus/milvus.py | 11 + .../backend/clients/oceanbase/oceanbase.py | 4 +- .../backend/clients/pgvector/pgvector.py | 1 + vectordb_bench/backend/clients/tidb/tidb.py | 10 +- vectordb_bench/backend/clients/vespa/vespa.py | 2 +- vectordb_bench/backend/runner/__init__.py | 2 + .../backend/runner/concurrent_runner.py | 278 ++++++++++++++++++ vectordb_bench/backend/runner/executor.py | 170 +++++++++++ .../backend/runner/serial_runner.py | 93 +----- vectordb_bench/backend/task_runner.py | 18 +- vectordb_bench/backend/utils.py | 41 +++ vectordb_bench/cli/cli.py | 26 +- .../components/run_test/submitTask.py | 11 +- vectordb_bench/interface.py | 31 +- vectordb_bench/models.py | 1 + 22 files changed, 730 insertions(+), 152 deletions(-) create mode 100644 tests/test_concurrent_runner.py create mode 100644 vectordb_bench/backend/runner/concurrent_runner.py create mode 100644 vectordb_bench/backend/runner/executor.py diff --git a/pyproject.toml b/pyproject.toml index d7bf42633..6706a3d4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,7 @@ lint.ignore = [ "INP001", # TODO "TID252", # TODO "N801", "N802", "N815", - "S101", "S108", "S603", "S311", + "S101", "S108", "S603", "S311", "S608", "PLR2004", "RUF017", "C416", diff --git a/tests/test_concurrent_runner.py b/tests/test_concurrent_runner.py new file mode 100644 index 000000000..c9e5d9267 --- /dev/null +++ b/tests/test_concurrent_runner.py @@ -0,0 +1,159 @@ +"""Tests for ConcurrentInsertRunner against a running Milvus instance. + +Includes: + - Correctness tests (threading & async backends) + - Parameterized benchmark: serial vs concurrent across (batch_size, workers) matrix + +NUM_PER_BATCH is set via os.environ before each run. Since runners execute +task() in a spawn subprocess that re-imports config, the env var takes effect. + +Requires: + - Milvus running at localhost:19530 + - Network access to download OpenAI 50K dataset + +Usage: + pytest tests/test_concurrent_runner.py -v -s # correctness tests only + python tests/test_concurrent_runner.py # full benchmark matrix +""" + +# ruff: noqa: T201 + +from __future__ import annotations + +import logging +import os +import time + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.milvus.config import FLATConfig +from vectordb_bench.backend.dataset import Dataset, DatasetSource +from vectordb_bench.backend.runner.concurrent_runner import ConcurrentInsertRunner, ExecutorBackend +from vectordb_bench.backend.runner.serial_runner import SerialInsertRunner + +log = logging.getLogger("vectordb_bench") +log.setLevel(logging.INFO) + +DATASET_SIZE = 50_000 + + +# ── Shared helpers ────────────────────────────────────────────────────── + + +def get_milvus_db(collection_name: str): + return DB.Milvus.init_cls( + dim=1536, + db_config={"uri": "http://localhost:19530", "user": "", "password": ""}, + db_case_config=FLATConfig(metric_type="COSINE"), + collection_name=collection_name, + drop_old=True, + ) + + +def prepare_dataset(): + dataset = Dataset.OPENAI.manager(DATASET_SIZE) + dataset.prepare(DatasetSource.AliyunOSS) + return dataset + + +def set_batch_size(batch_size: int) -> None: + os.environ["NUM_PER_BATCH"] = str(batch_size) + + +def timed_run(runner: SerialInsertRunner | ConcurrentInsertRunner) -> tuple[int, float]: + start = time.perf_counter() + count = runner.run() + return count, time.perf_counter() - start + + +# ── Correctness tests (pytest) ────────────────────────────────────────── + + +def test_concurrent_insert_threading(): + """Test concurrent insert with threading backend.""" + db = get_milvus_db("test_conc_threading") + runner = ConcurrentInsertRunner( + db=db, + dataset=prepare_dataset(), + normalize=False, + max_workers=4, + backend=ExecutorBackend.THREADING, + ) + count = runner.run() + assert count == DATASET_SIZE, f"Expected {DATASET_SIZE}, got {count}" + + +def test_concurrent_insert_async(): + """Test concurrent insert with async backend.""" + db = get_milvus_db("test_conc_async") + runner = ConcurrentInsertRunner( + db=db, + dataset=prepare_dataset(), + normalize=False, + max_workers=4, + backend=ExecutorBackend.ASYNC, + ) + count = runner.run() + assert count == DATASET_SIZE, f"Expected {DATASET_SIZE}, got {count}" + + +# ── Parameterized benchmark ──────────────────────────────────────────── + + +def run_serial(batch_size: int) -> tuple[int, float]: + set_batch_size(batch_size) + runner = SerialInsertRunner( + db=get_milvus_db(f"bench_serial_b{batch_size}"), + dataset=prepare_dataset(), + normalize=False, + ) + return timed_run(runner) + + +def run_concurrent(batch_size: int, workers: int) -> tuple[int, float]: + set_batch_size(batch_size) + runner = ConcurrentInsertRunner( + db=get_milvus_db(f"bench_conc_b{batch_size}_w{workers}"), + dataset=prepare_dataset(), + normalize=False, + max_workers=workers, + backend=ExecutorBackend.THREADING, + ) + return timed_run(runner) + + +def bench_matrix(): + batch_sizes = [100, 500, 1000, 5000] + worker_counts = [1, 2, 4, 8] + + conc_headers = [f"conc({w}w)" for w in worker_counts] + speedup_headers = [f"speedup({w}w)" for w in worker_counts] + print(f"\n{'Batch':>6} {'#Bat':>5} {'serial':>8}", end="") + for h in conc_headers: + print(f" {h:>10}", end="") + for h in speedup_headers: + print(f" {h:>12}", end="") + print() + print("-" * (22 + 10 * len(worker_counts) + 12 * len(worker_counts))) + + for bs in batch_sizes: + n_batches = DATASET_SIZE // bs + _, dur_s = run_serial(bs) + + conc_durs = [] + for w in worker_counts: + _, dur_c = run_concurrent(bs, w) + conc_durs.append(dur_c) + + print(f"{bs:>6} {n_batches:>5} {dur_s:>7.2f}s", end="") + for dur_c in conc_durs: + print(f" {dur_c:>9.2f}s", end="") + for dur_c in conc_durs: + print(f" {dur_s / dur_c:>11.2f}x", end="") + print() + + # restore default + set_batch_size(100) + + +if __name__ == "__main__": + bench_matrix() diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 07f77bb02..fc1813b38 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -20,6 +20,7 @@ class config: DATASET_SOURCE = env.str("DATASET_SOURCE", "S3") # Options "S3" or "AliyunOSS" DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") NUM_PER_BATCH = env.int("NUM_PER_BATCH", 100) + LOAD_CONCURRENCY = env.int("LOAD_CONCURRENCY", 0) # 0 = cpu_count TIME_PER_BATCH = 1 # 1s. for streaming insertion. MAX_INSERT_RETRY = 5 MAX_SEARCH_RETRY = 5 diff --git a/vectordb_bench/backend/clients/alisql/alisql.py b/vectordb_bench/backend/clients/alisql/alisql.py index f88cf9d88..6d1fbaefe 100644 --- a/vectordb_bench/backend/clients/alisql/alisql.py +++ b/vectordb_bench/backend/clients/alisql/alisql.py @@ -107,15 +107,13 @@ def init(self): self.cursor.execute(f"SET SESSION vidx_hnsw_ef_search = {search_param['ef_search']}") self.cursor.execute("COMMIT") - self.insert_sql = ( - f'INSERT INTO {self.db_config["database"]}.{self.table_name} (id, v) VALUES (%s, %s)' # noqa: S608 - ) + self.insert_sql = f'INSERT INTO {self.db_config["database"]}.{self.table_name} (id, v) VALUES (%s, %s)' self.select_sql = ( - f'SELECT id FROM {self.db_config["database"]}.{self.table_name} ' # noqa: S608 + f'SELECT id FROM {self.db_config["database"]}.{self.table_name} ' f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %s" ) self.select_sql_with_filter = ( - f'SELECT id FROM {self.db_config["database"]}.{self.table_name} WHERE id >= %s ' # noqa: S608 + f'SELECT id FROM {self.db_config["database"]}.{self.table_name} WHERE id >= %s ' f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %s" ) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 82eda1824..80709e8e3 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -140,6 +140,11 @@ class VectorDB(ABC): supported_filter_types: list[FilterOp] = [FilterOp.NonFilter] name: str = "" + # Whether the client can share a single connection across threads. + # If False, concurrent runners will deep-copy the instance and call + # init() per thread instead of sharing the parent connection. + thread_safe: bool = True + @classmethod def filter_supported(cls, filters: Filter) -> bool: """Ensure that the filters are supported before testing filtering cases.""" diff --git a/vectordb_bench/backend/clients/doris/doris.py b/vectordb_bench/backend/clients/doris/doris.py index 82b3a12da..01984d665 100644 --- a/vectordb_bench/backend/clients/doris/doris.py +++ b/vectordb_bench/backend/clients/doris/doris.py @@ -13,6 +13,8 @@ class Doris(VectorDB): + thread_safe: bool = False + def __init__( self, dim: int, diff --git a/vectordb_bench/backend/clients/mariadb/mariadb.py b/vectordb_bench/backend/clients/mariadb/mariadb.py index db3863c85..e6053a0d8 100644 --- a/vectordb_bench/backend/clients/mariadb/mariadb.py +++ b/vectordb_bench/backend/clients/mariadb/mariadb.py @@ -108,13 +108,13 @@ def init(self): self.cursor.execute(f"SET mhnsw_ef_search = {search_param['ef_search']}") self.cursor.execute("COMMIT") - self.insert_sql = f"INSERT INTO {self.db_name}.{self.table_name} (id, v) VALUES (%s, %s)" # noqa: S608 + self.insert_sql = f"INSERT INTO {self.db_name}.{self.table_name} (id, v) VALUES (%s, %s)" self.select_sql = ( - f"SELECT id FROM {self.db_name}.{self.table_name}" # noqa: S608 + f"SELECT id FROM {self.db_name}.{self.table_name}" f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %d" ) self.select_sql_with_filter = ( - f"SELECT id FROM {self.db_name}.{self.table_name} WHERE id >= %d " # noqa: S608 + f"SELECT id FROM {self.db_name}.{self.table_name} WHERE id >= %d " f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %d" ) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index ead2979ff..9e9dfb7f9 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -137,6 +137,16 @@ def init(self): self.client.close() self.client = None + def _wait_for_segments_sorted(self): + while True: + segments = self.client.list_persistent_segments(self.collection_name) + unsorted = [s for s in segments if not s.is_sorted] + if not unsorted: + log.info(f"{self.name} all persistent segments are sorted.") + break + log.debug(f"{self.name} waiting for {len(unsorted)} segments to be sorted...") + time.sleep(5) + def _wait_for_index(self): while True: info = self.client.describe_index(self.collection_name, self._vector_index_name) @@ -155,6 +165,7 @@ def _optimize(self): log.info(f"{self.name} optimizing before search") try: self.client.flush(self.collection_name) + self._wait_for_segments_sorted() self._wait_for_index() if self.case_config.is_gpu_index: log.debug("skip force merge compaction for gpu index type.") diff --git a/vectordb_bench/backend/clients/oceanbase/oceanbase.py b/vectordb_bench/backend/clients/oceanbase/oceanbase.py index 93c42aac1..bf615e4d0 100644 --- a/vectordb_bench/backend/clients/oceanbase/oceanbase.py +++ b/vectordb_bench/backend/clients/oceanbase/oceanbase.py @@ -186,7 +186,7 @@ def insert_embeddings( batch = [(metadata[i], embeddings[i]) for i in range(batch_start, batch_end)] values = ", ".join(f"({item_id}, '[{','.join(map(str, embedding))}]')" for item_id, embedding in batch) self._cursor.execute( - f"INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(32) */ INTO {self.table_name} VALUES {values}" # noqa: S608 + f"INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(32) */ INTO {self.table_name} VALUES {values}" ) insert_count += len(batch) except mysql.Error: @@ -217,7 +217,7 @@ def search_embedding( packed = struct.pack(f"<{len(query)}f", *query) hex_vec = packed.hex() query_str = ( - f"SELECT id FROM {self.table_name} " # noqa: S608 + f"SELECT id FROM {self.table_name} " f"{self.expr} ORDER BY " f"{self.db_case_config.parse_metric_func_str()}(embedding, X'{hex_vec}') " f"APPROXIMATE LIMIT {k}" diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 42fa7533d..30c797c38 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -21,6 +21,7 @@ class PgVector(VectorDB): """Use psycopg instructions""" + thread_safe: bool = False supported_filter_types: list[FilterOp] = [ FilterOp.NonFilter, FilterOp.NumGE, diff --git a/vectordb_bench/backend/clients/tidb/tidb.py b/vectordb_bench/backend/clients/tidb/tidb.py index a5c99bbe4..fba60d41c 100644 --- a/vectordb_bench/backend/clients/tidb/tidb.py +++ b/vectordb_bench/backend/clients/tidb/tidb.py @@ -119,7 +119,7 @@ def _optimize_check_tiflash_replica_progress(self): cursor.execute(f""" SELECT PROGRESS FROM information_schema.tiflash_replica WHERE TABLE_SCHEMA = "{database}" AND TABLE_NAME = "{self.table_name}" - """) # noqa: S608 + """) result = cursor.fetchone() return result[0] except Exception as e: @@ -131,7 +131,7 @@ def _optimize_wait_tiflash_catch_up(self): with self._get_connection() as (conn, cursor): cursor.execute('SET @@TIDB_ISOLATION_READ_ENGINES="tidb,tiflash"') conn.commit() - cursor.execute(f"SELECT COUNT(*) FROM {self.table_name}") # noqa: S608 + cursor.execute(f"SELECT COUNT(*) FROM {self.table_name}") result = cursor.fetchone() return result[0] except Exception as e: @@ -155,7 +155,7 @@ def _optimize_get_tiflash_index_pending_rows(self): SELECT SUM(ROWS_STABLE_NOT_INDEXED) FROM information_schema.tiflash_indexes WHERE TIDB_DATABASE = "{database}" AND TIDB_TABLE = "{self.table_name}" - """) # noqa: S608 + """) result = cursor.fetchone() return result[0] except Exception as e: @@ -172,7 +172,7 @@ def _insert_embeddings_serial( try: with self._get_connection() as (conn, cursor): buf = io.StringIO() - buf.write(f"INSERT INTO {self.table_name} (id, embedding) VALUES ") # noqa: S608 + buf.write(f"INSERT INTO {self.table_name} (id, embedding) VALUES ") for i in range(offset, offset + size): if i > offset: buf.write(",") @@ -220,6 +220,6 @@ def search_embedding( self.cursor.execute(f""" SELECT id FROM {self.table_name} ORDER BY {self.search_fn}(embedding, "{query!s}") LIMIT {k}; - """) # noqa: S608 + """) result = self.cursor.fetchall() return [int(i[0]) for i in result] diff --git a/vectordb_bench/backend/clients/vespa/vespa.py b/vectordb_bench/backend/clients/vespa/vespa.py index 5288bc04c..1f2e1b883 100644 --- a/vectordb_bench/backend/clients/vespa/vespa.py +++ b/vectordb_bench/backend/clients/vespa/vespa.py @@ -107,7 +107,7 @@ def search_embedding( embedding_field = "embedding" if self.case_config.quantization_type == "none" else "embedding_binary" yql = ( - f"select id from {self.schema_name} where " # noqa: S608 + f"select id from {self.schema_name} where " f"{{targetHits: {k}, hnsw.exploreAdditionalHits: {extra_ef}}}" f"nearestNeighbor({embedding_field}, query_embedding)" ) diff --git a/vectordb_bench/backend/runner/__init__.py b/vectordb_bench/backend/runner/__init__.py index 4af583773..d56fe0ff8 100644 --- a/vectordb_bench/backend/runner/__init__.py +++ b/vectordb_bench/backend/runner/__init__.py @@ -1,8 +1,10 @@ +from .concurrent_runner import ConcurrentInsertRunner from .mp_runner import MultiProcessingSearchRunner from .read_write_runner import ReadWriteRunner from .serial_runner import SerialInsertRunner, SerialSearchRunner __all__ = [ + "ConcurrentInsertRunner", "MultiProcessingSearchRunner", "ReadWriteRunner", "SerialInsertRunner", diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py new file mode 100644 index 000000000..6ed8e39fb --- /dev/null +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -0,0 +1,278 @@ +"""Concurrent insert runner with configurable executor backend. + +Replaces SerialInsertRunner for faster data loading in performance cases. + +Auto-detects thread-unsafe DBs via VectorDB.thread_safe and +falls back to single-worker mode. +""" + +from __future__ import annotations + +import concurrent.futures +import logging +import multiprocessing as mp +import threading +import time +from copy import deepcopy +from enum import StrEnum +from typing import TYPE_CHECKING + +import numpy as np + +from vectordb_bench.backend.filter import Filter, FilterOp, non_filter +from vectordb_bench.backend.utils import kill_proc_tree, time_it + +from ... import config +from ...models import PerformanceTimeoutError +from .executor import AsyncExecutor, ThreadExecutor + +if TYPE_CHECKING: + from vectordb_bench.backend.clients import api + from vectordb_bench.backend.dataset import DatasetManager + + from .executor import TaskExecutor + +log = logging.getLogger(__name__) + + +class ExecutorBackend(StrEnum): + THREADING = "threading" + ASYNC = "async" + + +class ConcurrentInsertRunner: + """Concurrent insert runner with pluggable executor backend. + + Thread-safety: If db.thread_safe is False, max_workers is clamped to 1 + and each worker thread gets a deep-copied DB instance with its own connection. + + Args: + db: VectorDB instance. + dataset: DatasetManager for batch iteration. + normalize: Whether to L2-normalize embeddings. + filters: Filter configuration. + timeout: Timeout in seconds for the overall operation. + max_workers: Number of concurrent workers (default: cpu_count). + backend: Executor backend to use ('threading' or 'async'). + """ + + def __init__( + self, + db: api.VectorDB, + dataset: DatasetManager, + normalize: bool, + filters: Filter = non_filter, + timeout: float | None = None, + max_workers: int | None = None, + backend: ExecutorBackend = ExecutorBackend.THREADING, + ): + self.timeout = timeout if isinstance(timeout, int | float) else None + self.dataset: DatasetManager = dataset + self.db = db + self.normalize = normalize + self.filters = filters + self.backend = backend + + effective_workers = max_workers or mp.cpu_count() + if not db.thread_safe: + log.info(f"DB {db.name} is not thread-safe, falling back to max_workers=1") + effective_workers = 1 + self.max_workers = effective_workers + + def __getstate__(self): + """Exclude unpicklable thread-local state for ProcessPoolExecutor(spawn).""" + state = self.__dict__.copy() + state.pop("_local", None) + state.pop("_ctx_lock", None) + state.pop("_thread_contexts", None) + state.pop("_iter_lock", None) + state.pop("_dataset_iter", None) + return state + + def __setstate__(self, state: dict): + self.__dict__.update(state) + self._local = threading.local() + self._ctx_lock = threading.Lock() + self._thread_contexts = [] + + def _create_executor(self) -> TaskExecutor: + if self.backend == ExecutorBackend.ASYNC: + return AsyncExecutor(max_workers=self.max_workers) + return ThreadExecutor(max_workers=self.max_workers) + + def _get_thread_db(self) -> api.VectorDB: + """Get or create a per-thread DB instance. + + Thread-safe DBs reuse self.db (connection opened in task()). + Non-thread-safe DBs get a deep-copied instance with its own connection, + cached in thread-local storage so it is created once per thread. + """ + if not hasattr(self._local, "db"): + if self.db.thread_safe: + self._local.db = self.db + else: + db = deepcopy(self.db) + # Manual __enter__/__exit__ because enter and exit happen in + # different scopes (here vs _cleanup_thread_contexts). + ctx = db.init() + ctx.__enter__() + self._local.db = db + with self._ctx_lock: + self._thread_contexts.append(ctx) + return self._local.db + + def _cleanup_thread_contexts(self) -> None: + """Close per-thread DB connections opened for non-thread-safe clients.""" + for ctx in self._thread_contexts: + try: + ctx.__exit__(None, None, None) + except Exception: + log.warning("Failed to close per-thread DB connection", exc_info=True) + self._thread_contexts.clear() + + def _insert_batch_with_retry( + self, + db: api.VectorDB, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + retry_idx: int = 0, + ) -> int: + """Insert a single batch with retry logic. Returns inserted count.""" + insert_count, error = db.insert_embeddings( + embeddings=embeddings, + metadata=metadata, + labels_data=labels_data, + ) + if error is not None: + log.warning(f"Insert failed, try_idx={retry_idx}, Exception: {error}") + retry_idx += 1 + if retry_idx <= config.MAX_INSERT_RETRY: + time.sleep(retry_idx) + return self._insert_batch_with_retry(db, embeddings, metadata, labels_data, retry_idx) + msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" + raise RuntimeError(msg) + return insert_count + + def _worker_insert( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + ) -> int: + """Worker function: insert a batch with retry. + + Thread-safe DBs: reuse self.db whose connection is already open + via task()'s `with self.db.init()` — all threads share it safely. + + Non-thread-safe DBs: use a per-thread deep-copied instance with + its own connection, cached via threading.local. + """ + db = self._get_thread_db() + return self._insert_batch_with_retry(db, embeddings, metadata, labels_data) + + def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None] | None: + """Pull the next batch from the shared dataset iterator. + + Thread-safe: only one thread reads from the iterator at a time. + Returns None when the iterator is exhausted. + """ + with self._iter_lock: + try: + data_df = next(self._dataset_iter) + except StopIteration: + return None + + all_metadata = data_df[self.dataset.data.train_id_field].tolist() + emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) + if self.normalize: + all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() + else: + all_embeddings = emb_np.tolist() + del emb_np + + labels_data = None + if self.filters.type == FilterOp.StrEqual: + if self.dataset.data.scalar_labels_file_separated: + labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() + else: + labels_data = data_df[self.filters.label_field].tolist() + + return all_embeddings, all_metadata, labels_data + + def _worker_loop(self) -> int: + """Worker loop: pull batches from the shared iterator and insert them.""" + total = 0 + while True: + batch = self._next_batch() + if batch is None: + break + embeddings, metadata, labels_data = batch + total += self._worker_insert(embeddings, metadata, labels_data) + return total + + def task(self) -> int: + """Insert entire dataset using concurrent executor. Runs in subprocess.""" + count = 0 + self._local = threading.local() + self._ctx_lock = threading.Lock() + self._thread_contexts = [] + self._iter_lock = threading.Lock() + self._dataset_iter = iter(self.dataset) + + with self.db.init(): + log.info( + f"({mp.current_process().name:16}) Start concurrent insert, " + f"batch_size={config.NUM_PER_BATCH}, max_workers={self.max_workers}" + ) + start = time.perf_counter() + + try: + with self._create_executor() as executor: + for _ in range(self.max_workers): + executor.submit(self._worker_loop) + + batch_results = executor.wait_all() + + # Log all errors, then raise the first one + errors = [r.error for r in batch_results if r.error is not None] + if errors: + for err in errors: + log.warning(f"Batch insert error: {err}") + raise errors[0] + + count = sum(r.value for r in batch_results) + finally: + self._cleanup_thread_contexts() + + log.info( + f"({mp.current_process().name:16}) Finish concurrent insert, " + f"count={count}, dur={time.perf_counter() - start:.2f}s" + ) + return count + + @time_it + def _insert_all_batches(self) -> int: + """Performance case only: run task() in subprocess with timeout.""" + with concurrent.futures.ProcessPoolExecutor( + mp_context=mp.get_context("spawn"), + max_workers=1, + ) as executor: + future = executor.submit(self.task) + try: + count = future.result(timeout=self.timeout) + except TimeoutError as e: + msg = f"VectorDB load dataset timeout in {self.timeout}" + log.warning(msg) + kill_proc_tree(pids=list(executor._processes.keys())) + raise PerformanceTimeoutError(msg) from e + except Exception as e: + log.warning(f"VectorDB load dataset error: {e}") + raise e from e + else: + return count + + def run(self) -> int: + """Insert full dataset concurrently. Returns total inserted count.""" + count, _ = self._insert_all_batches() + return count diff --git a/vectordb_bench/backend/runner/executor.py b/vectordb_bench/backend/runner/executor.py new file mode 100644 index 000000000..0bff2dc2a --- /dev/null +++ b/vectordb_bench/backend/runner/executor.py @@ -0,0 +1,170 @@ +"""Task executor abstraction with threading and async backends. + +Provides a unified interface for submitting callables with controlled +concurrency. Two implementations: + - ThreadExecutor: backed by ThreadPoolExecutor + - AsyncExecutor: backed by asyncio with semaphore-based concurrency control +""" + +from __future__ import annotations + +import asyncio +import logging +from abc import ABC, abstractmethod +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable + +log = logging.getLogger(__name__) + + +@dataclass +class TaskResult: + """Result of a single submitted task.""" + + value: Any = None + error: Exception | None = None + + @property + def success(self) -> bool: + return self.error is None + + +class TaskExecutor(ABC): + """Abstract executor that accepts callables and controls concurrency.""" + + @abstractmethod + def start(self) -> None: + """Initialize executor resources.""" + raise NotImplementedError + + @abstractmethod + def submit(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None: + """Submit a task for execution.""" + raise NotImplementedError + + @abstractmethod + def wait_all(self) -> list[TaskResult]: + """Block until all submitted tasks complete. Return results in submission order.""" + raise NotImplementedError + + @abstractmethod + def shutdown(self) -> None: + """Release executor resources. Safe to call multiple times.""" + raise NotImplementedError + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: object) -> bool: + self.shutdown() + return False + + +class ThreadExecutor(TaskExecutor): + """ThreadPoolExecutor-backed implementation.""" + + def __init__(self, max_workers: int): + self._max_workers = max(1, max_workers) + self._executor: ThreadPoolExecutor | None = None + self._futures: list[Future] = [] + + def start(self) -> None: + self._executor = ThreadPoolExecutor(max_workers=self._max_workers) + self._futures = [] + + def submit(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None: + if self._executor is None: + raise RuntimeError("Executor not started. Call start() or use as context manager.") + future = self._executor.submit(fn, *args, **kwargs) + self._futures.append(future) + + def wait_all(self) -> list[TaskResult]: + results = [] + for future in self._futures: + try: + value = future.result() + results.append(TaskResult(value=value)) + except Exception as e: + results.append(TaskResult(error=e)) + self._futures = [] + return results + + def shutdown(self) -> None: + if self._executor is not None: + self._executor.shutdown(wait=True) + self._executor = None + + +class AsyncExecutor(TaskExecutor): + """asyncio-backed implementation for async DB clients. + + Accepts coroutine functions (async def), runs them on a single event + loop thread with semaphore-based concurrency control. No thread pool. + """ + + def __init__(self, max_workers: int): + self._max_workers = max(1, max_workers) + self._loop: asyncio.AbstractEventLoop | None = None + self._semaphore: asyncio.Semaphore | None = None + self._coros: list = [] + self._owns_loop = False + + def start(self) -> None: + try: + self._loop = asyncio.get_running_loop() + except RuntimeError: + self._loop = asyncio.new_event_loop() + self._owns_loop = True + self._semaphore = asyncio.Semaphore(self._max_workers) + self._coros = [] + + def submit(self, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None: + """Submit a callable for execution. + + Accepts both coroutine functions (async def) and regular functions. + Sync functions are offloaded to a thread via run_in_executor. + """ + if self._loop is None or self._semaphore is None: + raise RuntimeError("Executor not started. Call start() or use as context manager.") + + async def _run(): + async with self._semaphore: + if asyncio.iscoroutinefunction(fn): + return await fn(*args, **kwargs) + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, lambda: fn(*args, **kwargs)) + + self._coros.append(_run()) + + def wait_all(self) -> list[TaskResult]: + if not self._coros: + return [] + + async def _gather(): + gathered = await asyncio.gather(*self._coros, return_exceptions=True) + results = [] + for item in gathered: + if isinstance(item, Exception): + results.append(TaskResult(error=item)) + else: + results.append(TaskResult(value=item)) + return results + + if self._owns_loop: + results = self._loop.run_until_complete(_gather()) + else: + results = asyncio.run_coroutine_threadsafe(_gather(), self._loop).result() + + self._coros = [] + return results + + def shutdown(self) -> None: + if self._owns_loop and self._loop is not None: + self._loop.close() + self._loop = None + self._semaphore = None diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 300553a4e..be0c6322d 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -1,4 +1,4 @@ -import concurrent +import concurrent.futures import logging import math import multiprocessing as mp @@ -6,14 +6,13 @@ import traceback import numpy as np -import psutil from vectordb_bench.backend.dataset import DatasetManager -from vectordb_bench.backend.filter import Filter, FilterOp, non_filter +from vectordb_bench.backend.filter import Filter, non_filter from ... import config from ...metric import calc_ndcg, calc_recall, get_ideal_dcg -from ...models import LoadTimeoutError, PerformanceTimeoutError +from ...models import LoadTimeoutError from .. import utils from ..clients import api @@ -38,66 +37,6 @@ def __init__( self.normalize = normalize self.filters = filters - def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): - _, error = db.insert_embeddings(**kwargs) - if error is not None: - log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") - retry_idx += 1 - if retry_idx <= config.MAX_INSERT_RETRY: - time.sleep(retry_idx) - self.retry_insert(db, retry_idx=retry_idx, **kwargs) - else: - msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" - raise RuntimeError(msg) from None - - def task(self) -> int: - count = 0 - with self.db.init(): - log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") - start = time.perf_counter() - for data_df in self.dataset: - all_metadata = data_df[self.dataset.data.train_id_field].tolist() - - emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) - if self.normalize: - log.debug("normalize the 100k train data") - all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() - else: - all_embeddings = emb_np.tolist() - del emb_np - log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") - - labels_data = None - if self.filters.type == FilterOp.StrEqual: - if self.dataset.data.scalar_labels_file_separated: - labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() - else: - labels_data = data_df[self.filters.label_field].tolist() - - insert_count, error = self.db.insert_embeddings( - embeddings=all_embeddings, - metadata=all_metadata, - labels_data=labels_data, - ) - if error is not None: - self.retry_insert( - self.db, - embeddings=all_embeddings, - metadata=all_metadata, - labels_data=labels_data, - ) - - assert insert_count == len(all_metadata) - count += insert_count - if count % 100_000 == 0: - log.info(f"({mp.current_process().name:16}) Loaded {count} embeddings into VectorDB") - - log.info( - f"({mp.current_process().name:16}) Finish loading all dataset into VectorDB, " - f"dur={time.perf_counter() - start}" - ) - return count - def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: int = 0) -> int: with self.db.init(): # unique id for endlessness insertion @@ -147,28 +86,6 @@ def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: ) return count - @utils.time_it - def _insert_all_batches(self) -> int: - """Performance case only""" - with concurrent.futures.ProcessPoolExecutor( - mp_context=mp.get_context("spawn"), - max_workers=1, - ) as executor: - future = executor.submit(self.task) - try: - count = future.result(timeout=self.timeout) - except TimeoutError as e: - msg = f"VectorDB load dataset timeout in {self.timeout}" - log.warning(msg) - for pid, _ in executor._processes.items(): - psutil.Process(pid).kill() - raise PerformanceTimeoutError(msg) from e - except Exception as e: - log.warning(f"VectorDB load dataset error: {e}") - raise e from e - else: - return count - def run_endlessness(self) -> int: """run forever util DB raises exception or crash""" # datasets for load tests are quite small, can fit into memory @@ -204,10 +121,6 @@ def run_endlessness(self) -> int: else: raise LoadTimeoutError(self.timeout) - def run(self) -> int: - count, _ = self._insert_all_batches() - return count - class SerialSearchRunner: def __init__( diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 8224a0415..6b51d1277 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -6,7 +6,6 @@ from enum import Enum, auto import numpy as np -import psutil from ..base import BaseModel from ..metric import Metric @@ -15,7 +14,14 @@ from .cases import Case, CaseLabel, StreamingPerformanceCase from .clients import DB, MetricType, api from .data_source import DatasetSource -from .runner import MultiProcessingSearchRunner, ReadWriteRunner, SerialInsertRunner, SerialSearchRunner +from .runner import ( + ConcurrentInsertRunner, + MultiProcessingSearchRunner, + ReadWriteRunner, + SerialInsertRunner, + SerialSearchRunner, +) +from .utils import kill_proc_tree log = logging.getLogger(__name__) @@ -241,14 +247,15 @@ def _run_streaming_case(self) -> Metric: @utils.time_it def _load_train_data(self): - """Insert train data and get the insert_duration""" + """Insert train data concurrently and get the insert_duration""" try: - runner = SerialInsertRunner( + runner = ConcurrentInsertRunner( self.db, self.ca.dataset, self.normalize, self.ca.filters, self.ca.load_timeout, + max_workers=self.config.load_concurrency or None, ) runner.run() except Exception as e: @@ -299,8 +306,7 @@ def _optimize(self) -> float: return future.result(timeout=self.ca.optimize_timeout)[1] except TimeoutError as e: log.warning(f"VectorDB optimize timeout in {self.ca.optimize_timeout}") - for pid, _ in executor._processes.items(): - psutil.Process(pid).kill() + kill_proc_tree(pids=list(executor._processes.keys())) raise PerformanceTimeoutError from e except Exception as e: log.warning(f"VectorDB optimize error: {e}") diff --git a/vectordb_bench/backend/utils.py b/vectordb_bench/backend/utils.py index 86c4faf5e..432f0d1d1 100644 --- a/vectordb_bench/backend/utils.py +++ b/vectordb_bench/backend/utils.py @@ -1,6 +1,47 @@ +import contextlib +import logging +import signal import time from functools import wraps +import psutil + +log = logging.getLogger(__name__) + + +def kill_proc_tree(pids: list[int] | None = None, grace: float = 2, timeout: float = 3): + """Kill child processes with SIGTERM, then SIGKILL for survivors. + + Args: + pids: Specific PIDs to kill. If None, kills all children of the + current process (recursive). + grace: Seconds to wait after SIGTERM before sending SIGKILL. + timeout: Seconds to wait for processes to fully exit after SIGKILL. + """ + if pids is not None: + targets = [] + for pid in pids: + with contextlib.suppress(psutil.NoSuchProcess): + targets.append(psutil.Process(pid)) + else: + targets = psutil.Process().children(recursive=True) + + for p in targets: + try: + log.warning(f"sending SIGTERM to child process: {p}") + p.send_signal(signal.SIGTERM) + except psutil.NoSuchProcess: + pass + + _, alive = psutil.wait_procs(targets, timeout=grace) + for p in alive: + try: + log.warning(f"force killing child process: {p}") + p.kill() + except psutil.NoSuchProcess: + pass + psutil.wait_procs(alive, timeout=timeout) + def numerize(n: int) -> str: """display positive number n for readability diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 12bb4be9b..94b13762a 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -1,7 +1,6 @@ import logging import time from collections.abc import Callable -from concurrent.futures import wait from datetime import datetime from pathlib import Path from pprint import pformat @@ -20,7 +19,7 @@ from .. import config from ..backend.clients import DB from ..backend.clients.api import MetricType -from ..interface import benchmark_runner, global_result_future +from ..interface import benchmark_runner from ..models import ( CaseConfig, CaseType, @@ -231,6 +230,16 @@ class CommonTypedDict(TypedDict): show_default=True, ), ] + load_concurrency: Annotated[ + int, + click.option( + "--load-concurrency", + type=int, + default=config.LOAD_CONCURRENCY, + show_default=True, + help="Number of concurrent workers for data loading in performance cases (0 = cpu_count)", + ), + ] search_serial: Annotated[ bool, click.option( @@ -643,15 +652,16 @@ def run( parameters["search_serial"], parameters["search_concurrent"], ), + load_concurrency=parameters["load_concurrency"], ) task_label = parameters["task_label"] log.info(f"Task:\n{pformat(task)}\n") if not parameters["dry_run"]: benchmark_runner.run([task], task_label) - time.sleep(5) - if global_result_future: - wait([global_result_future]) - - while benchmark_runner.has_running(): - time.sleep(1) + try: + while benchmark_runner.has_running(): + time.sleep(1) + except KeyboardInterrupt: + log.warning("Ctrl+C received, stopping benchmark...") + benchmark_runner.stop_running() diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index 01d0c5876..e5c2a1e42 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -61,11 +61,17 @@ def advancedSettings(st): "Concurrency Duration", value=config.CONCURRENCY_DURATION, label_visibility="collapsed" ) container[1].caption("concurrency duration for each concurrency search test") - return index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration + + container = st.columns([1, 2]) + load_concurrency = container[0].number_input( + "Load Concurrency", min_value=0, value=config.LOAD_CONCURRENCY, label_visibility="collapsed" + ) + container[1].caption("number of concurrent workers for data loading in performance cases (0 = cpu_count)") + return index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency def controlPanel(st, tasks: list[TaskConfig], taskLabel, isAllValid): - index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration = advancedSettings(st) + index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency = advancedSettings(st) def runHandler(): benchmark_runner.set_drop_old(not index_already_exists) @@ -80,6 +86,7 @@ def runHandler(): task.case_config.k = k task.case_config.concurrency_search_config.num_concurrency = concurrentInput_list task.case_config.concurrency_search_config.concurrency_duration = concurrency_duration + task.load_concurrency = load_concurrency benchmark_runner.set_download_address(use_aliyun) benchmark_runner.run(tasks, taskLabel) diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 42dc876b0..0d4119e93 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -2,20 +2,17 @@ import logging import multiprocessing as mp import pathlib -import signal import traceback import uuid -from collections.abc import Callable from enum import Enum from multiprocessing.connection import Connection -import psutil - from . import config from .backend.assembler import Assembler, FilterNotSupportedError from .backend.data_source import DatasetSource from .backend.result_collector import ResultCollector from .backend.task_runner import TaskRunner +from .backend.utils import kill_proc_tree from .metric import Metric from .models import ( CaseResult, @@ -240,7 +237,7 @@ def _clear_running_task(self): for r in self.running_task.case_runners: r.stop() - self.kill_proc_tree(timeout=5) + kill_proc_tree() self.running_task = None if self.receive_conn: @@ -261,29 +258,5 @@ def _run_async(self, conn: Connection) -> bool: return True - def kill_proc_tree( - self, - sig: int = signal.SIGTERM, - timeout: float | None = None, - on_terminate: Callable | None = None, - ): - """Kill a process tree (including grandchildren) with signal - "sig" and return a (gone, still_alive) tuple. - "on_terminate", if specified, is a callback function which is - called as soon as a child terminates. - """ - children = psutil.Process().children(recursive=True) - for p in children: - try: - log.warning(f"sending SIGTERM to child process: {p}") - p.send_signal(sig) - except psutil.NoSuchProcess: - pass - _, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate) - - for p in alive: - log.warning(f"force killing child process: {p}") - p.kill() - benchmark_runner = BenchMarkRunner() diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 0369b2e33..cdc64b9d7 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -243,6 +243,7 @@ class TaskConfig(BaseModel): db_case_config: DBCaseConfig case_config: CaseConfig stages: list[TaskStage] = ALL_TASK_STAGES + load_concurrency: int = config.LOAD_CONCURRENCY @property def db_name(self): From 243eb2e94c3f0c7298e3c41a2b9f3236071f10e6 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Thu, 2 Apr 2026 18:17:19 +0800 Subject: [PATCH 05/49] fix: Add back ujson in the requirements (#744) * fix: Add back ujson in the requirements * fix the coding style Signed-off-by: yangxuan --- pyproject.toml | 3 ++- vectordb_bench/backend/clients/polardb/polardb.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6706a3d4e..2baeb16e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,8 +39,9 @@ dependencies = [ "environs", "pydantic=0.10.1", + "ujson", ] dynamic = ["version"] diff --git a/vectordb_bench/backend/clients/polardb/polardb.py b/vectordb_bench/backend/clients/polardb/polardb.py index f42b6fca5..d53638b0e 100644 --- a/vectordb_bench/backend/clients/polardb/polardb.py +++ b/vectordb_bench/backend/clients/polardb/polardb.py @@ -109,14 +109,14 @@ def init(self): db_name = self.db_config["database"] hint = "/*+ SET_VAR(imci_enable_fast_vector_search=on) */" - self.insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" # noqa: S608 + self.insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" self.select_sql = ( - f"SELECT {hint} id FROM {db_name}.{self.table_name} " # noqa: S608 + f"SELECT {hint} id FROM {db_name}.{self.table_name} " f"ORDER BY DISTANCE(v, _binary %s, '{metric_type}') " f"LIMIT %s" ) self.select_sql_with_filter = ( - f"SELECT id FROM {db_name}.{self.table_name} " # noqa: S608 + f"SELECT id FROM {db_name}.{self.table_name} " f"WHERE id >= %s " f"ORDER BY DISTANCE(v, _binary %s, '{metric_type}') " f"LIMIT %s" @@ -218,7 +218,7 @@ def _insert_batch(self, embeddings: list[list[float]], metadata: list[int], offs conn, cursor = self._create_connection() try: db_name = self.db_config["database"] - insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" # noqa: S608 + insert_sql = f"INSERT INTO {db_name}.{self.table_name} (id, v) VALUES (%s, _binary %s)" batch_data = [] for i in range(offset, offset + size): batch_data.append((int(metadata[i]), self.vector_to_hex(embeddings[i]))) From 10ffbcb7060645e5dc5133b41e580108938fcf04 Mon Sep 17 00:00:00 2001 From: jamesgao-jpg Date: Wed, 1 Apr 2026 12:51:14 +0000 Subject: [PATCH 06/49] feat: add region parameter and fix SDK compatibility for turbopuffer client - Add --region CLI parameter (required) for region-based API routing - Change --api-base-url to optional override for private networking - Rename write(columns=...) to write(upsert_columns=...) per current SDK - Fix docstring referencing wrong database name Signed-off-by: jamesgao-jpg --- .../backend/clients/turbopuffer/cli.py | 18 ++++++++++++++---- .../backend/clients/turbopuffer/config.py | 4 +++- .../backend/clients/turbopuffer/turbopuffer.py | 16 +++++++++------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/vectordb_bench/backend/clients/turbopuffer/cli.py b/vectordb_bench/backend/clients/turbopuffer/cli.py index 6fd91f2a8..d510889a0 100644 --- a/vectordb_bench/backend/clients/turbopuffer/cli.py +++ b/vectordb_bench/backend/clients/turbopuffer/cli.py @@ -17,15 +17,24 @@ class TurboPufferTypedDict(TypedDict): str, click.option("--api-key", type=str, help="TurboPuffer API key", required=True), ] + region: Annotated[ + str, + click.option( + "--region", + type=str, + help="TurboPuffer region (e.g. aws-us-east-1, gcp-us-central1)", + required=True, + ), + ] api_base_url: Annotated[ str, click.option( "--api-base-url", type=str, - help="TurboPuffer API base URL", + help="Override the region-based API URL", required=False, - default="https://api.turbopuffer.com", - show_default=True, + default="", + show_default=False, ), ] namespace: Annotated[ @@ -54,7 +63,8 @@ def TurboPuffer(**parameters: Unpack[TurboPufferIndexTypedDict]): db_config=TurboPufferConfig( db_label=parameters["db_label"], api_key=SecretStr(parameters["api_key"]), - api_base_url=parameters["api_base_url"], + region=parameters["region"], + api_base_url=parameters["api_base_url"] or None, namespace=parameters["namespace"], ), db_case_config=TurboPufferIndexConfig(), diff --git a/vectordb_bench/backend/clients/turbopuffer/config.py b/vectordb_bench/backend/clients/turbopuffer/config.py index c552299e3..88e797351 100644 --- a/vectordb_bench/backend/clients/turbopuffer/config.py +++ b/vectordb_bench/backend/clients/turbopuffer/config.py @@ -5,12 +5,14 @@ class TurboPufferConfig(DBConfig): api_key: SecretStr - api_base_url: str + region: str + api_base_url: str | None = None namespace: str = "vdbbench_test" def to_dict(self) -> dict: return { "api_key": self.api_key.get_secret_value(), + "region": self.region, "api_base_url": self.api_base_url, "namespace": self.namespace, } diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py index 241551792..3e1f32af4 100644 --- a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -1,4 +1,4 @@ -"""Wrapper around the Pinecone vector database over VectorDB""" +"""Wrapper around the TurboPuffer vector database over VectorDB""" import logging import time @@ -30,9 +30,9 @@ def __init__( with_scalar_labels: bool = False, **kwargs, ): - """Initialize wrapper around the milvus vector database.""" self.api_key = db_config.get("api_key", "") - self.api_base_url = db_config.get("api_base_url", "") + self.region = db_config.get("region", "") + self.api_base_url = db_config.get("api_base_url") self.namespace = db_config.get("namespace", "") self.db_case_config = db_case_config self.metric = db_case_config.parse_metric() @@ -43,8 +43,10 @@ def __init__( self.with_scalar_labels = with_scalar_labels - # Initialize client with new SDK pattern - self.client = tpuf.Turbopuffer(api_key=self.api_key, base_url=self.api_base_url) + client_kwargs = {"api_key": self.api_key, "region": self.region} + if self.api_base_url: + client_kwargs["base_url"] = self.api_base_url + self.client = tpuf.Turbopuffer(**client_kwargs) if drop_old: log.info(f"Drop old. delete the namespace: {self.namespace}") @@ -78,7 +80,7 @@ def insert_embeddings( try: if self.with_scalar_labels: self.ns.write( - columns={ + upsert_columns={ self._scalar_id_field: metadata, self._vector_field: embeddings, self._scalar_label_field: labels_data, @@ -87,7 +89,7 @@ def insert_embeddings( ) else: self.ns.write( - columns={ + upsert_columns={ self._scalar_id_field: metadata, self._vector_field: embeddings, }, From 337d156f3b766609d234ff584b5ed47158a757e4 Mon Sep 17 00:00:00 2001 From: jamesgao-jpg Date: Wed, 1 Apr 2026 12:51:14 +0000 Subject: [PATCH 07/49] fix: turbopuffer client pickle/ID compatibility and add benchmark results - Defer tpuf.Turbopuffer client creation to init() to avoid pickle errors with ProcessPoolExecutor(spawn) - Cast search result IDs to int for ground truth recall comparison - Update leaderboard_v2.json with 20 TurboPuffer filter performance entries Signed-off-by: jamesgao-jpg --- .../clients/turbopuffer/turbopuffer.py | 18 +- vectordb_bench/results/leaderboard_v2.json | 200 ++++++++++++++++++ 2 files changed, 211 insertions(+), 7 deletions(-) diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py index 3e1f32af4..6de0df21d 100644 --- a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -43,21 +43,25 @@ def __init__( self.with_scalar_labels = with_scalar_labels - client_kwargs = {"api_key": self.api_key, "region": self.region} - if self.api_base_url: - client_kwargs["base_url"] = self.api_base_url - self.client = tpuf.Turbopuffer(**client_kwargs) - if drop_old: log.info(f"Drop old. delete the namespace: {self.namespace}") - ns = self.client.namespace(self.namespace) + tmp_client = self._create_client() + ns = tmp_client.namespace(self.namespace) try: ns.delete_all() except Exception as e: log.warning(f"Failed to delete all. Error: {e}") + tmp_client = None + + def _create_client(self) -> tpuf.Turbopuffer: + client_kwargs = {"api_key": self.api_key, "region": self.region} + if self.api_base_url: + client_kwargs["base_url"] = self.api_base_url + return tpuf.Turbopuffer(**client_kwargs) @contextmanager def init(self): + self.client = self._create_client() self.ns = self.client.namespace(self.namespace) yield @@ -110,7 +114,7 @@ def search_embedding( top_k=k, filters=self.expr, ) - return [row.id for row in res.rows] if res.rows is not None else [] + return [int(row.id) for row in res.rows] if res.rows is not None else [] def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index e4f8dece5..6dfb6a47d 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -2858,5 +2858,205 @@ "latency": 463.9, "recall": 0.8478, "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T10:01:48.528365", + "db_name": "TurboPuffer-2026-03-31T10:01:48.528365", + "qps": 346.5847, + "latency": 42.7, + "recall": 0.9631, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T11:53:58.907951", + "db_name": "TurboPuffer-2026-03-31T11:53:58.907951", + "qps": 369.4921, + "latency": 41.6, + "recall": 0.779, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:35:50.149485", + "db_name": "TurboPuffer-2026-03-31T09:35:50.149485", + "qps": 310.957, + "latency": 49.4, + "recall": 0.9698, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:18:27.391390", + "db_name": "TurboPuffer-2026-03-31T09:18:27.391390", + "qps": 798.328, + "latency": 56.7, + "recall": 0.8993, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T10:33:52.971649", + "db_name": "TurboPuffer-2026-03-31T10:33:52.971649", + "qps": 649.8781, + "latency": 55.2, + "recall": 0.8352, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:00:07.738985", + "db_name": "TurboPuffer-2026-03-31T12:00:07.738985", + "qps": 370.7241, + "latency": 49.6, + "recall": 0.7177, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:33:40.019128", + "db_name": "TurboPuffer-2026-03-31T12:33:40.019128", + "qps": 100.0554, + "latency": 69.3, + "recall": 0.9638, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T10:08:01.370057", + "db_name": "TurboPuffer-2026-03-31T10:08:01.370057", + "qps": 284.6367, + "latency": 47.6, + "recall": 0.9788, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:12:22.420710", + "db_name": "TurboPuffer-2026-03-31T12:12:22.420710", + "qps": 81.8678, + "latency": 105.6, + "recall": 0.8751, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:42:19.988467", + "db_name": "TurboPuffer-2026-03-31T09:42:19.988467", + "qps": 260.4031, + "latency": 48.3, + "recall": 0.9828, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T11:47:52.080875", + "db_name": "TurboPuffer-2026-03-31T11:47:52.080875", + "qps": 365.2505, + "latency": 34.9, + "recall": 0.8251, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T10:20:29.187645", + "db_name": "TurboPuffer-2026-03-31T10:20:29.187645", + "qps": 471.553, + "latency": 44.1, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:19:46.258746", + "db_name": "TurboPuffer-2026-03-31T12:19:46.258746", + "qps": 91.8612, + "latency": 85.7, + "recall": 0.8799, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:48:48.830799", + "db_name": "TurboPuffer-2026-03-31T09:48:48.830799", + "qps": 206.0934, + "latency": 56.7, + "recall": 0.9795, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T11:41:42.263276", + "db_name": "TurboPuffer-2026-03-31T11:41:42.263276", + "qps": 351.7114, + "latency": 46.7, + "recall": 0.8735, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:26:48.189944", + "db_name": "TurboPuffer-2026-03-31T12:26:48.189944", + "qps": 96.592, + "latency": 76.9, + "recall": 0.9178, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:29:40.287089", + "db_name": "TurboPuffer-2026-03-31T09:29:40.287089", + "qps": 802.6923, + "latency": 48.1, + "recall": 0.935, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T09:55:20.689312", + "db_name": "TurboPuffer-2026-03-31T09:55:20.689312", + "qps": 184.5363, + "latency": 53.4, + "recall": 0.9681, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31T10:14:19.197970", + "db_name": "TurboPuffer-2026-03-31T10:14:19.197970", + "qps": 323.0238, + "latency": 50.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31T12:06:15.579552", + "db_name": "TurboPuffer-2026-03-31T12:06:15.579552", + "qps": 382.5332, + "latency": 54.7, + "recall": 0.6135, + "filter_ratio": 0.98 } ] \ No newline at end of file From b39689bf3c030c1da97bbeda175325d286730d00 Mon Sep 17 00:00:00 2001 From: jamesgao-jpg Date: Wed, 1 Apr 2026 12:51:14 +0000 Subject: [PATCH 08/49] feat: add consolidated turbopuffer results and update streaming leaderboard Merge 22 individual TurboPuffer result files into single consolidated result file. Add streaming benchmark entries (500/1000 rows/s) to leaderboard_v2_streaming.json. Normalize TurboPuffer db_name and label in both leaderboard files. Signed-off-by: jamesgao-jpg --- ...lt_20260331_standard_2025_turbopuffer.json | 3072 +++++++++++++++++ vectordb_bench/results/leaderboard_v2.json | 80 +- .../results/leaderboard_v2_streaming.json | 18 + 3 files changed, 3130 insertions(+), 40 deletions(-) create mode 100644 vectordb_bench/results/TurboPuffer/result_20260331_standard_2025_turbopuffer.json diff --git a/vectordb_bench/results/TurboPuffer/result_20260331_standard_2025_turbopuffer.json b/vectordb_bench/results/TurboPuffer/result_20260331_standard_2025_turbopuffer.json new file mode 100644 index 000000000..3f6294f67 --- /dev/null +++ b/vectordb_bench/results/TurboPuffer/result_20260331_standard_2025_turbopuffer.json @@ -0,0 +1,3072 @@ +{ + "run_id": "45faeeb9909d4c20982712629b7109f8", + "task_label": "standard_2025", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3636.6248, + "optimize_duration": 60.1176, + "load_duration": 3696.7424, + "qps": 649.8781, + "serial_latency_p99": 0.0552, + "serial_latency_p95": 0.0323, + "recall": 0.8352, + "ndcg": 0.8489, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 5.0181, + 136.3428, + 401.9281, + 631.8033, + 649.8781, + 644.0239, + 638.2568, + 619.0102 + ], + "conc_latency_p99_list": [ + 0.7842317419981555, + 0.1055335910506619, + 0.04784800433084457, + 0.05269464588025578, + 0.1962103870000271, + 1.0594186752803216, + 1.0895923817619768, + 1.105607972859143 + ], + "conc_latency_p95_list": [ + 0.4166556664986274, + 0.06923620555135131, + 0.03216669879984694, + 0.04018809000044712, + 0.07153936980175785, + 0.08024858280041376, + 0.13536313344884537, + 1.0559816184473676 + ], + "conc_latency_avg_list": [ + 0.19879915163556802, + 0.03653489135292955, + 0.024727361365498506, + 0.031303927102332416, + 0.04542742658123844, + 0.06072973767027509, + 0.0906762097364659, + 0.12303716134739207 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 351.7114, + "serial_latency_p99": 0.0467, + "serial_latency_p95": 0.0306, + "recall": 0.8735, + "ndcg": 0.8847, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 39.0694, + 207.8572, + 315.6016, + 351.7114, + 350.2248, + 341.017, + 342.4504, + 333.8541 + ], + "conc_latency_p99_list": [ + 0.04308568556152748, + 0.03936693226052748, + 0.04930899777031294, + 0.08584493122052665, + 1.0702309198401785, + 1.1136302413583326, + 1.2335058578685432, + 2.3380776378013257 + ], + "conc_latency_p95_list": [ + 0.02841167169972323, + 0.02715096870197158, + 0.03799563039938221, + 0.07426544004883909, + 0.12973272005019681, + 0.21250388589869676, + 1.0903613543001485, + 1.1055554908001795 + ], + "conc_latency_avg_list": [ + 0.025532494878335706, + 0.02396293916721007, + 0.03149805626143546, + 0.056246979911343725, + 0.08423275960825198, + 0.11371272981539232, + 0.16889921984869344, + 0.22411732816279095 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 365.2505, + "serial_latency_p99": 0.0349, + "serial_latency_p95": 0.0247, + "recall": 0.8251, + "ndcg": 0.8424, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 41.7453, + 217.4248, + 327.3086, + 365.2505, + 360.5899, + 361.9818, + 353.6243, + 351.7834 + ], + "conc_latency_p99_list": [ + 0.03651399044829308, + 0.03795003006755009, + 0.04851697580088516, + 0.0817924941409364, + 1.069550116062965, + 1.1113606851896825, + 1.2188607804002098, + 2.3407695170007345 + ], + "conc_latency_p95_list": [ + 0.025673050000477815, + 0.025719283201397047, + 0.0362045420006325, + 0.07120951019969653, + 0.12899555909789345, + 0.17483308564860484, + 1.085536041801606, + 1.1010552038016612 + ], + "conc_latency_avg_list": [ + 0.023894811787483095, + 0.02290646905055157, + 0.03037167207873524, + 0.05417950100621118, + 0.08193541962097284, + 0.1079860384930871, + 0.1626031282242101, + 0.21517478888871755 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.8 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 369.4921, + "serial_latency_p99": 0.0416, + "serial_latency_p95": 0.026, + "recall": 0.779, + "ndcg": 0.8011, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 44.1762, + 217.1941, + 333.4918, + 368.3695, + 369.4921, + 361.0564, + 358.8268, + 351.6052 + ], + "conc_latency_p99_list": [ + 0.037339031600567986, + 0.04319219880198944, + 0.04170634405851157, + 0.08140748869991507, + 1.075197636150333, + 1.113729548148076, + 1.185450099499576, + 2.3347086974994453 + ], + "conc_latency_p95_list": [ + 0.025345681600447277, + 0.026001171001553303, + 0.035399802850224656, + 0.0708943822002766, + 0.11817662325156564, + 0.16750261040106082, + 1.0850413800017122, + 1.1004046580001159 + ], + "conc_latency_avg_list": [ + 0.022580311149034315, + 0.022936271871885103, + 0.02982460058318166, + 0.05370093295877385, + 0.07984126562141454, + 0.10845894286435549, + 0.16060839830850934, + 0.21328552188517172 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.9 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 370.7241, + "serial_latency_p99": 0.0496, + "serial_latency_p95": 0.0256, + "recall": 0.7177, + "ndcg": 0.7469, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 42.3641, + 213.6356, + 337.7677, + 367.4005, + 370.7241, + 352.4176, + 344.8198, + 354.0423 + ], + "conc_latency_p99_list": [ + 0.04112669600999651, + 0.04059317171937442, + 0.049354484229988935, + 0.08914589621916108, + 1.0625034953784052, + 1.1084485840195337, + 1.1761026821609268, + 2.329700628177888 + ], + "conc_latency_p95_list": [ + 0.026073782299499724, + 0.026734366401069565, + 0.03505219174985541, + 0.07247991124786494, + 0.11844929140133904, + 0.19264427510115556, + 1.0887599481509824, + 1.1004490541015912 + ], + "conc_latency_avg_list": [ + 0.023546151951321655, + 0.02331067781469216, + 0.02942534475558775, + 0.05384849155748122, + 0.07935110206611184, + 0.10908346295012428, + 0.16543306380794237, + 0.21503563146286667 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.95 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 382.5332, + "serial_latency_p99": 0.0547, + "serial_latency_p95": 0.0261, + "recall": 0.6135, + "ndcg": 0.6532, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 42.6797, + 211.9221, + 340.0303, + 381.3494, + 382.5332, + 377.5534, + 372.3789, + 370.273 + ], + "conc_latency_p99_list": [ + 0.045958382528879155, + 0.04791345044010085, + 0.050254700950063146, + 0.08182212736079236, + 1.0455512913525304, + 1.1066815286008933, + 1.1499066802006566, + 2.330022009398708 + ], + "conc_latency_p95_list": [ + 0.027506814953085268, + 0.030191246797767208, + 0.035157976998561935, + 0.06934378335081418, + 0.1208363270012342, + 0.17324353699950734, + 1.0816264840002987, + 1.0978374554006223 + ], + "conc_latency_avg_list": [ + 0.023371494825615652, + 0.023497298190272724, + 0.02923274826642055, + 0.051893872435525415, + 0.07698122536272771, + 0.1032508273402585, + 0.15527307758066816, + 0.20625486973445892 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.98 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 81.8678, + "serial_latency_p99": 0.1056, + "serial_latency_p95": 0.1021, + "recall": 0.8751, + "ndcg": 0.8923, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 10.0132, + 45.2108, + 69.7393, + 81.8678, + 81.5054, + 79.5748, + 80.1286, + 76.405 + ], + "conc_latency_p99_list": [ + 0.11135667605140044, + 0.12466917503959847, + 0.17545606489948118, + 0.4186298333996091, + 1.4131757669402578, + 2.561454358698395, + 4.161437169238486, + 6.53146800307746 + ], + "conc_latency_p95_list": [ + 0.10567549315092037, + 0.11936616160019184, + 0.15836084999864397, + 0.3964126334994944, + 1.2486386469990973, + 1.4115760019994923, + 2.527067465399887, + 2.6629810965998333 + ], + "conc_latency_avg_list": [ + 0.09962701216235118, + 0.11013113512517288, + 0.14239347650973677, + 0.24165083264509568, + 0.36060090070201456, + 0.4901049624103434, + 0.720209461182002, + 0.9583824911724366 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.99 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 91.8612, + "serial_latency_p99": 0.0857, + "serial_latency_p95": 0.0809, + "recall": 0.8799, + "ndcg": 0.8975, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 12.999, + 57.8487, + 82.2415, + 91.8612, + 90.3749, + 90.0362, + 89.1634, + 87.8603 + ], + "conc_latency_p99_list": [ + 0.08638456059743475, + 0.09575280532066244, + 0.13928119900861935, + 0.4007615396678755, + 1.3269691034200513, + 2.511851395880221, + 4.069974964441136, + 4.318752965519816 + ], + "conc_latency_p95_list": [ + 0.08070138149923878, + 0.09151377140078694, + 0.13317929814766102, + 0.33960816225135204, + 1.222629544099982, + 1.2648818186004065, + 2.463837313051044, + 2.5308212226009346 + ], + "conc_latency_avg_list": [ + 0.07674188703336754, + 0.08605192841317442, + 0.12095865782566981, + 0.2152256534213578, + 0.3258099637445097, + 0.4338084209719558, + 0.6348769175872554, + 0.8261389732993693 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.995 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 96.592, + "serial_latency_p99": 0.0769, + "serial_latency_p95": 0.0671, + "recall": 0.9178, + "ndcg": 0.9309, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 15.4173, + 68.7506, + 91.1578, + 95.9584, + 96.592, + 96.5377, + 92.9689, + 92.3363 + ], + "conc_latency_p99_list": [ + 0.07026230202154693, + 0.09216422941106428, + 0.1530633716819285, + 0.37221308532047254, + 1.379136629800488, + 2.4759424806782047, + 4.150151355230372, + 4.311918809250528 + ], + "conc_latency_p95_list": [ + 0.06769757184974878, + 0.07770936130109475, + 0.1223220942003536, + 0.3146231973501927, + 1.2210356577998025, + 1.2695832258003064, + 2.432397950649647, + 2.5293083527494673 + ], + "conc_latency_avg_list": [ + 0.06470390199121445, + 0.072410190861118, + 0.10899080468085105, + 0.20620261645924234, + 0.3044200164964746, + 0.40215828176873775, + 0.5961679787750713, + 0.7680528748023352 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.998 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 100.0554, + "serial_latency_p99": 0.0693, + "serial_latency_p95": 0.0649, + "recall": 0.9638, + "ndcg": 0.9702, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 16.4825, + 73.9278, + 94.48, + 100.0554, + 99.5101, + 96.945, + 94.0424, + 86.2962 + ], + "conc_latency_p99_list": [ + 0.06726060309974854, + 0.08376507117056463, + 0.12645121190198552, + 0.39609605101897616, + 1.3409943886408304, + 2.4752014657589463, + 4.116221232489005, + 4.324473266399982 + ], + "conc_latency_p95_list": [ + 0.06371058149852615, + 0.07187465289953253, + 0.11859881825057528, + 0.2999235360995954, + 1.2061096817997168, + 1.2548316742995667, + 2.412594861300204, + 2.5453477221013596 + ], + "conc_latency_avg_list": [ + 0.06052078211502204, + 0.06738729749599892, + 0.10525026012149993, + 0.19776199416695284, + 0.2947393509041799, + 0.39903049677303315, + 0.5802193109362205, + 0.8205917814459879 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_10m" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "filter_rate": 0.999 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 237.1288, + "optimize_duration": 60.1231, + "load_duration": 297.2519, + "qps": 798.328, + "serial_latency_p99": 0.0567, + "serial_latency_p95": 0.0373, + "recall": 0.8993, + "ndcg": 0.909, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 7.2334, + 233.3539, + 418.0091, + 779.6834, + 798.328, + 721.0519, + 665.1265, + 584.2167 + ], + "conc_latency_p99_list": [ + 0.16710825715950703, + 0.06532752437056843, + 0.06613579651964772, + 0.06110905407032987, + 0.14510449524073932, + 1.0521202659806477, + 1.0834973058097968, + 1.1199298221199219 + ], + "conc_latency_p95_list": [ + 0.1503542814499269, + 0.04019359780040751, + 0.03864274919997114, + 0.03302840365022349, + 0.06032294020042168, + 0.07371430989933284, + 0.12393024720067815, + 1.0562808335987939 + ], + "conc_latency_avg_list": [ + 0.13791148913299642, + 0.021343171084931285, + 0.023774816974899124, + 0.025365002319208883, + 0.03695695700041601, + 0.05426009768172259, + 0.08686200089993441, + 0.13066586199842864 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 802.6923, + "serial_latency_p99": 0.0481, + "serial_latency_p95": 0.0323, + "recall": 0.935, + "ndcg": 0.9417, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 35.1162, + 166.394, + 441.2273, + 754.7672, + 802.6923, + 758.6779, + 709.8788, + 667.6315 + ], + "conc_latency_p99_list": [ + 0.05474694928023395, + 0.05710166565986581, + 0.08122192963965079, + 0.053906371219891225, + 0.1059555222807467, + 1.0329836665192853, + 1.0822717395997459, + 1.1019392383902227 + ], + "conc_latency_p95_list": [ + 0.033523448400228514, + 0.03988007639854914, + 0.03580541750015981, + 0.03533476559914561, + 0.05827280570065341, + 0.07033409700015901, + 0.10542240839895384, + 1.0461306155005332 + ], + "conc_latency_avg_list": [ + 0.028406082829701498, + 0.0299218017148686, + 0.02252275923560204, + 0.02620027550993044, + 0.03675704353476552, + 0.051493407375862076, + 0.0814369894123999, + 0.114157400844387 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 310.957, + "serial_latency_p99": 0.0494, + "serial_latency_p95": 0.0478, + "recall": 0.9698, + "ndcg": 0.9735, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 22.9937, + 114.2135, + 201.3758, + 310.957, + 284.36, + 278.919, + 275.2043, + 268.6154 + ], + "conc_latency_p99_list": [ + 0.05187608798047224, + 0.0676384811005301, + 0.061836029370570034, + 0.10488168669999139, + 1.0839724454297168, + 1.1247000296189251, + 1.1835066912906216, + 2.350891308639548 + ], + "conc_latency_p95_list": [ + 0.04523930709929118, + 0.046784667498923224, + 0.05690364935007892, + 0.09261150339998493, + 0.13370714204984316, + 0.30792148500040484, + 1.109438732399667, + 1.1224438496010407 + ], + "conc_latency_avg_list": [ + 0.04338349602020482, + 0.04359157635044708, + 0.04934177400048448, + 0.06363149687529962, + 0.10389666149204274, + 0.13943306853059848, + 0.2093151750419116, + 0.28126611132382795 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.8 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 260.4031, + "serial_latency_p99": 0.0483, + "serial_latency_p95": 0.0461, + "recall": 0.9828, + "ndcg": 0.9852, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 23.2538, + 105.0145, + 165.942, + 260.4031, + 255.1092, + 249.9039, + 247.1553, + 222.8946 + ], + "conc_latency_p99_list": [ + 0.04928288878063535, + 0.05744525966034426, + 0.08332391123971328, + 0.12499408589974335, + 1.1001869340804298, + 1.1539904669602765, + 2.271939874058515, + 2.3963180435212417 + ], + "conc_latency_p95_list": [ + 0.04398724759921606, + 0.05287559080006758, + 0.06577553119968797, + 0.11041119500077912, + 0.1910506329497366, + 1.079135947499708, + 1.123120172900053, + 1.156959662200461 + ], + "conc_latency_avg_list": [ + 0.042898485931390104, + 0.0473868923309176, + 0.059896585720115844, + 0.07594923477056366, + 0.11576974236313346, + 0.15613794048676635, + 0.23384194864694827, + 0.34093762910663383 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.9 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 206.0934, + "serial_latency_p99": 0.0567, + "serial_latency_p95": 0.0493, + "recall": 0.9795, + "ndcg": 0.9827, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 23.9853, + 111.9502, + 167.2983, + 206.0934, + 202.6785, + 200.7624, + 198.7852, + 176.9053 + ], + "conc_latency_p99_list": [ + 0.04505532840961678, + 0.0549183023203659, + 0.07183914795945383, + 0.16050115711950638, + 1.1392679925593985, + 1.1783225936005692, + 2.391149005459738, + 2.4627247357385564 + ], + "conc_latency_p95_list": [ + 0.04352937004987325, + 0.049765212599595536, + 0.0665941942007521, + 0.14062045170048804, + 0.25411106819992707, + 1.1207731039994542, + 1.1548708326008637, + 1.2187981356008997 + ], + "conc_latency_avg_list": [ + 0.04158898759147668, + 0.044488593964723565, + 0.05943009694892961, + 0.09602951410439428, + 0.145682298229123, + 0.19324255140426388, + 0.2893373419111252, + 0.4256243749569817 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.95 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 184.5363, + "serial_latency_p99": 0.0534, + "serial_latency_p95": 0.0447, + "recall": 0.9681, + "ndcg": 0.9731, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 23.3311, + 105.7534, + 154.8527, + 184.5363, + 179.2525, + 179.9984, + 173.6043, + 172.2228 + ], + "conc_latency_p99_list": [ + 0.06133968915926745, + 0.05785074116065515, + 0.07778257604990356, + 0.1836007817494645, + 1.1589217726791685, + 1.2315237454791341, + 2.4227708060203077, + 2.5051683164208227 + ], + "conc_latency_p95_list": [ + 0.044938461699348405, + 0.053342464800516604, + 0.07181120149953131, + 0.1550450290005756, + 0.37187840719961957, + 1.1339543323992984, + 1.174229825800103, + 1.2181544718499933 + ], + "conc_latency_avg_list": [ + 0.04275554558972205, + 0.047119999140656205, + 0.06418707822424495, + 0.1072447022017609, + 0.1639824330546471, + 0.21581640828899204, + 0.3248775175758776, + 0.4320007027329427 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.98 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 346.5847, + "serial_latency_p99": 0.0427, + "serial_latency_p95": 0.0306, + "recall": 0.9631, + "ndcg": 0.9691, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 24.832, + 105.4399, + 268.1794, + 342.1315, + 346.5847, + 339.9961, + 331.4736, + 318.4889 + ], + "conc_latency_p99_list": [ + 0.0524532141194868, + 0.124523086280969, + 0.05130710799858207, + 0.09639830237927527, + 0.6372997376802239, + 1.1167713518602072, + 1.1833148257201527, + 2.346942362739901 + ], + "conc_latency_p95_list": [ + 0.04208997349924175, + 0.054497509400971464, + 0.04311577799853694, + 0.0820058475001133, + 0.15190685399975346, + 0.24409308345066127, + 1.094288076799785, + 1.113402110500283 + ], + "conc_latency_avg_list": [ + 0.040171743147231896, + 0.04722931638843159, + 0.03705184992629306, + 0.05782180853289151, + 0.0850777291892508, + 0.11466982697445866, + 0.17464619983809074, + 0.23867489188873692 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.99 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 284.6367, + "serial_latency_p99": 0.0476, + "serial_latency_p95": 0.0343, + "recall": 0.9788, + "ndcg": 0.9824, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 33.6033, + 154.3359, + 234.9368, + 284.6367, + 281.0354, + 282.1947, + 275.5716, + 254.759 + ], + "conc_latency_p99_list": [ + 0.04858522409849683, + 0.048488989419420224, + 0.05480865936007828, + 0.11453194788005944, + 1.0936976923593464, + 1.1293674966106844, + 2.3225755058392124, + 2.393918312400474 + ], + "conc_latency_p95_list": [ + 0.03065781050008809, + 0.037750776849679826, + 0.04890473074956389, + 0.09784386339888443, + 0.1834858106001775, + 0.5145842184007118, + 1.1164754623508997, + 1.13096353059982 + ], + "conc_latency_avg_list": [ + 0.02968536912167731, + 0.03225200436006818, + 0.04233185695698942, + 0.06951284309111169, + 0.10508535649574902, + 0.1382588987117255, + 0.2097049643681289, + 0.2930996819200318 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.995 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 323.0238, + "serial_latency_p99": 0.0504, + "serial_latency_p95": 0.0276, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 33.45, + 152.7353, + 227.8618, + 275.5285, + 291.2986, + 323.0238, + 318.2655, + 312.4427 + ], + "conc_latency_p99_list": [ + 0.04559309834949112, + 0.048093286539624296, + 0.05670377755039221, + 0.11922176679041513, + 1.0838092030408004, + 1.1185716670006514, + 1.246932903139767, + 2.348052036049994 + ], + "conc_latency_p95_list": [ + 0.031013834000077622, + 0.03764021150072949, + 0.05072674379944146, + 0.10094266020023496, + 0.18840500040023464, + 0.29478117500002554, + 1.0974889842496849, + 1.115618919398821 + ], + "conc_latency_avg_list": [ + 0.029821064443336008, + 0.03260137427613773, + 0.0436201719452207, + 0.07183805259952356, + 0.10142021420372482, + 0.12089794632971683, + 0.18129370678602072, + 0.24532422969222006 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.998 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 471.553, + "serial_latency_p99": 0.0441, + "serial_latency_p95": 0.0247, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 49.1528, + 228.9819, + 363.0592, + 468.8616, + 471.553, + 461.5481, + 437.5487, + 394.213 + ], + "conc_latency_p99_list": [ + 0.041978935619772575, + 0.04080429583897057, + 0.0448176010107636, + 0.08228810615008104, + 0.3177955880392612, + 1.090591485319801, + 1.134414788300455, + 2.3114677489898723 + ], + "conc_latency_p95_list": [ + 0.021382112400533514, + 0.02570402879937319, + 0.033227993899890854, + 0.06001253969943718, + 0.10880423700054961, + 0.13627656039943753, + 1.060789735999606, + 1.0911676228000942 + ], + "conc_latency_avg_list": [ + 0.020293962421219093, + 0.0217530582419813, + 0.02738142802709893, + 0.04220026652024836, + 0.06239387785636437, + 0.08461281376066486, + 0.13169568263657166, + 0.18839915684310343 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 400, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "filter_rate": 0.999 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 10000.231680193, + "optimize_duration": 60.23235460700016, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_p95_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 1001.3061, + 2032.9136, + 3065.0184, + 4086.7507, + 5107.3667, + 6128.9501, + 7149.9124, + 8171.4762, + 9192.0596, + 10223.3342, + 10444.8913 + ], + "st_max_qps_list_list": [ + 362.5803, + 334.5913, + 235.4548, + 319.031, + 459.0171, + 305.2895, + 336.8119, + 317.4398, + 442.5824, + 830.8648, + 837.1841 + ], + "st_recall_list": [ + 0.1028, + 0.1942, + 0.2856, + 0.3713, + 0.4495, + 0.5356, + 0.6176, + 0.7044, + 0.7795, + 0.8358, + 0.8358 + ], + "st_ndcg_list": [ + 0.1033, + 0.1954, + 0.2882, + 0.3754, + 0.4551, + 0.5423, + 0.626, + 0.7145, + 0.7914, + 0.849, + 0.849 + ], + "st_serial_latency_p99_list": [ + 0.162, + 0.2745, + 0.1655, + 0.154, + 0.1724, + 0.2706, + 0.2654, + 0.1929, + 0.0724, + 0.033, + 0.0413 + ], + "st_serial_latency_p95_list": [ + 0.138, + 0.1042, + 0.1335, + 0.1357, + 0.0747, + 0.1695, + 0.1519, + 0.1432, + 0.0571, + 0.0211, + 0.0213 + ], + "st_conc_failed_rate_list": [ + 1.621796951021732e-05, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 7.329873632978568e-06, + 0.0, + 0.0 + ], + "st_conc_num_list_list": [ + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ] + ], + "st_conc_qps_list_list": [ + [ + 362.5803, + 305.9887, + 211.6508 + ], + [ + 313.8881, + 334.5913, + 295.2759 + ], + [ + 235.4548, + 187.7903, + 223.1515 + ], + [ + 195.555, + 255.0244, + 319.031 + ], + [ + 459.0171, + 448.8463, + 326.9188 + ], + [ + 264.3766, + 302.7591, + 305.2895 + ], + [ + 212.8641, + 336.8119, + 265.5836 + ], + [ + 164.1314, + 174.9407, + 317.4398 + ], + [ + 280.9929, + 234.8017, + 442.5824 + ], + [ + 793.081, + 830.8648, + 830.5245 + ], + [ + 790.8788, + 837.1841, + 831.3462 + ] + ], + "st_conc_latency_p99_list_list": [ + [ + 0.125567, + 1.124351, + 2.306047 + ], + [ + 0.140031, + 1.105919, + 1.235967 + ], + [ + 0.201983, + 1.233919, + 2.289663 + ], + [ + 0.270847, + 1.172479, + 1.171455 + ], + [ + 0.098623, + 1.064959, + 1.157119 + ], + [ + 0.156671, + 1.128447, + 1.165311 + ], + [ + 0.239615, + 1.102847, + 1.522687 + ], + [ + 0.300031, + 1.263615, + 1.216511 + ], + [ + 0.163583, + 1.207295, + 1.197055 + ], + [ + 0.042719, + 0.201983, + 1.076223 + ], + [ + 0.046463, + 0.231167, + 1.074175 + ] + ], + "st_conc_latency_p95_list_list": [ + [ + 0.094463, + 0.180351, + 1.168383 + ], + [ + 0.106879, + 0.162175, + 1.087487 + ], + [ + 0.143487, + 1.043455, + 1.123327 + ], + [ + 0.193151, + 0.275455, + 1.065983 + ], + [ + 0.068607, + 0.115583, + 1.075199 + ], + [ + 0.118655, + 0.174463, + 1.084415 + ], + [ + 0.162047, + 0.147711, + 1.101823 + ], + [ + 0.225023, + 1.132543, + 1.075199 + ], + [ + 0.119295, + 0.299263, + 0.238591 + ], + [ + 0.032479, + 0.062495, + 0.081855 + ], + [ + 0.034303, + 0.059647, + 0.075775 + ] + ], + "st_conc_latency_avg_list_list": [ + [ + 0.05500603695334308, + 0.09764594385089817, + 0.18784171186688073 + ], + [ + 0.06353041564585622, + 0.08927120173312625, + 0.13474179712740827 + ], + [ + 0.08473482217393206, + 0.15917699015615014, + 0.17835078322515305 + ], + [ + 0.10202393731369927, + 0.11714498760979741, + 0.12473291585827614 + ], + [ + 0.043456775709144496, + 0.06659470207128572, + 0.12171512893789603 + ], + [ + 0.07546223394734103, + 0.09867369148091878, + 0.1303126478911198 + ], + [ + 0.0937178086011191, + 0.0887245961053375, + 0.14989714770693796 + ], + [ + 0.12154873281339651, + 0.17085041091090097, + 0.12538872906231163 + ], + [ + 0.07100562943422423, + 0.1272563421099916, + 0.08991327977599742 + ], + [ + 0.024614513704250777, + 0.03488576620155038, + 0.04598832608029421 + ], + [ + 0.024699256453538636, + 0.03461592000922972, + 0.04602557062363449 + ] + ] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_streaming_test3" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 1000, + "bulk_insert_ratio": 0.0 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 6900.174543077999, + "optimize_duration": 60.1439380869997, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_p95_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 90, + 100, + 110 + ], + "st_search_time_list": [ + 4901.1288, + 6922.6153, + 7155.3455 + ], + "st_max_qps_list_list": [ + 536.0198, + 1027.3522, + 1020.8349 + ], + "st_recall_list": [ + 0.7646, + 0.8365, + 0.836 + ], + "st_ndcg_list": [ + 0.7768, + 0.8503, + 0.8497 + ], + "st_serial_latency_p99_list": [ + 0.3132, + 0.0532, + 0.0406 + ], + "st_serial_latency_p95_list": [ + 0.2025, + 0.0385, + 0.0219 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0 + ], + "st_conc_num_list_list": [ + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ], + [ + 20, + 30, + 40 + ] + ], + "st_conc_qps_list_list": [ + [ + 404.5543, + 429.4512, + 536.0198 + ], + [ + 944.4992, + 1025.9923, + 1027.3522 + ], + [ + 930.6361, + 1020.5733, + 1020.8349 + ] + ], + "st_conc_latency_p99_list_list": [ + [ + 0.139007, + 1.069055, + 1.088511 + ], + [ + 0.038335, + 0.077695, + 0.569855 + ], + [ + 0.042943, + 0.078015, + 0.783871 + ] + ], + "st_conc_latency_p95_list_list": [ + [ + 0.103295, + 0.115455, + 0.121151 + ], + [ + 0.027967, + 0.044607, + 0.063839 + ], + [ + 0.028175, + 0.045951, + 0.065151 + ] + ], + "st_conc_latency_avg_list_list": [ + [ + 0.04936760449523388, + 0.06972336395202725, + 0.07444538541092975 + ], + [ + 0.02068985204345577, + 0.02825433607535321, + 0.037233642103957326 + ], + [ + 0.020995807770388515, + 0.028403830945016105, + 0.03740811619860986 + ] + ] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-03-31", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench_test_streaming_500" + }, + "db_case_config": { + "metric_type": "COSINE", + "use_multi_ns_for_filter": false, + "time_wait_warmup": 60 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 500, + "bulk_insert_ratio": 0.8 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1775001600.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index 6dfb6a47d..c463463fc 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -2862,8 +2862,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T10:01:48.528365", - "db_name": "TurboPuffer-2026-03-31T10:01:48.528365", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 346.5847, "latency": 42.7, "recall": 0.9631, @@ -2872,8 +2872,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T11:53:58.907951", - "db_name": "TurboPuffer-2026-03-31T11:53:58.907951", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 369.4921, "latency": 41.6, "recall": 0.779, @@ -2882,8 +2882,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:35:50.149485", - "db_name": "TurboPuffer-2026-03-31T09:35:50.149485", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 310.957, "latency": 49.4, "recall": 0.9698, @@ -2892,8 +2892,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:18:27.391390", - "db_name": "TurboPuffer-2026-03-31T09:18:27.391390", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 798.328, "latency": 56.7, "recall": 0.8993, @@ -2902,8 +2902,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T10:33:52.971649", - "db_name": "TurboPuffer-2026-03-31T10:33:52.971649", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 649.8781, "latency": 55.2, "recall": 0.8352, @@ -2912,8 +2912,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:00:07.738985", - "db_name": "TurboPuffer-2026-03-31T12:00:07.738985", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 370.7241, "latency": 49.6, "recall": 0.7177, @@ -2922,8 +2922,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:33:40.019128", - "db_name": "TurboPuffer-2026-03-31T12:33:40.019128", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 100.0554, "latency": 69.3, "recall": 0.9638, @@ -2932,8 +2932,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T10:08:01.370057", - "db_name": "TurboPuffer-2026-03-31T10:08:01.370057", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 284.6367, "latency": 47.6, "recall": 0.9788, @@ -2942,8 +2942,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:12:22.420710", - "db_name": "TurboPuffer-2026-03-31T12:12:22.420710", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 81.8678, "latency": 105.6, "recall": 0.8751, @@ -2952,8 +2952,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:42:19.988467", - "db_name": "TurboPuffer-2026-03-31T09:42:19.988467", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 260.4031, "latency": 48.3, "recall": 0.9828, @@ -2962,8 +2962,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T11:47:52.080875", - "db_name": "TurboPuffer-2026-03-31T11:47:52.080875", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 365.2505, "latency": 34.9, "recall": 0.8251, @@ -2972,8 +2972,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T10:20:29.187645", - "db_name": "TurboPuffer-2026-03-31T10:20:29.187645", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 471.553, "latency": 44.1, "recall": 1.0, @@ -2982,8 +2982,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:19:46.258746", - "db_name": "TurboPuffer-2026-03-31T12:19:46.258746", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 91.8612, "latency": 85.7, "recall": 0.8799, @@ -2992,8 +2992,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:48:48.830799", - "db_name": "TurboPuffer-2026-03-31T09:48:48.830799", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 206.0934, "latency": 56.7, "recall": 0.9795, @@ -3002,8 +3002,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T11:41:42.263276", - "db_name": "TurboPuffer-2026-03-31T11:41:42.263276", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 351.7114, "latency": 46.7, "recall": 0.8735, @@ -3012,8 +3012,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:26:48.189944", - "db_name": "TurboPuffer-2026-03-31T12:26:48.189944", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 96.592, "latency": 76.9, "recall": 0.9178, @@ -3022,8 +3022,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:29:40.287089", - "db_name": "TurboPuffer-2026-03-31T09:29:40.287089", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 802.6923, "latency": 48.1, "recall": 0.935, @@ -3032,8 +3032,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T09:55:20.689312", - "db_name": "TurboPuffer-2026-03-31T09:55:20.689312", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 184.5363, "latency": 53.4, "recall": 0.9681, @@ -3042,8 +3042,8 @@ { "dataset": "Cohere (Medium)", "db": "TurboPuffer", - "label": "2026-03-31T10:14:19.197970", - "db_name": "TurboPuffer-2026-03-31T10:14:19.197970", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 323.0238, "latency": 50.4, "recall": 1.0, @@ -3052,8 +3052,8 @@ { "dataset": "Cohere (Large)", "db": "TurboPuffer", - "label": "2026-03-31T12:06:15.579552", - "db_name": "TurboPuffer-2026-03-31T12:06:15.579552", + "label": "2026-03-31", + "db_name": "TurboPuffer", "qps": 382.5332, "latency": 54.7, "recall": 0.6135, diff --git a/vectordb_bench/results/leaderboard_v2_streaming.json b/vectordb_bench/results/leaderboard_v2_streaming.json index 224c78d12..222c68bf7 100644 --- a/vectordb_bench/results/leaderboard_v2_streaming.json +++ b/vectordb_bench/results/leaderboard_v2_streaming.json @@ -124,5 +124,23 @@ "insert_rate": 1000, "streaming_qps": 167.2689, "streaming_latency": 0.5048 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "insert_rate": 500, + "streaming_qps": 536.0198, + "streaming_latency": 0.3132 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "insert_rate": 1000, + "streaming_qps": 442.5824, + "streaming_latency": 0.0724 } ] \ No newline at end of file From 0fef7dd9ef8a5ee2a04dc9b4f17e90dccd7dc602 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 2 Apr 2026 07:35:42 +0000 Subject: [PATCH 09/49] feat: add SQ4U scalar quantization type for Milvus HNSW index Co-Authored-By: Claude Opus 4.6 (1M context) --- vectordb_bench/backend/clients/api.py | 1 + vectordb_bench/backend/clients/milvus/cli.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 80709e8e3..5511f18db 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -49,6 +49,7 @@ class IndexType(StrEnum): class SQType(StrEnum): + SQ4U = "SQ4U" SQ6 = "SQ6" SQ8 = "SQ8" BF16 = "BF16" diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index af31fe50d..2f2a286be 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -242,8 +242,8 @@ class MilvusHNSWSQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusHNSWTypedDic str | None, click.option( "--sq-type", - type=click.Choice(["SQ6", "SQ8", "BF16", "FP16", "FP32"], case_sensitive=False), - help="Scalar quantizer type. Supported values: SQ6,SQ8,BF16,FP16,FP32", + type=click.Choice(["SQ4U", "SQ6", "SQ8", "BF16", "FP16", "FP32"], case_sensitive=False), + help="Scalar quantizer type. Supported values: SQ4U,SQ6,SQ8,BF16,FP16,FP32", required=True, ), ] From 46cc146ce6c541acfb01c9c5b9d7a752e99f7616 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Apr 2026 09:06:08 +0000 Subject: [PATCH 10/49] Update benchmark results: Milvus 2.6.14, ElasticCloud 8.17, ZillizCloud Milvus results (16c64g, force_merge, v2.6.14): - 1M Cohere: SQ4U+FP16 (sweep refine_k) + SQ8 (sweep ef), 8 points each - 10M Cohere: SQ4U+FP16 + SQ8 (sweep ef), 8 points each - Total 32 benchmark configurations ElasticCloud and ZillizCloud results from standard benchmark runs. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...result_20260209_standard_elasticcloud.json | 1556 +++++ .../result_20260403_standard_milvus.json | 4074 +++++++++++++ .../result_20260209_standard_zillizcloud.json | 2814 +++++++++ vectordb_bench/results/leaderboard_v2.json | 5200 +++++++---------- 4 files changed, 10584 insertions(+), 3060 deletions(-) create mode 100644 vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json create mode 100644 vectordb_bench/results/Milvus/result_20260403_standard_milvus.json create mode 100644 vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json diff --git a/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json new file mode 100644 index 000000000..213c2d374 --- /dev/null +++ b/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json @@ -0,0 +1,1556 @@ +{ + "run_id": "80696b60e39749b295273db3cdba1b69", + "task_label": "standard_20260209", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 2030.4249, + "serial_latency_p99": 0.0106, + "serial_latency_p95": 0.0073, + "recall": 0.925, + "ndcg": 0.9306, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 139.9315, + 742.8751, + 1246.354, + 1773.6358, + 1944.4702, + 1997.4895, + 2030.4249, + 2030.061 + ], + "conc_latency_p99_list": [ + 0.011573358500827451, + 0.01055288627227128, + 0.013919824328950206, + 0.021638741448914516, + 0.035741090469564335, + 0.045256564997544046, + 0.06317665028094777, + 0.08284782179980536 + ], + "conc_latency_p95_list": [ + 0.00789401560141414, + 0.007500608301234024, + 0.010187717052576773, + 0.017150824350028415, + 0.026996734849308254, + 0.03459080500033451, + 0.04788784860138549, + 0.0629504940006882 + ], + "conc_latency_avg_list": [ + 0.0071435065296698895, + 0.006726900525604191, + 0.008019201478369918, + 0.011268319014980386, + 0.01541493476050156, + 0.02000300411234457, + 0.029459945652882687, + 0.039275078762117686 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 200 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 1804.8996, + "serial_latency_p99": 0.0123, + "serial_latency_p95": 0.0079, + "recall": 0.9365, + "ndcg": 0.9405, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 128.0906, + 646.8272, + 1197.2061, + 1629.7064, + 1728.435, + 1774.0467, + 1804.8996, + 1802.14 + ], + "conc_latency_p99_list": [ + 0.0134131232793152, + 0.012170731302467179, + 0.01435177448063773, + 0.023948891401232696, + 0.038592138251187846, + 0.04927967675830586, + 0.06325175963895165, + 0.08297362611905544 + ], + "conc_latency_p95_list": [ + 0.008920073449553456, + 0.008726374499019585, + 0.011022335800225845, + 0.019354423999175197, + 0.029856205349278752, + 0.03790496999936294, + 0.05230563919976702, + 0.06659373179936665 + ], + "conc_latency_avg_list": [ + 0.007804807226078364, + 0.007726593180097387, + 0.008348314213725788, + 0.01226467845434359, + 0.017345920270812453, + 0.022531855009387067, + 0.033063582489267676, + 0.04425664062925568 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 250 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 2353.8935, + "serial_latency_p99": 0.0171, + "serial_latency_p95": 0.0071, + "recall": 0.9056, + "ndcg": 0.9143, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 158.2504, + 791.7118, + 1438.1982, + 2003.6548, + 2210.8624, + 2291.9799, + 2345.97, + 2353.8935 + ], + "conc_latency_p99_list": [ + 0.010709055121405905, + 0.01030786765921221, + 0.01125256240178715, + 0.0189343996412208, + 0.02869696417925298, + 0.04078966366032545, + 0.05675600830181786, + 0.07136866949964316 + ], + "conc_latency_p95_list": [ + 0.0071951633981370815, + 0.00706048959873442, + 0.00878364119926118, + 0.014748687801329647, + 0.022636354199676134, + 0.031156333000035372, + 0.04301601800034405, + 0.05551979320152896 + ], + "conc_latency_avg_list": [ + 0.006316858815334563, + 0.006312276763547482, + 0.006949238477192474, + 0.009956655466073081, + 0.01355885582338353, + 0.017437786081889343, + 0.025496532582614126, + 0.033849936560945634 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 150 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 1623.8421, + "serial_latency_p99": 0.0118, + "serial_latency_p95": 0.0094, + "recall": 0.945, + "ndcg": 0.9479, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 129.0095, + 638.0261, + 1077.0653, + 1477.0665, + 1561.7802, + 1595.9667, + 1623.8421, + 1621.924 + ], + "conc_latency_p99_list": [ + 0.011765603999083404, + 0.012711298419817475, + 0.01651381758929347, + 0.026589661599427918, + 0.039944376738385454, + 0.049404421698636715, + 0.06832758593183824, + 0.08929936919903413 + ], + "conc_latency_p95_list": [ + 0.008827744000882376, + 0.008996219901018777, + 0.012427216298965503, + 0.021186420002777595, + 0.03210561599898938, + 0.04080166250059847, + 0.055962507547701525, + 0.07124235199989926 + ], + "conc_latency_avg_list": [ + 0.00774891642235306, + 0.007833377676777789, + 0.009280148463738688, + 0.01353283532887281, + 0.019196317583287527, + 0.025042119929708374, + 0.0368172028168529, + 0.04911914661560091 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 300 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 2808.2421, + "serial_latency_p99": 0.0095, + "serial_latency_p95": 0.0068, + "recall": 0.8674, + "ndcg": 0.8815, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 155.3245, + 826.7189, + 1600.1053, + 2225.9447, + 2513.1208, + 2695.4952, + 2765.1543, + 2808.2421 + ], + "conc_latency_p99_list": [ + 0.010928568999224795, + 0.010829480089960267, + 0.01071714987985615, + 0.01612071243016544, + 0.02644522499940649, + 0.032847231551932046, + 0.04966854284037254, + 0.06645170240146402 + ], + "conc_latency_p95_list": [ + 0.007179388998338254, + 0.006837483900562801, + 0.007670438798959367, + 0.012552767749366466, + 0.020286900500650518, + 0.02540200199928222, + 0.038052106797840664, + 0.05054814299946883 + ], + "conc_latency_avg_list": [ + 0.0064359595901721245, + 0.006044790967625746, + 0.0062457260062389625, + 0.008978903692601561, + 0.011928161713624801, + 0.01482735261744678, + 0.021619448376489415, + 0.028383759514242164 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3846.0365, + "optimize_duration": 1110.4687, + "load_duration": 4956.5052, + "qps": 1482.3772, + "serial_latency_p99": 0.0121, + "serial_latency_p95": 0.0094, + "recall": 0.9523, + "ndcg": 0.9546, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 123.3087, + 580.5824, + 1004.7055, + 1371.9591, + 1426.106, + 1442.8227, + 1482.3772, + 1482.2809 + ], + "conc_latency_p99_list": [ + 0.011828215117784533, + 0.01352971964137398, + 0.0166967718025262, + 0.02956553739913943, + 0.041614612400007904, + 0.05316882036993774, + 0.07466962000107744, + 0.09735826710020776 + ], + "conc_latency_p95_list": [ + 0.009158867201040264, + 0.009766673699778041, + 0.013278135998916696, + 0.023815533299421075, + 0.03488031349843368, + 0.04589065709897113, + 0.06176861299900338, + 0.07706757499909145 + ], + "conc_latency_avg_list": [ + 0.008107228449720227, + 0.008606959040297272, + 0.009948278934310932, + 0.014567306633654263, + 0.02102471552485447, + 0.027699906799067583, + 0.0403594785833045, + 0.05377204408997558 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 350 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1721.5416, + "serial_latency_p99": 0.0096, + "serial_latency_p95": 0.0084, + "recall": 0.876, + "ndcg": 0.8855, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 18.9602, + 421.7232, + 1110.2953, + 1558.5395, + 1655.8557, + 1687.2055, + 1707.2641, + 1721.5416 + ], + "conc_latency_p99_list": [ + 0.0890053563169205, + 0.06768262584373579, + 0.01639450403279624, + 0.025886146546399667, + 0.03685992147773504, + 0.04851100179657815, + 0.06766616894965408, + 0.08664867073734057 + ], + "conc_latency_p95_list": [ + 0.07462253859848716, + 0.04929285799589706, + 0.011980525049875722, + 0.02050993259763345, + 0.030178915952274107, + 0.039007512998068705, + 0.054956941800628524, + 0.06864605330047198 + ], + "conc_latency_avg_list": [ + 0.0527364081070003, + 0.011852286945072127, + 0.008997451744878971, + 0.012824499661199576, + 0.018105110192998403, + 0.023662544243058387, + 0.03503091284714358, + 0.04628931630721377 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 150 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1032.9696, + "serial_latency_p99": 0.0148, + "serial_latency_p95": 0.0127, + "recall": 0.9299, + "ndcg": 0.933, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 94.6407, + 456.8911, + 819.2779, + 986.623, + 1016.3699, + 1020.5033, + 1032.9696, + 1017.9882 + ], + "conc_latency_p99_list": [ + 0.01443464591226075, + 0.01668544119311263, + 0.020597075797559213, + 0.04234821415491753, + 0.055556238333374516, + 0.07370169115238243, + 0.10577300176868448, + 0.14174547339440327 + ], + "conc_latency_p95_list": [ + 0.012447767957200992, + 0.01300238400872331, + 0.016574700995988678, + 0.03295493289260776, + 0.04768081354850436, + 0.064796744252817, + 0.09333402040065265, + 0.12088888059952296 + ], + "conc_latency_avg_list": [ + 0.010563127625240488, + 0.010939355078447704, + 0.012200001814344278, + 0.020260497643984427, + 0.02949441625551025, + 0.03917002076625185, + 0.05792509511531056, + 0.07840660381120702 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 350 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1150.4393, + "serial_latency_p99": 0.0134, + "serial_latency_p95": 0.012, + "recall": 0.9225, + "ndcg": 0.9265, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 104.4067, + 517.3208, + 889.9421, + 1094.8411, + 1095.6488, + 1137.8826, + 1150.4393, + 1141.58 + ], + "conc_latency_p99_list": [ + 0.012909343038918442, + 0.01400524774362566, + 0.019253388400829866, + 0.038147892540728215, + 0.05472438236000016, + 0.06572198009467672, + 0.0952600136399269, + 0.1264052395534234 + ], + "conc_latency_p95_list": [ + 0.011359572803485206, + 0.011475628245534608, + 0.01540375875265454, + 0.03010115729921381, + 0.044985946398810484, + 0.05843431264365791, + 0.08308516180550213, + 0.10663665049651172 + ], + "conc_latency_avg_list": [ + 0.009575407694901247, + 0.009661187708110561, + 0.011232427720710226, + 0.018257383714569215, + 0.027368236626444314, + 0.035086554527816706, + 0.05202559829357753, + 0.06982070456346111 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 300 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1452.1536, + "serial_latency_p99": 0.0108, + "serial_latency_p95": 0.0092, + "recall": 0.8973, + "ndcg": 0.9042, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 49.0384, + 580.9947, + 986.4739, + 1351.5216, + 1402.1557, + 1419.7304, + 1450.6627, + 1452.1536 + ], + "conc_latency_p99_list": [ + 0.039148551175603646, + 0.013189356601214931, + 0.017704268741654233, + 0.030597097602731108, + 0.04256812395251478, + 0.05555199954222194, + 0.07699622272266429, + 0.0999706184824754 + ], + "conc_latency_p95_list": [ + 0.03537960539688356, + 0.010066490995814093, + 0.013690835254237753, + 0.024310099499416538, + 0.035451141749945236, + 0.04658339719389914, + 0.06417367161193396, + 0.08166611165797803 + ], + "conc_latency_avg_list": [ + 0.020388936871343147, + 0.008602396555117218, + 0.010126462637783804, + 0.014789400451352687, + 0.02138135761146721, + 0.028131951981904997, + 0.04123096160957269, + 0.0548839897278163 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 200 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 2181.3939, + "serial_latency_p99": 0.0094, + "serial_latency_p95": 0.0077, + "recall": 0.8353, + "ndcg": 0.8501, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 3.646, + 29.6904, + 333.5573, + 1900.0888, + 2079.4449, + 2131.3329, + 2181.3939, + 2019.3786 + ], + "conc_latency_p99_list": [ + 0.4185616793336521, + 0.3197556457712197, + 0.22267254341524675, + 0.020585235283360776, + 0.0324391679285327, + 0.04552019087132067, + 0.06043267271961665, + 0.08661333356372784 + ], + "conc_latency_p95_list": [ + 0.38934891536191574, + 0.2844508775517169, + 0.1684353517535783, + 0.01616000900394283, + 0.024561249790713186, + 0.0332476719981059, + 0.04531921409798087, + 0.06508466459927148 + ], + "conc_latency_avg_list": [ + 0.2742520264012538, + 0.1683271812554273, + 0.029971614569954105, + 0.010518879796956216, + 0.014417846781857802, + 0.018753533944310726, + 0.02740638227517153, + 0.039480719052758934 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 100 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 37808.2643, + "optimize_duration": 18845.2628, + "load_duration": 56653.5271, + "qps": 1295.5543, + "serial_latency_p99": 0.0112, + "serial_latency_p95": 0.0102, + "recall": 0.9126, + "ndcg": 0.9176, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 108.6988, + 554.0053, + 962.0715, + 1222.1509, + 1262.7418, + 1287.8623, + 1295.5543, + 1295.4142 + ], + "conc_latency_p99_list": [ + 0.012900208660430504, + 0.013385620156768804, + 0.017747372422018085, + 0.033394850781187424, + 0.04573169803712508, + 0.05876307546350308, + 0.08581843032181498, + 0.1129824651160743 + ], + "conc_latency_p95_list": [ + 0.010918107806355692, + 0.01064183129929006, + 0.014414189194212666, + 0.02686949290437041, + 0.03948758620244917, + 0.051792045756883454, + 0.0734679256027448, + 0.09282746819080781 + ], + "conc_latency_avg_list": [ + 0.009197126244932163, + 0.00902096815373196, + 0.010389166613132592, + 0.016355007809079263, + 0.02374101070396333, + 0.03104020946657135, + 0.046198905251943534, + 0.06155043230356708 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": true, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 250 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1770595200.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json new file mode 100644 index 000000000..e10404530 --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json @@ -0,0 +1,4074 @@ +{ + "run_id": "c11e83b51ff14060a08f06d58f801214", + "task_label": "standard_20260403", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, + "qps": 3917.2035, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0023, + "recall": 0.9203, + "ndcg": 0.9238, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 467.4942, + 1828.2004, + 2454.4495, + 2976.5539, + 3178.6052, + 3494.4787, + 3755.8025, + 3917.2035 + ], + "conc_latency_p99_list": [ + 0.0025411477516172455, + 0.0033511776782688685, + 0.005052069290541106, + 0.011894863871420993, + 0.016992485875962308, + 0.020195355401374387, + 0.026863074758439298, + 0.03365033164591297 + ], + "conc_latency_p95_list": [ + 0.0023739541989925782, + 0.00314145510783419, + 0.004699915152013999, + 0.009829005991923623, + 0.014131764802732504, + 0.017079706999356854, + 0.022949057801451987, + 0.02835416550078662 + ], + "conc_latency_avg_list": [ + 0.002136165356290855, + 0.002730801989459182, + 0.004067402028788586, + 0.006701902388970856, + 0.009401196782828802, + 0.011384491649831373, + 0.015778383442210466, + 0.020076245655094027 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3628.8527, + "serial_latency_p99": 0.0026, + "serial_latency_p95": 0.0024, + "recall": 0.9318, + "ndcg": 0.9346, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 456.622, + 1760.1234, + 2305.1866, + 2811.6811, + 3062.7639, + 3254.2934, + 3483.1948, + 3628.8527 + ], + "conc_latency_p99_list": [ + 0.0025775732805777807, + 0.003496330338239204, + 0.005392098429438191, + 0.012580982464569393, + 0.01744679359835574, + 0.02114546248485566, + 0.02862863955655484, + 0.03660006429607165 + ], + "conc_latency_p95_list": [ + 0.002428409402637044, + 0.003272143194044474, + 0.005011953243229073, + 0.010387669454212298, + 0.01447147800354287, + 0.017890810401149794, + 0.023953950349823568, + 0.030202264491526864 + ], + "conc_latency_avg_list": [ + 0.002187054468852311, + 0.002836289985030498, + 0.004330527318313066, + 0.007095944638723138, + 0.009756847450075634, + 0.012226813030036315, + 0.017037973309470753, + 0.021662613936531003 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 120, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3250.1112, + "serial_latency_p99": 0.0027, + "serial_latency_p95": 0.0025, + "recall": 0.9443, + "ndcg": 0.9463, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 434.6843, + 1686.1263, + 2163.5936, + 2555.03, + 2793.5899, + 2921.7137, + 3130.5251, + 3250.1112 + ], + "conc_latency_p99_list": [ + 0.0027417352329939604, + 0.003703351480362472, + 0.005958503000438213, + 0.013709455646312565, + 0.01829364382574568, + 0.022580389242502868, + 0.03056172190743382, + 0.037927262283337766 + ], + "conc_latency_p95_list": [ + 0.0025719269993714987, + 0.0034471726998162922, + 0.005409308793605305, + 0.011277809000603156, + 0.015221869490051177, + 0.019072343403240672, + 0.02558471505108173, + 0.03225500610860763 + ], + "conc_latency_avg_list": [ + 0.002297469730699608, + 0.0029608701952901183, + 0.004614075755959777, + 0.007808470701124476, + 0.010702304970719298, + 0.013606579627839316, + 0.018957654525715156, + 0.02420684028195088 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 150, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2762.4144, + "serial_latency_p99": 0.0031, + "serial_latency_p95": 0.0029, + "recall": 0.9556, + "ndcg": 0.9567, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 395.9308, + 1515.3208, + 1959.8545, + 2234.9654, + 2427.3893, + 2537.495, + 2657.5964, + 2762.4144 + ], + "conc_latency_p99_list": [ + 0.003096595037495717, + 0.00413693475740729, + 0.007250679123098961, + 0.01500825069088023, + 0.01980871459498304, + 0.024630421155015937, + 0.033659081743244314, + 0.041983861521002835 + ], + "conc_latency_p95_list": [ + 0.0028621868113987148, + 0.003837919446232263, + 0.006172719193273224, + 0.012415598499501357, + 0.0168672572079231, + 0.021068831244338074, + 0.028828703101316914, + 0.03613693019142374 + ], + "conc_latency_avg_list": [ + 0.002522697462250955, + 0.0032950399942241306, + 0.0050943009057731, + 0.008926197356642889, + 0.012320074532162776, + 0.01567786154805744, + 0.022343385743465827, + 0.028501836863104958 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 200, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2384.6245, + "serial_latency_p99": 0.0032, + "serial_latency_p95": 0.003, + "recall": 0.9627, + "ndcg": 0.9632, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 368.492, + 1356.984, + 1780.9947, + 1978.8754, + 2086.4087, + 2180.2788, + 2347.7647, + 2384.6245 + ], + "conc_latency_p99_list": [ + 0.0033670899452408775, + 0.004818720840266903, + 0.008331618664960839, + 0.01588158588972873, + 0.02218642996114793, + 0.02756158859701827, + 0.036660355595813585, + 0.047074296680802936 + ], + "conc_latency_p95_list": [ + 0.0031033190454763816, + 0.004360820600413717, + 0.00704125490374281, + 0.013423533007153307, + 0.019178588101203785, + 0.023883526999270543, + 0.03231190550286556, + 0.041070348300854674 + ], + "conc_latency_avg_list": [ + 0.0027103553220750067, + 0.0036796698451320694, + 0.005606053561698588, + 0.010084455073144883, + 0.01433518456445194, + 0.0182535198619555, + 0.025308713260092305, + 0.03299686872047841 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 250, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2134.1717, + "serial_latency_p99": 0.0038, + "serial_latency_p95": 0.0036, + "recall": 0.9671, + "ndcg": 0.9672, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 342.9302, + 1297.1692, + 1605.3373, + 1723.4931, + 1874.7201, + 1950.9837, + 2026.1279, + 2134.1717 + ], + "conc_latency_p99_list": [ + 0.003612513002881314, + 0.005075874237372772, + 0.009592544495098993, + 0.01732624325857614, + 0.02383941526291892, + 0.02994993883999997, + 0.04171322006412084, + 0.052276343395351435 + ], + "conc_latency_p95_list": [ + 0.00337228730058996, + 0.00455044719419675, + 0.008192990009411006, + 0.015064282899402313, + 0.02108702180557884, + 0.02638680679956451, + 0.036932101499405685, + 0.04595883999718353 + ], + "conc_latency_avg_list": [ + 0.0029124934026024873, + 0.0038495151919032532, + 0.0062189602467817824, + 0.011577066999218889, + 0.01595138406671783, + 0.02037925821852399, + 0.029283526202598515, + 0.03685219368742522 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 300, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1641.3478, + "serial_latency_p99": 0.0041, + "serial_latency_p95": 0.0039, + "recall": 0.9729, + "ndcg": 0.9726, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 300.7136, + 1134.4852, + 1338.9285, + 1408.0122, + 1523.2338, + 1540.9389, + 1631.3282, + 1641.3478 + ], + "conc_latency_p99_list": [ + 0.004302493912400677, + 0.006291621240816314, + 0.011818032589217181, + 0.020626583240227772, + 0.028327662804658774, + 0.03604436940222513, + 0.0508815360846347, + 0.06704054794870894 + ], + "conc_latency_p95_list": [ + 0.003943610056012403, + 0.0053649904970370695, + 0.010212573444005101, + 0.018129582199617286, + 0.02555973254638957, + 0.03308224900683854, + 0.04562628499115817, + 0.05833644095400814 + ], + "conc_latency_avg_list": [ + 0.0033214706270654664, + 0.004401974502845957, + 0.00745933768945177, + 0.014171396440932295, + 0.0196399944581788, + 0.025851743210982894, + 0.03640642504236049, + 0.048033687367785266 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 400, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1488.5841, + "serial_latency_p99": 0.0047, + "serial_latency_p95": 0.0043, + "recall": 0.9764, + "ndcg": 0.976, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 259.0154, + 1019.3401, + 1241.5382, + 1301.0531, + 1358.779, + 1399.0002, + 1425.3615, + 1488.5841 + ], + "conc_latency_p99_list": [ + 0.004889184155326802, + 0.007301273914636109, + 0.01256226149998838, + 0.02185779360006564, + 0.03058185266534565, + 0.03868378391998703, + 0.0577071424780297, + 0.07162654809217199 + ], + "conc_latency_p95_list": [ + 0.004537451700889505, + 0.006121107403305359, + 0.010819275506946724, + 0.019440887503151316, + 0.02824652445488027, + 0.035727200949622784, + 0.05152663829067023, + 0.0646723014942836 + ], + "conc_latency_avg_list": [ + 0.0038569616304989004, + 0.0048995726665189525, + 0.008043409835523815, + 0.015340665489496967, + 0.022014652133720863, + 0.02845767912486287, + 0.04164023567491625, + 0.05289991318053246 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 500, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, + "qps": 2747.3167, + "serial_latency_p99": 0.0033, + "serial_latency_p95": 0.003, + "recall": 0.9204, + "ndcg": 0.9262, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 412.8127, + 1516.5887, + 1934.7028, + 2345.2563, + 2531.5491, + 2627.1159, + 2747.3167, + 2733.8527 + ], + "conc_latency_p99_list": [ + 0.0030919767648447303, + 0.004120720853097738, + 0.006685402100338251, + 0.014304348317091375, + 0.019310851126065252, + 0.024482081828900833, + 0.03435190891788809, + 0.04423686145673856 + ], + "conc_latency_p95_list": [ + 0.0028119500013417563, + 0.003827417498541763, + 0.006025877799402224, + 0.011986060402705334, + 0.016484533800394274, + 0.020710799152220714, + 0.028733705398917665, + 0.037459268098609756 + ], + "conc_latency_avg_list": [ + 0.002419260628861797, + 0.003292419872587543, + 0.005160383001607833, + 0.00850752764269055, + 0.011804369542313493, + 0.01513907190518365, + 0.021623125977333776, + 0.028768996633767287 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2514.4481, + "serial_latency_p99": 0.0032, + "serial_latency_p95": 0.003, + "recall": 0.9303, + "ndcg": 0.9357, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 376.3101, + 1413.4044, + 1804.5963, + 2063.4126, + 2224.2985, + 2285.9557, + 2428.4015, + 2514.4481 + ], + "conc_latency_p99_list": [ + 0.003350270938608448, + 0.004419036731342202, + 0.007495493090173118, + 0.01604453914129406, + 0.021470454052177963, + 0.027058087129116764, + 0.03722573504091981, + 0.048193111100408685 + ], + "conc_latency_p95_list": [ + 0.003067891455066274, + 0.004094811100731022, + 0.006531247201928636, + 0.013379140298275157, + 0.018289522749910248, + 0.0233742457050539, + 0.03192761289792543, + 0.040280850498675136 + ], + "conc_latency_avg_list": [ + 0.002653342368273733, + 0.0035319332925499757, + 0.005532113228559582, + 0.009670992982743579, + 0.013421869019516723, + 0.017384391084778364, + 0.024445988357111505, + 0.03128177971695877 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 120, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2177.2345, + "serial_latency_p99": 0.0034, + "serial_latency_p95": 0.0031, + "recall": 0.9408, + "ndcg": 0.9456, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 354.703, + 1270.9673, + 1660.8118, + 1799.9874, + 1911.9995, + 2018.5829, + 2119.3303, + 2177.2345 + ], + "conc_latency_p99_list": [ + 0.0035591462109005084, + 0.0051245430819108154, + 0.008888348671316635, + 0.017485948742978506, + 0.023954762732464586, + 0.02942745841770374, + 0.04045989539969013, + 0.052306493496216695 + ], + "conc_latency_p95_list": [ + 0.003282636954463669, + 0.004633509999257512, + 0.0075649314512702395, + 0.014845420597703196, + 0.02076956499913649, + 0.025784255946928167, + 0.035505612393899356, + 0.045118153299335974 + ], + "conc_latency_avg_list": [ + 0.002815711151857267, + 0.003928179410303787, + 0.006011238757150647, + 0.011084966830100315, + 0.015638666422418603, + 0.019708495546663755, + 0.02799275543572887, + 0.03611913807969698 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 150, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1833.2575, + "serial_latency_p99": 0.0039, + "serial_latency_p95": 0.0035, + "recall": 0.951, + "ndcg": 0.9555, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 329.7676, + 1176.835, + 1426.5695, + 1546.3292, + 1656.2697, + 1716.0803, + 1790.6121, + 1833.2575 + ], + "conc_latency_p99_list": [ + 0.003868887719290797, + 0.005920497829865769, + 0.01097595489642117, + 0.01919553377847477, + 0.02603815700131236, + 0.03326150514345494, + 0.04663379819947293, + 0.05838948019809325 + ], + "conc_latency_p95_list": [ + 0.003534871701413067, + 0.005110333902484853, + 0.009471108402794925, + 0.01673110910487594, + 0.023566940006276127, + 0.02969893909685197, + 0.04118938999890815, + 0.05255015200236812 + ], + "conc_latency_avg_list": [ + 0.0030290844068619504, + 0.004243610521352684, + 0.007000278697486464, + 0.012908119052890047, + 0.018050427070200076, + 0.023198493359582885, + 0.03316593436696198, + 0.04298056582898082 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 200, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1552.4803, + "serial_latency_p99": 0.004, + "serial_latency_p95": 0.0037, + "recall": 0.9565, + "ndcg": 0.9605, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 303.2446, + 1078.6424, + 1299.206, + 1360.6235, + 1425.268, + 1465.3767, + 1520.0788, + 1552.4803 + ], + "conc_latency_p99_list": [ + 0.0040712328813970085, + 0.006725932629851741, + 0.012099390296789348, + 0.021067992354510352, + 0.02953478858660674, + 0.037649333699373524, + 0.05402578063920373, + 0.07030989054946984 + ], + "conc_latency_p95_list": [ + 0.0038339799008099357, + 0.005704106903795035, + 0.01050544159807032, + 0.01850984884877107, + 0.02700225284861517, + 0.034401998404064216, + 0.04840153990226099, + 0.06195298564853147 + ], + "conc_latency_avg_list": [ + 0.003293997160806082, + 0.004629127288807816, + 0.00768599232973781, + 0.014665701743116042, + 0.02097794137202591, + 0.027194933866063295, + 0.03908773235524631, + 0.050673599669096146 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 250, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1355.3121, + "serial_latency_p99": 0.0044, + "serial_latency_p95": 0.0042, + "recall": 0.9602, + "ndcg": 0.964, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 279.2851, + 975.7699, + 1150.3744, + 1187.2494, + 1247.0213, + 1283.8337, + 1302.6721, + 1355.3121 + ], + "conc_latency_p99_list": [ + 0.004451236832683208, + 0.0076792708405992016, + 0.01329627803701441, + 0.023810231103198035, + 0.033509768832373055, + 0.04278365236037641, + 0.06259672742118709, + 0.08097375019555315 + ], + "conc_latency_p95_list": [ + 0.004183817799275858, + 0.006473346400889567, + 0.011516073600796517, + 0.0212969502517808, + 0.030741239200142444, + 0.03883912599849282, + 0.056272352899395625, + 0.07115194579964736 + ], + "conc_latency_avg_list": [ + 0.003576748584184128, + 0.005118136384013237, + 0.008682482330775591, + 0.0168131587141993, + 0.023978672854972592, + 0.031007006045867456, + 0.04562714668667405, + 0.058088526012853484 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 300, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1079.2123, + "serial_latency_p99": 0.0053, + "serial_latency_p95": 0.0049, + "recall": 0.9648, + "ndcg": 0.9686, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 241.6992, + 799.4681, + 939.7848, + 975.2814, + 1001.75, + 1029.2059, + 1065.2548, + 1079.2123 + ], + "conc_latency_p99_list": [ + 0.005288620950886979, + 0.009748364380211563, + 0.015409245888222355, + 0.028667413447983563, + 0.04076468282910357, + 0.05202346699952616, + 0.07527647967857774, + 0.09870467029977587 + ], + "conc_latency_p95_list": [ + 0.004910965752060292, + 0.008243431097071152, + 0.013543863647282705, + 0.025803527399330048, + 0.03817865899909521, + 0.048153773248486686, + 0.06777279375382932, + 0.08822071449685609 + ], + "conc_latency_avg_list": [ + 0.004132684600647634, + 0.006247207634111248, + 0.010625959006772904, + 0.02046250279903178, + 0.02985517599151899, + 0.0386749397755516, + 0.055773659676497576, + 0.07301672729002262 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 400, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 876.5772, + "serial_latency_p99": 0.0063, + "serial_latency_p95": 0.0059, + "recall": 0.9676, + "ndcg": 0.9713, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 213.9083, + 704.2348, + 797.531, + 821.1706, + 825.533, + 797.2598, + 858.8645, + 876.5772 + ], + "conc_latency_p99_list": [ + 0.0060647999984212225, + 0.01099963705062691, + 0.017907913918024848, + 0.03398729390319204, + 0.049453794501459925, + 0.06846078443864825, + 0.09411621719773387, + 0.12295867912092945 + ], + "conc_latency_p95_list": [ + 0.005586325001786463, + 0.009474276846958667, + 0.015768809196015348, + 0.03062209299969254, + 0.046334437996847555, + 0.06321956860047066, + 0.08318591200077208, + 0.10695994940178935 + ], + "conc_latency_avg_list": [ + 0.004670567833335547, + 0.007091383770451675, + 0.01252405204148396, + 0.02430557731179879, + 0.036228697626746305, + 0.04987607783688658, + 0.06916092553517646, + 0.09005697652328008 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 500, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 10663.1231, + "serial_latency_p99": 0.002, + "serial_latency_p95": 0.0018, + "recall": 0.8405, + "ndcg": 0.8674, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 592.3235, + 2657.0693, + 4857.6653, + 7731.1954, + 9035.8688, + 9746.5046, + 10547.8051, + 10663.1231 + ], + "conc_latency_p99_list": [ + 0.0019481061986880378, + 0.002309793634340169, + 0.0025996991485590097, + 0.003907884397631278, + 0.005842388768505771, + 0.007729610755923204, + 0.011498715590278154, + 0.01563908824173267 + ], + "conc_latency_p95_list": [ + 0.0018476900004316121, + 0.0021391708025475962, + 0.0023516209941590203, + 0.0031451117574761156, + 0.004541207106376533, + 0.006081955801346338, + 0.008940340152912542, + 0.012104344801628029 + ], + "conc_latency_avg_list": [ + 0.0016856015045043139, + 0.0018777718208453423, + 0.0020532305285396453, + 0.0025771131947367317, + 0.0033031913740212363, + 0.0040746336526161255, + 0.005611474910509973, + 0.007361856948830913 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 10333.9072, + "serial_latency_p99": 0.002, + "serial_latency_p95": 0.0019, + "recall": 0.889, + "ndcg": 0.9058, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 568.5874, + 2594.6923, + 4693.0426, + 7429.976, + 8564.4439, + 9136.6911, + 9985.356, + 10333.9072 + ], + "conc_latency_p99_list": [ + 0.0020693473634310062, + 0.0023700819991063315, + 0.0026944835495669377, + 0.004012327503005508, + 0.006051957674062573, + 0.008206313409027643, + 0.012019852105004216, + 0.015895357320114278 + ], + "conc_latency_p95_list": [ + 0.0019474557979265227, + 0.0021944244945188984, + 0.002450034000503365, + 0.003276585503044771, + 0.004752537995955206, + 0.006455344991991296, + 0.009426155498658773, + 0.012384038396703547 + ], + "conc_latency_avg_list": [ + 0.0017559580574677634, + 0.0019229909223785742, + 0.0021257891514691474, + 0.0026812402175934932, + 0.0034838524012537763, + 0.00433970032247638, + 0.005933638012347279, + 0.007582591068540877 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.2 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 9575.6863, + "serial_latency_p99": 0.0023, + "serial_latency_p95": 0.0021, + "recall": 0.9189, + "ndcg": 0.93, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 535.4941, + 2437.6236, + 4429.593, + 6980.413, + 8035.6356, + 8594.0728, + 9041.3709, + 9575.6863 + ], + "conc_latency_p99_list": [ + 0.002218716748757288, + 0.002563155780226227, + 0.0028770195422111954, + 0.004238974919717297, + 0.006498919794103134, + 0.008666419192886682, + 0.013239830499514937, + 0.016983359853475096 + ], + "conc_latency_p95_list": [ + 0.002100886751577491, + 0.0023667280438530724, + 0.0026158143016800747, + 0.0034943125079735177, + 0.005161623004823923, + 0.006949284041184, + 0.010531335494306404, + 0.013410061004833551 + ], + "conc_latency_avg_list": [ + 0.0018646425533687709, + 0.0020474016047948205, + 0.00225211486396273, + 0.00285420751423595, + 0.003711133378722069, + 0.004623532072278411, + 0.006552865012667485, + 0.008189225702132959 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 1.5 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 8596.7694, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0022, + "recall": 0.9416, + "ndcg": 0.9493, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 497.6866, + 2259.6612, + 4114.0036, + 6455.3584, + 7309.2383, + 7686.907, + 8294.3018, + 8596.7694 + ], + "conc_latency_p99_list": [ + 0.002440318200387992, + 0.0028159428038634342, + 0.0031061971996678038, + 0.004590758396079762, + 0.00719438069412717, + 0.009845319669257151, + 0.014379469840059753, + 0.018956097210466392 + ], + "conc_latency_p95_list": [ + 0.0022961719951126724, + 0.002580139000201598, + 0.0028258040038053878, + 0.003789236750890268, + 0.005791235491051338, + 0.00791677404558868, + 0.011584387852053624, + 0.015152870106976477 + ], + "conc_latency_avg_list": [ + 0.0020063849993894605, + 0.002208682703794502, + 0.002425080457737401, + 0.003086665132816786, + 0.00408523815973775, + 0.005170043147020008, + 0.007138938063080818, + 0.009125464665924427 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 2.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 7704.3625, + "serial_latency_p99": 0.0027, + "serial_latency_p95": 0.0025, + "recall": 0.9541, + "ndcg": 0.96, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 464.498, + 2087.0727, + 3811.3426, + 5897.8764, + 6613.5685, + 6995.184, + 7496.7496, + 7704.3625 + ], + "conc_latency_p99_list": [ + 0.002632544774387497, + 0.0030649999552406367, + 0.003373088193475267, + 0.004990858241653769, + 0.00786220946611138, + 0.010790031323267592, + 0.015690509527339604, + 0.020764024818781757 + ], + "conc_latency_p95_list": [ + 0.002470412495313212, + 0.002798333394457586, + 0.0030606225445808377, + 0.004139978906459873, + 0.006436757100163957, + 0.008721163390146102, + 0.012695113198424209, + 0.016752441306016403 + ], + "conc_latency_avg_list": [ + 0.002149911288992391, + 0.0023915170520859654, + 0.002617775435786274, + 0.0033781344953593517, + 0.0045130488491076, + 0.005680457334630225, + 0.007906434767685636, + 0.010188877722018175 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 2.5 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 7023.6735, + "serial_latency_p99": 0.003, + "serial_latency_p95": 0.0028, + "recall": 0.962, + "ndcg": 0.9667, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 437.0253, + 1938.4624, + 3454.7847, + 5460.2663, + 6058.6283, + 6398.7693, + 6828.1471, + 7023.6735 + ], + "conc_latency_p99_list": [ + 0.0028729987128463105, + 0.0033484641878749246, + 0.0037352688424289217, + 0.0053953273908700725, + 0.008702208310278365, + 0.011712039967096652, + 0.0172460880043218, + 0.02189830483126571 + ], + "conc_latency_p95_list": [ + 0.00266674381273333, + 0.003031579001981299, + 0.0033792859961977225, + 0.004472345393151045, + 0.007129233997693518, + 0.009540854604711059, + 0.014027368000824936, + 0.018085699503717478 + ], + "conc_latency_avg_list": [ + 0.0022852531678980575, + 0.0025748727575029204, + 0.0028880249144625355, + 0.003650684004208795, + 0.004931203325481902, + 0.006211804309036586, + 0.008678531268037587, + 0.011200709193523 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 3.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 6031.3725, + "serial_latency_p99": 0.0033, + "serial_latency_p95": 0.003, + "recall": 0.971, + "ndcg": 0.9743, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 375.0077, + 1749.9239, + 3171.188, + 4757.8121, + 5337.8052, + 5585.8874, + 5830.76, + 6031.3725 + ], + "conc_latency_p99_list": [ + 0.0034118053846759725, + 0.00371807467265171, + 0.004087487124343162, + 0.006445972005603836, + 0.0099661920015933, + 0.013419415393291265, + 0.01954109726633761, + 0.02450512952229475 + ], + "conc_latency_p95_list": [ + 0.0031440883423783815, + 0.0033821857992734294, + 0.003687061999517027, + 0.0053249524033162745, + 0.008198342995456187, + 0.011055400453187756, + 0.01616062365719699, + 0.020329035600298084 + ], + "conc_latency_avg_list": [ + 0.002663293590200931, + 0.0028528372994040497, + 0.0031468442372841717, + 0.004191480480374251, + 0.005595629037850519, + 0.007119491187741896, + 0.010171118781282724, + 0.013044592076065837 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 4.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5258.1868, + "serial_latency_p99": 0.0036, + "serial_latency_p95": 0.0033, + "recall": 0.9768, + "ndcg": 0.9793, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 351.2889, + 1555.9976, + 2891.7532, + 4328.3041, + 4710.0472, + 4944.6567, + 5159.0143, + 5258.1868 + ], + "conc_latency_p99_list": [ + 0.0036150889145210364, + 0.004169191400287673, + 0.004474782356992364, + 0.00714936985692475, + 0.011309100479702464, + 0.014964991490269298, + 0.02106605595399745, + 0.02670282432547537 + ], + "conc_latency_p95_list": [ + 0.0032958149939076972, + 0.0038005771988537163, + 0.004026532001444138, + 0.0059051988064311445, + 0.0093500265997136, + 0.012421341851586474, + 0.017624663742026314, + 0.022111607256374555 + ], + "conc_latency_avg_list": [ + 0.0028433795541688497, + 0.003208622344277306, + 0.0034516157786571304, + 0.004607007276075149, + 0.006345752606166889, + 0.008043607402151223, + 0.01149538342406618, + 0.014941377751891149 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq4u-fp16-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ4U", + "refine": true, + "refine_type": "FP16", + "refine_k": 5.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5973.0024, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0023, + "recall": 0.9192, + "ndcg": 0.9299, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 192.2873, + 780.772, + 1243.1343, + 2209.7863, + 5135.3255, + 5268.6684, + 5595.5025, + 5973.0024 + ], + "conc_latency_p99_list": [ + 0.006922555523342447, + 0.011980964867980226, + 0.01822498522611567, + 0.021657125554483937, + 0.011112995611620146, + 0.015273953418945892, + 0.021470034882659094, + 0.025694710013340227 + ], + "conc_latency_p95_list": [ + 0.006153562400140799, + 0.009661811696423684, + 0.013727128539176193, + 0.01456564510299358, + 0.00905536999925971, + 0.012242838197562377, + 0.017436827097844797, + 0.020891863998258486 + ], + "conc_latency_avg_list": [ + 0.005195547580597013, + 0.00639603304659674, + 0.0080334040769494, + 0.009027100221592798, + 0.0058122537237125, + 0.007549590091448817, + 0.010597991937403054, + 0.01314884199753361 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5416.5758, + "serial_latency_p99": 0.0026, + "serial_latency_p95": 0.0024, + "recall": 0.9334, + "ndcg": 0.9421, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 468.5468, + 2023.0586, + 3273.7331, + 4338.5873, + 4722.177, + 4899.851, + 5215.5063, + 5416.5758 + ], + "conc_latency_p99_list": [ + 0.002587841608328745, + 0.0031619464718096405, + 0.003964070154324872, + 0.007774805837543681, + 0.012349921874993024, + 0.01622475358992233, + 0.022378725241142112, + 0.02744747619624828 + ], + "conc_latency_p95_list": [ + 0.0024390827456954867, + 0.0029180134042690042, + 0.0035825525046675466, + 0.006318293401272964, + 0.009966622248612111, + 0.013235174745204858, + 0.018063901497225743, + 0.02230343740739044 + ], + "conc_latency_avg_list": [ + 0.0021313683553780983, + 0.0024673246161570224, + 0.0030484556683506147, + 0.0045937434301476935, + 0.006330211445594135, + 0.008107756352776998, + 0.01136996968107513, + 0.014491805104902885 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 120, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 4771.4324, + "serial_latency_p99": 0.0028, + "serial_latency_p95": 0.0025, + "recall": 0.9479, + "ndcg": 0.9545, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 446.4305, + 1857.0762, + 3066.2065, + 3959.9145, + 4119.6984, + 4404.2396, + 4654.3656, + 4771.4324 + ], + "conc_latency_p99_list": [ + 0.0027465902952826583, + 0.003445194798405282, + 0.004290146040148099, + 0.008809843383787666, + 0.014246009238704564, + 0.017628938370035024, + 0.024053513460094107, + 0.029474502794328145 + ], + "conc_latency_p95_list": [ + 0.0025784781464608377, + 0.0031829175946768372, + 0.003835800604429096, + 0.0071477785022580065, + 0.011466846610710487, + 0.014443101210054009, + 0.019454453799698967, + 0.023967151201213708 + ], + "conc_latency_avg_list": [ + 0.002237092280444995, + 0.0026882400384546624, + 0.0032547429679731454, + 0.005034559716886033, + 0.007252702527297999, + 0.009024968635487724, + 0.012743531884434339, + 0.016486166204689928 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 150, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 4006.3994, + "serial_latency_p99": 0.0032, + "serial_latency_p95": 0.003, + "recall": 0.9609, + "ndcg": 0.966, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 396.0794, + 1682.937, + 2770.1408, + 3415.7329, + 3586.5656, + 3732.9793, + 3908.8395, + 4006.3994 + ], + "conc_latency_p99_list": [ + 0.0031631914011086342, + 0.003759940078307404, + 0.005007598159427288, + 0.010577416090527546, + 0.01645218172343448, + 0.0200205380024272, + 0.02614469664360513, + 0.03247302199597469 + ], + "conc_latency_p95_list": [ + 0.0029416235047392547, + 0.0034802541093085894, + 0.004256061747582862, + 0.008588075407897121, + 0.01299767559976317, + 0.016106622002553195, + 0.021291244098392777, + 0.026077511996845715 + ], + "conc_latency_avg_list": [ + 0.002521469348982409, + 0.0029665274119998614, + 0.0036030252400985627, + 0.005839950739688844, + 0.008331452198117406, + 0.010640699049339198, + 0.015161825352942679, + 0.019627049620518894 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 200, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3441.7597, + "serial_latency_p99": 0.0035, + "serial_latency_p95": 0.0032, + "recall": 0.9682, + "ndcg": 0.9725, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 363.4573, + 1530.4211, + 2519.3499, + 2941.0933, + 3130.5066, + 3239.8074, + 3357.9662, + 3441.7597 + ], + "conc_latency_p99_list": [ + 0.003447128901898394, + 0.004104390731372401, + 0.005795499211671998, + 0.012608604685810857, + 0.018390358952165124, + 0.02161771130544366, + 0.029535210086614822, + 0.03676844512228854 + ], + "conc_latency_p95_list": [ + 0.0032040981095633465, + 0.0037800333026098087, + 0.004730919548455857, + 0.010123601651866907, + 0.014520535804331302, + 0.017753759350307517, + 0.023507353749300814, + 0.029112758993869645 + ], + "conc_latency_avg_list": [ + 0.0027478315701259456, + 0.0032620836103758183, + 0.003962182206516774, + 0.006784177567489496, + 0.009538412010322416, + 0.012277581280440339, + 0.017678232976666337, + 0.022830987265851796 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 250, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3040.6216, + "serial_latency_p99": 0.0037, + "serial_latency_p95": 0.0035, + "recall": 0.9734, + "ndcg": 0.9771, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 323.1632, + 1408.5716, + 2291.0458, + 2658.8647, + 2809.4563, + 2889.303, + 2958.6865, + 3040.6216 + ], + "conc_latency_p99_list": [ + 0.0038972479960648343, + 0.004427719511440956, + 0.006464274920726896, + 0.013835774816980114, + 0.01854686296428553, + 0.023528500349348183, + 0.030925163915235258, + 0.037629884978523494 + ], + "conc_latency_p95_list": [ + 0.0036286949907662347, + 0.00409807980468031, + 0.005293481396074639, + 0.01114347539114533, + 0.015239884803304439, + 0.01894600450323196, + 0.02535856414833688, + 0.03149616299779154 + ], + "conc_latency_avg_list": [ + 0.0030909596088423805, + 0.0035447166881325863, + 0.004357504468202672, + 0.007503025706852371, + 0.010637931999990373, + 0.013780835663880903, + 0.02002760247152969, + 0.02587108250217688 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 300, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2446.7373, + "serial_latency_p99": 0.0043, + "serial_latency_p95": 0.004, + "recall": 0.9791, + "ndcg": 0.9822, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 290.4004, + 1199.0654, + 1943.7472, + 2207.5563, + 2272.2082, + 2361.8647, + 2425.272, + 2446.7373 + ], + "conc_latency_p99_list": [ + 0.004225126459205057, + 0.005225188090844314, + 0.007753408416756429, + 0.015723756006627808, + 0.021543593837413895, + 0.025226465429732312, + 0.0348595633450895, + 0.04409603113395864 + ], + "conc_latency_p95_list": [ + 0.003936908097239211, + 0.004869158701330889, + 0.006411225302144882, + 0.012806271501176525, + 0.017710478595108724, + 0.02138321524907951, + 0.02917539790214505, + 0.037459137551195454 + ], + "conc_latency_avg_list": [ + 0.003439496089142086, + 0.004164736080481502, + 0.005136606676356694, + 0.0090353538125237, + 0.013157439387184814, + 0.01685513374865052, + 0.024467911853133933, + 0.03215223014404708 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 400, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2084.6245, + "serial_latency_p99": 0.005, + "serial_latency_p95": 0.0046, + "recall": 0.9819, + "ndcg": 0.9847, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 257.7354, + 1074.5131, + 1699.1152, + 1906.3127, + 1964.552, + 1992.5973, + 2059.3939, + 2084.6245 + ], + "conc_latency_p99_list": [ + 0.004679669999168255, + 0.0057560302416095515, + 0.008956946645048449, + 0.016920548141788453, + 0.022776664053235435, + 0.02843890285366797, + 0.039300132958451285, + 0.0494756092831085 + ], + "conc_latency_p95_list": [ + 0.004424713403568603, + 0.005422145003103651, + 0.007496941405406687, + 0.014108828103053384, + 0.019311829509388187, + 0.024556438496802002, + 0.033556375399348325, + 0.04314066854931297 + ], + "conc_latency_avg_list": [ + 0.00387603491228106, + 0.004647482651951955, + 0.005875930555797928, + 0.010467915104743828, + 0.015220352811585824, + 0.019988946757462604, + 0.028783135202314097, + 0.037787432564769345 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-force_merge", + "version": "2.6.14", + "note": "", + "uri": "**********", + "user": null, + "password": null, + "num_shards": 1, + "replica_number": 1 + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "M": 16, + "efConstruction": 300, + "ef": 500, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + } + ] +} \ No newline at end of file diff --git a/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json new file mode 100644 index 000000000..619b5fbf4 --- /dev/null +++ b/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json @@ -0,0 +1,2814 @@ +{ + "run_id": "80696b60e39749b295273db3cdba1b69", + "task_label": "standard_20260209", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 9441.1235, + "serial_latency_p99": 0.0052, + "serial_latency_p95": 0.0039, + "recall": 0.9589, + "ndcg": 0.9658, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 306.3773, + 1519.6034, + 3129.1309, + 5457.1507, + 6585.7082, + 7420.0391, + 8468.6183, + 9441.1235 + ], + "conc_latency_p99_list": [ + 0.004086580532602967, + 0.00566743115196005, + 0.005243223664583638, + 0.0099503791576717, + 0.009588507658627355, + 0.011322382240905426, + 0.01665949933376398, + 0.018269318575912616 + ], + "conc_latency_p95_list": [ + 0.0036428007049835284, + 0.003802539707976394, + 0.003850769612472504, + 0.004741756187286226, + 0.006738374719861894, + 0.00850677301059477, + 0.01233988689491525, + 0.01412305135163478 + ], + "conc_latency_avg_list": [ + 0.0032598009878015348, + 0.003285434242748037, + 0.0031902432390411135, + 0.0036557288568108254, + 0.004539560740384104, + 0.005359882091644495, + 0.007008278288237143, + 0.008346175450269581 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 2, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 6125.6146, + "serial_latency_p99": 0.0049, + "serial_latency_p95": 0.0047, + "recall": 0.9919, + "ndcg": 0.9936, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 238.8712, + 1196.2193, + 2440.8391, + 4133.8486, + 4858.2331, + 5446.0047, + 6000.1652, + 6125.6146 + ], + "conc_latency_p99_list": [ + 0.004919703922932965, + 0.010525701696751709, + 0.006511175064661075, + 0.011745981796411804, + 0.01397663167037535, + 0.014958455199375772, + 0.02016973898542343, + 0.026595940839324612 + ], + "conc_latency_p95_list": [ + 0.0046619078202638775, + 0.004602618556236848, + 0.004652375826844946, + 0.006574279977940023, + 0.009700259550299961, + 0.011811009392840788, + 0.016332678495382422, + 0.02156840759853367 + ], + "conc_latency_avg_list": [ + 0.004181373075740294, + 0.004174027799032302, + 0.004089987135078621, + 0.004828215031413884, + 0.006158122789762511, + 0.007310655337680553, + 0.009892293927032914, + 0.012884722355037423 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 7, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 5502.1797, + "serial_latency_p99": 0.0038, + "serial_latency_p95": 0.0035, + "recall": 0.9452, + "ndcg": 0.9509, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 309.5267, + 1357.6945, + 2213.2564, + 3349.9157, + 3910.9875, + 4385.5899, + 5039.0299, + 5502.1797 + ], + "conc_latency_p99_list": [ + 0.004089714956353409, + 0.0056684831657912264, + 0.012667869632714424, + 0.01000201591959918, + 0.015246839204337462, + 0.01789945445081684, + 0.02318606456159614, + 0.02852195684099569 + ], + "conc_latency_p95_list": [ + 0.003446204590727575, + 0.00420360880671069, + 0.0055909310030983735, + 0.008043184356938578, + 0.011835442011943087, + 0.014750372216803953, + 0.01907540229440201, + 0.023490797984413794 + ], + "conc_latency_avg_list": [ + 0.0032266616589272513, + 0.0036776357130507064, + 0.004511172191326527, + 0.0059575964382477965, + 0.007649578970207299, + 0.009076414605964197, + 0.011790618147195132, + 0.01435249249351469 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 2, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 1827.5849, + "serial_latency_p99": 0.0054, + "serial_latency_p95": 0.0052, + "recall": 0.9903, + "ndcg": 0.9918, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 218.8679, + 861.6609, + 1195.6337, + 1339.6438, + 1475.742, + 1546.5953, + 1747.9077, + 1827.5849 + ], + "conc_latency_p99_list": [ + 0.006139998821017798, + 0.008449624522763766, + 0.014799785086070211, + 0.02650444130937102, + 0.03739181953598747, + 0.04722500271425814, + 0.05959104605717584, + 0.0715404962032335 + ], + "conc_latency_p95_list": [ + 0.005324425446451642, + 0.006872303040290717, + 0.011261535996163731, + 0.021800653115496966, + 0.03018094879225827, + 0.038646315991354645, + 0.04888677580165675, + 0.06127824939903802 + ], + "conc_latency_avg_list": [ + 0.004563713319136163, + 0.0057948937011398916, + 0.00835191496328548, + 0.014905808578982384, + 0.020272801297484298, + 0.025762202275047597, + 0.034040973878552726, + 0.043269632825280596 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 8, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 1938.1932, + "serial_latency_p99": 0.0056, + "serial_latency_p95": 0.0053, + "recall": 0.989, + "ndcg": 0.9906, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 252.1234, + 954.3561, + 1355.554, + 1490.2007, + 1619.1811, + 1720.5715, + 1860.9305, + 1938.1932 + ], + "conc_latency_p99_list": [ + 0.0050690684028086245, + 0.008166075175395238, + 0.011429371475242079, + 0.024431421170011097, + 0.03405256026599093, + 0.043166847461543534, + 0.05706310785026292, + 0.07251818811346307 + ], + "conc_latency_p95_list": [ + 0.004786731592321303, + 0.006139315699692815, + 0.009358109103050082, + 0.019912595101050083, + 0.028057811519829556, + 0.035778356752416585, + 0.04874245898681693, + 0.061324251000769436 + ], + "conc_latency_avg_list": [ + 0.003961802805693023, + 0.005232545384579334, + 0.007366828470110833, + 0.01339630708221878, + 0.018484241519041742, + 0.023149088452180534, + 0.03196686416892556, + 0.04073973314917765 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 7, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 3778.8811, + "serial_latency_p99": 0.0048, + "serial_latency_p95": 0.0042, + "recall": 0.9828, + "ndcg": 0.9851, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 275.1343, + 1220.1869, + 2080.6024, + 2544.5946, + 2934.4403, + 3222.4756, + 3520.6342, + 3778.8811 + ], + "conc_latency_p99_list": [ + 0.004224385871784762, + 0.006377705505292314, + 0.006628996630315663, + 0.01544506659847684, + 0.02085710293264129, + 0.02495263563963816, + 0.03397362035117112, + 0.0415226237432216 + ], + "conc_latency_p95_list": [ + 0.003899741142231505, + 0.004650917260732967, + 0.005831561920058448, + 0.011755315004847944, + 0.016476290803984734, + 0.020251012463995716, + 0.027687024004990225, + 0.03372844383848132 + ], + "conc_latency_avg_list": [ + 0.0036302553438832055, + 0.004091130600148307, + 0.0047987246661642296, + 0.007844535865702365, + 0.010192746767866709, + 0.012344825366337173, + 0.016888610032228992, + 0.020908843243531986 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 6, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 3974.8218, + "serial_latency_p99": 0.0048, + "serial_latency_p95": 0.0043, + "recall": 0.9396, + "ndcg": 0.9428, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 295.7118, + 1186.5002, + 1797.187, + 2449.5408, + 2857.3282, + 3164.2684, + 3660.3166, + 3974.8218 + ], + "conc_latency_p99_list": [ + 0.004793434205930709, + 0.005700148111791336, + 0.007426547166251105, + 0.014722349808434964, + 0.019732102921698247, + 0.02347602234221995, + 0.03035390855744481, + 0.03731970520602769 + ], + "conc_latency_p95_list": [ + 0.004169186984654516, + 0.004717364211683161, + 0.006335475675587076, + 0.01105007229198236, + 0.01624748620088212, + 0.019896189187420532, + 0.025662759994156657, + 0.031171760102733943 + ], + "conc_latency_avg_list": [ + 0.003377046875928882, + 0.004208154278338283, + 0.005554828192323107, + 0.00814848752272782, + 0.010461408703728187, + 0.012581798692238582, + 0.016239422653327416, + 0.019879490524355867 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 1, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 2971.1402, + "serial_latency_p99": 0.0116, + "serial_latency_p95": 0.0045, + "recall": 0.9729, + "ndcg": 0.9752, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 278.0971, + 1117.1061, + 1626.1624, + 2126.1398, + 2425.3155, + 2521.9392, + 2793.1709, + 2971.1402 + ], + "conc_latency_p99_list": [ + 0.004647255002055317, + 0.00566180575755425, + 0.011257058603223423, + 0.015976536156085786, + 0.023499950004043067, + 0.03039303767494857, + 0.04078626602888109, + 0.04914582587312907 + ], + "conc_latency_p95_list": [ + 0.004429338514455594, + 0.005089565094385761, + 0.007136018975870684, + 0.01292410076566739, + 0.019488130600075235, + 0.025593487400328737, + 0.03410469329974147, + 0.04188467259518802 + ], + "conc_latency_avg_list": [ + 0.003591256380123583, + 0.004470003286850064, + 0.00613923671987789, + 0.009389019339411041, + 0.012338699755027977, + 0.015788074921940728, + 0.02128022645956974, + 0.026573440291179154 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 4, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 8441.533, + "serial_latency_p99": 0.0069, + "serial_latency_p95": 0.0035, + "recall": 0.9785, + "ndcg": 0.9825, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 314.6659, + 1556.8466, + 3026.0164, + 4952.5941, + 6059.6492, + 6948.3916, + 7709.5146, + 8441.533 + ], + "conc_latency_p99_list": [ + 0.003662585185375065, + 0.003881094480166214, + 0.005496532852703231, + 0.012801527802657801, + 0.01098183460126168, + 0.011906764237210155, + 0.017680786546843585, + 0.02030978908645921 + ], + "conc_latency_p95_list": [ + 0.0033931825950276107, + 0.0034469130958314055, + 0.003594076508306898, + 0.00537703381414758, + 0.00756983550672885, + 0.009184699498291593, + 0.013430504295683926, + 0.015904919797321764 + ], + "conc_latency_avg_list": [ + 0.003173945048460723, + 0.003206842876612295, + 0.003299035623981205, + 0.004028295482036381, + 0.004934835427053102, + 0.005724545223453835, + 0.007703497705718975, + 0.009334748886407937 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 4, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 2703.5422, + "serial_latency_p99": 0.0129, + "serial_latency_p95": 0.0049, + "recall": 0.9903, + "ndcg": 0.992, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 230.8474, + 1033.6996, + 1653.7251, + 2021.6415, + 2247.5951, + 2433.7674, + 2579.1371, + 2703.5422 + ], + "conc_latency_p99_list": [ + 0.005364620329928583, + 0.008247331644815845, + 0.009261744469986306, + 0.019847353509976556, + 0.027152295518899337, + 0.0318573336204281, + 0.04668155071558431, + 0.053025491506559774 + ], + "conc_latency_p95_list": [ + 0.004779104294721037, + 0.005717401996662375, + 0.007679672696394844, + 0.015277760991011746, + 0.021049827802926295, + 0.025551227995310906, + 0.03599990590882953, + 0.045174946513725445 + ], + "conc_latency_avg_list": [ + 0.004327150692358546, + 0.004830543605956322, + 0.006038381999227765, + 0.009872808423519352, + 0.013310825758863387, + 0.016344593851929733, + 0.023038225918050447, + 0.02921974749485067 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 9, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 1628.2736, + "serial_latency_p99": 0.0056, + "serial_latency_p95": 0.0051, + "recall": 0.9913, + "ndcg": 0.9928, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 224.1263, + 854.2908, + 1191.923, + 1306.8314, + 1408.4071, + 1492.5651, + 1572.8756, + 1628.2736 + ], + "conc_latency_p99_list": [ + 0.005614027456322219, + 0.01122624498093481, + 0.013540994446957512, + 0.026513429321930727, + 0.0360522977943765, + 0.04583110647072318, + 0.06417124239553232, + 0.08320635374402624 + ], + "conc_latency_p95_list": [ + 0.0053050212460220795, + 0.006976215375470927, + 0.011254654199001379, + 0.022096098412293937, + 0.03048573801643215, + 0.037569552293280135, + 0.05408634888590313, + 0.06856599940219893 + ], + "conc_latency_avg_list": [ + 0.004456437869882418, + 0.005844962813409522, + 0.008378709673770517, + 0.01528081619265322, + 0.021239153420731547, + 0.026694518371955574, + 0.03784726188423443, + 0.04857833790231276 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 9, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 5019.5973, + "serial_latency_p99": 0.0057, + "serial_latency_p95": 0.0054, + "recall": 0.994, + "ndcg": 0.9954, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 222.5871, + 1150.5338, + 2195.8726, + 3676.709, + 4412.1372, + 4833.1625, + 5019.5973, + 5015.8037 + ], + "conc_latency_p99_list": [ + 0.005366289358644281, + 0.006616018440399769, + 0.0064373466832330474, + 0.010222766738734191, + 0.013315525980142408, + 0.016972876400686822, + 0.024728685971349477, + 0.031319626237964276 + ], + "conc_latency_p95_list": [ + 0.00513816498714732, + 0.004792370549694169, + 0.005205206610844471, + 0.007212611548311542, + 0.010267410500091497, + 0.013238379993708808, + 0.01984079669928178, + 0.02578976150834933 + ], + "conc_latency_avg_list": [ + 0.00448758533933214, + 0.0043394632993677415, + 0.0045465428246546005, + 0.0054275771136581205, + 0.006782299052349466, + 0.008242341170482134, + 0.011840508222988514, + 0.015742522812380217 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 9, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 7364.0396, + "serial_latency_p99": 0.004, + "serial_latency_p95": 0.0037, + "recall": 0.9893, + "ndcg": 0.9915, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 287.8688, + 1384.2897, + 2700.4491, + 4735.4478, + 5614.1579, + 6134.6648, + 6819.5071, + 7364.0396 + ], + "conc_latency_p99_list": [ + 0.00390698241040809, + 0.00731195724976711, + 0.00495785990206058, + 0.00734390948899089, + 0.012288954856630872, + 0.014986682942835616, + 0.01849594444676764, + 0.022832338945008815 + ], + "conc_latency_p95_list": [ + 0.0037214207914075814, + 0.003863213058502879, + 0.004057778001879342, + 0.005324349994771183, + 0.008072146945050915, + 0.010810172616038463, + 0.01463127658644225, + 0.01830196708324365 + ], + "conc_latency_avg_list": [ + 0.0034694395949774774, + 0.003606888217707231, + 0.003696606332343799, + 0.004213763141467857, + 0.0053274846319189195, + 0.006490250213165499, + 0.008702013864526974, + 0.010708946041863319 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 6, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 2724.9239, + "serial_latency_p99": 0.0124, + "serial_latency_p95": 0.0048, + "recall": 0.9811, + "ndcg": 0.9832, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 269.3651, + 1047.0123, + 1498.6024, + 1847.6803, + 2114.199, + 2284.3302, + 2537.2552, + 2724.9239 + ], + "conc_latency_p99_list": [ + 0.004742077292175964, + 0.00691745357704349, + 0.010720830453210511, + 0.01960868470487181, + 0.026337993171764537, + 0.032754160480981225, + 0.04255042473436333, + 0.05312397440138735 + ], + "conc_latency_p95_list": [ + 0.004494865804736036, + 0.0055081502971006556, + 0.007977579892030908, + 0.015719183007604443, + 0.022288659910555, + 0.02759612778027076, + 0.03666536140372045, + 0.04536134131485597 + ], + "conc_latency_avg_list": [ + 0.0037082374061278584, + 0.004769060159688272, + 0.0066635344327552045, + 0.010804306792214618, + 0.014153154028666374, + 0.017440448752945776, + 0.02343149585687298, + 0.029001097442020812 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 5, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 5514.815, + "serial_latency_p99": 0.0042, + "serial_latency_p95": 0.0036, + "recall": 0.9286, + "ndcg": 0.9355, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 313.799, + 1354.9874, + 2301.8011, + 3314.3036, + 3843.1259, + 4355.6117, + 5048.842, + 5514.815 + ], + "conc_latency_p99_list": [ + 0.0038773450409644284, + 0.008138228004099801, + 0.010878445263078868, + 0.011540972657094244, + 0.016388589342823227, + 0.01858296279708156, + 0.023569957185536613, + 0.02886486930365208 + ], + "conc_latency_p95_list": [ + 0.003412947052856907, + 0.004246345022693276, + 0.005220197608286979, + 0.008349375608668195, + 0.012340355810010787, + 0.0147909961087862, + 0.019336943980306387, + 0.023638575003133155 + ], + "conc_latency_avg_list": [ + 0.003182328343818464, + 0.003684439145729171, + 0.004337632718561462, + 0.006021014814140765, + 0.007777482876747058, + 0.009140085848755274, + 0.011767476800046897, + 0.014305157378590626 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 1, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 9901.1114, + "serial_latency_p99": 0.0039, + "serial_latency_p95": 0.0037, + "recall": 0.9385, + "ndcg": 0.9486, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 321.3357, + 1639.9924, + 3195.3018, + 5673.0951, + 6808.8856, + 7560.1348, + 9087.2499, + 9901.1114 + ], + "conc_latency_p99_list": [ + 0.00394074416602971, + 0.0041782407171558535, + 0.004392879804945551, + 0.010109323544893349, + 0.008123845955124116, + 0.014126944777672188, + 0.014278251143987291, + 0.01781155434960963 + ], + "conc_latency_p95_list": [ + 0.0034882987965829666, + 0.0035166182147804642, + 0.0035458351092529476, + 0.004310413988423532, + 0.006210639532946515, + 0.008767930739850271, + 0.010931958199944346, + 0.013749658003507645 + ], + "conc_latency_avg_list": [ + 0.0031001682930926035, + 0.0030430930397005885, + 0.0031233191875315565, + 0.0035155999207049896, + 0.004389041533959679, + 0.005263586471416786, + 0.006530556363972054, + 0.007958968690030266 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 1, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 3258.8508, + "serial_latency_p99": 0.0117, + "serial_latency_p95": 0.0045, + "recall": 0.9888, + "ndcg": 0.9906, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 244.7738, + 1130.2346, + 1809.9431, + 2312.2265, + 2588.2339, + 2788.0239, + 3069.518, + 3258.8508 + ], + "conc_latency_p99_list": [ + 0.005137932703364641, + 0.00555925140972249, + 0.008915685693500566, + 0.016561121140257453, + 0.022867804747074828, + 0.02806737951235845, + 0.036439670615363844, + 0.04337102690333264 + ], + "conc_latency_p95_list": [ + 0.004440846845682244, + 0.005053811200195923, + 0.0068235130165703595, + 0.01288393942813854, + 0.018162307806778695, + 0.02236156941507943, + 0.0296233788743848, + 0.03584610429388702 + ], + "conc_latency_avg_list": [ + 0.004080840300378187, + 0.004417934321996714, + 0.005516924373879563, + 0.008634555842136784, + 0.011560621919641356, + 0.014278915373578093, + 0.019378262406752788, + 0.024243179517198447 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 8, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 270.1267, + "optimize_duration": 437.4658, + "load_duration": 707.5925, + "qps": 5907.441, + "serial_latency_p99": 0.0057, + "serial_latency_p95": 0.0054, + "recall": 0.9931, + "ndcg": 0.9946, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 214.7436, + 1124.1591, + 2263.1224, + 3695.1435, + 4635.8817, + 5035.3873, + 5614.0659, + 5907.441 + ], + "conc_latency_p99_list": [ + 0.005838954218779679, + 0.009590839385055032, + 0.0072934928326867585, + 0.009990225688670754, + 0.013466239573317574, + 0.01589230435347418, + 0.021308113009436094, + 0.026848825681372538 + ], + "conc_latency_p95_list": [ + 0.005094571305380669, + 0.0048950490017887205, + 0.00495702201151289, + 0.007446101757523138, + 0.009852354813483545, + 0.01273485799174523, + 0.017310541804181415, + 0.021899056984693743 + ], + "conc_latency_avg_list": [ + 0.004651503297485146, + 0.004441934131452666, + 0.004411218431154767, + 0.005397841813638808, + 0.00645052096323231, + 0.007903104976167643, + 0.010583279479885773, + 0.013354355155792764 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 8, + "num_shards": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 5064.6982, + "serial_latency_p99": 0.0043, + "serial_latency_p95": 0.0036, + "recall": 0.9558, + "ndcg": 0.9606, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 306.5362, + 1368.7188, + 2310.9627, + 3255.5776, + 3791.6067, + 4073.62, + 4572.4511, + 5064.6982 + ], + "conc_latency_p99_list": [ + 0.0042355662427144124, + 0.0046483404963510114, + 0.009778717628214485, + 0.010217842382844546, + 0.015881566194002526, + 0.01989235390967224, + 0.025938013812992735, + 0.031185232510324568 + ], + "conc_latency_p95_list": [ + 0.0034556519065517934, + 0.004072950512636453, + 0.005186801168019884, + 0.008326760004274545, + 0.01228064175666077, + 0.016202768517541696, + 0.021469047002028674, + 0.025539752503391355 + ], + "conc_latency_avg_list": [ + 0.003258234978368169, + 0.0036474964905585366, + 0.004320538177772956, + 0.006130977769455615, + 0.007887396918013946, + 0.009779561744223083, + 0.013001510996033932, + 0.015590046073741476 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 3, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 3468.5933, + "serial_latency_p99": 0.0045, + "serial_latency_p95": 0.0043, + "recall": 0.9634, + "ndcg": 0.9661, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 274.7753, + 1086.6444, + 1657.8469, + 2230.3556, + 2576.5151, + 2840.9102, + 3207.9495, + 3468.5933 + ], + "conc_latency_p99_list": [ + 0.007525063498178497, + 0.00871218444284751, + 0.009115133894665613, + 0.01669143169856398, + 0.021790961647639086, + 0.026465490460395806, + 0.03498110753978835, + 0.042080024706956466 + ], + "conc_latency_p95_list": [ + 0.004381819497211836, + 0.00529403816035483, + 0.007012556104746182, + 0.01246311155118746, + 0.018361228803405537, + 0.022542869101744148, + 0.029249506900669076, + 0.03583758800959913 + ], + "conc_latency_avg_list": [ + 0.003635002963577244, + 0.0045950829257419635, + 0.006023093199217258, + 0.008949508158072033, + 0.011605007790743322, + 0.014003128689967868, + 0.018541925579661857, + 0.02278122412115334 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 3, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3205.4829, + "optimize_duration": 1275.7844, + "load_duration": 4481.2673, + "qps": 2401.3204, + "serial_latency_p99": 0.0107, + "serial_latency_p95": 0.0047, + "recall": 0.9866, + "ndcg": 0.9884, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 261.8215, + 985.1764, + 1468.0087, + 1749.415, + 1931.5307, + 2090.0742, + 2278.9704, + 2401.3204 + ], + "conc_latency_p99_list": [ + 0.004836413672601339, + 0.009077261193306102, + 0.010354967679304536, + 0.02026946535654133, + 0.029030042108206543, + 0.03499512374750339, + 0.046797644338803394, + 0.057348916197370266 + ], + "conc_latency_p95_list": [ + 0.004629465389007237, + 0.0058674284955486655, + 0.0082442566199461, + 0.01657543244800763, + 0.02401039500546176, + 0.029254749882966277, + 0.03907370918313973, + 0.04936321394779952 + ], + "conc_latency_avg_list": [ + 0.0038149669483554033, + 0.00506845096702024, + 0.006802651714890116, + 0.01140782706843602, + 0.015489598411640663, + 0.019035290587860968, + 0.02608886756739919, + 0.03292382999700411 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 6, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 3568.396, + "serial_latency_p99": 0.0048, + "serial_latency_p95": 0.0043, + "recall": 0.9863, + "ndcg": 0.9883, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 249.5646, + 1116.5066, + 1828.4446, + 2389.4823, + 2645.8957, + 2879.4104, + 3356.3991, + 3568.396 + ], + "conc_latency_p99_list": [ + 0.004949838446336795, + 0.010793446648749472, + 0.011542035994352773, + 0.015714827887131834, + 0.023098365282639866, + 0.028270509486901574, + 0.03385155899741221, + 0.0405952908913605 + ], + "conc_latency_p95_list": [ + 0.004326387906621675, + 0.005205007104086689, + 0.0069263952391338535, + 0.012388522701803593, + 0.01838434826204321, + 0.022779258753871545, + 0.02818173549894709, + 0.034205201656732236 + ], + "conc_latency_avg_list": [ + 0.004002170059550175, + 0.0044719544014582705, + 0.005461118931996274, + 0.008354930265743018, + 0.011307457646718058, + 0.013821107916470069, + 0.01770926681597559, + 0.022161158532467397 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 7, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2923.0628, + "optimize_duration": 2272.8723, + "load_duration": 5195.935, + "qps": 4674.1861, + "serial_latency_p99": 0.0045, + "serial_latency_p95": 0.0038, + "recall": 0.967, + "ndcg": 0.9705, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 299.441, + 1293.8246, + 2204.8398, + 2998.8928, + 3416.6029, + 3814.1905, + 4403.9058, + 4674.1861 + ], + "conc_latency_p99_list": [ + 0.00402290889178403, + 0.007907517027342683, + 0.006291847964457707, + 0.013353673194069417, + 0.017164103323593728, + 0.021020031592343, + 0.027193114216788664, + 0.033575675918255006 + ], + "conc_latency_p95_list": [ + 0.00356991050648503, + 0.004438751634734216, + 0.005453104941989295, + 0.00961559300776571, + 0.013857692398596555, + 0.017230113997356966, + 0.02220476679212879, + 0.02798210658947937 + ], + "conc_latency_avg_list": [ + 0.003335578729612294, + 0.0038557001362392825, + 0.0045284253144977004, + 0.006655331635103421, + 0.008756953741340391, + 0.010436090944337196, + 0.013487455343736422, + 0.016883584811310505 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-force_merge", + "version": "v2026.1", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********", + "collection_name": "ZillizCloudVDBBench" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "level": 4, + "num_shards": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": {}, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1770595200.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index c463463fc..566ce4285 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -1,3062 +1,2142 @@ [ - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 597.3641, - "latency": 12.1, - "recall": 0.9221, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 551.1757, - "latency": 10.7, - "recall": 0.9327, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 492.9765, - "latency": 13.4, - "recall": 0.9443, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 431.0155, - "latency": 15.2, - "recall": 0.954, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 377.3634, - "latency": 16.2, - "recall": 0.9615, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 337.5819, - "latency": 15.0, - "recall": 0.9666, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 279.7257, - "latency": 18.1, - "recall": 0.9729, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 3033.786, - "latency": 8.7, - "recall": 0.9934, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 3019.2416, - "latency": 9.5, - "recall": 0.9765, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2890.9523, - "latency": 9.4, - "recall": 0.9625, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2789.7212, - "latency": 8.2, - "recall": 0.9538, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2457.2628, - "latency": 9.0, - "recall": 0.9378, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2209.4973, - "latency": 13.7, - "recall": 0.9228, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1960.388, - "latency": 11.0, - "recall": 0.9076, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1725.092, - "latency": 11.7, - "recall": 0.8969, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1307.419, - "latency": 12.3, - "recall": 0.8925, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1520.4145, - "latency": 12.5, - "recall": 0.9028, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1273.3452, - "latency": 12.3, - "recall": 0.9242, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1011.7943, - "latency": 15.2, - "recall": 0.945, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 824.5097, - "latency": 15.5, - "recall": 0.9558, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 350.0132, - "latency": 29.7, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 179.5204, - "latency": 51.4, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 72.99, - "latency": 111.4, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 42.9877, - "latency": 201.9, - "recall": 0.9912, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 96.4987, - "latency": 113.1, - "recall": 0.9296, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 189.3789, - "latency": 58.8, - "recall": 0.9149, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 246.7071, - "latency": 45.1, - "recall": 0.9018, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 229.0379, - "latency": 43.0, - "recall": 0.8908, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 125.6164, - "latency": 69.8, - "recall": 0.8746, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 376.3752, - "latency": 14.5, - "recall": 0.9039, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 341.2325, - "latency": 13.4, - "recall": 0.9136, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 300.7678, - "latency": 12.8, - "recall": 0.922, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 257.0398, - "latency": 15.4, - "recall": 0.9303, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 228.7734, - "latency": 16.2, - "recall": 0.9374, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 204.3654, - "latency": 18.2, - "recall": 0.9424, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 167.5075, - "latency": 18.0, - "recall": 0.9501, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 146.339, - "latency": 20.9, - "recall": 0.9557, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "qps": 129.0705, - "latency": 24.3, - "recall": 0.96, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2095.7067, - "latency": 12.4, - "recall": 0.8961, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1925.3019, - "latency": 11.3, - "recall": 0.9141, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1707.8841, - "latency": 10.0, - "recall": 0.9314, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1442.0638, - "latency": 10.1, - "recall": 0.9482, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1115.106, - "latency": 13.1, - "recall": 0.9662, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 910.4322, - "latency": 14.2, - "recall": 0.9748, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2175.2694, - "latency": 9.8, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1430.0244, - "latency": 12.6, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 692.5751, - "latency": 18.7, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 364.3516, - "latency": 26.4, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 190.3777, - "latency": 47.9, - "recall": 1.0, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 249.3519, - "latency": 44.7, - "recall": 0.9446, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 437.8735, - "latency": 27.1, - "recall": 0.9364, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 669.9441, - "latency": 19.1, - "recall": 0.9227, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 899.3114, - "latency": 14.9, - "recall": 0.9072, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3465.1696, - "latency": 2.2, - "recall": 0.9528, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3102.1518, - "latency": 2.3, - "recall": 0.9608, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 2681.2848, - "latency": 2.5, - "recall": 0.9681, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 2232.9105, - "latency": 2.9, - "recall": 0.9757, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1913.17, - "latency": 3.2, - "recall": 0.9797, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1575.8137, - "latency": 3.5, - "recall": 0.9822, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1344.5652, - "latency": 4.2, - "recall": 0.9849, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1148.7813, - "latency": 4.8, - "recall": 0.9861, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3680.6045, - "latency": 2.2, - "recall": 0.9954, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3407.9972, - "latency": 2.2, - "recall": 0.994, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3062.6755, - "latency": 2.4, - "recall": 0.9932, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 2568.3371, - "latency": 2.7, - "recall": 0.9927, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1886.2891, - "latency": 3.3, - "recall": 0.992, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 923.8685, - "latency": 6.7, - "recall": 0.9919, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 1442.1076, - "latency": 4.1, - "recall": 0.9517, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 2002.9865, - "latency": 3.2, - "recall": 0.9467, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 3201.9438, - "latency": 2.3, - "recall": 0.922, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11763.5538, - "latency": 1.5, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11803.1944, - "latency": 1.5, - "recall": 0.9778, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11520.9234, - "latency": 1.5, - "recall": 0.9634, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11280.0849, - "latency": 1.6, - "recall": 0.9507, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10671.8925, - "latency": 1.7, - "recall": 0.9339, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10258.2661, - "latency": 1.7, - "recall": 0.9139, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 9681.5656, - "latency": 1.9, - "recall": 0.9008, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 8945.4041, - "latency": 1.9, - "recall": 0.8894, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 5436.8907, - "latency": 2.0, - "recall": 0.929, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 437.1695, - "latency": 5.1, - "recall": 0.951, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 400.0053, - "latency": 5.6, - "recall": 0.9558, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 343.938, - "latency": 6.5, - "recall": 0.9605, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 275.2271, - "latency": 7.7, - "recall": 0.9649, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 234.9937, - "latency": 8.7, - "recall": 0.9677, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 202.6975, - "latency": 9.9, - "recall": 0.9696, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 164.6073, - "latency": 12.1, - "recall": 0.972, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 135.9624, - "latency": 13.9, - "recall": 0.9733, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 432.8374, - "latency": 5.0, - "recall": 0.9865, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 368.6042, - "latency": 5.4, - "recall": 0.9843, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 318.8159, - "latency": 7.1, - "recall": 0.9836, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 217.0963, - "latency": 9.1, - "recall": 0.9822, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 150.9989, - "latency": 12.5, - "recall": 0.9814, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 76.0341, - "latency": 23.1, - "recall": 0.9797, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 166.5611, - "latency": 11.5, - "recall": 0.9675, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 248.9625, - "latency": 8.3, - "recall": 0.9608, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "qps": 435.3358, - "latency": 8.2, - "recall": 0.9417, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11397.7043, - "latency": 1.6, - "recall": 0.9597, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10891.7531, - "latency": 1.7, - "recall": 0.9408, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10276.7451, - "latency": 1.7, - "recall": 0.9159, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 9664.2855, - "latency": 1.8, - "recall": 0.899, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 8936.7962, - "latency": 2.0, - "recall": 0.8835, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 5671.2562, - "latency": 2.1, - "recall": 0.903, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 3157.707, - "latency": 2.3, - "recall": 0.9347, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 1985.8124, - "latency": 2.6, - "recall": 0.9407, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 920.9627, - "latency": 3.4, - "recall": 0.9488, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1146.5286, - "latency": 13.7, - "recall": 0.9262, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1148.1735, - "latency": 8.9, - "recall": 0.9801, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1149.1219, - "latency": 10.3, - "recall": 0.9764, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1140.4099, - "latency": 13.5, - "recall": 0.9716, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1123.5147, - "latency": 18.5, - "recall": 0.9688, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 487.8343, - "latency": 25.4, - "recall": 0.9668, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 264.9324, - "latency": 49.6, - "recall": 0.936, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 492.4887, - "latency": 29.6, - "recall": 0.9269, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 823.1775, - "latency": 20.5, - "recall": 0.9148, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1147.1977, - "latency": 13.3, - "recall": 0.8999, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1131.3087, - "latency": 14.1, - "recall": 0.9024, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1114.952, - "latency": 12.7, - "recall": 0.97, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 583.5009, - "latency": 23.0, - "recall": 0.9668, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 31.4779, - "latency": 351.0, - "recall": 0.9414, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 57.8988, - "latency": 200.1, - "recall": 0.9332, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 101.1774, - "latency": 116.1, - "recall": 0.9241, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 212.7466, - "latency": 58.7, - "recall": 0.9099, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 372.2462, - "latency": 35.9, - "recall": 0.8977, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 617.0881, - "latency": 22.4, - "recall": 0.8844, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1094.5967, - "latency": 14.3, - "recall": 0.8659, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 4318.9697, - "latency": 4.3, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 4250.2894, - "latency": 4.6, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 2997.4391, - "latency": 6.1, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1494.5334, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1108.6473, - "latency": 7.4, - "recall": 0.995, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1289.5164, - "latency": 6.4, - "recall": 0.9906, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1059.3394, - "latency": 7.8, - "recall": 0.9856, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 987.0795, - "latency": 7.1, - "recall": 0.9804, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1591.7055, - "latency": 7.8, - "recall": 0.8506, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1202.8677, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 639.3991, - "latency": 7.3, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 274.8559, - "latency": 9.9, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 441.4152, - "latency": 8.3, - "recall": 0.997, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 358.8949, - "latency": 9.5, - "recall": 0.995, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 325.2245, - "latency": 10.3, - "recall": 0.9909, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 273.4174, - "latency": 13.3, - "recall": 0.9789, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 262.8314, - "latency": 11.3, - "recall": 0.9808, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 434.5481, - "latency": 8.5, - "recall": 0.7237, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 446.9116, - "latency": 9.2, - "recall": 0.9357, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 388.3028, - "latency": 9.6, - "recall": 0.9431, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 323.3964, - "latency": 9.8, - "recall": 0.9507, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 256.4668, - "latency": 11.3, - "recall": 0.9588, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 145.5316, - "latency": 18.4, - "recall": 0.9726, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 1242.428, - "latency": 6.4, - "recall": 0.9474, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 1111.3633, - "latency": 7.0, - "recall": 0.955, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 955.4701, - "latency": 7.2, - "recall": 0.9629, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 783.5207, - "latency": 7.7, - "recall": 0.971, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 470.8546, - "latency": 9.5, - "recall": 0.9835, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 950.6332, - "latency": 13.2, - "recall": 0.914, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 823.2224, - "latency": 13.5, - "recall": 0.9434, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 743.9815, - "latency": 14.8, - "recall": 0.9583, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 683.1873, - "latency": 15.7, - "recall": 0.9677, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 619.7468, - "latency": 17.2, - "recall": 0.9738, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 537.4082, - "latency": 18.8, - "recall": 0.9809, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 474.9941, - "latency": 20.9, - "recall": 0.9848, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 505.7458, - "latency": 20.7, - "recall": 0.9068, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 433.9034, - "latency": 23.1, - "recall": 0.931, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 381.7737, - "latency": 25.7, - "recall": 0.9431, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 342.1123, - "latency": 29.0, - "recall": 0.951, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 308.2216, - "latency": 31.3, - "recall": 0.9561, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 257.7928, - "latency": 36.4, - "recall": 0.9626, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 223.8166, - "latency": 42.1, - "recall": 0.9666, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3055.0123, - "latency": 7.2, - "recall": 0.9066, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3013.4439, - "latency": 6.9, - "recall": 0.9268, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2801.7241, - "latency": 7.4, - "recall": 0.9476, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2590.3809, - "latency": 8.6, - "recall": 0.9679, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2291.2159, - "latency": 8.9, - "recall": 0.9764, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3099.4124, - "latency": 6.2, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3014.2483, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2073.2153, - "latency": 11.0, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1507.6899, - "latency": 12.8, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 942.2296, - "latency": 18.2, - "recall": 1.0, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 677.1414, - "latency": 33.5, - "recall": 0.7655, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2685.6654, - "latency": 7.6, - "recall": 0.4914, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2604.4444, - "latency": 7.8, - "recall": 0.63, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2159.051, - "latency": 9.4, - "recall": 0.801, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2251.1274, - "latency": 8.7, - "recall": 0.8848, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3103.0539, - "latency": 5.6, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3086.1957, - "latency": 6.7, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3090.0478, - "latency": 6.4, - "recall": 0.9628, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3064.6288, - "latency": 6.5, - "recall": 0.9507, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3065.6134, - "latency": 6.2, - "recall": 0.9328, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3028.858, - "latency": 6.7, - "recall": 0.9133, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2935.9403, - "latency": 6.8, - "recall": 0.8992, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2771.2009, - "latency": 7.6, - "recall": 0.889, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1610.9496, - "latency": 10.8, - "recall": 0.9, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1557.3623, - "latency": 10.8, - "recall": 0.9244, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1473.9256, - "latency": 11.7, - "recall": 0.9484, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1388.5547, - "latency": 12.5, - "recall": 0.9597, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1022.2696, - "latency": 17.9, - "recall": 0.936, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 696.9777, - "latency": 24.6, - "recall": 0.997, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 353.7862, - "latency": 45.2, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 210.3227, - "latency": 71.4, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 114.8061, - "latency": 126.6, - "recall": 0.9985, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 504.9179, - "latency": 272.6, - "recall": 0.4664, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1053.1495, - "latency": 17.7, - "recall": 0.5673, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 808.3294, - "latency": 22.2, - "recall": 0.7016, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 8.0584, - "latency": 1757.9, - "recall": 1.0, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3033.5491, - "latency": 6.4, - "recall": 0.9844, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2988.4205, - "latency": 7.6, - "recall": 0.9741, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2950.717, - "latency": 6.9, - "recall": 0.9558, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2782.0274, - "latency": 7.4, - "recall": 0.9466, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2708.6752, - "latency": 8.4, - "recall": 0.9337, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2275.2854, - "latency": 9.1, - "recall": 0.917, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 1844.8918, - "latency": 10.6, - "recall": 0.9085, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 1301.4102, - "latency": 14.7, - "recall": 0.9011, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 13.0379, - "latency": 1063.5, - "recall": 1.0, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9704.4214, - "latency": 2.5, - "recall": 0.9169, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9463.4991, - "latency": 2.6, - "recall": 0.9393, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9194.9922, - "latency": 2.7, - "recall": 0.9543, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8779.8779, - "latency": 2.9, - "recall": 0.9685, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8153.4648, - "latency": 3.0, - "recall": 0.9757, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6848.5254, - "latency": 4.4, - "recall": 0.9835, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6124.4431, - "latency": 3.8, - "recall": 0.9873, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5186.8135, - "latency": 5.3, - "recall": 0.9893, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 4898.2048, - "latency": 5.6, - "recall": 0.9904, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9773.6593, - "latency": 3.7, - "recall": 0.9955, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9081.1518, - "latency": 3.0, - "recall": 0.9943, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8455.2896, - "latency": 4.0, - "recall": 0.9921, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7610.0519, - "latency": 3.3, - "recall": 0.9903, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7589.664, - "latency": 3.8, - "recall": 0.9235, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6750.2495, - "latency": 4.4, - "recall": 0.9105, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5506.1808, - "latency": 5.5, - "recall": 0.9193, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6860.8577, - "latency": 4.7, - "recall": 0.9226, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8468.4611, - "latency": 3.1, - "recall": 0.8925, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10089.4308, - "latency": 2.6, - "recall": 0.9934, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10557.4373, - "latency": 2.7, - "recall": 0.9393, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9805.0401, - "latency": 2.6, - "recall": 0.9257, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10020.5299, - "latency": 2.6, - "recall": 0.9788, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10041.0338, - "latency": 2.7, - "recall": 0.9693, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9861.9686, - "latency": 2.6, - "recall": 0.955, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9507.9991, - "latency": 2.8, - "recall": 0.9453, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9428.4531, - "latency": 2.6, - "recall": 0.9331, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9048.6431, - "latency": 3.9, - "recall": 0.9216, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 8695.2765, - "latency": 4.3, - "recall": 0.9603, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9244.1135, - "latency": 4.2, - "recall": 0.9724, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9289.0118, - "latency": 4.2, - "recall": 0.9574, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9374.8941, - "latency": 4.2, - "recall": 0.9425, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9368.1325, - "latency": 3.8, - "recall": 0.9292, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9220.3627, - "latency": 3.8, - "recall": 0.9081, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 8633.8949, - "latency": 4.1, - "recall": 0.8928, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 6820.6863, - "latency": 3.2, - "recall": 0.9159, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 3938.6004, - "latency": 3.7, - "recall": 0.9196, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3957.0757, - "latency": 2.7, - "recall": 0.932, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3539.2869, - "latency": 4.3, - "recall": 0.9471, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3154.6501, - "latency": 3.9, - "recall": 0.9565, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2743.4561, - "latency": 4.3, - "recall": 0.9681, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2318.7835, - "latency": 3.1, - "recall": 0.9763, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1763.2054, - "latency": 5.0, - "recall": 0.9829, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1454.0462, - "latency": 4.0, - "recall": 0.9863, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1251.1255, - "latency": 4.5, - "recall": 0.9884, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1076.8329, - "latency": 4.4, - "recall": 0.9897, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3411.0934, - "latency": 3.3, - "recall": 0.995, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2838.356, - "latency": 3.8, - "recall": 0.9946, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1826.0672, - "latency": 5.3, - "recall": 0.9938, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1234.6534, - "latency": 6.4, - "recall": 0.9942, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1773.0919, - "latency": 5.3, - "recall": 0.9699, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1454.8382, - "latency": 4.6, - "recall": 0.9659, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1373.0307, - "latency": 5.7, - "recall": 0.9716, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2039.8673, - "latency": 3.8, - "recall": 0.9559, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2950.8165, - "latency": 3.3, - "recall": 0.9147, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.4972, - "latency": 337.1, - "recall": 0.8717, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 192.1164, - "latency": 345.6, - "recall": 0.4276, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 197.4455, - "latency": 349.3, - "recall": 0.5314, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 196.9391, - "latency": 263.4, - "recall": 0.6549, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 201.5401, - "latency": 282.4, - "recall": 0.7086, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 202.2424, - "latency": 301.7, - "recall": 0.7592, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 198.599, - "latency": 358.8, - "recall": 0.8085, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.0349, - "latency": 275.3, - "recall": 0.8325, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 202.1405, - "latency": 282.6, - "recall": 0.8492, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 201.1282, - "latency": 269.2, - "recall": 0.8637, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 194.8021, - "latency": 559.8, - "recall": 0.86, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 187.4268, - "latency": 453.7, - "recall": 0.4692, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 198.397, - "latency": 506.9, - "recall": 0.5409, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 174.3549, - "latency": 496.9, - "recall": 0.6279, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 172.95, - "latency": 515.6, - "recall": 0.7004, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 190.9747, - "latency": 517.4, - "recall": 0.7398, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 186.0237, - "latency": 474.0, - "recall": 0.7847, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 192.1458, - "latency": 480.5, - "recall": 0.8103, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 179.4203, - "latency": 497.5, - "recall": 0.8273, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.5444, - "latency": 463.9, - "recall": 0.8478, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 346.5847, - "latency": 42.7, - "recall": 0.9631, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 369.4921, - "latency": 41.6, - "recall": 0.779, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 310.957, - "latency": 49.4, - "recall": 0.9698, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 798.328, - "latency": 56.7, - "recall": 0.8993, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 649.8781, - "latency": 55.2, - "recall": 0.8352, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 370.7241, - "latency": 49.6, - "recall": 0.7177, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 100.0554, - "latency": 69.3, - "recall": 0.9638, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 284.6367, - "latency": 47.6, - "recall": 0.9788, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 81.8678, - "latency": 105.6, - "recall": 0.8751, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 260.4031, - "latency": 48.3, - "recall": 0.9828, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 365.2505, - "latency": 34.9, - "recall": 0.8251, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 471.553, - "latency": 44.1, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 91.8612, - "latency": 85.7, - "recall": 0.8799, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 206.0934, - "latency": 56.7, - "recall": 0.9795, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 351.7114, - "latency": 46.7, - "recall": 0.8735, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 96.592, - "latency": 76.9, - "recall": 0.9178, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 802.6923, - "latency": 48.1, - "recall": 0.935, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 184.5363, - "latency": 53.4, - "recall": 0.9681, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 323.0238, - "latency": 50.4, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 382.5332, - "latency": 54.7, - "recall": 0.6135, - "filter_ratio": 0.98 - } + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1146.5286, + "latency": 13.7, + "recall": 0.9262, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1148.1735, + "latency": 8.9, + "recall": 0.9801, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1149.1219, + "latency": 10.3, + "recall": 0.9764, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1140.4099, + "latency": 13.5, + "recall": 0.9716, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1123.5147, + "latency": 18.5, + "recall": 0.9688, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 487.8343, + "latency": 25.4, + "recall": 0.9668, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 264.9324, + "latency": 49.6, + "recall": 0.936, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 492.4887, + "latency": 29.6, + "recall": 0.9269, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 823.1775, + "latency": 20.5, + "recall": 0.9148, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1147.1977, + "latency": 13.3, + "recall": 0.8999, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1131.3087, + "latency": 14.1, + "recall": 0.9024, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1114.952, + "latency": 12.7, + "recall": 0.97, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 583.5009, + "latency": 23.0, + "recall": 0.9668, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 31.4779, + "latency": 351.0, + "recall": 0.9414, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 57.8988, + "latency": 200.1, + "recall": 0.9332, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 101.1774, + "latency": 116.1, + "recall": 0.9241, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 212.7466, + "latency": 58.7, + "recall": 0.9099, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 372.2462, + "latency": 35.9, + "recall": 0.8977, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 617.0881, + "latency": 22.4, + "recall": 0.8844, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1094.5967, + "latency": 14.3, + "recall": 0.8659, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4318.9697, + "latency": 4.3, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4250.2894, + "latency": 4.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 2997.4391, + "latency": 6.1, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1494.5334, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1108.6473, + "latency": 7.4, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1289.5164, + "latency": 6.4, + "recall": 0.9906, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1059.3394, + "latency": 7.8, + "recall": 0.9856, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 987.0795, + "latency": 7.1, + "recall": 0.9804, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1591.7055, + "latency": 7.8, + "recall": 0.8506, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1202.8677, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 639.3991, + "latency": 7.3, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 274.8559, + "latency": 9.9, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 441.4152, + "latency": 8.3, + "recall": 0.997, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 358.8949, + "latency": 9.5, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 325.2245, + "latency": 10.3, + "recall": 0.9909, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 273.4174, + "latency": 13.3, + "recall": 0.9789, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 262.8314, + "latency": 11.3, + "recall": 0.9808, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 434.5481, + "latency": 8.5, + "recall": 0.7237, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 446.9116, + "latency": 9.2, + "recall": 0.9357, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 388.3028, + "latency": 9.6, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 323.3964, + "latency": 9.8, + "recall": 0.9507, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 256.4668, + "latency": 11.3, + "recall": 0.9588, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 145.5316, + "latency": 18.4, + "recall": 0.9726, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1242.428, + "latency": 6.4, + "recall": 0.9474, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1111.3633, + "latency": 7.0, + "recall": 0.955, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 955.4701, + "latency": 7.2, + "recall": 0.9629, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 783.5207, + "latency": 7.7, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 470.8546, + "latency": 9.5, + "recall": 0.9835, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 950.6332, + "latency": 13.2, + "recall": 0.914, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 823.2224, + "latency": 13.5, + "recall": 0.9434, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 743.9815, + "latency": 14.8, + "recall": 0.9583, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 683.1873, + "latency": 15.7, + "recall": 0.9677, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 619.7468, + "latency": 17.2, + "recall": 0.9738, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 537.4082, + "latency": 18.8, + "recall": 0.9809, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 474.9941, + "latency": 20.9, + "recall": 0.9848, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 505.7458, + "latency": 20.7, + "recall": 0.9068, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 433.9034, + "latency": 23.1, + "recall": 0.931, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 381.7737, + "latency": 25.7, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 342.1123, + "latency": 29.0, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 308.2216, + "latency": 31.3, + "recall": 0.9561, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 257.7928, + "latency": 36.4, + "recall": 0.9626, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 223.8166, + "latency": 42.1, + "recall": 0.9666, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3055.0123, + "latency": 7.2, + "recall": 0.9066, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3013.4439, + "latency": 6.9, + "recall": 0.9268, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2801.7241, + "latency": 7.4, + "recall": 0.9476, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2590.3809, + "latency": 8.6, + "recall": 0.9679, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2291.2159, + "latency": 8.9, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3099.4124, + "latency": 6.2, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3014.2483, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2073.2153, + "latency": 11.0, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1507.6899, + "latency": 12.8, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 942.2296, + "latency": 18.2, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 677.1414, + "latency": 33.5, + "recall": 0.7655, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2685.6654, + "latency": 7.6, + "recall": 0.4914, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2604.4444, + "latency": 7.8, + "recall": 0.63, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2159.051, + "latency": 9.4, + "recall": 0.801, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2251.1274, + "latency": 8.7, + "recall": 0.8848, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3103.0539, + "latency": 5.6, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3086.1957, + "latency": 6.7, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3090.0478, + "latency": 6.4, + "recall": 0.9628, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3064.6288, + "latency": 6.5, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3065.6134, + "latency": 6.2, + "recall": 0.9328, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3028.858, + "latency": 6.7, + "recall": 0.9133, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2935.9403, + "latency": 6.8, + "recall": 0.8992, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2771.2009, + "latency": 7.6, + "recall": 0.889, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1610.9496, + "latency": 10.8, + "recall": 0.9, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1557.3623, + "latency": 10.8, + "recall": 0.9244, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1473.9256, + "latency": 11.7, + "recall": 0.9484, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1388.5547, + "latency": 12.5, + "recall": 0.9597, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1022.2696, + "latency": 17.9, + "recall": 0.936, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 696.9777, + "latency": 24.6, + "recall": 0.997, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 353.7862, + "latency": 45.2, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 210.3227, + "latency": 71.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 114.8061, + "latency": 126.6, + "recall": 0.9985, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 504.9179, + "latency": 272.6, + "recall": 0.4664, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1053.1495, + "latency": 17.7, + "recall": 0.5673, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 808.3294, + "latency": 22.2, + "recall": 0.7016, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 8.0584, + "latency": 1757.9, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3033.5491, + "latency": 6.4, + "recall": 0.9844, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2988.4205, + "latency": 7.6, + "recall": 0.9741, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2950.717, + "latency": 6.9, + "recall": 0.9558, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2782.0274, + "latency": 7.4, + "recall": 0.9466, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2708.6752, + "latency": 8.4, + "recall": 0.9337, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2275.2854, + "latency": 9.1, + "recall": 0.917, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1844.8918, + "latency": 10.6, + "recall": 0.9085, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1301.4102, + "latency": 14.7, + "recall": 0.9011, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 13.0379, + "latency": 1063.5, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.4972, + "latency": 337.1, + "recall": 0.8717, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1164, + "latency": 345.6, + "recall": 0.4276, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 197.4455, + "latency": 349.3, + "recall": 0.5314, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 196.9391, + "latency": 263.4, + "recall": 0.6549, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.5401, + "latency": 282.4, + "recall": 0.7086, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.2424, + "latency": 301.7, + "recall": 0.7592, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.599, + "latency": 358.8, + "recall": 0.8085, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.0349, + "latency": 275.3, + "recall": 0.8325, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.1405, + "latency": 282.6, + "recall": 0.8492, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.1282, + "latency": 269.2, + "recall": 0.8637, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 194.8021, + "latency": 559.8, + "recall": 0.86, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 187.4268, + "latency": 453.7, + "recall": 0.4692, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.397, + "latency": 506.9, + "recall": 0.5409, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 174.3549, + "latency": 496.9, + "recall": 0.6279, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 172.95, + "latency": 515.6, + "recall": 0.7004, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 190.9747, + "latency": 517.4, + "recall": 0.7398, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 186.0237, + "latency": 474.0, + "recall": 0.7847, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1458, + "latency": 480.5, + "recall": 0.8103, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 179.4203, + "latency": 497.5, + "recall": 0.8273, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.5444, + "latency": 463.9, + "recall": 0.8478, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 346.5847, + "latency": 42.7, + "recall": 0.9631, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 369.4921, + "latency": 41.6, + "recall": 0.779, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 310.957, + "latency": 49.4, + "recall": 0.9698, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 798.328, + "latency": 56.7, + "recall": 0.8993, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 649.8781, + "latency": 55.2, + "recall": 0.8352, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 370.7241, + "latency": 49.6, + "recall": 0.7177, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 100.0554, + "latency": 69.3, + "recall": 0.9638, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 284.6367, + "latency": 47.6, + "recall": 0.9788, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 81.8678, + "latency": 105.6, + "recall": 0.8751, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 260.4031, + "latency": 48.3, + "recall": 0.9828, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 365.2505, + "latency": 34.9, + "recall": 0.8251, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 471.553, + "latency": 44.1, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 91.8612, + "latency": 85.7, + "recall": 0.8799, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 206.0934, + "latency": 56.7, + "recall": 0.9795, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 351.7114, + "latency": 46.7, + "recall": 0.8735, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 96.592, + "latency": 76.9, + "recall": 0.9178, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 802.6923, + "latency": 48.1, + "recall": 0.935, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 184.5363, + "latency": 53.4, + "recall": 0.9681, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 323.0238, + "latency": 50.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 382.5332, + "latency": 54.7, + "recall": 0.6135, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2030.4249, + "latency": 10.6, + "recall": 0.925, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1804.8996, + "latency": 12.3, + "recall": 0.9365, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2353.8935, + "latency": 17.1, + "recall": 0.9056, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1623.8421, + "latency": 11.8, + "recall": 0.945, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2808.2421, + "latency": 9.5, + "recall": 0.8674, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1482.3772, + "latency": 12.1, + "recall": 0.9523, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1721.5416, + "latency": 9.6, + "recall": 0.876, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1032.9696, + "latency": 14.8, + "recall": 0.9299, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1150.4393, + "latency": 13.4, + "recall": 0.9225, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1452.1536, + "latency": 10.8, + "recall": 0.8973, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2181.3939, + "latency": 9.4, + "recall": 0.8353, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1295.5543, + "latency": 11.2, + "recall": 0.9126, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9441.1235, + "latency": 5.2, + "recall": 0.9589, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6125.6146, + "latency": 4.9, + "recall": 0.9919, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5502.1797, + "latency": 3.8, + "recall": 0.9452, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1827.5849, + "latency": 5.4, + "recall": 0.9903, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1938.1932, + "latency": 5.6, + "recall": 0.989, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3778.8811, + "latency": 4.8, + "recall": 0.9828, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3974.8218, + "latency": 4.8, + "recall": 0.9396, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2971.1402, + "latency": 11.6, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8441.533, + "latency": 6.9, + "recall": 0.9785, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 2703.5422, + "latency": 12.9, + "recall": 0.9903, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1628.2736, + "latency": 5.6, + "recall": 0.9913, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5019.5973, + "latency": 5.7, + "recall": 0.994, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7364.0396, + "latency": 4.0, + "recall": 0.9893, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2724.9239, + "latency": 12.4, + "recall": 0.9811, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5514.815, + "latency": 4.2, + "recall": 0.9286, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9901.1114, + "latency": 3.9, + "recall": 0.9385, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3258.8508, + "latency": 11.7, + "recall": 0.9888, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5907.441, + "latency": 5.7, + "recall": 0.9931, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5064.6982, + "latency": 4.3, + "recall": 0.9558, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3468.5933, + "latency": 4.5, + "recall": 0.9634, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2401.3204, + "latency": 10.7, + "recall": 0.9866, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3568.396, + "latency": 4.8, + "recall": 0.9863, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 4674.1861, + "latency": 4.5, + "recall": 0.967, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3917.2035, + "latency": 2.4, + "recall": 0.9203, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3628.8527, + "latency": 2.6, + "recall": 0.9318, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3250.1112, + "latency": 2.7, + "recall": 0.9443, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2762.4144, + "latency": 3.1, + "recall": 0.9556, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2384.6245, + "latency": 3.2, + "recall": 0.9627, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2134.1717, + "latency": 3.8, + "recall": 0.9671, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1641.3478, + "latency": 4.1, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1488.5841, + "latency": 4.7, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2747.3167, + "latency": 3.3, + "recall": 0.9204, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2514.4481, + "latency": 3.2, + "recall": 0.9303, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2177.2345, + "latency": 3.4, + "recall": 0.9408, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1833.2575, + "latency": 3.9, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1552.4803, + "latency": 4.0, + "recall": 0.9565, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1355.3121, + "latency": 4.4, + "recall": 0.9602, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1079.2123, + "latency": 5.3, + "recall": 0.9648, + "filter_ratio": 0.0 + }, + { + "dataset": "Unknown", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 876.5772, + "latency": 6.3, + "recall": 0.9676, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10663.1231, + "latency": 2.0, + "recall": 0.8405, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10333.9072, + "latency": 2.0, + "recall": 0.889, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 9575.6863, + "latency": 2.3, + "recall": 0.9189, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 8596.7694, + "latency": 2.4, + "recall": 0.9416, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7704.3625, + "latency": 2.7, + "recall": 0.9541, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7023.6735, + "latency": 3.0, + "recall": 0.962, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 6031.3725, + "latency": 3.3, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 5258.1868, + "latency": 3.6, + "recall": 0.9768, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5973.0024, + "latency": 2.4, + "recall": 0.9192, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5416.5758, + "latency": 2.6, + "recall": 0.9334, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4771.4324, + "latency": 2.8, + "recall": 0.9479, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4006.3994, + "latency": 3.2, + "recall": 0.9609, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3441.7597, + "latency": 3.5, + "recall": 0.9682, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3040.6216, + "latency": 3.7, + "recall": 0.9734, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2446.7373, + "latency": 4.3, + "recall": 0.9791, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2084.6245, + "latency": 5.0, + "recall": 0.9819, + "filter_ratio": 0.0 + } ] \ No newline at end of file From 1c771cc17a3927ca13a0309a4beda8316b2e5e6b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Apr 2026 09:26:14 +0000 Subject: [PATCH 11/49] Unify result timestamps to standard_20260403 Rename ElasticCloud and ZillizCloud result files from 20260209 to 20260403 and update task_label to standard_20260403 for consistency with Milvus results. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...ticcloud.json => result_20260403_standard_elasticcloud.json} | 2 +- ...llizcloud.json => result_20260403_standard_zillizcloud.json} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename vectordb_bench/results/ElasticCloud/{result_20260209_standard_elasticcloud.json => result_20260403_standard_elasticcloud.json} (99%) rename vectordb_bench/results/ZillizCloud/{result_20260209_standard_zillizcloud.json => result_20260403_standard_zillizcloud.json} (99%) diff --git a/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json similarity index 99% rename from vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json rename to vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json index 213c2d374..54f9065d0 100644 --- a/vectordb_bench/results/ElasticCloud/result_20260209_standard_elasticcloud.json +++ b/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json @@ -1,6 +1,6 @@ { "run_id": "80696b60e39749b295273db3cdba1b69", - "task_label": "standard_20260209", + "task_label": "standard_20260403", "results": [ { "metrics": { diff --git a/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json similarity index 99% rename from vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json rename to vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index 619b5fbf4..e118d1f1c 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260209_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -1,6 +1,6 @@ { "run_id": "80696b60e39749b295273db3cdba1b69", - "task_label": "standard_20260209", + "task_label": "standard_20260403", "results": [ { "metrics": { From 8f7d6bb5ea1e41bf05a6746437222cda0d32c628 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Apr 2026 09:34:44 +0000 Subject: [PATCH 12/49] fix: unify run_id across all result files Co-Authored-By: Claude Opus 4.6 (1M context) --- ...result_20260403_standard_elasticcloud.json | 2 +- .../result_20260403_standard_zillizcloud.json | 2 +- vectordb_bench/results/leaderboard_v2.json | 1626 ++++++++++++----- 3 files changed, 1220 insertions(+), 410 deletions(-) diff --git a/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json index 54f9065d0..2ec557926 100644 --- a/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json +++ b/vectordb_bench/results/ElasticCloud/result_20260403_standard_elasticcloud.json @@ -1,5 +1,5 @@ { - "run_id": "80696b60e39749b295273db3cdba1b69", + "run_id": "c11e83b51ff14060a08f06d58f801214", "task_label": "standard_20260403", "results": [ { diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index e118d1f1c..75760ff71 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -1,5 +1,5 @@ { - "run_id": "80696b60e39749b295273db3cdba1b69", + "run_id": "c11e83b51ff14060a08f06d58f801214", "task_label": "standard_20260403", "results": [ { diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index 566ce4285..bbc1e918b 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -1469,6 +1469,776 @@ "recall": 0.6135, "filter_ratio": 0.98 }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3917.2035, + "latency": 2.4, + "recall": 0.9203, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3628.8527, + "latency": 2.6, + "recall": 0.9318, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3250.1112, + "latency": 2.7, + "recall": 0.9443, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2762.4144, + "latency": 3.1, + "recall": 0.9556, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2384.6245, + "latency": 3.2, + "recall": 0.9627, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2134.1717, + "latency": 3.8, + "recall": 0.9671, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1641.3478, + "latency": 4.1, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1488.5841, + "latency": 4.7, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2747.3167, + "latency": 3.3, + "recall": 0.9204, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2514.4481, + "latency": 3.2, + "recall": 0.9303, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2177.2345, + "latency": 3.4, + "recall": 0.9408, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1833.2575, + "latency": 3.9, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1552.4803, + "latency": 4.0, + "recall": 0.9565, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1355.3121, + "latency": 4.4, + "recall": 0.9602, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1079.2123, + "latency": 5.3, + "recall": 0.9648, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 876.5772, + "latency": 6.3, + "recall": 0.9676, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10663.1231, + "latency": 2.0, + "recall": 0.8405, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10333.9072, + "latency": 2.0, + "recall": 0.889, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 9575.6863, + "latency": 2.3, + "recall": 0.9189, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 8596.7694, + "latency": 2.4, + "recall": 0.9416, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7704.3625, + "latency": 2.7, + "recall": 0.9541, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7023.6735, + "latency": 3.0, + "recall": 0.962, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 6031.3725, + "latency": 3.3, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 5258.1868, + "latency": 3.6, + "recall": 0.9768, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5973.0024, + "latency": 2.4, + "recall": 0.9192, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5416.5758, + "latency": 2.6, + "recall": 0.9334, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4771.4324, + "latency": 2.8, + "recall": 0.9479, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4006.3994, + "latency": 3.2, + "recall": 0.9609, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3441.7597, + "latency": 3.5, + "recall": 0.9682, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3040.6216, + "latency": 3.7, + "recall": 0.9734, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2446.7373, + "latency": 4.3, + "recall": 0.9791, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2084.6245, + "latency": 5.0, + "recall": 0.9819, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11763.5538, + "latency": 1.5, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11803.1944, + "latency": 1.5, + "recall": 0.9778, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11520.9234, + "latency": 1.5, + "recall": 0.9634, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11280.0849, + "latency": 1.6, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10671.8925, + "latency": 1.7, + "recall": 0.9339, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10258.2661, + "latency": 1.7, + "recall": 0.9139, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9681.5656, + "latency": 1.9, + "recall": 0.9008, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8945.4041, + "latency": 1.9, + "recall": 0.8894, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5436.8907, + "latency": 2.0, + "recall": 0.929, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11397.7043, + "latency": 1.6, + "recall": 0.9597, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10891.7531, + "latency": 1.7, + "recall": 0.9408, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10276.7451, + "latency": 1.7, + "recall": 0.9159, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9664.2855, + "latency": 1.8, + "recall": 0.899, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8936.7962, + "latency": 2.0, + "recall": 0.8835, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5671.2562, + "latency": 2.1, + "recall": 0.903, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 3157.707, + "latency": 2.3, + "recall": 0.9347, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 1985.8124, + "latency": 2.6, + "recall": 0.9407, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 920.9627, + "latency": 3.4, + "recall": 0.9488, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3033.786, + "latency": 8.7, + "recall": 0.9934, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3019.2416, + "latency": 9.5, + "recall": 0.9765, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2890.9523, + "latency": 9.4, + "recall": 0.9625, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2789.7212, + "latency": 8.2, + "recall": 0.9538, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2457.2628, + "latency": 9.0, + "recall": 0.9378, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2209.4973, + "latency": 13.7, + "recall": 0.9228, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1960.388, + "latency": 11.0, + "recall": 0.9076, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1725.092, + "latency": 11.7, + "recall": 0.8969, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1307.419, + "latency": 12.3, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 350.0132, + "latency": 29.7, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 179.5204, + "latency": 51.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 72.99, + "latency": 111.4, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 42.9877, + "latency": 201.9, + "recall": 0.9912, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 96.4987, + "latency": 113.1, + "recall": 0.9296, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 189.3789, + "latency": 58.8, + "recall": 0.9149, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 246.7071, + "latency": 45.1, + "recall": 0.9018, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 229.0379, + "latency": 43.0, + "recall": 0.8908, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 125.6164, + "latency": 69.8, + "recall": 0.8746, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2175.2694, + "latency": 9.8, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1430.0244, + "latency": 12.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 692.5751, + "latency": 18.7, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 364.3516, + "latency": 26.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 190.3777, + "latency": 47.9, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 249.3519, + "latency": 44.7, + "recall": 0.9446, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 437.8735, + "latency": 27.1, + "recall": 0.9364, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 669.9441, + "latency": 19.1, + "recall": 0.9227, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 899.3114, + "latency": 14.9, + "recall": 0.9072, + "filter_ratio": 0.5 + }, { "dataset": "Cohere (Medium)", "db": "ElasticCloud", @@ -1476,7 +2246,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 2030.4249, "latency": 10.6, - "recall": 0.925, + "recall": 0.9306, "filter_ratio": 0.0 }, { @@ -1486,7 +2256,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1804.8996, "latency": 12.3, - "recall": 0.9365, + "recall": 0.9405, "filter_ratio": 0.0 }, { @@ -1496,7 +2266,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 2353.8935, "latency": 17.1, - "recall": 0.9056, + "recall": 0.9143, "filter_ratio": 0.0 }, { @@ -1506,7 +2276,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1623.8421, "latency": 11.8, - "recall": 0.945, + "recall": 0.9479, "filter_ratio": 0.0 }, { @@ -1516,7 +2286,7 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 2808.2421, "latency": 9.5, - "recall": 0.8674, + "recall": 0.8815, "filter_ratio": 0.0 }, { @@ -1526,67 +2296,67 @@ "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1482.3772, "latency": 12.1, - "recall": 0.9523, + "recall": 0.9546, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1721.5416, "latency": 9.6, - "recall": 0.876, + "recall": 0.8855, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1032.9696, "latency": 14.8, - "recall": 0.9299, + "recall": 0.933, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1150.4393, "latency": 13.4, - "recall": 0.9225, + "recall": 0.9265, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1452.1536, "latency": 10.8, - "recall": 0.8973, + "recall": 0.9042, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 2181.3939, "latency": 9.4, - "recall": 0.8353, + "recall": 0.8501, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ElasticCloud", "label": "8c60g-force_merge", "db_name": "ElasticCloud-8c60g-force_merge", "qps": 1295.5543, "latency": 11.2, - "recall": 0.9126, + "recall": 0.9176, "filter_ratio": 0.0 }, { @@ -1594,549 +2364,589 @@ "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 9441.1235, - "latency": 5.2, - "recall": 0.9589, - "filter_ratio": 0.0 + "qps": 9773.6593, + "latency": 3.7, + "recall": 0.9955, + "filter_ratio": 0.999 }, { "dataset": "Cohere (Medium)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 6125.6146, - "latency": 4.9, - "recall": 0.9919, - "filter_ratio": 0.0 + "qps": 9081.1518, + "latency": 3.0, + "recall": 0.9943, + "filter_ratio": 0.998 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5502.1797, - "latency": 3.8, - "recall": 0.9452, - "filter_ratio": 0.0 + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8455.2896, + "latency": 4.0, + "recall": 0.9921, + "filter_ratio": 0.995 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 1827.5849, - "latency": 5.4, + "qps": 7610.0519, + "latency": 3.3, "recall": 0.9903, - "filter_ratio": 0.0 + "filter_ratio": 0.99 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 1938.1932, - "latency": 5.6, - "recall": 0.989, - "filter_ratio": 0.0 + "qps": 7589.664, + "latency": 3.8, + "recall": 0.9235, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6750.2495, + "latency": 4.4, + "recall": 0.9105, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5506.1808, + "latency": 5.5, + "recall": 0.9193, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6860.8577, + "latency": 4.7, + "recall": 0.9226, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8468.4611, + "latency": 3.1, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10089.4308, + "latency": 2.6, + "recall": 0.9934, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10557.4373, + "latency": 2.7, + "recall": 0.9393, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9805.0401, + "latency": 2.6, + "recall": 0.9257, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10020.5299, + "latency": 2.6, + "recall": 0.9788, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10041.0338, + "latency": 2.7, + "recall": 0.9693, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9861.9686, + "latency": 2.6, + "recall": 0.955, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9507.9991, + "latency": 2.8, + "recall": 0.9453, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9428.4531, + "latency": 2.6, + "recall": 0.9331, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9048.6431, + "latency": 3.9, + "recall": 0.9216, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8695.2765, + "latency": 4.3, + "recall": 0.9603, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9244.1135, + "latency": 4.2, + "recall": 0.9724, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9289.0118, + "latency": 4.2, + "recall": 0.9574, + "filter_ratio": 0.995 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3778.8811, - "latency": 4.8, - "recall": 0.9828, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9374.8941, + "latency": 4.2, + "recall": 0.9425, + "filter_ratio": 0.99 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3974.8218, - "latency": 4.8, - "recall": 0.9396, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9368.1325, + "latency": 3.8, + "recall": 0.9292, + "filter_ratio": 0.98 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2971.1402, - "latency": 11.6, - "recall": 0.9729, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9220.3627, + "latency": 3.8, + "recall": 0.9081, + "filter_ratio": 0.95 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8441.533, - "latency": 6.9, - "recall": 0.9785, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8633.8949, + "latency": 4.1, + "recall": 0.8928, + "filter_ratio": 0.9 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 2703.5422, - "latency": 12.9, - "recall": 0.9903, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 6820.6863, + "latency": 3.2, + "recall": 0.9159, + "filter_ratio": 0.8 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1628.2736, - "latency": 5.6, - "recall": 0.9913, - "filter_ratio": 0.0 + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 3938.6004, + "latency": 3.7, + "recall": 0.9196, + "filter_ratio": 0.5 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 5019.5973, - "latency": 5.7, - "recall": 0.994, - "filter_ratio": 0.0 + "qps": 3411.0934, + "latency": 3.3, + "recall": 0.995, + "filter_ratio": 0.999 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 7364.0396, - "latency": 4.0, - "recall": 0.9893, - "filter_ratio": 0.0 + "qps": 2838.356, + "latency": 3.8, + "recall": 0.9946, + "filter_ratio": 0.998 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 2724.9239, - "latency": 12.4, - "recall": 0.9811, - "filter_ratio": 0.0 + "qps": 1826.0672, + "latency": 5.3, + "recall": 0.9938, + "filter_ratio": 0.995 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5514.815, - "latency": 4.2, - "recall": 0.9286, - "filter_ratio": 0.0 + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1234.6534, + "latency": 6.4, + "recall": 0.9942, + "filter_ratio": 0.99 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 9901.1114, - "latency": 3.9, - "recall": 0.9385, - "filter_ratio": 0.0 + "qps": 1773.0919, + "latency": 5.3, + "recall": 0.9699, + "filter_ratio": 0.98 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3258.8508, - "latency": 11.7, - "recall": 0.9888, - "filter_ratio": 0.0 + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1454.8382, + "latency": 4.6, + "recall": 0.9659, + "filter_ratio": 0.95 }, { - "dataset": "Cohere (Medium)", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 5907.441, + "qps": 1373.0307, "latency": 5.7, - "recall": 0.9931, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5064.6982, - "latency": 4.3, - "recall": 0.9558, - "filter_ratio": 0.0 + "recall": 0.9716, + "filter_ratio": 0.9 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 3468.5933, - "latency": 4.5, - "recall": 0.9634, - "filter_ratio": 0.0 + "qps": 2039.8673, + "latency": 3.8, + "recall": 0.9559, + "filter_ratio": 0.8 }, { - "dataset": "Unknown", + "dataset": "Cohere (Large)", "db": "ZillizCloud", "label": "8cu-perf", "db_name": "ZillizCloud-8cu-perf", - "qps": 2401.3204, - "latency": 10.7, - "recall": 0.9866, - "filter_ratio": 0.0 + "qps": 2950.8165, + "latency": 3.3, + "recall": 0.9147, + "filter_ratio": 0.5 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3568.396, - "latency": 4.8, - "recall": 0.9863, + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9441.1235, + "latency": 5.2, + "recall": 0.9658, "filter_ratio": 0.0 }, { - "dataset": "Unknown", + "dataset": "Cohere (Medium)", "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 4674.1861, - "latency": 4.5, - "recall": 0.967, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3917.2035, - "latency": 2.4, - "recall": 0.9203, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3628.8527, - "latency": 2.6, - "recall": 0.9318, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3250.1112, - "latency": 2.7, - "recall": 0.9443, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2762.4144, - "latency": 3.1, - "recall": 0.9556, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2384.6245, - "latency": 3.2, - "recall": 0.9627, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2134.1717, - "latency": 3.8, - "recall": 0.9671, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 1641.3478, - "latency": 4.1, - "recall": 0.9729, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 1488.5841, - "latency": 4.7, - "recall": 0.9764, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2747.3167, - "latency": 3.3, - "recall": 0.9204, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2514.4481, - "latency": 3.2, - "recall": 0.9303, - "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2177.2345, - "latency": 3.4, - "recall": 0.9408, + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6125.6146, + "latency": 4.9, + "recall": 0.9936, "filter_ratio": 0.0 - }, - { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1833.2575, - "latency": 3.9, - "recall": 0.951, + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5502.1797, + "latency": 3.8, + "recall": 0.9509, "filter_ratio": 0.0 }, { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1552.4803, - "latency": 4.0, - "recall": 0.9565, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1827.5849, + "latency": 5.4, + "recall": 0.9918, "filter_ratio": 0.0 }, { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1355.3121, - "latency": 4.4, - "recall": 0.9602, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1938.1932, + "latency": 5.6, + "recall": 0.9906, "filter_ratio": 0.0 }, { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1079.2123, - "latency": 5.3, - "recall": 0.9648, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3778.8811, + "latency": 4.8, + "recall": 0.9851, "filter_ratio": 0.0 }, { - "dataset": "Unknown", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 876.5772, - "latency": 6.3, - "recall": 0.9676, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3974.8218, + "latency": 4.8, + "recall": 0.9428, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 10663.1231, - "latency": 2.0, - "recall": 0.8405, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2971.1402, + "latency": 11.6, + "recall": 0.9752, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 10333.9072, - "latency": 2.0, - "recall": 0.889, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8441.533, + "latency": 6.9, + "recall": 0.9825, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 9575.6863, - "latency": 2.3, - "recall": 0.9189, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 2703.5422, + "latency": 12.9, + "recall": 0.992, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 8596.7694, - "latency": 2.4, - "recall": 0.9416, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1628.2736, + "latency": 5.6, + "recall": 0.9928, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 7704.3625, - "latency": 2.7, - "recall": 0.9541, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5019.5973, + "latency": 5.7, + "recall": 0.9954, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 7023.6735, - "latency": 3.0, - "recall": 0.962, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7364.0396, + "latency": 4.0, + "recall": 0.9915, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 6031.3725, - "latency": 3.3, - "recall": 0.971, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2724.9239, + "latency": 12.4, + "recall": 0.9832, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 5258.1868, - "latency": 3.6, - "recall": 0.9768, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5514.815, + "latency": 4.2, + "recall": 0.9355, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 5973.0024, - "latency": 2.4, - "recall": 0.9192, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9901.1114, + "latency": 3.9, + "recall": 0.9486, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 5416.5758, - "latency": 2.6, - "recall": 0.9334, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3258.8508, + "latency": 11.7, + "recall": 0.9906, "filter_ratio": 0.0 }, { "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 4771.4324, - "latency": 2.8, - "recall": 0.9479, + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5907.441, + "latency": 5.7, + "recall": 0.9946, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 4006.3994, - "latency": 3.2, - "recall": 0.9609, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 5064.6982, + "latency": 4.3, + "recall": 0.9606, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 3441.7597, - "latency": 3.5, - "recall": 0.9682, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3468.5933, + "latency": 4.5, + "recall": 0.9661, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 3040.6216, - "latency": 3.7, - "recall": 0.9734, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2401.3204, + "latency": 10.7, + "recall": 0.9884, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2446.7373, - "latency": 4.3, - "recall": 0.9791, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 3568.396, + "latency": 4.8, + "recall": 0.9883, "filter_ratio": 0.0 }, { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2084.6245, - "latency": 5.0, - "recall": 0.9819, + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-force_merge", + "db_name": "ZillizCloud-8cu-perf-force_merge", + "qps": 4674.1861, + "latency": 4.5, + "recall": 0.9705, "filter_ratio": 0.0 } ] \ No newline at end of file From 7c2a4b79734420b97e9b6e0625a2f50994bf6122 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 3 Apr 2026 13:09:09 +0000 Subject: [PATCH 13/49] fix: align streaming leaderboard labels with vector search results Update db_name/label in leaderboard_v2_streaming.json to match leaderboard_v2.json after force_merge became the default: - Milvus: 16c64g-sq8 -> 16c64g-sq8-force_merge - ElasticCloud: 8c60g -> 8c60g-force_merge This fixes the website failing to associate streaming and vector search results due to mismatched db_name keys. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../results/leaderboard_v2_streaming.json | 288 +++++++++--------- 1 file changed, 144 insertions(+), 144 deletions(-) diff --git a/vectordb_bench/results/leaderboard_v2_streaming.json b/vectordb_bench/results/leaderboard_v2_streaming.json index 222c68bf7..73075e8a8 100644 --- a/vectordb_bench/results/leaderboard_v2_streaming.json +++ b/vectordb_bench/results/leaderboard_v2_streaming.json @@ -1,146 +1,146 @@ [ - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "insert_rate": 500, - "streaming_qps": 61.6708, - "streaming_latency": 0.0794 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g", - "db_name": "ElasticCloud-8c60g", - "insert_rate": 1000, - "streaming_qps": 61.8172, - "streaming_latency": 0.2223 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "insert_rate": 500, - "streaming_qps": 305.9971, - "streaming_latency": 0.005 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8", - "db_name": "Milvus-16c64g-sq8", - "insert_rate": 1000, - "streaming_qps": 155.9613, - "streaming_latency": 0.0203 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "insert_rate": 500, - "streaming_qps": 367.4299, - "streaming_latency": 1.8286 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "insert_rate": 1000, - "streaming_qps": 369.6771, - "streaming_latency": 5.992 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "insert_rate": 500, - "streaming_qps": 393.753, - "streaming_latency": 0.0162 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "insert_rate": 1000, - "streaming_qps": 347.5774, - "streaming_latency": 0.0118 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "insert_rate": 1000, - "streaming_qps": 149.7168, - "streaming_latency": 0.098 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "insert_rate": 500, - "streaming_qps": 161.6694, - "streaming_latency": 0.052 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "insert_rate": 500, - "streaming_qps": 2118.7516, - "streaming_latency": 0.0068 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "insert_rate": 1000, - "streaming_qps": 1860.2575, - "streaming_latency": 0.0101 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "insert_rate": 500, - "streaming_qps": 180.9549, - "streaming_latency": 0.4204 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "insert_rate": 1000, - "streaming_qps": 167.2689, - "streaming_latency": 0.5048 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "insert_rate": 500, - "streaming_qps": 536.0198, - "streaming_latency": 0.3132 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "insert_rate": 1000, - "streaming_qps": 442.5824, - "streaming_latency": 0.0724 - } + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "insert_rate": 500, + "streaming_qps": 61.6708, + "streaming_latency": 0.0794 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "insert_rate": 1000, + "streaming_qps": 61.8172, + "streaming_latency": 0.2223 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "insert_rate": 500, + "streaming_qps": 305.9971, + "streaming_latency": 0.005 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "insert_rate": 1000, + "streaming_qps": 155.9613, + "streaming_latency": 0.0203 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "insert_rate": 500, + "streaming_qps": 367.4299, + "streaming_latency": 1.8286 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "insert_rate": 1000, + "streaming_qps": 369.6771, + "streaming_latency": 5.992 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "insert_rate": 500, + "streaming_qps": 393.753, + "streaming_latency": 0.0162 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "insert_rate": 1000, + "streaming_qps": 347.5774, + "streaming_latency": 0.0118 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "insert_rate": 1000, + "streaming_qps": 149.7168, + "streaming_latency": 0.098 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "insert_rate": 500, + "streaming_qps": 161.6694, + "streaming_latency": 0.052 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "insert_rate": 500, + "streaming_qps": 2118.7516, + "streaming_latency": 0.0068 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "insert_rate": 1000, + "streaming_qps": 1860.2575, + "streaming_latency": 0.0101 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "insert_rate": 500, + "streaming_qps": 180.9549, + "streaming_latency": 0.4204 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "insert_rate": 1000, + "streaming_qps": 167.2689, + "streaming_latency": 0.5048 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "insert_rate": 500, + "streaming_qps": 536.0198, + "streaming_latency": 0.3132 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "insert_rate": 1000, + "streaming_qps": 442.5824, + "streaming_latency": 0.0724 + } ] \ No newline at end of file From 51c5158506cd09e6a19dbcb6029f60aa8071cda6 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 8 Apr 2026 02:41:36 +0000 Subject: [PATCH 14/49] fix: update ZillizCloud benchmark with Cardinal backend results Replace ZillizCloud-8cu-perf case_id=4/5 data with new Cardinal backend benchmark results (level 1-9, 1M and 10M datasets, v2026.4). Remove force_merge entries as Cardinal uses unified 4-segment architecture for 10M. New results show significant QPS improvement: - 1M: 13,316 QPS (was 9,704) at recall 0.938 - 10M: 7,385 QPS (was 3,957) at recall 0.938 Sort all leaderboard entries by (db_name, dataset, filter_ratio, qps DESC) to fix line chart rendering. Remove one SQ4U 1M outlier (recall=0.84). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../result_20260403_standard_zillizcloud.json | 2264 +++---- vectordb_bench/results/leaderboard_v2.json | 5840 ++++++++--------- 2 files changed, 3690 insertions(+), 4414 deletions(-) diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index 75760ff71..e02463ebd 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -5,14 +5,14 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 9441.1235, - "serial_latency_p99": 0.0052, - "serial_latency_p95": 0.0039, - "recall": 0.9589, - "ndcg": 0.9658, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 13316.2336, + "serial_latency_p99": 0.002, + "serial_latency_p95": 0.0019, + "recall": 0.9383, + "ndcg": 0.9484, "conc_num_list": [ 1, 5, @@ -24,44 +24,44 @@ 80 ], "conc_qps_list": [ - 306.3773, - 1519.6034, - 3129.1309, - 5457.1507, - 6585.7082, - 7420.0391, - 8468.6183, - 9441.1235 + 557.5439, + 2665.8166, + 5047.6131, + 8707.1886, + 10381.4448, + 11435.3821, + 12833.6289, + 13316.2336 ], "conc_latency_p99_list": [ - 0.004086580532602967, - 0.00566743115196005, - 0.005243223664583638, - 0.0099503791576717, - 0.009588507658627355, - 0.011322382240905426, - 0.01665949933376398, - 0.018269318575912616 + 0.001960459982510656, + 0.002204382496420288, + 0.0025926707847975195, + 0.003785052003804594, + 0.005270074445288626, + 0.006814960413612425, + 0.009530203816248103, + 0.01240227443340701 ], "conc_latency_p95_list": [ - 0.0036428007049835284, - 0.003802539707976394, - 0.003850769612472504, - 0.004741756187286226, - 0.006738374719861894, - 0.00850677301059477, - 0.01233988689491525, - 0.01412305135163478 + 0.0018868701998144388, + 0.0020324485929450018, + 0.0021866516792215405, + 0.002869376807939261, + 0.004040546333999372, + 0.0052645970426965505, + 0.007324896720820107, + 0.009507914245477877 ], "conc_latency_avg_list": [ - 0.0032598009878015348, - 0.003285434242748037, - 0.0031902432390411135, - 0.0036557288568108254, - 0.004539560740384104, - 0.005359882091644495, - 0.007008278288237143, - 0.008346175450269581 + 0.0017909009081054225, + 0.0018719543108110278, + 0.001976279835245286, + 0.002288533358936022, + 0.0028752306497382865, + 0.0034729409658481964, + 0.004612408416183334, + 0.0058844483816759795 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -82,10 +82,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -93,7 +93,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 2, + "level": 1, "num_shards": 1 }, "case_config": { @@ -116,25 +116,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 6125.6146, - "serial_latency_p99": 0.0049, - "serial_latency_p95": 0.0047, - "recall": 0.9919, - "ndcg": 0.9936, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 12837.5287, + "serial_latency_p99": 0.0021, + "serial_latency_p95": 0.002, + "recall": 0.9588, + "ndcg": 0.9657, "conc_num_list": [ 1, 5, @@ -146,44 +143,44 @@ 80 ], "conc_qps_list": [ - 238.8712, - 1196.2193, - 2440.8391, - 4133.8486, - 4858.2331, - 5446.0047, - 6000.1652, - 6125.6146 + 539.7039, + 2582.4122, + 4917.6589, + 8479.8846, + 10116.1707, + 11068.2155, + 12229.2208, + 12837.5287 ], "conc_latency_p99_list": [ - 0.004919703922932965, - 0.010525701696751709, - 0.006511175064661075, - 0.011745981796411804, - 0.01397663167037535, - 0.014958455199375772, - 0.02016973898542343, - 0.026595940839324612 + 0.0020551326422719287, + 0.002271793531253935, + 0.0026150090881856096, + 0.00380311052547768, + 0.005449252330581657, + 0.0069998929975554295, + 0.010041847208049142, + 0.012852289543952794 ], "conc_latency_p95_list": [ - 0.0046619078202638775, - 0.004602618556236848, - 0.004652375826844946, - 0.006574279977940023, - 0.009700259550299961, - 0.011811009392840788, - 0.016332678495382422, - 0.02156840759853367 + 0.0019641239661723374, + 0.0021013408317230643, + 0.002235009270953014, + 0.002924549448653124, + 0.004163405022700318, + 0.0054554910166189075, + 0.007754770980682224, + 0.009949398809112607 ], "conc_latency_avg_list": [ - 0.004181373075740294, - 0.004174027799032302, - 0.004089987135078621, - 0.004828215031413884, - 0.006158122789762511, - 0.007310655337680553, - 0.009892293927032914, - 0.012884722355037423 + 0.0018501577211713547, + 0.001932346768512776, + 0.0020285184508130197, + 0.0023497830070767353, + 0.002947098563192882, + 0.003586949439268913, + 0.0048461200425519, + 0.006100907381850077 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -204,132 +201,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 7, - "num_shards": 1 - }, - "case_config": { - "case_id": 5, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 5502.1797, - "serial_latency_p99": 0.0038, - "serial_latency_p95": 0.0035, - "recall": 0.9452, - "ndcg": 0.9509, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 309.5267, - 1357.6945, - 2213.2564, - 3349.9157, - 3910.9875, - 4385.5899, - 5039.0299, - 5502.1797 - ], - "conc_latency_p99_list": [ - 0.004089714956353409, - 0.0056684831657912264, - 0.012667869632714424, - 0.01000201591959918, - 0.015246839204337462, - 0.01789945445081684, - 0.02318606456159614, - 0.02852195684099569 - ], - "conc_latency_p95_list": [ - 0.003446204590727575, - 0.00420360880671069, - 0.0055909310030983735, - 0.008043184356938578, - 0.011835442011943087, - 0.014750372216803953, - 0.01907540229440201, - 0.023490797984413794 - ], - "conc_latency_avg_list": [ - 0.0032266616589272513, - 0.0036776357130507064, - 0.004511172191326527, - 0.0059575964382477965, - 0.007649578970207299, - 0.009076414605964197, - 0.011790618147195132, - 0.01435249249351469 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -341,495 +216,7 @@ "num_shards": 1 }, "case_config": { - "case_id": 4, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 1827.5849, - "serial_latency_p99": 0.0054, - "serial_latency_p95": 0.0052, - "recall": 0.9903, - "ndcg": 0.9918, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 218.8679, - 861.6609, - 1195.6337, - 1339.6438, - 1475.742, - 1546.5953, - 1747.9077, - 1827.5849 - ], - "conc_latency_p99_list": [ - 0.006139998821017798, - 0.008449624522763766, - 0.014799785086070211, - 0.02650444130937102, - 0.03739181953598747, - 0.04722500271425814, - 0.05959104605717584, - 0.0715404962032335 - ], - "conc_latency_p95_list": [ - 0.005324425446451642, - 0.006872303040290717, - 0.011261535996163731, - 0.021800653115496966, - 0.03018094879225827, - 0.038646315991354645, - 0.04888677580165675, - 0.06127824939903802 - ], - "conc_latency_avg_list": [ - 0.004563713319136163, - 0.0057948937011398916, - 0.00835191496328548, - 0.014905808578982384, - 0.020272801297484298, - 0.025762202275047597, - 0.034040973878552726, - 0.043269632825280596 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 8, - "num_shards": 1 - }, - "case_config": { - "case_id": 4, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 1938.1932, - "serial_latency_p99": 0.0056, - "serial_latency_p95": 0.0053, - "recall": 0.989, - "ndcg": 0.9906, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 252.1234, - 954.3561, - 1355.554, - 1490.2007, - 1619.1811, - 1720.5715, - 1860.9305, - 1938.1932 - ], - "conc_latency_p99_list": [ - 0.0050690684028086245, - 0.008166075175395238, - 0.011429371475242079, - 0.024431421170011097, - 0.03405256026599093, - 0.043166847461543534, - 0.05706310785026292, - 0.07251818811346307 - ], - "conc_latency_p95_list": [ - 0.004786731592321303, - 0.006139315699692815, - 0.009358109103050082, - 0.019912595101050083, - 0.028057811519829556, - 0.035778356752416585, - 0.04874245898681693, - 0.061324251000769436 - ], - "conc_latency_avg_list": [ - 0.003961802805693023, - 0.005232545384579334, - 0.007366828470110833, - 0.01339630708221878, - 0.018484241519041742, - 0.023149088452180534, - 0.03196686416892556, - 0.04073973314917765 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 7, - "num_shards": 1 - }, - "case_config": { - "case_id": 4, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 3778.8811, - "serial_latency_p99": 0.0048, - "serial_latency_p95": 0.0042, - "recall": 0.9828, - "ndcg": 0.9851, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 275.1343, - 1220.1869, - 2080.6024, - 2544.5946, - 2934.4403, - 3222.4756, - 3520.6342, - 3778.8811 - ], - "conc_latency_p99_list": [ - 0.004224385871784762, - 0.006377705505292314, - 0.006628996630315663, - 0.01544506659847684, - 0.02085710293264129, - 0.02495263563963816, - 0.03397362035117112, - 0.0415226237432216 - ], - "conc_latency_p95_list": [ - 0.003899741142231505, - 0.004650917260732967, - 0.005831561920058448, - 0.011755315004847944, - 0.016476290803984734, - 0.020251012463995716, - 0.027687024004990225, - 0.03372844383848132 - ], - "conc_latency_avg_list": [ - 0.0036302553438832055, - 0.004091130600148307, - 0.0047987246661642296, - 0.007844535865702365, - 0.010192746767866709, - 0.012344825366337173, - 0.016888610032228992, - 0.020908843243531986 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 6, - "num_shards": 1 - }, - "case_config": { - "case_id": 4, - "custom_case": {}, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ] - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 3974.8218, - "serial_latency_p99": 0.0048, - "serial_latency_p95": 0.0043, - "recall": 0.9396, - "ndcg": 0.9428, - "conc_num_list": [ - 1, - 5, - 10, - 20, - 30, - 40, - 60, - 80 - ], - "conc_qps_list": [ - 295.7118, - 1186.5002, - 1797.187, - 2449.5408, - 2857.3282, - 3164.2684, - 3660.3166, - 3974.8218 - ], - "conc_latency_p99_list": [ - 0.004793434205930709, - 0.005700148111791336, - 0.007426547166251105, - 0.014722349808434964, - 0.019732102921698247, - 0.02347602234221995, - 0.03035390855744481, - 0.03731970520602769 - ], - "conc_latency_p95_list": [ - 0.004169186984654516, - 0.004717364211683161, - 0.006335475675587076, - 0.01105007229198236, - 0.01624748620088212, - 0.019896189187420532, - 0.025662759994156657, - 0.031171760102733943 - ], - "conc_latency_avg_list": [ - 0.003377046875928882, - 0.004208154278338283, - 0.005554828192323107, - 0.00814848752272782, - 0.010461408703728187, - 0.012581798692238582, - 0.016239422653327416, - 0.019879490524355867 - ], - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2026.1", - "note": "", - "uri": "**********", - "user": "db_admin", - "password": "**********", - "collection_name": "ZillizCloudVDBBench" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE", - "use_partition_key": false, - "level": 1, - "num_shards": 1 - }, - "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -848,25 +235,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 2971.1402, - "serial_latency_p99": 0.0116, - "serial_latency_p95": 0.0045, - "recall": 0.9729, - "ndcg": 0.9752, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 12248.9154, + "serial_latency_p99": 0.0022, + "serial_latency_p95": 0.0021, + "recall": 0.9687, + "ndcg": 0.9742, "conc_num_list": [ 1, 5, @@ -878,44 +262,44 @@ 80 ], "conc_qps_list": [ - 278.0971, - 1117.1061, - 1626.1624, - 2126.1398, - 2425.3155, - 2521.9392, - 2793.1709, - 2971.1402 + 526.2995, + 2512.1902, + 4765.4856, + 8188.3548, + 9485.6147, + 10432.6026, + 11820.0344, + 12248.9154 ], "conc_latency_p99_list": [ - 0.004647255002055317, - 0.00566180575755425, - 0.011257058603223423, - 0.015976536156085786, - 0.023499950004043067, - 0.03039303767494857, - 0.04078626602888109, - 0.04914582587312907 + 0.0021030055452138188, + 0.0023121346672996877, + 0.0026887326262658466, + 0.003929865991231055, + 0.005846867947839196, + 0.007567329368903298, + 0.01027507190126926, + 0.013403068128973255 ], "conc_latency_p95_list": [ - 0.004429338514455594, - 0.005089565094385761, - 0.007136018975870684, - 0.01292410076566739, - 0.019488130600075235, - 0.025593487400328737, - 0.03410469329974147, - 0.04188467259518802 + 0.002018616924760863, + 0.00216002133092843, + 0.0023071649571647867, + 0.003043863424682058, + 0.004536616246332414, + 0.005861608259147033, + 0.008004442710080184, + 0.010495775798335672 ], "conc_latency_avg_list": [ - 0.003591256380123583, - 0.004470003286850064, - 0.00613923671987789, - 0.009389019339411041, - 0.012338699755027977, - 0.015788074921940728, - 0.02128022645956974, - 0.026573440291179154 + 0.0018973754593796914, + 0.001986612150848518, + 0.0020934129605979786, + 0.0024318237152467248, + 0.0031476610878461374, + 0.00380841888066139, + 0.005011055927151683, + 0.006389479412319173 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -936,10 +320,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -947,11 +331,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 4, + "level": 3, "num_shards": 1 }, "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -970,23 +354,20 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 8441.533, - "serial_latency_p99": 0.0069, - "serial_latency_p95": 0.0035, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 11501.6652, + "serial_latency_p99": 0.0022, + "serial_latency_p95": 0.0022, "recall": 0.9785, "ndcg": 0.9825, "conc_num_list": [ @@ -1000,44 +381,44 @@ 80 ], "conc_qps_list": [ - 314.6659, - 1556.8466, - 3026.0164, - 4952.5941, - 6059.6492, - 6948.3916, - 7709.5146, - 8441.533 + 498.953, + 2391.8074, + 4531.3051, + 7692.4049, + 8962.935, + 9790.7556, + 10733.1679, + 11501.6652 ], "conc_latency_p99_list": [ - 0.003662585185375065, - 0.003881094480166214, - 0.005496532852703231, - 0.012801527802657801, - 0.01098183460126168, - 0.011906764237210155, - 0.017680786546843585, - 0.02030978908645921 + 0.002246916518197395, + 0.002433281935518607, + 0.0027782795106759276, + 0.004184300593915395, + 0.006221411311416887, + 0.007918057614006104, + 0.011295880685211153, + 0.013962703556753681 ], "conc_latency_p95_list": [ - 0.0033931825950276107, - 0.0034469130958314055, - 0.003594076508306898, - 0.00537703381414758, - 0.00756983550672885, - 0.009184699498291593, - 0.013430504295683926, - 0.015904919797321764 + 0.002148064094944857, + 0.002272934094071388, + 0.00242809351766482, + 0.0032858662889339025, + 0.0048346503434004255, + 0.006268709182040765, + 0.008884398700320158, + 0.011115554475691165 ], "conc_latency_avg_list": [ - 0.003173945048460723, - 0.003206842876612295, - 0.003299035623981205, - 0.004028295482036381, - 0.004934835427053102, - 0.005724545223453835, - 0.007703497705718975, - 0.009334748886407937 + 0.0020014255442295315, + 0.002086812448327015, + 0.0022017174912128874, + 0.0025910515942768643, + 0.0033292378315385594, + 0.0040559394841899535, + 0.005528263390600377, + 0.006816899394463919 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1058,10 +439,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1092,25 +473,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 2703.5422, - "serial_latency_p99": 0.0129, - "serial_latency_p95": 0.0049, - "recall": 0.9903, - "ndcg": 0.992, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 10566.6823, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0023, + "recall": 0.9838, + "ndcg": 0.9868, "conc_num_list": [ 1, 5, @@ -1122,44 +500,44 @@ 80 ], "conc_qps_list": [ - 230.8474, - 1033.6996, - 1653.7251, - 2021.6415, - 2247.5951, - 2433.7674, - 2579.1371, - 2703.5422 + 473.2503, + 2266.5174, + 4325.3899, + 7263.7246, + 8352.1369, + 9198.3778, + 10210.3429, + 10566.6823 ], "conc_latency_p99_list": [ - 0.005364620329928583, - 0.008247331644815845, - 0.009261744469986306, - 0.019847353509976556, - 0.027152295518899337, - 0.0318573336204281, - 0.04668155071558431, - 0.053025491506559774 + 0.0023700519639533015, + 0.0025644300674321128, + 0.0029225554369622857, + 0.0044386597932316255, + 0.00657977487426251, + 0.008452395589556558, + 0.011632206621579828, + 0.015279923828202287 ], "conc_latency_p95_list": [ - 0.004779104294721037, - 0.005717401996662375, - 0.007679672696394844, - 0.015277760991011746, - 0.021049827802926295, - 0.025551227995310906, - 0.03599990590882953, - 0.045174946513725445 + 0.0022803591098636386, + 0.002403592297923751, + 0.002544367348309606, + 0.0034909876121673724, + 0.005244998395210129, + 0.0067171422066167, + 0.009315501491073517, + 0.01212728061946109 ], "conc_latency_avg_list": [ - 0.004327150692358546, - 0.004830543605956322, - 0.006038381999227765, - 0.009872808423519352, - 0.013310825758863387, - 0.016344593851929733, - 0.023038225918050447, - 0.02921974749485067 + 0.0021102672976180386, + 0.0022023083714165855, + 0.0023067257051930207, + 0.0027442442136128803, + 0.003573842319898506, + 0.004316896363141895, + 0.005797124719790328, + 0.007427947098361652 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1179,11 +557,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1191,11 +569,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 9, + "level": 5, "num_shards": 1 }, "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -1214,25 +592,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 1628.2736, - "serial_latency_p99": 0.0056, - "serial_latency_p95": 0.0051, - "recall": 0.9913, - "ndcg": 0.9928, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 9227.318, + "serial_latency_p99": 0.0027, + "serial_latency_p95": 0.0026, + "recall": 0.9893, + "ndcg": 0.9915, "conc_num_list": [ 1, 5, @@ -1244,44 +619,44 @@ 80 ], "conc_qps_list": [ - 224.1263, - 854.2908, - 1191.923, - 1306.8314, - 1408.4071, - 1492.5651, - 1572.8756, - 1628.2736 + 427.7206, + 2082.4408, + 3978.3343, + 6561.5406, + 7528.8901, + 8016.929, + 8877.3559, + 9227.318 ], "conc_latency_p99_list": [ - 0.005614027456322219, - 0.01122624498093481, - 0.013540994446957512, - 0.026513429321930727, - 0.0360522977943765, - 0.04583110647072318, - 0.06417124239553232, - 0.08320635374402624 + 0.002635034592822194, + 0.002782075599534437, + 0.003112639953033066, + 0.0048944263125304125, + 0.00728108051931485, + 0.009557482128147962, + 0.013117193853249772, + 0.017160092100966726 ], "conc_latency_p95_list": [ - 0.0053050212460220795, - 0.006976215375470927, - 0.011254654199001379, - 0.022096098412293937, - 0.03048573801643215, - 0.037569552293280135, - 0.05408634888590313, - 0.06856599940219893 + 0.0025485639926046133, + 0.0026276092685293406, + 0.002766418919782154, + 0.003919153648894278, + 0.005883905396331093, + 0.007771545994910409, + 0.010690590104786672, + 0.013842354586813595 ], "conc_latency_avg_list": [ - 0.004456437869882418, - 0.005844962813409522, - 0.008378709673770517, - 0.01528081619265322, - 0.021239153420731547, - 0.026694518371955574, - 0.03784726188423443, - 0.04857833790231276 + 0.002334789800170702, + 0.0023967380922842478, + 0.002508067757169752, + 0.0030384944076291527, + 0.003962180586705604, + 0.004954530890871444, + 0.006679939886193658, + 0.00850275890671233 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1302,10 +677,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1313,11 +688,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 9, + "level": 6, "num_shards": 1 }, "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -1336,25 +711,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 5019.5973, - "serial_latency_p99": 0.0057, - "serial_latency_p95": 0.0054, - "recall": 0.994, - "ndcg": 0.9954, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 8320.4606, + "serial_latency_p99": 0.0029, + "serial_latency_p95": 0.0028, + "recall": 0.9919, + "ndcg": 0.9936, "conc_num_list": [ 1, 5, @@ -1366,44 +738,44 @@ 80 ], "conc_qps_list": [ - 222.5871, - 1150.5338, - 2195.8726, - 3676.709, - 4412.1372, - 4833.1625, - 5019.5973, - 5015.8037 + 396.8576, + 1911.4443, + 3643.8766, + 5864.3359, + 6725.025, + 7313.0633, + 7887.1755, + 8320.4606 ], "conc_latency_p99_list": [ - 0.005366289358644281, - 0.006616018440399769, - 0.0064373466832330474, - 0.010222766738734191, - 0.013315525980142408, - 0.016972876400686822, - 0.024728685971349477, - 0.031319626237964276 + 0.002857399091590195, + 0.0030318665952654557, + 0.003405487750424072, + 0.00554735084413551, + 0.00816060005221516, + 0.010382328977575536, + 0.014569980294909328, + 0.019027257570996884 ], "conc_latency_p95_list": [ - 0.00513816498714732, - 0.004792370549694169, - 0.005205206610844471, - 0.007212611548311542, - 0.010267410500091497, - 0.013238379993708808, - 0.01984079669928178, - 0.02578976150834933 + 0.002767942799255252, + 0.0028832201816840096, + 0.0030418253620155154, + 0.004497910931240768, + 0.006637068011332303, + 0.008489592577097936, + 0.011977570212911815, + 0.015408052015118301 ], "conc_latency_avg_list": [ - 0.00448758533933214, - 0.0043394632993677415, - 0.0045465428246546005, - 0.0054275771136581205, - 0.006782299052349466, - 0.008242341170482134, - 0.011840508222988514, - 0.015742522812380217 + 0.0025167798504936094, + 0.0026116185368739234, + 0.0027388090118584786, + 0.003400314036561817, + 0.0044390573829028, + 0.005437121205254614, + 0.0075142181236358555, + 0.009432022524963333 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1424,10 +796,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1435,7 +807,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 9, + "level": 7, "num_shards": 1 }, "case_config": { @@ -1458,25 +830,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 7364.0396, - "serial_latency_p99": 0.004, - "serial_latency_p95": 0.0037, - "recall": 0.9893, - "ndcg": 0.9915, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 7524.9879, + "serial_latency_p99": 0.0032, + "serial_latency_p95": 0.0031, + "recall": 0.9931, + "ndcg": 0.9946, "conc_num_list": [ 1, 5, @@ -1488,44 +857,44 @@ 80 ], "conc_qps_list": [ - 287.8688, - 1384.2897, - 2700.4491, - 4735.4478, - 5614.1579, - 6134.6648, - 6819.5071, - 7364.0396 + 363.7387, + 1761.1002, + 3372.7075, + 5421.2445, + 6091.3075, + 6579.0629, + 7061.7035, + 7524.9879 ], "conc_latency_p99_list": [ - 0.00390698241040809, - 0.00731195724976711, - 0.00495785990206058, - 0.00734390948899089, - 0.012288954856630872, - 0.014986682942835616, - 0.01849594444676764, - 0.022832338945008815 + 0.0031524766457732764, + 0.0032982631359482185, + 0.0036804132821271207, + 0.006010826830170116, + 0.008973520016297695, + 0.011436859995592378, + 0.016333586145192387, + 0.020569344094838005 ], "conc_latency_p95_list": [ - 0.0037214207914075814, - 0.003863213058502879, - 0.004057778001879342, - 0.005324349994771183, - 0.008072146945050915, - 0.010810172616038463, - 0.01463127658644225, - 0.01830196708324365 + 0.0030544937413651495, + 0.0031513541325693946, + 0.0033083077374612907, + 0.004909411718836053, + 0.007369707978796214, + 0.009455612045712769, + 0.013346620369702576, + 0.01678576849226374 ], "conc_latency_avg_list": [ - 0.0034694395949774774, - 0.003606888217707231, - 0.003696606332343799, - 0.004213763141467857, - 0.0053274846319189195, - 0.006490250213165499, - 0.008702013864526974, - 0.010708946041863319 + 0.0027460860977463145, + 0.0028345832649254296, + 0.0029595516874138823, + 0.003678043908615719, + 0.004902225199747265, + 0.006042755015168581, + 0.008396028243715591, + 0.010450248464720832 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1546,10 +915,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1557,7 +926,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 6, + "level": 8, "num_shards": 1 }, "case_config": { @@ -1580,25 +949,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 2724.9239, - "serial_latency_p99": 0.0124, - "serial_latency_p95": 0.0048, - "recall": 0.9811, - "ndcg": 0.9832, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 6813.2439, + "serial_latency_p99": 0.0035, + "serial_latency_p95": 0.0034, + "recall": 0.9939, + "ndcg": 0.9953, "conc_num_list": [ 1, 5, @@ -1610,44 +976,44 @@ 80 ], "conc_qps_list": [ - 269.3651, - 1047.0123, - 1498.6024, - 1847.6803, - 2114.199, - 2284.3302, - 2537.2552, - 2724.9239 + 336.5978, + 1642.6462, + 3138.7921, + 4969.8163, + 5673.7771, + 6090.0396, + 6493.0362, + 6813.2439 ], "conc_latency_p99_list": [ - 0.004742077292175964, - 0.00691745357704349, - 0.010720830453210511, - 0.01960868470487181, - 0.026337993171764537, - 0.032754160480981225, - 0.04255042473436333, - 0.05312397440138735 + 0.003431524743209593, + 0.003548497949959711, + 0.0039453561976552, + 0.006659876625053583, + 0.009535451157134956, + 0.012163680926314556, + 0.017726751818554484, + 0.022734694816172126 ], "conc_latency_p95_list": [ - 0.004494865804736036, - 0.0055081502971006556, - 0.007977579892030908, - 0.015719183007604443, - 0.022288659910555, - 0.02759612778027076, - 0.03666536140372045, - 0.04536134131485597 + 0.0033281981566688048, + 0.0034026612091111017, + 0.0035765988053753973, + 0.005437539250124246, + 0.007899306231411173, + 0.0101619676919654, + 0.014541133219609037, + 0.01846120802219957 ], "conc_latency_avg_list": [ - 0.0037082374061278584, - 0.004769060159688272, - 0.0066635344327552045, - 0.010804306792214618, - 0.014153154028666374, - 0.017440448752945776, - 0.02343149585687298, - 0.029001097442020812 + 0.0029676403704890024, + 0.0030391423281907995, + 0.003179950204961118, + 0.0040130850174019085, + 0.005263782041679775, + 0.006520409861910488, + 0.009132446429719516, + 0.011525998827399965 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1668,10 +1034,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1679,11 +1045,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 5, + "level": 9, "num_shards": 1 }, "case_config": { - "case_id": 4, + "case_id": 5, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -1702,25 +1068,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 5514.815, - "serial_latency_p99": 0.0042, - "serial_latency_p95": 0.0036, - "recall": 0.9286, - "ndcg": 0.9355, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 7385.2066, + "serial_latency_p99": 0.0021, + "serial_latency_p95": 0.002, + "recall": 0.9384, + "ndcg": 0.9441, "conc_num_list": [ 1, 5, @@ -1732,44 +1095,44 @@ 80 ], "conc_qps_list": [ - 313.799, - 1354.9874, - 2301.8011, - 3314.3036, - 3843.1259, - 4355.6117, - 5048.842, - 5514.815 + 505.8686, + 2189.7181, + 3241.0071, + 4661.5936, + 5380.3716, + 5932.9765, + 6767.3029, + 7385.2066 ], "conc_latency_p99_list": [ - 0.0038773450409644284, - 0.008138228004099801, - 0.010878445263078868, - 0.011540972657094244, - 0.016388589342823227, - 0.01858296279708156, - 0.023569957185536613, - 0.02886486930365208 + 0.0022006261744536458, + 0.002760201054625213, + 0.0037472858920227733, + 0.0067906589363701635, + 0.010014167174231262, + 0.012363597416551783, + 0.016403084830380974, + 0.01993567283730954 ], "conc_latency_p95_list": [ - 0.003412947052856907, - 0.004246345022693276, - 0.005220197608286979, - 0.008349375608668195, - 0.012340355810010787, - 0.0147909961087862, - 0.019336943980306387, - 0.023638575003133155 + 0.002098581724567339, + 0.002593909669667482, + 0.0035126500617479904, + 0.005520994003745727, + 0.008371637808158994, + 0.010467902664095164, + 0.01375423202989623, + 0.016782393981702625 ], "conc_latency_avg_list": [ - 0.003182328343818464, - 0.003684439145729171, - 0.004337632718561462, - 0.006021014814140765, - 0.007777482876747058, - 0.009140085848755274, - 0.011767476800046897, - 0.014305157378590626 + 0.001973905573358728, + 0.0022795015833306665, + 0.0030794282993742857, + 0.004278176924915009, + 0.005550848216261312, + 0.0066920424895898075, + 0.008768764746552707, + 0.010642102641863554 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1789,11 +1152,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1824,25 +1187,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 9901.1114, - "serial_latency_p99": 0.0039, - "serial_latency_p95": 0.0037, - "recall": 0.9385, - "ndcg": 0.9486, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 6793.8443, + "serial_latency_p99": 0.0022, + "serial_latency_p95": 0.0021, + "recall": 0.9522, + "ndcg": 0.9568, "conc_num_list": [ 1, 5, @@ -1854,44 +1214,44 @@ 80 ], "conc_qps_list": [ - 321.3357, - 1639.9924, - 3195.3018, - 5673.0951, - 6808.8856, - 7560.1348, - 9087.2499, - 9901.1114 + 503.9478, + 2114.1911, + 3099.2869, + 4416.7094, + 5150.037, + 5691.2226, + 6289.7121, + 6793.8443 ], "conc_latency_p99_list": [ - 0.00394074416602971, - 0.0041782407171558535, - 0.004392879804945551, - 0.010109323544893349, - 0.008123845955124116, - 0.014126944777672188, - 0.014278251143987291, - 0.01781155434960963 + 0.00215859985910356, + 0.002847761964658275, + 0.003923368623363787, + 0.007356263077235778, + 0.010528954989276825, + 0.01292447085143066, + 0.01769668879453093, + 0.021927826073952026 ], "conc_latency_p95_list": [ - 0.0034882987965829666, - 0.0035166182147804642, - 0.0035458351092529476, - 0.004310413988423532, - 0.006210639532946515, - 0.008767930739850271, - 0.010931958199944346, - 0.013749658003507645 + 0.002082229100051336, + 0.0026818424608791246, + 0.0036669230728875847, + 0.005921932504861616, + 0.00885509350337088, + 0.010938107722904525, + 0.014863643678836526, + 0.018132617010269306 ], "conc_latency_avg_list": [ - 0.0031001682930926035, - 0.0030430930397005885, - 0.0031233191875315565, - 0.0035155999207049896, - 0.004389041533959679, - 0.005263586471416786, - 0.006530556363972054, - 0.007958968690030266 + 0.001981333490019148, + 0.002360769086835183, + 0.003220095266754697, + 0.004516460962995695, + 0.005803644141310085, + 0.006980248662568344, + 0.009430996778203294, + 0.011560318047614999 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1912,10 +1272,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -1923,11 +1283,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 1, + "level": 2, "num_shards": 1 }, "case_config": { - "case_id": 5, + "case_id": 4, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -1946,25 +1306,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 3258.8508, - "serial_latency_p99": 0.0117, - "serial_latency_p95": 0.0045, - "recall": 0.9888, - "ndcg": 0.9906, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 6242.5346, + "serial_latency_p99": 0.0023, + "serial_latency_p95": 0.0022, + "recall": 0.961, + "ndcg": 0.965, "conc_num_list": [ 1, 5, @@ -1976,44 +1333,44 @@ 80 ], "conc_qps_list": [ - 244.7738, - 1130.2346, - 1809.9431, - 2312.2265, - 2588.2339, - 2788.0239, - 3069.518, - 3258.8508 + 446.4866, + 2002.9326, + 2953.3925, + 4164.4285, + 4805.5397, + 5240.789, + 5870.8363, + 6242.5346 ], "conc_latency_p99_list": [ - 0.005137932703364641, - 0.00555925140972249, - 0.008915685693500566, - 0.016561121140257453, - 0.022867804747074828, - 0.02806737951235845, - 0.036439670615363844, - 0.04337102690333264 + 0.002856350458459926, + 0.0029789366439217715, + 0.004114359063096344, + 0.007887981488602236, + 0.011494717993773522, + 0.014308837521821265, + 0.019093695696210486, + 0.023705490870634095 ], "conc_latency_p95_list": [ - 0.004440846845682244, - 0.005053811200195923, - 0.0068235130165703595, - 0.01288393942813854, - 0.018162307806778695, - 0.02236156941507943, - 0.0296233788743848, - 0.03584610429388702 + 0.0025365250825416293, + 0.0028055029979441315, + 0.0038476928253658115, + 0.0064046091894852, + 0.009612656582612544, + 0.012057665473548695, + 0.01607741924817674, + 0.019808521203231066 ], "conc_latency_avg_list": [ - 0.004080840300378187, - 0.004417934321996714, - 0.005516924373879563, - 0.008634555842136784, - 0.011560621919641356, - 0.014278915373578093, - 0.019378262406752788, - 0.024243179517198447 + 0.0022367817378751067, + 0.0024923140599496687, + 0.003379658321121232, + 0.0047897960603506306, + 0.0062167658101995584, + 0.007583969576561268, + 0.010102243992110408, + 0.012617357519441232 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2033,11 +1390,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2045,7 +1402,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 8, + "level": 3, "num_shards": 1 }, "case_config": { @@ -2068,25 +1425,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 270.1267, - "optimize_duration": 437.4658, - "load_duration": 707.5925, - "qps": 5907.441, - "serial_latency_p99": 0.0057, - "serial_latency_p95": 0.0054, - "recall": 0.9931, - "ndcg": 0.9946, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5779.119, + "serial_latency_p99": 0.0023, + "serial_latency_p95": 0.0023, + "recall": 0.971, + "ndcg": 0.9742, "conc_num_list": [ 1, 5, @@ -2098,44 +1452,44 @@ 80 ], "conc_qps_list": [ - 214.7436, - 1124.1591, - 2263.1224, - 3695.1435, - 4635.8817, - 5035.3873, - 5614.0659, - 5907.441 + 476.6126, + 1920.6379, + 2828.6202, + 3905.7911, + 4495.2986, + 4903.9634, + 5352.3634, + 5779.119 ], "conc_latency_p99_list": [ - 0.005838954218779679, - 0.009590839385055032, - 0.0072934928326867585, - 0.009990225688670754, - 0.013466239573317574, - 0.01589230435347418, - 0.021308113009436094, - 0.026848825681372538 + 0.002284108918393031, + 0.003114189200568944, + 0.004329544047359377, + 0.008613102727103977, + 0.012333529423922296, + 0.015281949284835726, + 0.020651346329832433, + 0.025012934936676175 ], "conc_latency_p95_list": [ - 0.005094571305380669, - 0.0048950490017887205, - 0.00495702201151289, - 0.007446101757523138, - 0.009852354813483545, - 0.01273485799174523, - 0.017310541804181415, - 0.021899056984693743 + 0.0022121331770904363, + 0.0029337720014154913, + 0.004045496415346861, + 0.007010379375424236, + 0.010350381198804826, + 0.012867126709898001, + 0.01739263068011496, + 0.021097359794657676 ], "conc_latency_avg_list": [ - 0.004651503297485146, - 0.004441934131452666, - 0.004411218431154767, - 0.005397841813638808, - 0.00645052096323231, - 0.007903104976167643, - 0.010583279479885773, - 0.013354355155792764 + 0.002095217670483641, + 0.0025990775656036295, + 0.003529053754031682, + 0.005105319901460065, + 0.006648317818211828, + 0.008106390568584667, + 0.011082872888195554, + 0.01359851345251022 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2156,10 +1510,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2167,11 +1521,11 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 8, + "level": 4, "num_shards": 1 }, "case_config": { - "case_id": 5, + "case_id": 4, "custom_case": {}, "k": 100, "concurrency_search_config": { @@ -2190,25 +1544,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 5064.6982, - "serial_latency_p99": 0.0043, - "serial_latency_p95": 0.0036, - "recall": 0.9558, - "ndcg": 0.9606, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 5183.5843, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0023, + "recall": 0.9784, + "ndcg": 0.981, "conc_num_list": [ 1, 5, @@ -2220,44 +1571,44 @@ 80 ], "conc_qps_list": [ - 306.5362, - 1368.7188, - 2310.9627, - 3255.5776, - 3791.6067, - 4073.62, - 4572.4511, - 5064.6982 + 456.5329, + 1821.263, + 2677.2938, + 3617.961, + 4097.4679, + 4447.7515, + 4863.2316, + 5183.5843 ], "conc_latency_p99_list": [ - 0.0042355662427144124, - 0.0046483404963510114, - 0.009778717628214485, - 0.010217842382844546, - 0.015881566194002526, - 0.01989235390967224, - 0.025938013812992735, - 0.031185232510324568 + 0.002395828692242503, + 0.0033142844232497736, + 0.004636971018044278, + 0.009431733234669086, + 0.013644934630719942, + 0.01654091664124283, + 0.022408935175044466, + 0.027210572804324325 ], "conc_latency_p95_list": [ - 0.0034556519065517934, - 0.004072950512636453, - 0.005186801168019884, - 0.008326760004274545, - 0.01228064175666077, - 0.016202768517541696, - 0.021469047002028674, - 0.025539752503391355 + 0.00232212619157508, + 0.0031105756846955047, + 0.0043217132915742695, + 0.007749295464600434, + 0.011374025803525001, + 0.01404695180244743, + 0.01901522108237258, + 0.022979849949479103 ], "conc_latency_avg_list": [ - 0.003258234978368169, - 0.0036474964905585366, - 0.004320538177772956, - 0.006130977769455615, - 0.007887396918013946, - 0.009779561744223083, - 0.013001510996033932, - 0.015590046073741476 + 0.0021874608272499383, + 0.0027410038616561545, + 0.003728350102575746, + 0.005515110815448404, + 0.0072920914670729624, + 0.008931282929280606, + 0.012187039287903855, + 0.015173242426486785 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2277,11 +1628,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2289,7 +1640,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 3, + "level": 5, "num_shards": 1 }, "case_config": { @@ -2312,25 +1663,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 3468.5933, - "serial_latency_p99": 0.0045, - "serial_latency_p95": 0.0043, - "recall": 0.9634, - "ndcg": 0.9661, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 4259.5572, + "serial_latency_p99": 0.0026, + "serial_latency_p95": 0.0026, + "recall": 0.9845, + "ndcg": 0.9866, "conc_num_list": [ 1, 5, @@ -2342,44 +1690,44 @@ 80 ], "conc_qps_list": [ - 274.7753, - 1086.6444, - 1657.8469, - 2230.3556, - 2576.5151, - 2840.9102, - 3207.9495, - 3468.5933 + 424.077, + 1676.9478, + 2434.6516, + 3184.2011, + 3571.9199, + 3767.5544, + 4061.1698, + 4259.5572 ], "conc_latency_p99_list": [ - 0.007525063498178497, - 0.00871218444284751, - 0.009115133894665613, - 0.01669143169856398, - 0.021790961647639086, - 0.026465490460395806, - 0.03498110753978835, - 0.042080024706956466 + 0.002595295326318592, + 0.003621646617539228, + 0.005367588647641237, + 0.011171487247338519, + 0.0155316284415312, + 0.019644083841703817, + 0.02703630503267048, + 0.03310978163208347 ], "conc_latency_p95_list": [ - 0.004381819497211836, - 0.00529403816035483, - 0.007012556104746182, - 0.01246311155118746, - 0.018361228803405537, - 0.022542869101744148, - 0.029249506900669076, - 0.03583758800959913 + 0.002518506458727643, + 0.0034135113819502294, + 0.004873151995707303, + 0.009142277223872952, + 0.013012332818470895, + 0.016382047021761527, + 0.022542749025160444, + 0.027544378107995725 ], "conc_latency_avg_list": [ - 0.003635002963577244, - 0.0045950829257419635, - 0.006023093199217258, - 0.008949508158072033, - 0.011605007790743322, - 0.014003128689967868, - 0.018541925579661857, - 0.02278122412115334 + 0.00235500177392571, + 0.0029770619258879757, + 0.00410082317131261, + 0.006266042699984648, + 0.008367804647869575, + 0.010561791085210772, + 0.01460435324245386, + 0.018471206884305532 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2400,10 +1748,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2411,7 +1759,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 3, + "level": 6, "num_shards": 1 }, "case_config": { @@ -2434,25 +1782,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 3205.4829, - "optimize_duration": 1275.7844, - "load_duration": 4481.2673, - "qps": 2401.3204, - "serial_latency_p99": 0.0107, - "serial_latency_p95": 0.0047, - "recall": 0.9866, - "ndcg": 0.9884, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3614.3118, + "serial_latency_p99": 0.0029, + "serial_latency_p95": 0.0028, + "recall": 0.9877, + "ndcg": 0.9894, "conc_num_list": [ 1, 5, @@ -2464,44 +1809,44 @@ 80 ], "conc_qps_list": [ - 261.8215, - 985.1764, - 1468.0087, - 1749.415, - 1931.5307, - 2090.0742, - 2278.9704, - 2401.3204 + 397.2298, + 1557.1023, + 2236.3239, + 2845.1058, + 3146.0016, + 3289.0762, + 3520.8116, + 3614.3118 ], "conc_latency_p99_list": [ - 0.004836413672601339, - 0.009077261193306102, - 0.010354967679304536, - 0.02026946535654133, - 0.029030042108206543, - 0.03499512374750339, - 0.046797644338803394, - 0.057348916197370266 + 0.0027946092444472016, + 0.003967489110655152, + 0.00615017178060953, + 0.0128463427053066, + 0.017453623053152116, + 0.022870610732934445, + 0.029518100013956432, + 0.03869990034552756 ], "conc_latency_p95_list": [ - 0.004629465389007237, - 0.0058674284955486655, - 0.0082442566199461, - 0.01657543244800763, - 0.02401039500546176, - 0.029254749882966277, - 0.03907370918313973, - 0.04936321394779952 + 0.0027128490241011606, + 0.003724151346250437, + 0.0054281810007523745, + 0.01044328572170343, + 0.01436160156154074, + 0.01821845376980491, + 0.024413908016867936, + 0.031903167749987915 ], "conc_latency_avg_list": [ - 0.0038149669483554033, - 0.00506845096702024, - 0.006802651714890116, - 0.01140782706843602, - 0.015489598411640663, - 0.019035290587860968, - 0.02608886756739919, - 0.03292382999700411 + 0.0025143936207342324, + 0.003206745084255346, + 0.004464954355791834, + 0.007013879446545796, + 0.009503234791544602, + 0.012096279585844553, + 0.01685092675726034, + 0.021781507836163158 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2522,10 +1867,10 @@ "db": "ZillizCloud", "db_config": { "db_label": "8cu-perf", - "version": "v2026.1", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2533,7 +1878,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 6, + "level": 7, "num_shards": 1 }, "case_config": { @@ -2556,25 +1901,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 3568.396, - "serial_latency_p99": 0.0048, - "serial_latency_p95": 0.0043, - "recall": 0.9863, - "ndcg": 0.9883, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3181.0537, + "serial_latency_p99": 0.003, + "serial_latency_p95": 0.0029, + "recall": 0.9892, + "ndcg": 0.9908, "conc_num_list": [ 1, 5, @@ -2586,44 +1928,44 @@ 80 ], "conc_qps_list": [ - 249.5646, - 1116.5066, - 1828.4446, - 2389.4823, - 2645.8957, - 2879.4104, - 3356.3991, - 3568.396 + 373.2168, + 1461.7612, + 2063.4765, + 2583.7979, + 2781.1167, + 2908.4205, + 3073.9858, + 3181.0537 ], "conc_latency_p99_list": [ - 0.004949838446336795, - 0.010793446648749472, - 0.011542035994352773, - 0.015714827887131834, - 0.023098365282639866, - 0.028270509486901574, - 0.03385155899741221, - 0.0405952908913605 + 0.003002641891944222, + 0.004314229479641651, + 0.007060362810152582, + 0.01453221907839178, + 0.01971446685201954, + 0.024677592392545203, + 0.032532848112750784, + 0.04069755139702464 ], "conc_latency_p95_list": [ - 0.004326387906621675, - 0.005205007104086689, - 0.0069263952391338535, - 0.012388522701803593, - 0.01838434826204321, - 0.022779258753871545, - 0.02818173549894709, - 0.034205201656732236 + 0.0029138700163457544, + 0.00403262600011658, + 0.0060169674427015705, + 0.011576333804987366, + 0.01587381720310077, + 0.019931912398897106, + 0.027404746599495412, + 0.03406435703218449 ], "conc_latency_avg_list": [ - 0.004002170059550175, - 0.0044719544014582705, - 0.005461118931996274, - 0.008354930265743018, - 0.011307457646718058, - 0.013821107916470069, - 0.01770926681597559, - 0.022161158532467397 + 0.002676176823972926, + 0.0034160777606854604, + 0.004838368703871575, + 0.0077237947842559935, + 0.010749740541723023, + 0.013688260722494459, + 0.01930447747175536, + 0.024747977200758154 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2643,11 +1985,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2655,7 +1997,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 7, + "level": 8, "num_shards": 1 }, "case_config": { @@ -2678,25 +2020,22 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } }, { "metrics": { "max_load_count": 0, - "insert_duration": 2923.0628, - "optimize_duration": 2272.8723, - "load_duration": 5195.935, - "qps": 4674.1861, - "serial_latency_p99": 0.0045, - "serial_latency_p95": 0.0038, - "recall": 0.967, - "ndcg": 0.9705, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2804.8357, + "serial_latency_p99": 0.0033, + "serial_latency_p95": 0.0032, + "recall": 0.9903, + "ndcg": 0.9918, "conc_num_list": [ 1, 5, @@ -2708,44 +2047,44 @@ 80 ], "conc_qps_list": [ - 299.441, - 1293.8246, - 2204.8398, - 2998.8928, - 3416.6029, - 3814.1905, - 4403.9058, - 4674.1861 + 352.7729, + 1367.945, + 1917.0391, + 2338.0313, + 2510.7421, + 2617.9646, + 2727.8716, + 2804.8357 ], "conc_latency_p99_list": [ - 0.00402290889178403, - 0.007907517027342683, - 0.006291847964457707, - 0.013353673194069417, - 0.017164103323593728, - 0.021020031592343, - 0.027193114216788664, - 0.033575675918255006 + 0.00323100520123262, + 0.004680905669229106, + 0.00799012375646271, + 0.01590309410821647, + 0.021047461275593433, + 0.025625948917586353, + 0.036803203278686886, + 0.0467973963287659 ], "conc_latency_p95_list": [ - 0.00356991050648503, - 0.004438751634734216, - 0.005453104941989295, - 0.00961559300776571, - 0.013857692398596555, - 0.017230113997356966, - 0.02220476679212879, - 0.02798210658947937 + 0.0031083543435670435, + 0.0043611170200165365, + 0.0067188970482675355, + 0.0125031745119486, + 0.017026124306721607, + 0.021185590000823137, + 0.03006786260521039, + 0.03819325279910117 ], "conc_latency_avg_list": [ - 0.003335578729612294, - 0.0038557001362392825, - 0.0045284253144977004, - 0.006655331635103421, - 0.008756953741340391, - 0.010436090944337196, - 0.013487455343736422, - 0.016883584811310505 + 0.002831514758067566, + 0.003649909442997153, + 0.00520858526610239, + 0.00853725560951756, + 0.011906267087115343, + 0.015190065559178532, + 0.021789805178996344, + 0.0280490530042168 ], "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -2765,11 +2104,11 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "8cu-perf-force_merge", - "version": "v2026.1", + "db_label": "8cu-perf", + "version": "v2026.4", "note": "", "uri": "**********", - "user": "db_admin", + "user": "root", "password": "**********", "collection_name": "ZillizCloudVDBBench" }, @@ -2777,7 +2116,7 @@ "index": "AUTOINDEX", "metric_type": "COSINE", "use_partition_key": false, - "level": 4, + "level": 9, "num_shards": 1 }, "case_config": { @@ -2800,13 +2139,10 @@ } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ] - }, - "label": ":)" + } } ], "file_fmt": "result_{}_{}_{}.json", diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json index bbc1e918b..c987ac9e7 100644 --- a/vectordb_bench/results/leaderboard_v2.json +++ b/vectordb_bench/results/leaderboard_v2.json @@ -1,2952 +1,2892 @@ [ - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1146.5286, - "latency": 13.7, - "recall": 0.9262, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1148.1735, - "latency": 8.9, - "recall": 0.9801, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1149.1219, - "latency": 10.3, - "recall": 0.9764, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1140.4099, - "latency": 13.5, - "recall": 0.9716, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1123.5147, - "latency": 18.5, - "recall": 0.9688, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 487.8343, - "latency": 25.4, - "recall": 0.9668, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 264.9324, - "latency": 49.6, - "recall": 0.936, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 492.4887, - "latency": 29.6, - "recall": 0.9269, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 823.1775, - "latency": 20.5, - "recall": 0.9148, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1147.1977, - "latency": 13.3, - "recall": 0.8999, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1131.3087, - "latency": 14.1, - "recall": 0.9024, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1114.952, - "latency": 12.7, - "recall": 0.97, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 583.5009, - "latency": 23.0, - "recall": 0.9668, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 31.4779, - "latency": 351.0, - "recall": 0.9414, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 57.8988, - "latency": 200.1, - "recall": 0.9332, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 101.1774, - "latency": 116.1, - "recall": 0.9241, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 212.7466, - "latency": 58.7, - "recall": 0.9099, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 372.2462, - "latency": 35.9, - "recall": 0.8977, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 617.0881, - "latency": 22.4, - "recall": 0.8844, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Pinecone", - "label": "p2.x8-1node", - "db_name": "Pinecone-p2.x8-1node", - "qps": 1094.5967, - "latency": 14.3, - "recall": 0.8659, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 4318.9697, - "latency": 4.3, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 4250.2894, - "latency": 4.6, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 2997.4391, - "latency": 6.1, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1494.5334, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1108.6473, - "latency": 7.4, - "recall": 0.995, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1289.5164, - "latency": 6.4, - "recall": 0.9906, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1059.3394, - "latency": 7.8, - "recall": 0.9856, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 987.0795, - "latency": 7.1, - "recall": 0.9804, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1591.7055, - "latency": 7.8, - "recall": 0.8506, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 1202.8677, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 639.3991, - "latency": 7.3, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 274.8559, - "latency": 9.9, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 441.4152, - "latency": 8.3, - "recall": 0.997, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 358.8949, - "latency": 9.5, - "recall": 0.995, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 325.2245, - "latency": 10.3, - "recall": 0.9909, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 273.4174, - "latency": 13.3, - "recall": 0.9789, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 262.8314, - "latency": 11.3, - "recall": 0.9808, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g-payload-index", - "db_name": "QdrantCloud-16c64g-payload-index", - "qps": 434.5481, - "latency": 8.5, - "recall": 0.7237, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 446.9116, - "latency": 9.2, - "recall": 0.9357, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 388.3028, - "latency": 9.6, - "recall": 0.9431, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 323.3964, - "latency": 9.8, - "recall": 0.9507, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 256.4668, - "latency": 11.3, - "recall": 0.9588, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 145.5316, - "latency": 18.4, - "recall": 0.9726, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 1242.428, - "latency": 6.4, - "recall": 0.9474, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 1111.3633, - "latency": 7.0, - "recall": 0.955, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 955.4701, - "latency": 7.2, - "recall": 0.9629, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 783.5207, - "latency": 7.7, - "recall": 0.971, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "QdrantCloud", - "label": "16c64g", - "db_name": "QdrantCloud-16c64g", - "qps": 470.8546, - "latency": 9.5, - "recall": 0.9835, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 950.6332, - "latency": 13.2, - "recall": 0.914, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 823.2224, - "latency": 13.5, - "recall": 0.9434, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 743.9815, - "latency": 14.8, - "recall": 0.9583, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 683.1873, - "latency": 15.7, - "recall": 0.9677, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 619.7468, - "latency": 17.2, - "recall": 0.9738, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 537.4082, - "latency": 18.8, - "recall": 0.9809, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 474.9941, - "latency": 20.9, - "recall": 0.9848, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 505.7458, - "latency": 20.7, - "recall": 0.9068, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 433.9034, - "latency": 23.1, - "recall": 0.931, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 381.7737, - "latency": 25.7, - "recall": 0.9431, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 342.1123, - "latency": 29.0, - "recall": 0.951, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 308.2216, - "latency": 31.3, - "recall": 0.9561, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 257.7928, - "latency": 36.4, - "recall": 0.9626, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g", - "db_name": "OpenSearch-16c128g", - "qps": 223.8166, - "latency": 42.1, - "recall": 0.9666, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3055.0123, - "latency": 7.2, - "recall": 0.9066, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3013.4439, - "latency": 6.9, - "recall": 0.9268, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2801.7241, - "latency": 7.4, - "recall": 0.9476, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2590.3809, - "latency": 8.6, - "recall": 0.9679, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2291.2159, - "latency": 8.9, - "recall": 0.9764, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3099.4124, - "latency": 6.2, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 3014.2483, - "latency": 7.0, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2073.2153, - "latency": 11.0, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1507.6899, - "latency": 12.8, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 942.2296, - "latency": 18.2, - "recall": 1.0, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 677.1414, - "latency": 33.5, - "recall": 0.7655, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2685.6654, - "latency": 7.6, - "recall": 0.4914, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2604.4444, - "latency": 7.8, - "recall": 0.63, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 2159.051, - "latency": 9.4, - "recall": 0.801, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2251.1274, - "latency": 8.7, - "recall": 0.8848, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3103.0539, - "latency": 5.6, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3086.1957, - "latency": 6.7, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3090.0478, - "latency": 6.4, - "recall": 0.9628, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3064.6288, - "latency": 6.5, - "recall": 0.9507, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3065.6134, - "latency": 6.2, - "recall": 0.9328, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3028.858, - "latency": 6.7, - "recall": 0.9133, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2935.9403, - "latency": 6.8, - "recall": 0.8992, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2771.2009, - "latency": 7.6, - "recall": 0.889, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1610.9496, - "latency": 10.8, - "recall": 0.9, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1557.3623, - "latency": 10.8, - "recall": 0.9244, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1473.9256, - "latency": 11.7, - "recall": 0.9484, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1388.5547, - "latency": 12.5, - "recall": 0.9597, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1022.2696, - "latency": 17.9, - "recall": 0.936, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 696.9777, - "latency": 24.6, - "recall": 0.997, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 353.7862, - "latency": 45.2, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 210.3227, - "latency": 71.4, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 114.8061, - "latency": 126.6, - "recall": 0.9985, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 504.9179, - "latency": 272.6, - "recall": 0.4664, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 1053.1495, - "latency": 17.7, - "recall": 0.5673, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 808.3294, - "latency": 22.2, - "recall": 0.7016, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-force_merge", - "db_name": "OpenSearch-16c128g-force_merge", - "qps": 8.0584, - "latency": 1757.9, - "recall": 1.0, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 3033.5491, - "latency": 6.4, - "recall": 0.9844, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2988.4205, - "latency": 7.6, - "recall": 0.9741, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2950.717, - "latency": 6.9, - "recall": 0.9558, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2782.0274, - "latency": 7.4, - "recall": 0.9466, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2708.6752, - "latency": 8.4, - "recall": 0.9337, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 2275.2854, - "latency": 9.1, - "recall": 0.917, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 1844.8918, - "latency": 10.6, - "recall": 0.9085, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 1301.4102, - "latency": 14.7, - "recall": 0.9011, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "OpenSearch", - "label": "16c128g-routing-64shard-force_merge", - "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", - "qps": 13.0379, - "latency": 1063.5, - "recall": 1.0, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.4972, - "latency": 337.1, - "recall": 0.8717, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 192.1164, - "latency": 345.6, - "recall": 0.4276, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 197.4455, - "latency": 349.3, - "recall": 0.5314, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 196.9391, - "latency": 263.4, - "recall": 0.6549, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 201.5401, - "latency": 282.4, - "recall": 0.7086, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 202.2424, - "latency": 301.7, - "recall": 0.7592, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 198.599, - "latency": 358.8, - "recall": 0.8085, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.0349, - "latency": 275.3, - "recall": 0.8325, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 202.1405, - "latency": 282.6, - "recall": 0.8492, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 201.1282, - "latency": 269.2, - "recall": 0.8637, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 194.8021, - "latency": 559.8, - "recall": 0.86, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 187.4268, - "latency": 453.7, - "recall": 0.4692, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 198.397, - "latency": 506.9, - "recall": 0.5409, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 174.3549, - "latency": 496.9, - "recall": 0.6279, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 172.95, - "latency": 515.6, - "recall": 0.7004, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 190.9747, - "latency": 517.4, - "recall": 0.7398, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 186.0237, - "latency": 474.0, - "recall": 0.7847, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 192.1458, - "latency": 480.5, - "recall": 0.8103, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 179.4203, - "latency": 497.5, - "recall": 0.8273, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "S3Vectors", - "label": "", - "db_name": "S3Vectors", - "qps": 199.5444, - "latency": 463.9, - "recall": 0.8478, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 346.5847, - "latency": 42.7, - "recall": 0.9631, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 369.4921, - "latency": 41.6, - "recall": 0.779, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 310.957, - "latency": 49.4, - "recall": 0.9698, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 798.328, - "latency": 56.7, - "recall": 0.8993, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 649.8781, - "latency": 55.2, - "recall": 0.8352, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 370.7241, - "latency": 49.6, - "recall": 0.7177, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 100.0554, - "latency": 69.3, - "recall": 0.9638, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 284.6367, - "latency": 47.6, - "recall": 0.9788, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 81.8678, - "latency": 105.6, - "recall": 0.8751, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 260.4031, - "latency": 48.3, - "recall": 0.9828, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 365.2505, - "latency": 34.9, - "recall": 0.8251, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 471.553, - "latency": 44.1, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 91.8612, - "latency": 85.7, - "recall": 0.8799, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 206.0934, - "latency": 56.7, - "recall": 0.9795, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 351.7114, - "latency": 46.7, - "recall": 0.8735, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 96.592, - "latency": 76.9, - "recall": 0.9178, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 802.6923, - "latency": 48.1, - "recall": 0.935, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 184.5363, - "latency": 53.4, - "recall": 0.9681, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 323.0238, - "latency": 50.4, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "TurboPuffer", - "label": "2026-03-31", - "db_name": "TurboPuffer", - "qps": 382.5332, - "latency": 54.7, - "recall": 0.6135, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3917.2035, - "latency": 2.4, - "recall": 0.9203, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3628.8527, - "latency": 2.6, - "recall": 0.9318, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 3250.1112, - "latency": 2.7, - "recall": 0.9443, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2762.4144, - "latency": 3.1, - "recall": 0.9556, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2384.6245, - "latency": 3.2, - "recall": 0.9627, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 2134.1717, - "latency": 3.8, - "recall": 0.9671, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 1641.3478, - "latency": 4.1, - "recall": 0.9729, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 1488.5841, - "latency": 4.7, - "recall": 0.9764, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2747.3167, - "latency": 3.3, - "recall": 0.9204, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2514.4481, - "latency": 3.2, - "recall": 0.9303, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2177.2345, - "latency": 3.4, - "recall": 0.9408, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1833.2575, - "latency": 3.9, - "recall": 0.951, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1552.4803, - "latency": 4.0, - "recall": 0.9565, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1355.3121, - "latency": 4.4, - "recall": 0.9602, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 1079.2123, - "latency": 5.3, - "recall": 0.9648, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 876.5772, - "latency": 6.3, - "recall": 0.9676, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 10663.1231, - "latency": 2.0, - "recall": 0.8405, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 10333.9072, - "latency": 2.0, - "recall": 0.889, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 9575.6863, - "latency": 2.3, - "recall": 0.9189, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 8596.7694, - "latency": 2.4, - "recall": 0.9416, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 7704.3625, - "latency": 2.7, - "recall": 0.9541, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 7023.6735, - "latency": 3.0, - "recall": 0.962, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 6031.3725, - "latency": 3.3, - "recall": 0.971, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq4u-fp16-force_merge", - "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", - "qps": 5258.1868, - "latency": 3.6, - "recall": 0.9768, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 5973.0024, - "latency": 2.4, - "recall": 0.9192, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 5416.5758, - "latency": 2.6, - "recall": 0.9334, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 4771.4324, - "latency": 2.8, - "recall": 0.9479, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 4006.3994, - "latency": 3.2, - "recall": 0.9609, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 3441.7597, - "latency": 3.5, - "recall": 0.9682, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 3040.6216, - "latency": 3.7, - "recall": 0.9734, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2446.7373, - "latency": 4.3, - "recall": 0.9791, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-force_merge", - "db_name": "Milvus-16c64g-sq8-force_merge", - "qps": 2084.6245, - "latency": 5.0, - "recall": 0.9819, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11763.5538, - "latency": 1.5, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11803.1944, - "latency": 1.5, - "recall": 0.9778, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11520.9234, - "latency": 1.5, - "recall": 0.9634, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11280.0849, - "latency": 1.6, - "recall": 0.9507, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10671.8925, - "latency": 1.7, - "recall": 0.9339, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10258.2661, - "latency": 1.7, - "recall": 0.9139, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 9681.5656, - "latency": 1.9, - "recall": 0.9008, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 8945.4041, - "latency": 1.9, - "recall": 0.8894, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 5436.8907, - "latency": 2.0, - "recall": 0.929, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 11397.7043, - "latency": 1.6, - "recall": 0.9597, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10891.7531, - "latency": 1.7, - "recall": 0.9408, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 10276.7451, - "latency": 1.7, - "recall": 0.9159, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 9664.2855, - "latency": 1.8, - "recall": 0.899, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 8936.7962, - "latency": 2.0, - "recall": 0.8835, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 5671.2562, - "latency": 2.1, - "recall": 0.903, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 3157.707, - "latency": 2.3, - "recall": 0.9347, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 1985.8124, - "latency": 2.6, - "recall": 0.9407, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "Milvus", - "label": "16c64g-sq8-partition_key", - "db_name": "Milvus-16c64g-sq8-partition_key", - "qps": 920.9627, - "latency": 3.4, - "recall": 0.9488, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 3033.786, - "latency": 8.7, - "recall": 0.9934, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 3019.2416, - "latency": 9.5, - "recall": 0.9765, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2890.9523, - "latency": 9.4, - "recall": 0.9625, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2789.7212, - "latency": 8.2, - "recall": 0.9538, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2457.2628, - "latency": 9.0, - "recall": 0.9378, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 2209.4973, - "latency": 13.7, - "recall": 0.9228, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1960.388, - "latency": 11.0, - "recall": 0.9076, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1725.092, - "latency": 11.7, - "recall": 0.8969, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-routing-64shard", - "db_name": "ElasticCloud-8c60g-routing-64shard", - "qps": 1307.419, - "latency": 12.3, - "recall": 0.8925, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 350.0132, - "latency": 29.7, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 179.5204, - "latency": 51.4, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 72.99, - "latency": 111.4, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 42.9877, - "latency": 201.9, - "recall": 0.9912, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 96.4987, - "latency": 113.1, - "recall": 0.9296, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 189.3789, - "latency": 58.8, - "recall": 0.9149, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 246.7071, - "latency": 45.1, - "recall": 0.9018, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 229.0379, - "latency": 43.0, - "recall": 0.8908, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 125.6164, - "latency": 69.8, - "recall": 0.8746, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2175.2694, - "latency": 9.8, - "recall": 1.0, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1430.0244, - "latency": 12.6, - "recall": 1.0, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 692.5751, - "latency": 18.7, - "recall": 1.0, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 364.3516, - "latency": 26.4, - "recall": 1.0, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 190.3777, - "latency": 47.9, - "recall": 1.0, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 249.3519, - "latency": 44.7, - "recall": 0.9446, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 437.8735, - "latency": 27.1, - "recall": 0.9364, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 669.9441, - "latency": 19.1, - "recall": 0.9227, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 899.3114, - "latency": 14.9, - "recall": 0.9072, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2030.4249, - "latency": 10.6, - "recall": 0.9306, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1804.8996, - "latency": 12.3, - "recall": 0.9405, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2353.8935, - "latency": 17.1, - "recall": 0.9143, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1623.8421, - "latency": 11.8, - "recall": 0.9479, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2808.2421, - "latency": 9.5, - "recall": 0.8815, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1482.3772, - "latency": 12.1, - "recall": 0.9546, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1721.5416, - "latency": 9.6, - "recall": 0.8855, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1032.9696, - "latency": 14.8, - "recall": 0.933, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1150.4393, - "latency": 13.4, - "recall": 0.9265, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1452.1536, - "latency": 10.8, - "recall": 0.9042, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 2181.3939, - "latency": 9.4, - "recall": 0.8501, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ElasticCloud", - "label": "8c60g-force_merge", - "db_name": "ElasticCloud-8c60g-force_merge", - "qps": 1295.5543, - "latency": 11.2, - "recall": 0.9176, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9773.6593, - "latency": 3.7, - "recall": 0.9955, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9081.1518, - "latency": 3.0, - "recall": 0.9943, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8455.2896, - "latency": 4.0, - "recall": 0.9921, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7610.0519, - "latency": 3.3, - "recall": 0.9903, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7589.664, - "latency": 3.8, - "recall": 0.9235, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6750.2495, - "latency": 4.4, - "recall": 0.9105, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5506.1808, - "latency": 5.5, - "recall": 0.9193, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6860.8577, - "latency": 4.7, - "recall": 0.9226, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8468.4611, - "latency": 3.1, - "recall": 0.8925, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10089.4308, - "latency": 2.6, - "recall": 0.9934, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10557.4373, - "latency": 2.7, - "recall": 0.9393, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9805.0401, - "latency": 2.6, - "recall": 0.9257, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10020.5299, - "latency": 2.6, - "recall": 0.9788, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 10041.0338, - "latency": 2.7, - "recall": 0.9693, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9861.9686, - "latency": 2.6, - "recall": 0.955, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9507.9991, - "latency": 2.8, - "recall": 0.9453, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9428.4531, - "latency": 2.6, - "recall": 0.9331, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9048.6431, - "latency": 3.9, - "recall": 0.9216, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 8695.2765, - "latency": 4.3, - "recall": 0.9603, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9244.1135, - "latency": 4.2, - "recall": 0.9724, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9289.0118, - "latency": 4.2, - "recall": 0.9574, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9374.8941, - "latency": 4.2, - "recall": 0.9425, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9368.1325, - "latency": 3.8, - "recall": 0.9292, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 9220.3627, - "latency": 3.8, - "recall": 0.9081, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 8633.8949, - "latency": 4.1, - "recall": 0.8928, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 6820.6863, - "latency": 3.2, - "recall": 0.9159, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-partition_key", - "db_name": "ZillizCloud-8cu-perf-partition_key", - "qps": 3938.6004, - "latency": 3.7, - "recall": 0.9196, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3411.0934, - "latency": 3.3, - "recall": 0.995, - "filter_ratio": 0.999 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2838.356, - "latency": 3.8, - "recall": 0.9946, - "filter_ratio": 0.998 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1826.0672, - "latency": 5.3, - "recall": 0.9938, - "filter_ratio": 0.995 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1234.6534, - "latency": 6.4, - "recall": 0.9942, - "filter_ratio": 0.99 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1773.0919, - "latency": 5.3, - "recall": 0.9699, - "filter_ratio": 0.98 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1454.8382, - "latency": 4.6, - "recall": 0.9659, - "filter_ratio": 0.95 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1373.0307, - "latency": 5.7, - "recall": 0.9716, - "filter_ratio": 0.9 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2039.8673, - "latency": 3.8, - "recall": 0.9559, - "filter_ratio": 0.8 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2950.8165, - "latency": 3.3, - "recall": 0.9147, - "filter_ratio": 0.5 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9441.1235, - "latency": 5.2, - "recall": 0.9658, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 6125.6146, - "latency": 4.9, - "recall": 0.9936, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5502.1797, - "latency": 3.8, - "recall": 0.9509, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1827.5849, - "latency": 5.4, - "recall": 0.9918, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1938.1932, - "latency": 5.6, - "recall": 0.9906, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3778.8811, - "latency": 4.8, - "recall": 0.9851, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3974.8218, - "latency": 4.8, - "recall": 0.9428, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2971.1402, - "latency": 11.6, - "recall": 0.9752, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 8441.533, - "latency": 6.9, - "recall": 0.9825, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 2703.5422, - "latency": 12.9, - "recall": 0.992, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 1628.2736, - "latency": 5.6, - "recall": 0.9928, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5019.5973, - "latency": 5.7, - "recall": 0.9954, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 7364.0396, - "latency": 4.0, - "recall": 0.9915, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2724.9239, - "latency": 12.4, - "recall": 0.9832, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5514.815, - "latency": 4.2, - "recall": 0.9355, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 9901.1114, - "latency": 3.9, - "recall": 0.9486, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3258.8508, - "latency": 11.7, - "recall": 0.9906, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Medium)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 5907.441, - "latency": 5.7, - "recall": 0.9946, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 5064.6982, - "latency": 4.3, - "recall": 0.9606, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 3468.5933, - "latency": 4.5, - "recall": 0.9661, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf", - "db_name": "ZillizCloud-8cu-perf", - "qps": 2401.3204, - "latency": 10.7, - "recall": 0.9884, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 3568.396, - "latency": 4.8, - "recall": 0.9883, - "filter_ratio": 0.0 - }, - { - "dataset": "Cohere (Large)", - "db": "ZillizCloud", - "label": "8cu-perf-force_merge", - "db_name": "ZillizCloud-8cu-perf-force_merge", - "qps": 4674.1861, - "latency": 4.5, - "recall": 0.9705, - "filter_ratio": 0.0 - } + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2181.3939, + "latency": 9.4, + "recall": 0.8501, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1721.5416, + "latency": 9.6, + "recall": 0.8855, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1452.1536, + "latency": 10.8, + "recall": 0.9042, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1295.5543, + "latency": 11.2, + "recall": 0.9176, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1150.4393, + "latency": 13.4, + "recall": 0.9265, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1032.9696, + "latency": 14.8, + "recall": 0.933, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 125.6164, + "latency": 69.8, + "recall": 0.8746, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 229.0379, + "latency": 43.0, + "recall": 0.8908, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 246.7071, + "latency": 45.1, + "recall": 0.9018, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 189.3789, + "latency": 58.8, + "recall": 0.9149, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 96.4987, + "latency": 113.1, + "recall": 0.9296, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 42.9877, + "latency": 201.9, + "recall": 0.9912, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 72.99, + "latency": 111.4, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 179.5204, + "latency": 51.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 350.0132, + "latency": 29.7, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2808.2421, + "latency": 9.5, + "recall": 0.8815, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2353.8935, + "latency": 17.1, + "recall": 0.9143, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2030.4249, + "latency": 10.6, + "recall": 0.9306, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1804.8996, + "latency": 12.3, + "recall": 0.9405, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1623.8421, + "latency": 11.8, + "recall": 0.9479, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1482.3772, + "latency": 12.1, + "recall": 0.9546, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 899.3114, + "latency": 14.9, + "recall": 0.9072, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 669.9441, + "latency": 19.1, + "recall": 0.9227, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 437.8735, + "latency": 27.1, + "recall": 0.9364, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 249.3519, + "latency": 44.7, + "recall": 0.9446, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 190.3777, + "latency": 47.9, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 364.3516, + "latency": 26.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 692.5751, + "latency": 18.7, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1430.0244, + "latency": 12.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2175.2694, + "latency": 9.8, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1307.419, + "latency": 12.3, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1725.092, + "latency": 11.7, + "recall": 0.8969, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1960.388, + "latency": 11.0, + "recall": 0.9076, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2209.4973, + "latency": 13.7, + "recall": 0.9228, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2457.2628, + "latency": 9.0, + "recall": 0.9378, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2789.7212, + "latency": 8.2, + "recall": 0.9538, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2890.9523, + "latency": 9.4, + "recall": 0.9625, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3019.2416, + "latency": 9.5, + "recall": 0.9765, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3033.786, + "latency": 8.7, + "recall": 0.9934, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3917.2035, + "latency": 2.4, + "recall": 0.9203, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3628.8527, + "latency": 2.6, + "recall": 0.9318, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 3250.1112, + "latency": 2.7, + "recall": 0.9443, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2762.4144, + "latency": 3.1, + "recall": 0.9556, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2384.6245, + "latency": 3.2, + "recall": 0.9627, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 2134.1717, + "latency": 3.8, + "recall": 0.9671, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1641.3478, + "latency": 4.1, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 1488.5841, + "latency": 4.7, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 10333.9072, + "latency": 2.0, + "recall": 0.889, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 9575.6863, + "latency": 2.3, + "recall": 0.9189, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 8596.7694, + "latency": 2.4, + "recall": 0.9416, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7704.3625, + "latency": 2.7, + "recall": 0.9541, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 7023.6735, + "latency": 3.0, + "recall": 0.962, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 6031.3725, + "latency": 3.3, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq4u-fp16-force_merge", + "db_name": "Milvus-16c64g-sq4u-fp16-force_merge", + "qps": 5258.1868, + "latency": 3.6, + "recall": 0.9768, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2747.3167, + "latency": 3.3, + "recall": 0.9204, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2514.4481, + "latency": 3.2, + "recall": 0.9303, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2177.2345, + "latency": 3.4, + "recall": 0.9408, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1833.2575, + "latency": 3.9, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1552.4803, + "latency": 4.0, + "recall": 0.9565, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1355.3121, + "latency": 4.4, + "recall": 0.9602, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 1079.2123, + "latency": 5.3, + "recall": 0.9648, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 876.5772, + "latency": 6.3, + "recall": 0.9676, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5973.0024, + "latency": 2.4, + "recall": 0.9192, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 5416.5758, + "latency": 2.6, + "recall": 0.9334, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4771.4324, + "latency": 2.8, + "recall": 0.9479, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 4006.3994, + "latency": 3.2, + "recall": 0.9609, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3441.7597, + "latency": 3.5, + "recall": 0.9682, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 3040.6216, + "latency": 3.7, + "recall": 0.9734, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2446.7373, + "latency": 4.3, + "recall": 0.9791, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-force_merge", + "db_name": "Milvus-16c64g-sq8-force_merge", + "qps": 2084.6245, + "latency": 5.0, + "recall": 0.9819, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 920.9627, + "latency": 3.4, + "recall": 0.9488, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 1985.8124, + "latency": 2.6, + "recall": 0.9407, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 3157.707, + "latency": 2.3, + "recall": 0.9347, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5671.2562, + "latency": 2.1, + "recall": 0.903, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8936.7962, + "latency": 2.0, + "recall": 0.8835, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9664.2855, + "latency": 1.8, + "recall": 0.899, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10276.7451, + "latency": 1.7, + "recall": 0.9159, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10891.7531, + "latency": 1.7, + "recall": 0.9408, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11397.7043, + "latency": 1.6, + "recall": 0.9597, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5436.8907, + "latency": 2.0, + "recall": 0.929, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8945.4041, + "latency": 1.9, + "recall": 0.8894, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9681.5656, + "latency": 1.9, + "recall": 0.9008, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10258.2661, + "latency": 1.7, + "recall": 0.9139, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10671.8925, + "latency": 1.7, + "recall": 0.9339, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11280.0849, + "latency": 1.6, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11520.9234, + "latency": 1.5, + "recall": 0.9634, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11803.1944, + "latency": 1.5, + "recall": 0.9778, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11763.5538, + "latency": 1.5, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 505.7458, + "latency": 20.7, + "recall": 0.9068, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 433.9034, + "latency": 23.1, + "recall": 0.931, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 381.7737, + "latency": 25.7, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 342.1123, + "latency": 29.0, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 308.2216, + "latency": 31.3, + "recall": 0.9561, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 257.7928, + "latency": 36.4, + "recall": 0.9626, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 223.8166, + "latency": 42.1, + "recall": 0.9666, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 950.6332, + "latency": 13.2, + "recall": 0.914, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 823.2224, + "latency": 13.5, + "recall": 0.9434, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 743.9815, + "latency": 14.8, + "recall": 0.9583, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 683.1873, + "latency": 15.7, + "recall": 0.9677, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 619.7468, + "latency": 17.2, + "recall": 0.9738, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 537.4082, + "latency": 18.8, + "recall": 0.9809, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 474.9941, + "latency": 20.9, + "recall": 0.9848, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1610.9496, + "latency": 10.8, + "recall": 0.9, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1557.3623, + "latency": 10.8, + "recall": 0.9244, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1473.9256, + "latency": 11.7, + "recall": 0.9484, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1388.5547, + "latency": 12.5, + "recall": 0.9597, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 8.0584, + "latency": 1757.9, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 808.3294, + "latency": 22.2, + "recall": 0.7016, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1053.1495, + "latency": 17.7, + "recall": 0.5673, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 504.9179, + "latency": 272.6, + "recall": 0.4664, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 114.8061, + "latency": 126.6, + "recall": 0.9985, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 210.3227, + "latency": 71.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 353.7862, + "latency": 45.2, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 696.9777, + "latency": 24.6, + "recall": 0.997, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1022.2696, + "latency": 17.9, + "recall": 0.936, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3055.0123, + "latency": 7.2, + "recall": 0.9066, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3013.4439, + "latency": 6.9, + "recall": 0.9268, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2801.7241, + "latency": 7.4, + "recall": 0.9476, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2590.3809, + "latency": 8.6, + "recall": 0.9679, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2291.2159, + "latency": 8.9, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2159.051, + "latency": 9.4, + "recall": 0.801, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2604.4444, + "latency": 7.8, + "recall": 0.63, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2685.6654, + "latency": 7.6, + "recall": 0.4914, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 677.1414, + "latency": 33.5, + "recall": 0.7655, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 942.2296, + "latency": 18.2, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1507.6899, + "latency": 12.8, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2073.2153, + "latency": 11.0, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3014.2483, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3099.4124, + "latency": 6.2, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 13.0379, + "latency": 1063.5, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1301.4102, + "latency": 14.7, + "recall": 0.9011, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1844.8918, + "latency": 10.6, + "recall": 0.9085, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2275.2854, + "latency": 9.1, + "recall": 0.917, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2708.6752, + "latency": 8.4, + "recall": 0.9337, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2782.0274, + "latency": 7.4, + "recall": 0.9466, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2950.717, + "latency": 6.9, + "recall": 0.9558, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2988.4205, + "latency": 7.6, + "recall": 0.9741, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3033.5491, + "latency": 6.4, + "recall": 0.9844, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2251.1274, + "latency": 8.7, + "recall": 0.8848, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2771.2009, + "latency": 7.6, + "recall": 0.889, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2935.9403, + "latency": 6.8, + "recall": 0.8992, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3028.858, + "latency": 6.7, + "recall": 0.9133, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3065.6134, + "latency": 6.2, + "recall": 0.9328, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3064.6288, + "latency": 6.5, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3090.0478, + "latency": 6.4, + "recall": 0.9628, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3086.1957, + "latency": 6.7, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3103.0539, + "latency": 5.6, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1131.3087, + "latency": 14.1, + "recall": 0.9024, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1094.5967, + "latency": 14.3, + "recall": 0.8659, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 617.0881, + "latency": 22.4, + "recall": 0.8844, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 372.2462, + "latency": 35.9, + "recall": 0.8977, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 212.7466, + "latency": 58.7, + "recall": 0.9099, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 101.1774, + "latency": 116.1, + "recall": 0.9241, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 57.8988, + "latency": 200.1, + "recall": 0.9332, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 31.4779, + "latency": 351.0, + "recall": 0.9414, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 583.5009, + "latency": 23.0, + "recall": 0.9668, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1114.952, + "latency": 12.7, + "recall": 0.97, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1146.5286, + "latency": 13.7, + "recall": 0.9262, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1147.1977, + "latency": 13.3, + "recall": 0.8999, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 823.1775, + "latency": 20.5, + "recall": 0.9148, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 492.4887, + "latency": 29.6, + "recall": 0.9269, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 264.9324, + "latency": 49.6, + "recall": 0.936, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 487.8343, + "latency": 25.4, + "recall": 0.9668, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1123.5147, + "latency": 18.5, + "recall": 0.9688, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1140.4099, + "latency": 13.5, + "recall": 0.9716, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1149.1219, + "latency": 10.3, + "recall": 0.9764, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1148.1735, + "latency": 8.9, + "recall": 0.9801, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 446.9116, + "latency": 9.2, + "recall": 0.9357, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 388.3028, + "latency": 9.6, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 323.3964, + "latency": 9.8, + "recall": 0.9507, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 256.4668, + "latency": 11.3, + "recall": 0.9588, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 145.5316, + "latency": 18.4, + "recall": 0.9726, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1242.428, + "latency": 6.4, + "recall": 0.9474, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1111.3633, + "latency": 7.0, + "recall": 0.955, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 955.4701, + "latency": 7.2, + "recall": 0.9629, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 783.5207, + "latency": 7.7, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 470.8546, + "latency": 9.5, + "recall": 0.9835, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 434.5481, + "latency": 8.5, + "recall": 0.7237, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 262.8314, + "latency": 11.3, + "recall": 0.9808, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 273.4174, + "latency": 13.3, + "recall": 0.9789, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 325.2245, + "latency": 10.3, + "recall": 0.9909, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 358.8949, + "latency": 9.5, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 441.4152, + "latency": 8.3, + "recall": 0.997, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 274.8559, + "latency": 9.9, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 639.3991, + "latency": 7.3, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1202.8677, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1591.7055, + "latency": 7.8, + "recall": 0.8506, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 987.0795, + "latency": 7.1, + "recall": 0.9804, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1059.3394, + "latency": 7.8, + "recall": 0.9856, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1289.5164, + "latency": 6.4, + "recall": 0.9906, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1108.6473, + "latency": 7.4, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1494.5334, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 2997.4391, + "latency": 6.1, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4250.2894, + "latency": 4.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4318.9697, + "latency": 4.3, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 194.8021, + "latency": 559.8, + "recall": 0.86, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.5444, + "latency": 463.9, + "recall": 0.8478, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 179.4203, + "latency": 497.5, + "recall": 0.8273, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1458, + "latency": 480.5, + "recall": 0.8103, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 186.0237, + "latency": 474.0, + "recall": 0.7847, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 190.9747, + "latency": 517.4, + "recall": 0.7398, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 172.95, + "latency": 515.6, + "recall": 0.7004, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 174.3549, + "latency": 496.9, + "recall": 0.6279, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.397, + "latency": 506.9, + "recall": 0.5409, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 187.4268, + "latency": 453.7, + "recall": 0.4692, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.4972, + "latency": 337.1, + "recall": 0.8717, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.1282, + "latency": 269.2, + "recall": 0.8637, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.1405, + "latency": 282.6, + "recall": 0.8492, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.0349, + "latency": 275.3, + "recall": 0.8325, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.599, + "latency": 358.8, + "recall": 0.8085, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.2424, + "latency": 301.7, + "recall": 0.7592, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.5401, + "latency": 282.4, + "recall": 0.7086, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 196.9391, + "latency": 263.4, + "recall": 0.6549, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 197.4455, + "latency": 349.3, + "recall": 0.5314, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1164, + "latency": 345.6, + "recall": 0.4276, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 649.8781, + "latency": 55.2, + "recall": 0.8352, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 351.7114, + "latency": 46.7, + "recall": 0.8735, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 365.2505, + "latency": 34.9, + "recall": 0.8251, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 369.4921, + "latency": 41.6, + "recall": 0.779, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 370.7241, + "latency": 49.6, + "recall": 0.7177, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 382.5332, + "latency": 54.7, + "recall": 0.6135, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 81.8678, + "latency": 105.6, + "recall": 0.8751, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 91.8612, + "latency": 85.7, + "recall": 0.8799, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 96.592, + "latency": 76.9, + "recall": 0.9178, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 100.0554, + "latency": 69.3, + "recall": 0.9638, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 798.328, + "latency": 56.7, + "recall": 0.8993, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 802.6923, + "latency": 48.1, + "recall": 0.935, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 310.957, + "latency": 49.4, + "recall": 0.9698, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 260.4031, + "latency": 48.3, + "recall": 0.9828, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 206.0934, + "latency": 56.7, + "recall": 0.9795, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 184.5363, + "latency": 53.4, + "recall": 0.9681, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 346.5847, + "latency": 42.7, + "recall": 0.9631, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 284.6367, + "latency": 47.6, + "recall": 0.9788, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 323.0238, + "latency": 50.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "TurboPuffer", + "label": "2026-03-31", + "db_name": "TurboPuffer", + "qps": 471.553, + "latency": 44.1, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7385.2066, + "latency": 2.1, + "recall": 0.9384, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6793.8443, + "latency": 2.2, + "recall": 0.9522, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6242.5346, + "latency": 2.3, + "recall": 0.961, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5779.119, + "latency": 2.3, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5183.5843, + "latency": 2.4, + "recall": 0.9784, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 4259.5572, + "latency": 2.6, + "recall": 0.9845, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3614.3118, + "latency": 2.9, + "recall": 0.9877, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3181.0537, + "latency": 3.0, + "recall": 0.9892, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2804.8357, + "latency": 3.3, + "recall": 0.9903, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 13316.2336, + "latency": 2.0, + "recall": 0.9383, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 12837.5287, + "latency": 2.1, + "recall": 0.9588, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 12248.9154, + "latency": 2.2, + "recall": 0.9687, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 11501.6652, + "latency": 2.2, + "recall": 0.9785, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 10566.6823, + "latency": 2.4, + "recall": 0.9838, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9227.318, + "latency": 2.7, + "recall": 0.9893, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8320.4606, + "latency": 2.9, + "recall": 0.9919, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7524.9879, + "latency": 3.2, + "recall": 0.9931, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6813.2439, + "latency": 3.5, + "recall": 0.9939, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2950.8165, + "latency": 3.3, + "recall": 0.9147, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2039.8673, + "latency": 3.8, + "recall": 0.9559, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1373.0307, + "latency": 5.7, + "recall": 0.9716, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1454.8382, + "latency": 4.6, + "recall": 0.9659, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1773.0919, + "latency": 5.3, + "recall": 0.9699, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1234.6534, + "latency": 6.4, + "recall": 0.9942, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1826.0672, + "latency": 5.3, + "recall": 0.9938, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2838.356, + "latency": 3.8, + "recall": 0.9946, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3411.0934, + "latency": 3.3, + "recall": 0.995, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8468.4611, + "latency": 3.1, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6860.8577, + "latency": 4.7, + "recall": 0.9226, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5506.1808, + "latency": 5.5, + "recall": 0.9193, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6750.2495, + "latency": 4.4, + "recall": 0.9105, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7589.664, + "latency": 3.8, + "recall": 0.9235, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7610.0519, + "latency": 3.3, + "recall": 0.9903, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8455.2896, + "latency": 4.0, + "recall": 0.9921, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9081.1518, + "latency": 3.0, + "recall": 0.9943, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9773.6593, + "latency": 3.7, + "recall": 0.9955, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 3938.6004, + "latency": 3.7, + "recall": 0.9196, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 6820.6863, + "latency": 3.2, + "recall": 0.9159, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8633.8949, + "latency": 4.1, + "recall": 0.8928, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9220.3627, + "latency": 3.8, + "recall": 0.9081, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9368.1325, + "latency": 3.8, + "recall": 0.9292, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9374.8941, + "latency": 4.2, + "recall": 0.9425, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9289.0118, + "latency": 4.2, + "recall": 0.9574, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9244.1135, + "latency": 4.2, + "recall": 0.9724, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8695.2765, + "latency": 4.3, + "recall": 0.9603, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9048.6431, + "latency": 3.9, + "recall": 0.9216, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9428.4531, + "latency": 2.6, + "recall": 0.9331, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9507.9991, + "latency": 2.8, + "recall": 0.9453, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9861.9686, + "latency": 2.6, + "recall": 0.955, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10041.0338, + "latency": 2.7, + "recall": 0.9693, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10020.5299, + "latency": 2.6, + "recall": 0.9788, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9805.0401, + "latency": 2.6, + "recall": 0.9257, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10557.4373, + "latency": 2.7, + "recall": 0.9393, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10089.4308, + "latency": 2.6, + "recall": 0.9934, + "filter_ratio": 0.999 + } ] \ No newline at end of file From dad3c3d7227d9810fe0172a1f117a46d18ed1a29 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Thu, 9 Apr 2026 10:16:52 +0800 Subject: [PATCH 15/49] feat: Upgrade pydantic to v2 (#750) 1. Upgrade pydantic to 2.x 2. Remove results/ from .gitignore, those files need to track 3. fix the coding styles in the results Signed-off-by: yangxuan --- .gitignore | 1 - install/requirements_py3.11.txt | 2 +- pyproject.toml | 2 +- .../backend/clients/alisql/config.py | 4 +- .../backend/clients/alloydb/config.py | 22 ++++----- vectordb_bench/backend/clients/api.py | 21 +++++--- .../backend/clients/aws_opensearch/config.py | 25 +++++----- .../backend/clients/chroma/config.py | 2 +- .../backend/clients/cockroachdb/config.py | 8 +-- .../backend/clients/doris/config.py | 9 ++-- .../backend/clients/lindorm/config.py | 28 +++++------ .../backend/clients/mariadb/config.py | 6 +-- .../backend/clients/milvus/config.py | 25 +++++----- .../backend/clients/oss_opensearch/config.py | 49 ++++++++++--------- .../backend/clients/pgdiskann/config.py | 12 ++--- .../backend/clients/pgvecto_rs/config.py | 6 +-- .../backend/clients/pgvector/config.py | 14 +++--- .../backend/clients/pgvectorscale/config.py | 16 +++--- .../backend/clients/polardb/config.py | 2 +- .../backend/clients/qdrant_cloud/config.py | 19 ++++--- vectordb_bench/backend/clients/tidb/config.py | 21 +++++--- vectordb_bench/backend/dataset.py | 26 +++++----- vectordb_bench/base.py | 5 +- .../components/custom/getCustomConfig.py | 6 ++- .../frontend/config/dbCaseConfigs.py | 6 +-- vectordb_bench/frontend/pages/qps_recall.py | 5 +- vectordb_bench/frontend/pages/results.py | 2 +- vectordb_bench/models.py | 6 +-- vectordb_bench/restful/format_res.py | 4 +- vectordb_bench/results/getLeaderboardData.py | 29 ++++++----- .../results/getLeaderboardDataV2.py | 15 +++--- 31 files changed, 213 insertions(+), 185 deletions(-) diff --git a/.gitignore b/.gitignore index 8985eeb4d..cea1306b0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ build/ venv/ .venv/ .idea/ -results/ logs/ # Worktrees diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 4214267a3..130745816 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -20,7 +20,7 @@ psutil polars plotly environs -pydantic=2.0,<3 scikit-learn pymilvus clickhouse_connect diff --git a/pyproject.toml b/pyproject.toml index 2baeb16e3..e72be9697 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "polars", "plotly", "environs", - "pydantic=2.0,<3", "scikit-learn", "pymilvus", # with pandas, numpy "hdrhistogram>=0.10.1", diff --git a/vectordb_bench/backend/clients/alisql/config.py b/vectordb_bench/backend/clients/alisql/config.py index f942c30f1..16c56e90f 100644 --- a/vectordb_bench/backend/clients/alisql/config.py +++ b/vectordb_bench/backend/clients/alisql/config.py @@ -49,8 +49,8 @@ def parse_metric(self) -> str: class AliSQLHNSWConfig(AliSQLIndexConfig, DBCaseConfig): - M: int | None - ef_search: int | None + M: int | None = None + ef_search: int | None = None index: IndexType = IndexType.HNSW def index_param(self) -> dict: diff --git a/vectordb_bench/backend/clients/alloydb/config.py b/vectordb_bench/backend/clients/alloydb/config.py index d6e54e487..11e65084e 100644 --- a/vectordb_bench/backend/clients/alloydb/config.py +++ b/vectordb_bench/backend/clients/alloydb/config.py @@ -43,8 +43,8 @@ class AlloyDBIndexParam(TypedDict): metric: str index_type: str index_creation_with_options: Sequence[dict[str, Any]] - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None class AlloyDBSearchParam(TypedDict): @@ -120,15 +120,15 @@ def _optionally_build_set_options( class AlloyDBScaNNConfig(AlloyDBIndexConfig): index: IndexType = IndexType.SCANN - num_leaves: int | None - quantizer: str | None - enable_pca: str | None - max_num_levels: int | None - num_leaves_to_search: int | None - max_top_neighbors_buffer_size: int | None - pre_reordering_num_neighbors: int | None - num_search_threads: int | None - max_num_prefetch_datasets: int | None + num_leaves: int | None = None + quantizer: str | None = None + enable_pca: str | None = None + max_num_levels: int | None = None + num_leaves_to_search: int | None = None + max_top_neighbors_buffer_size: int | None = None + pre_reordering_num_neighbors: int | None = None + num_search_threads: int | None = None + max_num_prefetch_datasets: int | None = None maintenance_work_mem: str | None = None max_parallel_workers: int | None = None diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 5511f18db..0f8103597 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -2,7 +2,7 @@ from contextlib import contextmanager from enum import StrEnum -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, model_validator from vectordb_bench.backend.filter import Filter, FilterOp @@ -90,13 +90,18 @@ def common_long_configs() -> list[str]: def to_dict(self) -> dict: raise NotImplementedError - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in cls.common_short_configs() or field.name in cls.common_long_configs(): - return v - if not v and isinstance(v, str | SecretStr): - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class DBCaseConfig(ABC): diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index 5ab63010d..7742d421d 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -1,7 +1,7 @@ import logging from enum import Enum -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -32,17 +32,18 @@ def to_dict(self) -> dict: "timeout": 600, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password", "host"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class AWSOS_Engine(Enum): diff --git a/vectordb_bench/backend/clients/chroma/config.py b/vectordb_bench/backend/clients/chroma/config.py index cd3e01ecc..a1d9903d1 100644 --- a/vectordb_bench/backend/clients/chroma/config.py +++ b/vectordb_bench/backend/clients/chroma/config.py @@ -6,7 +6,7 @@ class ChromaConfig(DBConfig): user: str | None = None - password: SecretStr | None + password: SecretStr | None = None host: SecretStr = "localhost" port: int = 8000 diff --git a/vectordb_bench/backend/clients/cockroachdb/config.py b/vectordb_bench/backend/clients/cockroachdb/config.py index 0d608da8f..88ec0e5ea 100644 --- a/vectordb_bench/backend/clients/cockroachdb/config.py +++ b/vectordb_bench/backend/clients/cockroachdb/config.py @@ -75,16 +75,16 @@ class CockroachDBIndexParam(TypedDict): metric: str index_creation_with_options: Sequence[dict[str, Any]] - min_partition_size: int | None - max_partition_size: int | None - build_beam_size: int | None + min_partition_size: int | None = None + max_partition_size: int | None = None + build_beam_size: int | None = None class CockroachDBSearchParam(TypedDict): """Search parameters for CockroachDB vector queries.""" metric_fun_op: LiteralString - vector_search_beam_size: int | None + vector_search_beam_size: int | None = None class CockroachDBSessionCommands(TypedDict): diff --git a/vectordb_bench/backend/clients/doris/config.py b/vectordb_bench/backend/clients/doris/config.py index a15309922..7c79ba728 100644 --- a/vectordb_bench/backend/clients/doris/config.py +++ b/vectordb_bench/backend/clients/doris/config.py @@ -1,6 +1,6 @@ import logging -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -17,9 +17,10 @@ class DorisConfig(DBConfig): db_name: str = "test" ssl: bool = False - @validator("*") - def not_empty_field(cls, v: any, field: any): - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + return data def to_dict(self) -> dict: pwd_str = self.password.get_secret_value() diff --git a/vectordb_bench/backend/clients/lindorm/config.py b/vectordb_bench/backend/clients/lindorm/config.py index 367f369e3..0e0d4aae6 100644 --- a/vectordb_bench/backend/clients/lindorm/config.py +++ b/vectordb_bench/backend/clients/lindorm/config.py @@ -43,9 +43,9 @@ def parse_metric(self) -> str: class HNSWConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.HNSW - M: int | None - efConstruction: int | None - efSearch: int | None + M: int | None = None + efConstruction: int | None = None + efSearch: int | None = None filter_type: str | None = "efficient_filter" k_expand_scope: int | None = 1000 @@ -72,12 +72,12 @@ def search_param(self, do_filter: bool = False) -> dict: # first layer searching for cluster centroids is hnsw class IVFPQConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.IVFPQ - nlist: int | None - nprobe: int | None + nlist: int | None = None + nprobe: int | None = None # search parameters - centroids_hnsw_M: int | None - centroids_hnsw_efConstruction: int | None - centroids_hnsw_efSearch: int | None + centroids_hnsw_M: int | None = None + centroids_hnsw_efConstruction: int | None = None + centroids_hnsw_efSearch: int | None = None filter_type: str | None = "efficient_filter" reorder_factor: int | None = 10 @@ -116,13 +116,13 @@ def search_param(self, do_filter: bool = False) -> dict: class IVFBQConfig(LindormIndexConfig, DBCaseConfig): index: IndexType = IndexType.IVFBQ - nlist: int | None - exbits: int | None - nprobe: int | None + nlist: int | None = None + exbits: int | None = None + nprobe: int | None = None # search parameters - centroids_hnsw_M: int | None - centroids_hnsw_efConstruction: int | None - centroids_hnsw_efSearch: int | None + centroids_hnsw_M: int | None = None + centroids_hnsw_efConstruction: int | None = None + centroids_hnsw_efSearch: int | None = None filter_type: str | None = "efficient_filter" reorder_factor: int | None = 10 diff --git a/vectordb_bench/backend/clients/mariadb/config.py b/vectordb_bench/backend/clients/mariadb/config.py index d183adc76..21ea9ac2e 100644 --- a/vectordb_bench/backend/clients/mariadb/config.py +++ b/vectordb_bench/backend/clients/mariadb/config.py @@ -46,11 +46,11 @@ def parse_metric(self) -> str: class MariaDBHNSWConfig(MariaDBIndexConfig, DBCaseConfig): - M: int | None - ef_search: int | None + M: int | None = None + ef_search: int | None = None index: IndexType = IndexType.HNSW storage_engine: str = "InnoDB" - max_cache_size: int | None + max_cache_size: int | None = None def index_param(self) -> dict: return { diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 9ffbdcece..98118c6df 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, IndexType, MetricType, SQType @@ -19,17 +19,18 @@ def to_dict(self) -> dict: "replica_number": self.replica_number, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class MilvusIndexConfig(BaseModel): diff --git a/vectordb_bench/backend/clients/oss_opensearch/config.py b/vectordb_bench/backend/clients/oss_opensearch/config.py index 83fed3d58..a5d69459a 100644 --- a/vectordb_bench/backend/clients/oss_opensearch/config.py +++ b/vectordb_bench/backend/clients/oss_opensearch/config.py @@ -1,7 +1,7 @@ import logging from enum import Enum -from pydantic import BaseModel, SecretStr, root_validator, validator +from pydantic import BaseModel, SecretStr, field_validator, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -32,17 +32,18 @@ def to_dict(self) -> dict: "timeout": 600, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if ( - field.name in cls.common_short_configs() - or field.name in cls.common_long_configs() - or field.name in ["user", "password", "host"] - ): - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class OSSOS_Engine(Enum): @@ -111,7 +112,8 @@ class OSSOpenSearchIndexConfig(BaseModel, DBCaseConfig): compression_level: str = CompressionLevel.LEVEL_32X oversample_factor: float = 1.0 - @validator("quantization_type", pre=True, always=True) + @field_validator("quantization_type", mode="before") + @classmethod def validate_quantization_type(cls, value: any): """Convert string values to enum""" if not value: @@ -128,19 +130,22 @@ def validate_quantization_type(cls, value: any): return mapping.get(value, OSSOpenSearchQuantization.NONE) - @root_validator - def validate_engine_name(cls, values: dict): - """Map engine_name string from UI to engine enum""" - if values.get("engine_name"): - engine_name = values["engine_name"].lower() + @model_validator(mode="before") + @classmethod + def validate_engine_name(cls, data: any) -> any: + if not isinstance(data, dict): + return data + # Map engine_name to engine enum + if data.get("engine_name"): + engine_name = data["engine_name"].lower() if engine_name == "faiss": - values["engine"] = OSSOS_Engine.faiss + data["engine"] = OSSOS_Engine.faiss elif engine_name == "lucene": - values["engine"] = OSSOS_Engine.lucene + data["engine"] = OSSOS_Engine.lucene else: log.warning(f"Unknown engine_name: {engine_name}, defaulting to faiss") - values["engine"] = OSSOS_Engine.faiss - return values + data["engine"] = OSSOS_Engine.faiss + return data def __eq__(self, obj: any): return ( diff --git a/vectordb_bench/backend/clients/pgdiskann/config.py b/vectordb_bench/backend/clients/pgdiskann/config.py index 7f83a05c8..8715b1e42 100644 --- a/vectordb_bench/backend/clients/pgdiskann/config.py +++ b/vectordb_bench/backend/clients/pgdiskann/config.py @@ -43,8 +43,8 @@ class PgDiskANNIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None create_index_before_load: bool = False create_index_after_load: bool = True - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None def parse_metric(self) -> str: if self.metric_type == MetricType.L2: @@ -120,10 +120,10 @@ def _optionally_build_set_options( class PgDiskANNImplConfig(PgDiskANNIndexConfig): index: IndexType = IndexType.DISKANN - max_neighbors: int | None - l_value_ib: int | None - pq_param_num_chunks: int | None - l_value_is: float | None + max_neighbors: int | None = None + l_value_ib: int | None = None + pq_param_num_chunks: int | None = None + l_value_is: float | None = None reranking: bool | None = None reranking_metric: str | None = None quantized_fetch_limit: int | None = None diff --git a/vectordb_bench/backend/clients/pgvecto_rs/config.py b/vectordb_bench/backend/clients/pgvecto_rs/config.py index fbb7c5d81..73c2573a0 100644 --- a/vectordb_bench/backend/clients/pgvecto_rs/config.py +++ b/vectordb_bench/backend/clients/pgvecto_rs/config.py @@ -78,7 +78,7 @@ def session_param(self) -> dict[str, str | int]: ... class PgVectoRSHNSWConfig(PgVectoRSIndexConfig): index: IndexType = IndexType.HNSW m: int | None = None - ef_search: int | None + ef_search: int | None = None ef_construction: int | None = None def index_param(self) -> dict[str, str]: @@ -106,8 +106,8 @@ def session_param(self) -> dict[str, str | int]: class PgVectoRSIVFFlatConfig(PgVectoRSIndexConfig): index: IndexType = IndexType.IVFFlat - probes: int | None - lists: int | None + probes: int | None = None + lists: int | None = None def index_param(self) -> dict[str, str]: if self.quantization_type is None: diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index 98e82f1c2..7da238a4b 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -47,8 +47,8 @@ class PgVectorIndexParam(TypedDict): metric: str index_type: str index_creation_with_options: Sequence[dict[str, Any]] - maintenance_work_mem: str | None - max_parallel_workers: int | None + maintenance_work_mem: str | None = None + max_parallel_workers: int | None = None class PgVectorSearchParam(TypedDict): @@ -175,13 +175,13 @@ class PgVectorIVFFlatConfig(PgVectorIndexConfig): a good place to start is sqrt(lists) """ - lists: int | None - probes: int | None + lists: int | None = None + probes: int | None = None index: IndexType = IndexType.ES_IVFFlat maintenance_work_mem: str | None = None max_parallel_workers: int | None = None quantization_type: str | None = None - table_quantization_type: str | None + table_quantization_type: str | None = None reranking: bool | None = None quantized_fetch_limit: int | None = None reranking_metric: str | None = None @@ -226,12 +226,12 @@ class PgVectorHNSWConfig(PgVectorIndexConfig): m: int | None # DETAIL: Valid values are between "2" and "100". ef_construction: int | None # ef_construction must be greater than or equal to 2 * m - ef_search: int | None + ef_search: int | None = None index: IndexType = IndexType.ES_HNSW maintenance_work_mem: str | None = None max_parallel_workers: int | None = None quantization_type: str | None = None - table_quantization_type: str | None + table_quantization_type: str | None = None reranking: bool | None = None quantized_fetch_limit: int | None = None reranking_metric: str | None = None diff --git a/vectordb_bench/backend/clients/pgvectorscale/config.py b/vectordb_bench/backend/clients/pgvectorscale/config.py index e22c45c8d..07750cffb 100644 --- a/vectordb_bench/backend/clients/pgvectorscale/config.py +++ b/vectordb_bench/backend/clients/pgvectorscale/config.py @@ -70,14 +70,14 @@ def session_param(self) -> dict: ... class PgVectorScaleStreamingDiskANNConfig(PgVectorScaleIndexConfig): index: IndexType = IndexType.STREAMING_DISKANN - storage_layout: str | None - num_neighbors: int | None - search_list_size: int | None - max_alpha: float | None - num_dimensions: int | None - num_bits_per_dimension: int | None - query_search_list_size: int | None - query_rescore: int | None + storage_layout: str | None = None + num_neighbors: int | None = None + search_list_size: int | None = None + max_alpha: float | None = None + num_dimensions: int | None = None + num_bits_per_dimension: int | None = None + query_search_list_size: int | None = None + query_rescore: int | None = None def index_param(self) -> dict: return { diff --git a/vectordb_bench/backend/clients/polardb/config.py b/vectordb_bench/backend/clients/polardb/config.py index c75448c49..5121c1e19 100644 --- a/vectordb_bench/backend/clients/polardb/config.py +++ b/vectordb_bench/backend/clients/polardb/config.py @@ -11,7 +11,7 @@ class PolarDBConfigDict(TypedDict): host: str port: int database: str - unix_socket: str | None + unix_socket: str | None = None class PolarDBConfig(DBConfig): diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index b2eeb2ce6..06543aaab 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,6 +1,6 @@ from typing import TypeVar -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -25,11 +25,18 @@ def to_dict(self) -> dict: "url": self.url.get_secret_value(), } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in ["api_key"]: - return v - return super().not_empty_field(v, field) + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"api_key"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class QdrantIndexConfig(BaseModel, DBCaseConfig): diff --git a/vectordb_bench/backend/clients/tidb/config.py b/vectordb_bench/backend/clients/tidb/config.py index 71fdbad66..93098ede1 100644 --- a/vectordb_bench/backend/clients/tidb/config.py +++ b/vectordb_bench/backend/clients/tidb/config.py @@ -1,6 +1,6 @@ from typing import TypedDict -from pydantic import BaseModel, SecretStr, validator +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, MetricType @@ -35,13 +35,18 @@ def to_dict(self) -> TiDBConfigDict: "ssl_verify_identity": self.ssl, } - @validator("*") - def not_empty_field(cls, v: any, field: any): - if field.name in ["password", "db_label"]: - return v - if isinstance(v, str | SecretStr) and len(v) == 0: - raise ValueError("Empty string!") - return v + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"password"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + raise ValueError("Empty string!") + return data class TiDBIndexConfig(BaseModel, DBCaseConfig): diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index d1de9e328..94216532f 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -7,12 +7,12 @@ import logging import pathlib from enum import Enum -from typing import Any, NamedTuple +from typing import Any, ClassVar, NamedTuple import pandas as pd import polars as pl from pyarrow.parquet import ParquetFile -from pydantic import PrivateAttr, validator +from pydantic import field_validator from vectordb_bench import config from vectordb_bench.base import BaseModel @@ -38,7 +38,7 @@ class BaseDataset(BaseModel): metric_type: MetricType use_shuffled: bool with_gt: bool = False - _size_label: dict[int, SizeLabel] = PrivateAttr() + _size_label: ClassVar[dict[int, SizeLabel]] is_custom: bool = False with_remote_resource: bool = True # for label filter cases @@ -57,7 +57,8 @@ class BaseDataset(BaseModel): gt_id_field: str = "id" gt_neighbors_field: str = "neighbors_id" - @validator("size") + @field_validator("size") + @classmethod def verify_size(cls, v: int): if v not in cls._size_label: msg = f"Size {v} not supported for the dataset, expected: {cls._size_label.keys()}" @@ -102,7 +103,8 @@ class CustomDataset(BaseDataset): scalar_labels_file: str = "scalar_labels.parquet" label_percentages: list[float] = [] - @validator("size") + @field_validator("size") + @classmethod def verify_size(cls, v: int): return v @@ -136,7 +138,7 @@ class LAION(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000_000: SizeLabel(100_000_000, "LARGE", 100), } @@ -146,7 +148,7 @@ class GIST(BaseDataset): dim: int = 960 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), } @@ -158,7 +160,7 @@ class Cohere(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), @@ -196,7 +198,7 @@ class Bioasq(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), } @@ -232,7 +234,7 @@ class Glove(BaseDataset): dim: int = 200 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = False - _size_label: dict = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} + _size_label: ClassVar[dict] = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} class SIFT(BaseDataset): @@ -240,7 +242,7 @@ class SIFT(BaseDataset): dim: int = 128 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: dict = { + _size_label: ClassVar[dict] = { 500_000: SizeLabel( 500_000, "SMALL", @@ -257,7 +259,7 @@ class OpenAI(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: dict = { + _size_label: ClassVar[dict] = { 50_000: SizeLabel(50_000, "SMALL", 1), 500_000: SizeLabel(500_000, "MEDIUM", 1), 5_000_000: SizeLabel(5_000_000, "LARGE", 10), diff --git a/vectordb_bench/base.py b/vectordb_bench/base.py index 502d5fa49..401d2086d 100644 --- a/vectordb_bench/base.py +++ b/vectordb_bench/base.py @@ -1,5 +1,6 @@ from pydantic import BaseModel as PydanticBaseModel +from pydantic import ConfigDict -class BaseModel(PydanticBaseModel, arbitrary_types_allowed=True): - pass +class BaseModel(PydanticBaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/vectordb_bench/frontend/components/custom/getCustomConfig.py b/vectordb_bench/frontend/components/custom/getCustomConfig.py index a1ddfb737..03831a83c 100644 --- a/vectordb_bench/frontend/components/custom/getCustomConfig.py +++ b/vectordb_bench/frontend/components/custom/getCustomConfig.py @@ -62,14 +62,16 @@ def get_custom_streaming_configs(): def save_custom_configs(custom_configs: list[CustomDatasetConfig]): with open(config.CUSTOM_CONFIG_DIR, "w") as f: - json.dump([custom_config.dict() for custom_config in custom_configs], f, indent=4) + json.dump([custom_config.model_dump() for custom_config in custom_configs], f, indent=4) def save_all_custom_configs( performance_configs: list[CustomCaseConfig], streaming_configs: list[CustomStreamingCaseConfig] ): """Save both performance and streaming configs to the same JSON file""" - all_configs = [config.dict() for config in performance_configs] + [config.dict() for config in streaming_configs] + all_configs = [config.model_dump() for config in performance_configs] + [ + config.model_dump() for config in streaming_configs + ] with open(config.CUSTOM_CONFIG_DIR, "w") as f: json.dump(all_configs, f, indent=4) diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 387f7fb4a..d15c4e7ee 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -119,7 +119,7 @@ def get_custom_case_items() -> list[UICaseItem]: CaseConfig( case_id=CaseType.PerformanceCustomDataset, custom_case={ - **custom_config.dict(), + **custom_config.model_dump(), "use_filter": False, }, ) @@ -140,7 +140,7 @@ def get_custom_case_items() -> list[UICaseItem]: CaseConfig( case_id=CaseType.PerformanceCustomDataset, custom_case={ - **custom_config.dict(), + **custom_config.model_dump(), "use_filter": True, "label_percentage": label_percentage, }, @@ -174,7 +174,7 @@ def get_custom_streaming_case_items() -> list[UICaseItem]: case_id=CaseType.StreamingCustomDataset, custom_case={ "description": custom_config.description, - "dataset_config": custom_config.dataset_config.dict(), + "dataset_config": custom_config.dataset_config.model_dump(), }, ) ], diff --git a/vectordb_bench/frontend/pages/qps_recall.py b/vectordb_bench/frontend/pages/qps_recall.py index 27f9c4691..fb8f680c5 100644 --- a/vectordb_bench/frontend/pages/qps_recall.py +++ b/vectordb_bench/frontend/pages/qps_recall.py @@ -43,7 +43,10 @@ def case_results_filter(case_result: CaseResult) -> bool: case = case_result.task_config.case_config.case return case.label == CaseLabel.Performance and case.filters.type == FilterOp.NonFilter - default_selected_task_labels = ["standard_2025"] + default_selected_task_labels = ["standard_20260403", "standard_20250519"] + # Filter defaults to only include labels that exist in results + available_labels = {r.task_label for r in allResults} + default_selected_task_labels = [l for l in default_selected_task_labels if l in available_labels] shownData, failedTasks, showCaseNames = getshownData( resultSelectorContainer, allResults, diff --git a/vectordb_bench/frontend/pages/results.py b/vectordb_bench/frontend/pages/results.py index a146f2fdc..216029bb1 100644 --- a/vectordb_bench/frontend/pages/results.py +++ b/vectordb_bench/frontend/pages/results.py @@ -32,7 +32,7 @@ def main(): st.caption( "Choose your desired test results to display from the sidebar. " "For your reference, we've included two standard benchmarks tested by our team. " - "Note that `standard_2025` was tested in 2025; the others in 2023. " + "Note that `standard_20260403` is the latest benchmark; the others were tested in 2023-2025. " "Unless explicitly labeled as distributed multi-node, test with single-node mode by default." ) st.caption("We welcome community contributions for better results, parameter configurations, and optimizations.") diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index cdc64b9d7..a7e7c09f1 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -205,7 +205,7 @@ def k(self, value): ''' def __hash__(self) -> int: - return hash(self.json()) + return hash(self.model_dump_json()) @property def case(self) -> Case: @@ -314,7 +314,7 @@ def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): log.info(f"write results to disk {result_file}") with pathlib.Path(result_file).open("w") as f: - b = partial.json(exclude={"db_config": {"password", "api_key"}}) + b = partial.model_dump_json(exclude={"db_config": {"password", "api_key"}}) f.write(b) def get_case_config(case_config: CaseConfig) -> dict[CaseConfig]: @@ -381,7 +381,7 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: else: # Default to 0 for older result files that don't have P95 data case_result["metrics"]["serial_latency_p95"] = 0.0 - return TestResult.validate(test_result) + return TestResult.model_validate(test_result) def display(self, dbs: list[DB] | None = None): filter_list = dbs if dbs and isinstance(dbs, list) else None diff --git a/vectordb_bench/restful/format_res.py b/vectordb_bench/restful/format_res.py index 2e289ec3b..326986319 100644 --- a/vectordb_bench/restful/format_res.py +++ b/vectordb_bench/restful/format_res.py @@ -63,7 +63,7 @@ def format_results(test_results: list[TestResult], task_label: str) -> list[dict db_label=task_config.db_config.db_label, version=task_config.db_config.version, note=task_config.db_config.note, - params=task_config.db_case_config.dict(), + params=task_config.db_case_config.model_dump(), case_name=case.name, dataset=dataset.full_name, dim=dataset.dim, @@ -71,6 +71,6 @@ def format_results(test_results: list[TestResult], task_label: str) -> list[dict filter_rate=filter_.filter_rate, k=task_config.case_config.k, **metrics, - ).dict() + ).model_dump() ) return results diff --git a/vectordb_bench/results/getLeaderboardData.py b/vectordb_bench/results/getLeaderboardData.py index aef024bdc..1650a6f87 100644 --- a/vectordb_bench/results/getLeaderboardData.py +++ b/vectordb_bench/results/getLeaderboardData.py @@ -1,14 +1,16 @@ -from vectordb_bench import config -import ujson import pathlib +from datetime import datetime + +import ujson + +from vectordb_bench import config from vectordb_bench.backend.cases import CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.frontend.config.dbPrices import DB_DBLABEL_TO_PRICE from vectordb_bench.interface import benchMarkRunner from vectordb_bench.models import ResultLabel, TestResult -from datetime import datetime -taskLabelToCode = { +task_label_to_code = { ResultLabel.FAILED: -1, ResultLabel.OUTOFRANGE: -2, ResultLabel.NORMAL: 1, @@ -18,15 +20,14 @@ def format_time(ts: float) -> str: default_standard_test_time = datetime(2023, 8, 1) t = datetime.fromtimestamp(ts) - if t < default_standard_test_time: - t = default_standard_test_time + t = max(t, default_standard_test_time) return t.strftime("%Y-%m") def main(): - allResults: list[TestResult] = benchMarkRunner.get_results() + all_results: list[TestResult] = benchMarkRunner.get_results() - if allResults is not None: + if all_results is not None: data = [ { "db": d.task_config.db.value, @@ -36,18 +37,16 @@ def main(): "qps": d.metrics.qps, "latency": d.metrics.serial_latency_p99, "recall": d.metrics.recall, - "label": taskLabelToCode[d.label], + "label": task_label_to_code[d.label], "note": d.task_config.db_config.note, "version": d.task_config.db_config.version, "test_time": format_time(test_result.timestamp), } - for test_result in allResults + for test_result in all_results if "standard" in test_result.task_label for d in test_result.results - if d.task_config.case_config.case_id != CaseType.CapacityDim128 - and d.task_config.case_config.case_id != CaseType.CapacityDim960 - if d.task_config.db != DB.ZillizCloud - or test_result.timestamp >= datetime(2024, 1, 1).timestamp() + if d.task_config.case_config.case_id not in {CaseType.CapacityDim128, CaseType.CapacityDim960} + if d.task_config.db != DB.ZillizCloud or test_result.timestamp >= datetime(2024, 1, 1).timestamp() ] # compute qp$ @@ -58,7 +57,7 @@ def main(): price = DB_DBLABEL_TO_PRICE.get(db, {}).get(db_label, 0) d["qp$"] = (qps / price * 3600) if price > 0 else 0.0 - with open(pathlib.Path(config.RESULTS_LOCAL_DIR, "leaderboard.json"), "w") as f: + with pathlib.Path(config.RESULTS_LOCAL_DIR, "leaderboard.json").open("w") as f: ujson.dump(data, f) diff --git a/vectordb_bench/results/getLeaderboardDataV2.py b/vectordb_bench/results/getLeaderboardDataV2.py index 62440886f..188d9876f 100644 --- a/vectordb_bench/results/getLeaderboardDataV2.py +++ b/vectordb_bench/results/getLeaderboardDataV2.py @@ -1,17 +1,14 @@ import json import logging +import pathlib - +from vectordb_bench import config from vectordb_bench.backend.cases import CaseType, StreamingPerformanceCase -from vectordb_bench.backend.clients import DB +from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import CaseResult -from vectordb_bench import config -import numpy as np logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s") -from vectordb_bench.interface import BenchMarkRunner - def get_standard_2025_results() -> list[CaseResult]: all_results = BenchMarkRunner.get_results() @@ -23,7 +20,7 @@ def get_standard_2025_results() -> list[CaseResult]: def save_to_json(data: list[dict], file_name: str): - with open(file_name, "w") as f: + with pathlib.Path(file_name).open("w") as f: json.dump(data, f, indent=4) @@ -56,11 +53,11 @@ def main(): } ) else: - case: StreamingPerformanceCase = case + streaming_case: StreamingPerformanceCase = case # use 90p search stage results to represent streaming performance qps_90p = metrics.st_max_qps_list_list[metrics.st_search_stage_list.index(90)] latency_90p = metrics.st_serial_latency_p99_list[metrics.st_search_stage_list.index(90)] - insert_rate = case.insert_rate + insert_rate = streaming_case.insert_rate streaming_data.append( { "dataset": dataset, From cf09d634edafcd0379a707b72873b7a8879e30dc Mon Sep 17 00:00:00 2001 From: ChenLiqing Date: Thu, 9 Apr 2026 11:01:38 +0800 Subject: [PATCH 16/49] fix: fill missing build durations in Milvus and ZillizCloud results (#751) Populate insert_duration, optimize_duration, load_duration for all entries in result_20260403 files. Previously only the first entry per index had values while the rest were 0.0. Milvus (re-measured on 2.6-opt-v2): - 1M SQ4U: insert=129.8s, optimize=152.2s, load=282.0s - 1M SQ8: insert=119.5s, optimize=235.9s, load=355.4s - 10M SQ4U/SQ8: copied from existing first-entry values ZillizCloud (from prior build runs): - 1M: insert=246.7s, optimize=101.2s, load=347.9s - 10M: insert=2450.8s, optimize=136.9s, load=2587.8s Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.6 (1M context) --- .../result_20260403_standard_milvus.json | 180 +++++++++--------- .../result_20260403_standard_zillizcloud.json | 108 +++++------ 2 files changed, 144 insertions(+), 144 deletions(-) diff --git a/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json index e10404530..56219eb1d 100644 --- a/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json +++ b/vectordb_bench/results/Milvus/result_20260403_standard_milvus.json @@ -134,9 +134,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 3628.8527, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0024, @@ -261,9 +261,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 3250.1112, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0025, @@ -388,9 +388,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2762.4144, "serial_latency_p99": 0.0031, "serial_latency_p95": 0.0029, @@ -515,9 +515,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2384.6245, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -642,9 +642,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 2134.1717, "serial_latency_p99": 0.0038, "serial_latency_p95": 0.0036, @@ -769,9 +769,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 1641.3478, "serial_latency_p99": 0.0041, "serial_latency_p95": 0.0039, @@ -896,9 +896,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1444.847, + "optimize_duration": 7859.1353, + "load_duration": 9303.9823, "qps": 1488.5841, "serial_latency_p99": 0.0047, "serial_latency_p95": 0.0043, @@ -1152,9 +1152,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 2514.4481, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -1279,9 +1279,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 2177.2345, "serial_latency_p99": 0.0034, "serial_latency_p95": 0.0031, @@ -1406,9 +1406,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1833.2575, "serial_latency_p99": 0.0039, "serial_latency_p95": 0.0035, @@ -1533,9 +1533,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1552.4803, "serial_latency_p99": 0.004, "serial_latency_p95": 0.0037, @@ -1660,9 +1660,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1355.3121, "serial_latency_p99": 0.0044, "serial_latency_p95": 0.0042, @@ -1787,9 +1787,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 1079.2123, "serial_latency_p99": 0.0053, "serial_latency_p95": 0.0049, @@ -1914,9 +1914,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 1459.7483, + "optimize_duration": 7524.2304, + "load_duration": 8983.9787, "qps": 876.5772, "serial_latency_p99": 0.0063, "serial_latency_p95": 0.0059, @@ -2041,9 +2041,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 10663.1231, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0018, @@ -2168,9 +2168,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 10333.9072, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -2295,9 +2295,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 9575.6863, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0021, @@ -2422,9 +2422,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 8596.7694, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0022, @@ -2549,9 +2549,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 7704.3625, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0025, @@ -2676,9 +2676,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 7023.6735, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0028, @@ -2803,9 +2803,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 6031.3725, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.003, @@ -2930,9 +2930,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 129.799, + "optimize_duration": 152.2479, + "load_duration": 282.0468, "qps": 5258.1868, "serial_latency_p99": 0.0036, "serial_latency_p95": 0.0033, @@ -3057,9 +3057,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 5973.0024, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -3184,9 +3184,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 5416.5758, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0024, @@ -3311,9 +3311,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 4771.4324, "serial_latency_p99": 0.0028, "serial_latency_p95": 0.0025, @@ -3438,9 +3438,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 4006.3994, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.003, @@ -3565,9 +3565,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 3441.7597, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0032, @@ -3692,9 +3692,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 3040.6216, "serial_latency_p99": 0.0037, "serial_latency_p95": 0.0035, @@ -3819,9 +3819,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 2446.7373, "serial_latency_p99": 0.0043, "serial_latency_p95": 0.004, @@ -3946,9 +3946,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 119.5399, + "optimize_duration": 235.8939, + "load_duration": 355.4338, "qps": 2084.6245, "serial_latency_p99": 0.005, "serial_latency_p95": 0.0046, diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index e02463ebd..fe81ffe05 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -5,9 +5,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 13316.2336, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -124,9 +124,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 12837.5287, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -243,9 +243,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 12248.9154, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -362,9 +362,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 11501.6652, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0022, @@ -481,9 +481,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 10566.6823, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -600,9 +600,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 9227.318, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0026, @@ -719,9 +719,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 8320.4606, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -838,9 +838,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 7524.9879, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.0031, @@ -957,9 +957,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 246.6523, + "optimize_duration": 101.2089, + "load_duration": 347.8612, "qps": 6813.2439, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0034, @@ -1076,9 +1076,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 7385.2066, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -1195,9 +1195,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 6793.8443, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -1314,9 +1314,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 6242.5346, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0022, @@ -1433,9 +1433,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 5779.119, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0023, @@ -1552,9 +1552,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 5183.5843, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -1671,9 +1671,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 4259.5572, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0026, @@ -1790,9 +1790,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 3614.3118, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -1909,9 +1909,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 3181.0537, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0029, @@ -2028,9 +2028,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 0.0, - "optimize_duration": 0.0, - "load_duration": 0.0, + "insert_duration": 2450.8286, + "optimize_duration": 136.9261, + "load_duration": 2587.7547, "qps": 2804.8357, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.0032, From 268e7ab3d8e282af3f422c772f1db81f12c3d16b Mon Sep 17 00:00:00 2001 From: Alexander Guzhva Date: Thu, 9 Apr 2026 07:03:18 +0000 Subject: [PATCH 17/49] Introduce Intel SVS (#749) Signed-off-by: Alexandr Guzhva --- vectordb_bench/backend/clients/api.py | 3 + vectordb_bench/backend/clients/milvus/cli.py | 163 ++++++++++++++++++ .../backend/clients/milvus/config.py | 62 +++++++ 3 files changed, 228 insertions(+) diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 0f8103597..ecbddef39 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -43,6 +43,9 @@ class IndexType(StrEnum): GPU_CAGRA = "GPU_CAGRA" SCANN = "scann" SCANN_MILVUS = "SCANN_MILVUS" + SVS_VAMANA = "SVS_VAMANA" + SVS_VAMANA_LVQ = "SVS_VAMANA_LVQ" + SVS_VAMANA_LEANVEC = "SVS_VAMANA_LEANVEC" Hologres_HGraph = "HGraph" Hologres_Graph = "Graph" NONE = "NONE" diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 2f2a286be..ae7269801 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -485,6 +485,169 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): ) +class MilvusSVSVamanaTypedDict(CommonTypedDict, MilvusTypedDict): + svs_graph_max_degree: Annotated[ + int, + click.option( + "--svs-graph-max-degree", + type=int, + help="Maximum degree of the Vamana graph (4-256).", + required=True, + ), + ] + svs_construction_window_size: Annotated[ + int, + click.option( + "--svs-construction-window-size", + type=int, + help="Window size for graph construction.", + required=False, + default=40, + show_default=True, + ), + ] + svs_alpha: Annotated[ + float | None, + click.option( + "--svs-alpha", + type=float, + help="Pruning parameter (default: 1.2 for L2, 0.95 for IP/COSINE).", + required=False, + default=None, + ), + ] + svs_storage_kind: Annotated[ + str, + click.option( + "--svs-storage-kind", + type=click.Choice( + ["fp32", "fp16", "sqi8", "lvq4x0", "lvq4x4", "lvq4x8", "leanvec4x4", "leanvec4x8", "leanvec8x8"], + case_sensitive=False, + ), + help="Data storage format.", + required=False, + default="fp32", + show_default=True, + ), + ] + svs_search_window_size: Annotated[ + int | None, + click.option( + "--svs-search-window-size", + type=int, + help="Window size for search (1-10000).", + required=False, + default=None, + ), + ] + svs_search_buffer_capacity: Annotated[ + int | None, + click.option( + "--svs-search-buffer-capacity", + type=int, + help="Buffer capacity for search priority queue (1-10000).", + required=False, + default=None, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaTypedDict) +def MilvusSVSVamana(**parameters: Unpack[MilvusSVSVamanaTypedDict]): + from .config import MilvusConfig, SVSVamanaConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaTypedDict) +def MilvusSVSVamanaLVQ(**parameters: Unpack[MilvusSVSVamanaTypedDict]): + from .config import MilvusConfig, SVSVamanaLVQConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaLVQConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + **parameters, + ) + + +class MilvusSVSVamanaLeanVecTypedDict(MilvusSVSVamanaTypedDict): + svs_leanvec_dim: Annotated[ + int, + click.option( + "--svs-leanvec-dim", + type=int, + help="Dimensionality for LeanVec compression (0 = d/2).", + required=False, + default=0, + show_default=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusSVSVamanaLeanVecTypedDict) +def MilvusSVSVamanaLeanVec(**parameters: Unpack[MilvusSVSVamanaLeanVecTypedDict]): + from .config import MilvusConfig, SVSVamanaLeanVecConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=SVSVamanaLeanVecConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + svs_leanvec_dim=parameters["svs_leanvec_dim"], + ), + **parameters, + ) + + class MilvusGPUIVFPQTypedDict( CommonTypedDict, MilvusTypedDict, diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 98118c6df..620a6b484 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -442,6 +442,65 @@ def search_param(self) -> dict: } +class SVSVamanaConfig(MilvusIndexConfig, DBCaseConfig): + svs_graph_max_degree: int + svs_construction_window_size: int = 40 + svs_alpha: float | None = None + svs_storage_kind: str = "fp32" + svs_search_window_size: int | None = None + svs_search_buffer_capacity: int | None = None + index: IndexType = IndexType.SVS_VAMANA + + def index_param(self) -> dict: + params = { + "svs_graph_max_degree": self.svs_graph_max_degree, + "svs_construction_window_size": self.svs_construction_window_size, + "svs_storage_kind": self.svs_storage_kind, + } + if self.svs_alpha is not None: + params["svs_alpha"] = self.svs_alpha + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": params, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": { + "svs_search_window_size": self.svs_search_window_size, + "svs_search_buffer_capacity": self.svs_search_buffer_capacity, + }, + } + + +class SVSVamanaLVQConfig(SVSVamanaConfig): + svs_storage_kind: str = "lvq4x4" + index: IndexType = IndexType.SVS_VAMANA_LVQ + + +class SVSVamanaLeanVecConfig(SVSVamanaConfig): + svs_storage_kind: str = "leanvec4x4" + svs_leanvec_dim: int = 0 + index: IndexType = IndexType.SVS_VAMANA_LEANVEC + + def index_param(self) -> dict: + params = { + "svs_graph_max_degree": self.svs_graph_max_degree, + "svs_construction_window_size": self.svs_construction_window_size, + "svs_storage_kind": self.svs_storage_kind, + "svs_leanvec_dim": self.svs_leanvec_dim, + } + if self.svs_alpha is not None: + params["svs_alpha"] = self.svs_alpha + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": params, + } + + _milvus_case_config = { IndexType.AUTOINDEX: AutoIndexConfig, IndexType.HNSW: HNSWConfig, @@ -459,4 +518,7 @@ def search_param(self) -> dict: IndexType.GPU_CAGRA: GPUCAGRAConfig, IndexType.GPU_BRUTE_FORCE: GPUBruteForceConfig, IndexType.SCANN_MILVUS: SCANNConfig, + IndexType.SVS_VAMANA: SVSVamanaConfig, + IndexType.SVS_VAMANA_LVQ: SVSVamanaLVQConfig, + IndexType.SVS_VAMANA_LEANVEC: SVSVamanaLeanVecConfig, } From 619ce1bb82bec5aa05956f45ec0c70ba59c8366c Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Thu, 9 Apr 2026 15:37:04 +0800 Subject: [PATCH 18/49] fix: Skip compaction when encouters permission error (#753) Signed-off-by: yangxuan --- vectordb_bench/backend/clients/milvus/milvus.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 9e9dfb7f9..d36a15c24 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -165,23 +165,27 @@ def _optimize(self): log.info(f"{self.name} optimizing before search") try: self.client.flush(self.collection_name) - self._wait_for_segments_sorted() - self._wait_for_index() + if self.case_config.is_gpu_index: log.debug("skip force merge compaction for gpu index type.") else: try: + # wait for sort, index, compact + self._wait_for_segments_sorted() + self._wait_for_index() compaction_id = self.client.compact(self.collection_name, target_size=(2**63 - 1)) if compaction_id > 0: self._wait_for_compaction(compaction_id) log.info(f"{self.name} force merge compaction completed.") - self._wait_for_index() except Exception as e: - log.warning(f"{self.name} compact error: {e}") + log.warning(f"{self.name} compact or list segments error: {e}") if hasattr(e, "code") and e.code().name == "PERMISSION_DENIED": - log.warning("Skip compact due to permission denied.") + log.warning("Skip compact due to list segments or compact permission denied.") else: raise e from None + + # wait for index no matter what + self._wait_for_index() self.client.refresh_load(self.collection_name) except Exception as e: log.warning(f"{self.name} optimize error: {e}") From f0a8d031ecd3b9692617a7f20a13f83d8a742882 Mon Sep 17 00:00:00 2001 From: ChenLiqing Date: Tue, 14 Apr 2026 10:55:36 +0800 Subject: [PATCH 19/49] fix: refresh ZillizCloud build durations for 1M and 10M baselines (#754) Update result_20260403_standard_zillizcloud.json to use the latest validated build timings from recent reruns for case_id=5 (1M) and case_id=4 (10M), including insert_duration, optimize_duration, and load_duration. Co-authored-by: Ubuntu Co-authored-by: Claude Sonnet 4.6 --- .../result_20260403_standard_zillizcloud.json | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json index fe81ffe05..1f18a729a 100644 --- a/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json +++ b/vectordb_bench/results/ZillizCloud/result_20260403_standard_zillizcloud.json @@ -5,9 +5,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 13316.2336, "serial_latency_p99": 0.002, "serial_latency_p95": 0.0019, @@ -124,9 +124,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 12837.5287, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -243,9 +243,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 12248.9154, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -362,9 +362,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 11501.6652, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0022, @@ -481,9 +481,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 10566.6823, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -600,9 +600,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 9227.318, "serial_latency_p99": 0.0027, "serial_latency_p95": 0.0026, @@ -719,9 +719,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 8320.4606, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -838,9 +838,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 7524.9879, "serial_latency_p99": 0.0032, "serial_latency_p95": 0.0031, @@ -957,9 +957,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 246.6523, - "optimize_duration": 101.2089, - "load_duration": 347.8612, + "insert_duration": 246.6425, + "optimize_duration": 3135.662, + "load_duration": 3382.3046, "qps": 6813.2439, "serial_latency_p99": 0.0035, "serial_latency_p95": 0.0034, @@ -1076,9 +1076,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 7385.2066, "serial_latency_p99": 0.0021, "serial_latency_p95": 0.002, @@ -1195,9 +1195,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 6793.8443, "serial_latency_p99": 0.0022, "serial_latency_p95": 0.0021, @@ -1314,9 +1314,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 6242.5346, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0022, @@ -1433,9 +1433,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 5779.119, "serial_latency_p99": 0.0023, "serial_latency_p95": 0.0023, @@ -1552,9 +1552,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 5183.5843, "serial_latency_p99": 0.0024, "serial_latency_p95": 0.0023, @@ -1671,9 +1671,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 4259.5572, "serial_latency_p99": 0.0026, "serial_latency_p95": 0.0026, @@ -1790,9 +1790,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 3614.3118, "serial_latency_p99": 0.0029, "serial_latency_p95": 0.0028, @@ -1909,9 +1909,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 3181.0537, "serial_latency_p99": 0.003, "serial_latency_p95": 0.0029, @@ -2028,9 +2028,9 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 2450.8286, - "optimize_duration": 136.9261, - "load_duration": 2587.7547, + "insert_duration": 2451.8899, + "optimize_duration": 4862.256, + "load_duration": 7314.1459, "qps": 2804.8357, "serial_latency_p99": 0.0033, "serial_latency_p95": 0.0032, From 77d76ab9d7c612737c5e22bcf2698ccb8cc48893 Mon Sep 17 00:00:00 2001 From: SongYoungUk Date: Tue, 14 Apr 2026 18:24:22 +0900 Subject: [PATCH 20/49] feat: add VectorChord benchmark support (#745) * feat: add VectorChord support and VCHORDRQ index type * feat: add VectorChordRQ command to CLI * feat: add VectorChord support to README * feat: add VectorChordGraph support and configuration * feat: add max_scan_tuples parameter to VectorChordGraph * feat: enhance VectorChord with improved type safety and search functionality * feat: add vectorchord extension creation on connection Co-authored-by: edgar-p --- README.md | 33 +- vectordb_bench/backend/clients/__init__.py | 14 + vectordb_bench/backend/clients/api.py | 2 + .../backend/clients/vectorchord/__init__.py | 0 .../backend/clients/vectorchord/cli.py | 267 ++++++++++++++ .../backend/clients/vectorchord/config.py | 196 +++++++++++ .../clients/vectorchord/vectorchord.py | 325 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 3 + 8 files changed, 838 insertions(+), 2 deletions(-) create mode 100644 vectordb_bench/backend/clients/vectorchord/__init__.py create mode 100644 vectordb_bench/backend/clients/vectorchord/cli.py create mode 100644 vectordb_bench/backend/clients/vectorchord/config.py create mode 100644 vectordb_bench/backend/clients/vectorchord/vectorchord.py diff --git a/README.md b/README.md index 1b0f46309..685e37a47 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ All the database client supported | pinecone | `pip install vectordb-bench[pinecone]` | | weaviate | `pip install vectordb-bench[weaviate]` | | elastic, aliyun_elasticsearch| `pip install vectordb-bench[elastic]` | -| pgvector, pgvectorscale, pgdiskann, alloydb | `pip install vectordb-bench[pgvector]` | +| pgvector, pgvectorscale, pgdiskann, alloydb, vectorchord | `pip install vectordb-bench[pgvector]` | | pgvecto.rs | `pip install vectordb-bench[pgvecto_rs]` | | redis | `pip install vectordb-bench[redis]` | | memorydb | `pip install vectordb-bench[memorydb]` | @@ -86,6 +86,7 @@ Options: Commands: pgvectorhnsw pgvectorivfflat + vectorchordrq test weaviate ``` @@ -179,6 +180,34 @@ Options: --help Show this message and exit. ``` +### Run VectorChord (vchordrq) from command line + +VectorChord is a PostgreSQL extension for scalable vector similarity search using IVF + RaBitQ indexing. +It is fully compatible with pgvector data types and provides faster queries and index builds. + +```shell +vectordbbench vectorchordrq \ + --user-name postgres --password '' \ + --host localhost --port 5432 --db-name vectordb \ + --case-type Performance1536D50K \ + --lists 1000 --probes 10 --epsilon 1.9 \ + --spherical-centroids --build-threads 8 \ + --max-parallel-workers 15 +``` + +Key VectorChord-specific options: +| Option | Description | +|--------|-------------| +| `--lists` | Number of IVF lists for vchordrq index | +| `--probes` | Number of probes during search (default: 10) | +| `--epsilon` | Reranking precision factor, 0.0-4.0 (default: 1.9) | +| `--residual-quantization` | Enable residual quantization | +| `--spherical-centroids` | L2-normalize centroids (recommended for cosine/IP) | +| `--build-threads` | Number of threads for index building (1-255) | +| `--degree-of-parallelism` | Degree of parallelism for index build (1-256) | +| `--max-parallel-workers` | Sets max_parallel_workers & max_parallel_maintenance_workers | +| `--max-scan-tuples` | Max tuples to scan before stopping (-1 for unlimited) | + ### Run awsopensearch from command line ```shell @@ -756,7 +785,7 @@ Now we can only run one task at the same time. ### Code Structure ![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8c06512e-5419-4381-b084-9c93aed59639) ### Client -Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, Redis, Chroma, CockroachDB, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. +Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, VectorChord, Redis, Chroma, CockroachDB, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. ### Benchmark Cases We've developed lots of comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into four main types: #### Capacity Case diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 214d85e96..8437a3458 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -59,6 +59,7 @@ class DB(Enum): Zvec = "Zvec" Endee = "Endee" Lindorm = "Lindorm" + VectorChord = "VectorChord" PolarDB = "PolarDB" @property @@ -247,6 +248,10 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormVector + if self == DB.VectorChord: + from .vectorchord.vectorchord import VectorChord + + return VectorChord if self == DB.PolarDB: from .polardb.polardb import PolarDB @@ -441,6 +446,10 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return LindormConfig + if self == DB.VectorChord: + from .vectorchord.config import VectorChordConfig + + return VectorChordConfig if self == DB.PolarDB: from .polardb.config import PolarDBConfig @@ -617,6 +626,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _lindorm_vector_case_config.get(index_type) + if self == DB.VectorChord: + from .vectorchord.config import _vectorchord_case_config + + return _vectorchord_case_config.get(index_type) + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index ecbddef39..f507abe33 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -42,6 +42,8 @@ class IndexType(StrEnum): GPU_IVF_PQ = "GPU_IVF_PQ" GPU_CAGRA = "GPU_CAGRA" SCANN = "scann" + VCHORDRQ = "vchordrq" + VCHORDG = "vchordg" SCANN_MILVUS = "SCANN_MILVUS" SVS_VAMANA = "SVS_VAMANA" SVS_VAMANA_LVQ = "SVS_VAMANA_LVQ" diff --git a/vectordb_bench/backend/clients/vectorchord/__init__.py b/vectordb_bench/backend/clients/vectorchord/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/vectorchord/cli.py b/vectordb_bench/backend/clients/vectorchord/cli.py new file mode 100644 index 000000000..2b3e3862b --- /dev/null +++ b/vectordb_bench/backend/clients/vectorchord/cli.py @@ -0,0 +1,267 @@ +import os +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class VectorChordTypedDict(CommonTypedDict): + user_name: Annotated[ + str, + click.option("--user-name", type=str, help="Db username", required=True), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), + ] + + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + port: Annotated[ + int, + click.option( + "--port", + type=int, + help="Postgres database port", + default=5432, + show_default=True, + required=False, + ), + ] + db_name: Annotated[str, click.option("--db-name", type=str, help="Db name", required=True)] + max_parallel_workers: Annotated[ + int | None, + click.option( + "--max-parallel-workers", + type=int, + help="Sets the maximum number of parallel workers for index creation", + required=False, + ), + ] + quantization_type: Annotated[ + str | None, + click.option( + "--quantization-type", + type=click.Choice(["vector", "halfvec", "rabitq8", "rabitq4"]), + help="Quantization type for vectors", + default="vector", + show_default=True, + ), + ] + + +class VectorChordRQTypedDict(VectorChordTypedDict): + lists: Annotated[ + int | None, + click.option( + "--lists", + type=int, + help="Number of IVF lists for vchordrq index", + ), + ] + probes: Annotated[ + int | None, + click.option( + "--probes", + type=int, + help="Number of probes during search", + default=10, + show_default=True, + ), + ] + epsilon: Annotated[ + float | None, + click.option( + "--epsilon", + type=float, + help="Reranking precision factor (0.0-4.0, higher is more accurate but slower)", + default=1.9, + show_default=True, + ), + ] + residual_quantization: Annotated[ + bool, + click.option( + "--residual-quantization/--no-residual-quantization", + type=bool, + help="Enable residual quantization for improved accuracy", + default=False, + show_default=True, + ), + ] + rerank_in_table: Annotated[ + bool, + click.option( + "--rerank-in-table/--no-rerank-in-table", + type=bool, + help="Read vectors from table instead of storing in index (saves storage, degrades query performance)", + default=False, + show_default=True, + ), + ] + spherical_centroids: Annotated[ + bool, + click.option( + "--spherical-centroids/--no-spherical-centroids", + type=bool, + help="L2-normalize centroids during K-means (recommended for cosine/IP)", + default=False, + show_default=True, + ), + ] + build_threads: Annotated[ + int | None, + click.option( + "--build-threads", + type=int, + help="Number of threads for index building (range: 1-255)", + ), + ] + degree_of_parallelism: Annotated[ + int | None, + click.option( + "--degree-of-parallelism", + type=int, + help="Degree of parallelism for index build (range: 1-256, default: 32)", + ), + ] + max_scan_tuples: Annotated[ + int | None, + click.option( + "--max-scan-tuples", + type=int, + help="Max tuples to scan before stopping (-1 for unlimited)", + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(VectorChordRQTypedDict) +def VectorChordRQ( + **parameters: Unpack[VectorChordRQTypedDict], +): + from .config import VectorChordConfig, VectorChordRQConfig + + run( + db=DB.VectorChord, + db_config=VectorChordConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + db_name=parameters["db_name"], + ), + db_case_config=VectorChordRQConfig( + quantization_type=parameters["quantization_type"], + lists=parameters["lists"], + probes=parameters["probes"], + epsilon=parameters["epsilon"], + residual_quantization=parameters["residual_quantization"], + rerank_in_table=parameters["rerank_in_table"], + spherical_centroids=parameters["spherical_centroids"], + build_threads=parameters["build_threads"], + degree_of_parallelism=parameters["degree_of_parallelism"], + max_scan_tuples=parameters["max_scan_tuples"], + max_parallel_workers=parameters["max_parallel_workers"], + ), + **parameters, + ) + + +class VectorChordGraphTypedDict(VectorChordTypedDict): + m: Annotated[ + int | None, + click.option( + "--m", + type=int, + help="Max neighbors per vertex (default: 32)", + ), + ] + ef_construction: Annotated[ + int | None, + click.option( + "--ef-construction", + type=int, + help="Dynamic list size during insertion (default: 64)", + ), + ] + bits: Annotated[ + int | None, + click.option( + "--bits", + type=int, + help="RaBitQ quantization ratio (1 or 2, default: 2)", + ), + ] + ef_search: Annotated[ + int | None, + click.option( + "--ef-search", + type=int, + help="Dynamic list size for search (default: 64)", + default=64, + show_default=True, + ), + ] + beam_search: Annotated[ + int | None, + click.option( + "--beam-search", + type=int, + help="Batch vertex access width during search (default: 1)", + ), + ] + max_scan_tuples: Annotated[ + int | None, + click.option( + "--max-scan-tuples", + type=int, + help="Max tuples to scan before stopping (-1 for unlimited)", + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(VectorChordGraphTypedDict) +def VectorChordGraph( + **parameters: Unpack[VectorChordGraphTypedDict], +): + from .config import VectorChordConfig, VectorChordGraphConfig + + run( + db=DB.VectorChord, + db_config=VectorChordConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + db_name=parameters["db_name"], + ), + db_case_config=VectorChordGraphConfig( + quantization_type=parameters["quantization_type"], + m=parameters["m"], + ef_construction=parameters["ef_construction"], + bits=parameters["bits"], + ef_search=parameters["ef_search"], + beam_search=parameters["beam_search"], + max_parallel_workers=parameters["max_parallel_workers"], + max_scan_tuples=parameters["max_scan_tuples"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/vectorchord/config.py b/vectordb_bench/backend/clients/vectorchord/config.py new file mode 100644 index 000000000..95916eb06 --- /dev/null +++ b/vectordb_bench/backend/clients/vectorchord/config.py @@ -0,0 +1,196 @@ +from abc import abstractmethod +from typing import Literal, LiteralString, TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class VectorChordConfigDict(TypedDict): + """These keys will be directly used as kwargs in psycopg connection string, + so the names must match exactly psycopg API""" + + user: str + password: str + host: str + port: int + dbname: str + + +class VectorChordConfig(DBConfig): + user_name: SecretStr = SecretStr("postgres") + password: SecretStr + host: str = "localhost" + port: int = 5432 + db_name: str = "vectordb" + + def to_dict(self) -> VectorChordConfigDict: + user_str = self.user_name.get_secret_value() + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + } + + +_METRIC_OPS = { + "vector": { + MetricType.L2: "vector_l2_ops", + MetricType.IP: "vector_ip_ops", + MetricType.COSINE: "vector_cosine_ops", + }, + "halfvec": { + MetricType.L2: "halfvec_l2_ops", + MetricType.IP: "halfvec_ip_ops", + MetricType.COSINE: "halfvec_cosine_ops", + }, + "rabitq8": { + MetricType.L2: "rabitq8_l2_ops", + MetricType.IP: "rabitq8_ip_ops", + MetricType.COSINE: "rabitq8_cosine_ops", + }, + "rabitq4": { + MetricType.L2: "rabitq4_l2_ops", + MetricType.IP: "rabitq4_ip_ops", + MetricType.COSINE: "rabitq4_cosine_ops", + }, +} + + +class VectorChordIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + create_index_before_load: bool = False + create_index_after_load: bool = True + quantization_type: Literal["vector", "halfvec", "rabitq8", "rabitq4"] = "vector" + + def parse_metric(self) -> str: + ops = _METRIC_OPS.get(self.quantization_type, _METRIC_OPS["vector"]) + return ops.get(self.metric_type, ops[MetricType.COSINE]) + + def parse_metric_fun_op(self) -> LiteralString: + if self.metric_type == MetricType.L2: + return "<->" + if self.metric_type == MetricType.IP: + return "<#>" + return "<=>" + + @abstractmethod + def index_param(self) -> dict: ... + + @abstractmethod + def search_param(self) -> dict: ... + + @abstractmethod + def session_param(self) -> dict: ... + + +class VectorChordRQConfig(VectorChordIndexConfig): + index: IndexType = IndexType.VCHORDRQ + # Build parameters (top-level options) + residual_quantization: bool = False + rerank_in_table: bool = False + degree_of_parallelism: int | None = None # default 32, range [1, 256] + # Build parameters ([build.internal] section) + lists: int | None = None + spherical_centroids: bool = False + build_threads: int | None = None # range [1, 255] + # PostgreSQL tuning parameter + max_parallel_workers: int | None = None # sets max_parallel_workers & max_parallel_maintenance_workers + # Search parameters (GUCs) + probes: int | None = 10 + epsilon: float | None = 1.9 # range [0.0, 4.0] + max_scan_tuples: int | None = None # default -1, range [-1, 2147483647] + + def index_param(self) -> dict: + options_parts = [] + if self.rerank_in_table: + options_parts.append("rerank_in_table = true") + if self.residual_quantization: + options_parts.append("residual_quantization = true") + if self.degree_of_parallelism is not None: + options_parts.append(f"degree_of_parallelism = {self.degree_of_parallelism}") + options_parts.append("[build.internal]") + if self.lists is not None: + options_parts.append(f"lists = [{self.lists}]") + if self.spherical_centroids: + options_parts.append("spherical_centroids = true") + if self.build_threads is not None: + options_parts.append(f"build_threads = {self.build_threads}") + + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "quantization_type": self.quantization_type, + "options": "\n".join(options_parts), + "max_parallel_workers": self.max_parallel_workers, + } + + def search_param(self) -> dict: + return { + "metric_fun_op": self.parse_metric_fun_op(), + } + + def session_param(self) -> dict: + params = {} + if self.probes is not None: + params["vchordrq.probes"] = str(self.probes) + if self.epsilon is not None: + params["vchordrq.epsilon"] = str(self.epsilon) + if self.max_scan_tuples is not None: + params["vchordrq.max_scan_tuples"] = str(self.max_scan_tuples) + return params + + +class VectorChordGraphConfig(VectorChordIndexConfig): + index: IndexType = IndexType.VCHORDG + # Build parameters + m: int | None = None # default 32, max neighbors per vertex + ef_construction: int | None = None # default 64 + bits: int | None = None # default 2, quantization ratio (1 or 2) + # PostgreSQL tuning parameter + max_parallel_workers: int | None = None + # Search parameters (GUCs) + ef_search: int | None = 64 # range [1, 65535] + beam_search: int | None = None # default 1 + max_scan_tuples: int | None = None # default -1, range [-1, 2147483647] + + def index_param(self) -> dict: + options_parts = [] + if self.m is not None: + options_parts.append(f"m = {self.m}") + if self.ef_construction is not None: + options_parts.append(f"ef_construction = {self.ef_construction}") + if self.bits is not None: + options_parts.append(f"bits = {self.bits}") + + return { + "metric": self.parse_metric(), + "index_type": self.index.value, + "quantization_type": self.quantization_type, + "options": "\n".join(options_parts), + "max_parallel_workers": self.max_parallel_workers, + } + + def search_param(self) -> dict: + return { + "metric_fun_op": self.parse_metric_fun_op(), + } + + def session_param(self) -> dict: + params = {} + if self.ef_search is not None: + params["vchordg.ef_search"] = str(self.ef_search) + if self.beam_search is not None: + params["vchordg.beam_search"] = str(self.beam_search) + if self.max_scan_tuples is not None: + params["vchordg.max_scan_tuples"] = str(self.max_scan_tuples) + return params + + +_vectorchord_case_config = { + IndexType.VCHORDRQ: VectorChordRQConfig, + IndexType.VCHORDG: VectorChordGraphConfig, +} diff --git a/vectordb_bench/backend/clients/vectorchord/vectorchord.py b/vectordb_bench/backend/clients/vectorchord/vectorchord.py new file mode 100644 index 000000000..77a462e75 --- /dev/null +++ b/vectordb_bench/backend/clients/vectorchord/vectorchord.py @@ -0,0 +1,325 @@ +"""Wrapper around the VectorChord vector database over VectorDB""" + +import logging +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any + +import numpy as np +import psycopg +from pgvector.psycopg import register_vector +from psycopg import Connection, Cursor, sql + +from ...filter import Filter, FilterOp +from ..api import VectorDB +from .config import VectorChordConfigDict, VectorChordIndexConfig + +log = logging.getLogger(__name__) + + +class VectorChord(VectorDB): + """Use psycopg instructions""" + + thread_safe: bool = False + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + + conn: psycopg.Connection[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None + + _search: sql.Composed + where_clause: str = "" + + def __init__( + self, + dim: int, + db_config: VectorChordConfigDict, + db_case_config: VectorChordIndexConfig, + collection_name: str = "vectorchord_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "VectorChord" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + self._index_name = "vectorchord_index" + self._primary_field = "id" + self._vector_field = "embedding" + + index_param = self.case_config.index_param() + self._quantization_type = index_param["quantization_type"] + self._index_method = index_param["index_type"] + + self.conn, self.cursor = self._create_connection(**self.db_config) + + # create vectorchord extension if not exists + self.cursor.execute("CREATE EXTENSION IF NOT EXISTS vchord CASCADE") + self.conn.commit() + + log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ), + ): + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + f"{self.name} config values: {self.db_config}\n{self.case_config}" + ) + log.error(msg) + raise RuntimeError(msg) + + if drop_old: + self._drop_index() + self._drop_table() + self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + @staticmethod + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + register_vector(conn) + conn.autocommit = False + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + + @contextmanager + def init(self) -> Generator[None, None, None]: + self.conn, self.cursor = self._create_connection(**self.db_config) + + # index configuration may have commands defined that we should set during each client session + session_options: dict[str, Any] = self.case_config.session_param() + + if len(session_options) > 0: + for setting_name, setting_val in session_options.items(): + command = sql.SQL("SET {setting_name} " + "= {setting_val};").format( + setting_name=sql.Identifier(setting_name), + setting_val=sql.Literal(str(setting_val)), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop table : {self.table_name}") + + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( + table_name=sql.Identifier(self.table_name), + ), + ) + self.conn.commit() + + def optimize(self, data_size: int | None = None): + self._post_insert() + + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + if self.case_config.create_index_after_load: + self._drop_index() + self._create_index() + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop index : {self._index_name}") + + drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( + index_name=sql.Identifier(self._index_name), + ) + log.debug(drop_index_sql.as_string(self.cursor)) + self.cursor.execute(drop_index_sql) + self.conn.commit() + + def _set_parallel_index_build_param(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + if index_param["max_parallel_workers"] is not None: + self.cursor.execute( + sql.SQL("SET max_parallel_workers TO '{}';").format( + index_param["max_parallel_workers"], + ), + ) + self.cursor.execute( + sql.SQL("SET max_parallel_maintenance_workers TO '{}';").format( + index_param["max_parallel_workers"], + ), + ) + self.cursor.execute( + sql.SQL("ALTER TABLE {} SET (parallel_workers = {});").format( + sql.Identifier(self.table_name), + index_param["max_parallel_workers"], + ), + ) + self.conn.commit() + + results = self.cursor.execute(sql.SQL("SHOW max_parallel_workers;")).fetchall() + results.extend(self.cursor.execute(sql.SQL("SHOW max_parallel_maintenance_workers;")).fetchall()) + log.info(f"{self.name} parallel index creation parameters: {results}") + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client create index : {self._index_name}") + + index_param: dict[str, Any] = self.case_config.index_param() + self._set_parallel_index_build_param() + + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING {index_method} (embedding {embedding_metric}) + """, + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + index_method=sql.SQL(self._index_method), + embedding_metric=sql.Identifier(index_param["metric"]), + ) + + options_str = index_param.get("options", "") + if options_str: + with_clause = sql.SQL( + "WITH (options = $vchord$\n{options}\n$vchord$);", + ).format(options=sql.SQL(options_str)) + else: + with_clause = sql.SQL(";") + + full_sql = index_create_sql + sql.SQL(" ") + with_clause + log.debug(full_sql.as_string(self.cursor)) + self.cursor.execute(full_sql) + self.conn.commit() + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + log.info(f"{self.name} client create table : {self.table_name}") + + col_type = self._quantization_type + if col_type in ("rabitq8", "rabitq4"): + # rabitq types need vector column + quantization during insert + col_type = "vector" + + self.cursor.execute( + sql.SQL( + "CREATE TABLE IF NOT EXISTS public.{table_name} " + "(id BIGINT PRIMARY KEY, embedding {col_type}({dim}));", + ).format( + table_name=sql.Identifier(self.table_name), + col_type=sql.SQL(col_type), + dim=dim, + ), + ) + self.conn.commit() + except Exception as e: + log.warning(f"Failed to create vectorchord table: {self.table_name} error: {e}") + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + if self._quantization_type == "halfvec": + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name), + ), + ) as copy: + copy.set_types(["bigint", "halfvec"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, np.float16(embeddings_arr[i]))) + else: + # vector, rabitq8, rabitq4 all store as vector column + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name), + ), + ) as copy: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) + self.conn.commit() + + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into vectorchord table ({self.table_name}), error: {e}") + return 0, e + + def _generate_search_query(self) -> sql.Composed: + # Search query cast type: rabitq8/rabitq4 queries still accept ::vector input + cast_type = "vector" + return sql.Composed( + [ + sql.SQL("SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding ").format( + table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL(self.where_clause), + ), + sql.SQL(self.case_config.search_param()["metric_fun_op"]), + sql.SQL(f" %s::{cast_type} LIMIT %s::int"), + ], + ) + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.where_clause = "" + elif filters.type == FilterOp.NumGE: + self.where_clause = f"WHERE {self._primary_field} >= {filters.int_value}" + else: + msg = f"Not support Filter for VectorChord - {filters}" + raise ValueError(msg) + + self._search = self._generate_search_query() + + def search_embedding( + self, + query: list[float], + k: int = 100, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + q = np.asarray(query) + result = self.cursor.execute(self._search, (q, k), prepare=True, binary=True) + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index b48d5900c..3e21746aa 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -38,6 +38,7 @@ from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB from ..backend.clients.turbopuffer.cli import TurboPuffer +from ..backend.clients.vectorchord.cli import VectorChordGraph, VectorChordRQ from ..backend.clients.vespa.cli import Vespa from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex @@ -87,6 +88,8 @@ cli.add_command(LindormHNSW) cli.add_command(LindormIVFBQ) cli.add_command(Pinecone) +cli.add_command(VectorChordRQ) +cli.add_command(VectorChordGraph) cli.add_command(PolarDBHNSWFlat) cli.add_command(PolarDBHNSWPQ) cli.add_command(PolarDBHNSWSQ) From 5a9173e90d1cd9b010580ffc5a6e507ff792e5eb Mon Sep 17 00:00:00 2001 From: shaohuasong-fang Date: Mon, 20 Apr 2026 10:37:12 +0800 Subject: [PATCH 21/49] =?UTF-8?q?fix(pgvector):=20normalize=20index=5Ftype?= =?UTF-8?q?=20to=20lowercase=20in=20=5Fcreate=5Findex=20to=20=E2=80=A6=20(?= =?UTF-8?q?#760)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(pgvector): normalize index_type to lowercase in _create_index to match PostgreSQL access method names PostgreSQL pgvector extension registers index access methods in lowercase (e.g. "hnsw", "ivfflat"), but the frontend passes IndexType.HNSW.value which is uppercase "HNSW", causing "access method HNSW does not exist" error. * Fix index type usage in pgvector.py Replaced index_param['index_type'] with index_type_lower for consistency. * add comment sign '#' I have added the # before [FIX] --- .../backend/clients/pgvector/pgvector.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 30c797c38..0b0750172 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -335,6 +335,12 @@ def _create_index(self): index_param = self.case_config.index_param() self._set_parallel_index_build_param() + # [FIX] The index access method name registered by the PostgreSQL pgvector extension is in lowercase (e.g., "hnsw", "ivfflat"), + # but the index type passed from the frontend UI is uppercase "HNSW" via IndexType.HNSW.value, causing SQL syntax "USING 'HNSW'" + # to fail with error "access method HNSW does not exist". Here we uniformly convert it to lowercase to match PostgreSQL's access method name. + index_type_lower = index_param["index_type"].lower() + log.info(f"index_type (original={index_param['index_type']}, normalized={index_type_lower})") + options = [] for option in index_param["index_creation_with_options"]: if option["val"] is not None: @@ -360,7 +366,9 @@ def _create_index(self): if index_param["quantization_type"] == "bit" else sql.Identifier("embedding") ), - index_type=sql.Identifier(index_param["index_type"]), + # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] + index_type=sql.Identifier(index_type_lower), + # index_type=sql.Identifier(index_param["index_type"]), # This assumes that the quantization_type value matches the quantization function name quantization_type=sql.SQL(index_param["quantization_type"]), dim=self.dim, @@ -375,7 +383,9 @@ def _create_index(self): ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), - index_type=sql.Identifier(index_param["index_type"]), + # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] + index_type=sql.Identifier(index_type_lower), + # index_type=sql.Identifier(index_param["index_type"]), embedding_metric=sql.Identifier(index_param["metric"]), ) From c4083f31d37ea3d73af5e2bdef8b223d418fe889 Mon Sep 17 00:00:00 2001 From: Xiang Fu Date: Sun, 19 Apr 2026 23:18:51 -0700 Subject: [PATCH 22/49] feat: add Apache Pinot vector search client (#757) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a complete Apache Pinot client for VectorDBBench. Index types: HNSW (Lucene), IVF_FLAT, IVF_PQ, IVF_ON_DISK Metrics: L2, IP, COSINE Filters: NumGE, StrEqual Optional dep: pip install "vectordb-bench[pinot]" Parallel loading: thread_safe=True — each worker thread maintains its own row buffer and flushes to Pinot via a fresh HTTP session. Since Pinot's ingestFromFile is synchronous (blocks until HNSW index is built, ~6 min per 100K×768D segment), concurrent flushes across threads reduce load time significantly vs sequential flushing. Benchmark results: Small dataset (OpenAI 50K, 768D, L2): HNSW: 798 QPS, recall=1.000 IVF_FLAT: 800 QPS, recall=1.000 IVF_PQ: 795 QPS, recall=1.000 IVF_ON_DISK: 691 QPS, recall=1.000 Large dataset (Cohere 1M, 768D, COSINE): HNSW m=16: 74 QPS, recall=0.982 Filter benchmark (Cohere 1M, COSINE, HNSW m=32): 1% NumGE: 71 QPS, recall=0.977 99% NumGE: 97 QPS, recall=0.649 Co-authored-by: Claude Sonnet 4.6 --- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 20 + .../backend/clients/pinot/__init__.py | 0 vectordb_bench/backend/clients/pinot/cli.py | 202 ++++++++ .../backend/clients/pinot/config.py | 94 ++++ vectordb_bench/backend/clients/pinot/pinot.py | 449 ++++++++++++++++++ vectordb_bench/cli/vectordbbench.py | 2 + 7 files changed, 768 insertions(+) create mode 100644 vectordb_bench/backend/clients/pinot/__init__.py create mode 100644 vectordb_bench/backend/clients/pinot/cli.py create mode 100644 vectordb_bench/backend/clients/pinot/config.py create mode 100644 vectordb_bench/backend/clients/pinot/pinot.py diff --git a/pyproject.toml b/pyproject.toml index e72be9697..2ed18b115 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ turbopuffer = [ "turbopuffer" ] zvec = [ "zvec" ] endee = [ "endee==0.1.10" ] lindorm = [ "opensearch-py" ] +pinot = [ "requests" ] [project.urls] Repository = "https://github.com/zilliztech/VectorDBBench" diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 8437a3458..9029be05f 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -61,6 +61,7 @@ class DB(Enum): Lindorm = "Lindorm" VectorChord = "VectorChord" PolarDB = "PolarDB" + Pinot = "Pinot" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -257,6 +258,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return PolarDB + if self == DB.Pinot: + from .pinot.pinot import Pinot + + return Pinot + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -455,6 +461,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return PolarDBConfig + if self == DB.Pinot: + from .pinot.config import PinotConfig + + return PinotConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -631,6 +642,15 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _vectorchord_case_config.get(index_type) + if self == DB.Pinot: + from .pinot.config import PinotHNSWConfig, PinotIVFFlatConfig, PinotIVFPQConfig + + return { + IndexType.HNSW: PinotHNSWConfig, + IndexType.IVFFlat: PinotIVFFlatConfig, + IndexType.IVFPQ: PinotIVFPQConfig, + }.get(index_type, PinotHNSWConfig) + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/pinot/__init__.py b/vectordb_bench/backend/clients/pinot/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/pinot/cli.py b/vectordb_bench/backend/clients/pinot/cli.py new file mode 100644 index 000000000..e51160b05 --- /dev/null +++ b/vectordb_bench/backend/clients/pinot/cli.py @@ -0,0 +1,202 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor2, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class PinotTypedDict(TypedDict): + controller_host: Annotated[ + str, + click.option("--controller-host", type=str, default="localhost", help="Pinot Controller host"), + ] + controller_port: Annotated[ + int, + click.option("--controller-port", type=int, default=9000, help="Pinot Controller port"), + ] + broker_host: Annotated[ + str, + click.option("--broker-host", type=str, default="localhost", help="Pinot Broker host"), + ] + broker_port: Annotated[ + int, + click.option("--broker-port", type=int, default=8099, help="Pinot Broker port"), + ] + username: Annotated[ + str, + click.option("--username", type=str, default=None, help="Pinot username (optional)"), + ] + password: Annotated[ + str, + click.option("--password", type=str, default=None, help="Pinot password (optional)"), + ] + ingest_batch_size: Annotated[ + int, + click.option( + "--ingest-batch-size", + type=int, + default=100_000, + show_default=True, + help=( + "Rows buffered before flushing one Pinot segment (one ingestFromFile call). " + "Larger values mean fewer segments and better IVF training / query performance. " + "Reduce if memory is constrained (100K x 768-dim float32 ~= 300 MB)." + ), + ), + ] + + +def _pinot_db_config(parameters: dict): + from .config import PinotConfig + + return PinotConfig( + db_label=parameters["db_label"], + controller_host=parameters["controller_host"], + controller_port=parameters["controller_port"], + broker_host=parameters["broker_host"], + broker_port=parameters["broker_port"], + username=parameters.get("username"), + password=SecretStr(parameters["password"]) if parameters.get("password") else None, + ingest_batch_size=parameters["ingest_batch_size"], + ) + + +@click.group() +def Pinot(): + """Apache Pinot vector search benchmarks.""" + + +# --------------------------------------------------------------------------- +# HNSW +# --------------------------------------------------------------------------- + + +class PinotHNSWTypedDict(CommonTypedDict, PinotTypedDict, HNSWFlavor2): ... + + +@Pinot.command("hnsw") +@click_parameter_decorators_from_typed_dict(PinotHNSWTypedDict) +def pinot_hnsw(**parameters: Unpack[PinotHNSWTypedDict]): + from .config import PinotHNSWConfig + + run( + db=DB.Pinot, + db_config=_pinot_db_config(parameters), + db_case_config=PinotHNSWConfig( + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef=parameters["ef_runtime"], + ), + **parameters, + ) + + +# --------------------------------------------------------------------------- +# IVF_FLAT +# --------------------------------------------------------------------------- + + +class PinotIVFFlatTypedDict(CommonTypedDict, PinotTypedDict): + nlist: Annotated[ + int, + click.option("--nlist", type=int, default=128, help="Number of Voronoi cells (IVF nlist)"), + ] + quantizer: Annotated[ + str, + click.option( + "--quantizer", + type=click.Choice(["FLAT", "SQ8", "SQ4"]), + default="FLAT", + help="Quantizer type for IVF_FLAT", + ), + ] + nprobe: Annotated[ + int, + click.option("--nprobe", type=int, default=8, help="Number of cells to probe at query time"), + ] + train_sample_size: Annotated[ + int, + click.option( + "--train-sample-size", + type=int, + default=None, + help="Training sample size (defaults to max(nlist*50, 1000) if not set)", + ), + ] + + +@Pinot.command("ivf-flat") +@click_parameter_decorators_from_typed_dict(PinotIVFFlatTypedDict) +def pinot_ivf_flat(**parameters: Unpack[PinotIVFFlatTypedDict]): + from .config import PinotIVFFlatConfig + + run( + db=DB.Pinot, + db_config=_pinot_db_config(parameters), + db_case_config=PinotIVFFlatConfig( + nlist=parameters["nlist"], + quantizer=parameters["quantizer"], + nprobe=parameters["nprobe"], + train_sample_size=parameters.get("train_sample_size"), + ), + **parameters, + ) + + +# --------------------------------------------------------------------------- +# IVF_PQ +# --------------------------------------------------------------------------- + + +class PinotIVFPQTypedDict(CommonTypedDict, PinotTypedDict): + nlist: Annotated[ + int, + click.option("--nlist", type=int, default=128, help="Number of Voronoi cells (IVF nlist)"), + ] + pq_m: Annotated[ + int, + click.option("--pq-m", type=int, default=8, help="Number of PQ sub-quantizers (must divide dimension)"), + ] + pq_nbits: Annotated[ + int, + click.option( + "--pq-nbits", + type=click.Choice(["4", "6", "8"]), + default="8", + help="Bits per PQ code (4, 6, or 8)", + ), + ] + train_sample_size: Annotated[ + int, + click.option("--train-sample-size", type=int, default=6400, help="Training sample size (must be >= nlist)"), + ] + nprobe: Annotated[ + int, + click.option("--nprobe", type=int, default=8, help="Number of cells to probe at query time"), + ] + + +@Pinot.command("ivf-pq") +@click_parameter_decorators_from_typed_dict(PinotIVFPQTypedDict) +def pinot_ivf_pq(**parameters: Unpack[PinotIVFPQTypedDict]): + from .config import PinotIVFPQConfig + + run( + db=DB.Pinot, + db_config=_pinot_db_config(parameters), + db_case_config=PinotIVFPQConfig( + nlist=parameters["nlist"], + pq_m=parameters["pq_m"], + pq_nbits=int(parameters["pq_nbits"]), + train_sample_size=parameters["train_sample_size"], + nprobe=parameters["nprobe"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/pinot/config.py b/vectordb_bench/backend/clients/pinot/config.py new file mode 100644 index 000000000..e29db0bec --- /dev/null +++ b/vectordb_bench/backend/clients/pinot/config.py @@ -0,0 +1,94 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType + + +class PinotConfig(DBConfig): + controller_host: str = "localhost" + controller_port: int = 9000 + broker_host: str = "localhost" + broker_port: int = 8099 + username: str | None = None + password: SecretStr | None = None + # Rows buffered before flushing one Pinot segment (one ingestFromFile call). + # Larger values → fewer segments → better IVF training & query perf. + # 100_000 rows x 768-dim float32 ~= 300 MB in-memory. + ingest_batch_size: int = 100_000 + + def to_dict(self) -> dict: + return { + "controller_host": self.controller_host, + "controller_port": self.controller_port, + "broker_host": self.broker_host, + "broker_port": self.broker_port, + "username": self.username, + "password": self.password.get_secret_value() if self.password else None, + "ingest_batch_size": self.ingest_batch_size, + } + + +class PinotHNSWConfig(BaseModel, DBCaseConfig): + """HNSW vector index config for Apache Pinot (Lucene-based).""" + + metric_type: MetricType | None = None + m: int = 16 # maxCon: max connections per node + ef_construction: int = 100 # beamWidth: construction beam width + ef: int | None = None # ef_search: HNSW candidate list size at query time (default=k) + + def index_param(self) -> dict: + return { + "vectorIndexType": "HNSW", + "maxCon": str(self.m), + "beamWidth": str(self.ef_construction), + } + + def search_param(self) -> dict: + # ef controls the HNSW candidate list during search via vectorSimilarity(col, q, ef). + # Larger ef → better recall, slightly higher latency. Defaults to k if not set. + return {"ef": self.ef} if self.ef is not None else {} + + +class PinotIVFFlatConfig(BaseModel, DBCaseConfig): + """IVF_FLAT vector index config for Apache Pinot.""" + + metric_type: MetricType | None = None + nlist: int = 128 # number of Voronoi cells (centroids) + quantizer: str = "FLAT" # FLAT, SQ8, or SQ4 + train_sample_size: int | None = None # defaults to max(nlist*50, 1000) if None + nprobe: int = 8 # number of cells to probe at query time + + def index_param(self) -> dict: + params: dict = { + "vectorIndexType": "IVF_FLAT", + "nlist": str(self.nlist), + "quantizer": self.quantizer, + } + if self.train_sample_size is not None: + params["trainSampleSize"] = str(self.train_sample_size) + return params + + def search_param(self) -> dict: + return {"nprobe": self.nprobe} + + +class PinotIVFPQConfig(BaseModel, DBCaseConfig): + """IVF_PQ vector index config for Apache Pinot (residual product quantization).""" + + metric_type: MetricType | None = None + nlist: int = 128 # number of Voronoi cells (centroids) + pq_m: int = 8 # number of sub-quantizers (must divide vectorDimension) + pq_nbits: int = 8 # bits per sub-quantizer code: 4, 6, or 8 + train_sample_size: int = 6400 # training sample size (must be >= nlist) + nprobe: int = 8 # number of cells to probe at query time + + def index_param(self) -> dict: + return { + "vectorIndexType": "IVF_PQ", + "nlist": str(self.nlist), + "pqM": str(self.pq_m), + "pqNbits": str(self.pq_nbits), + "trainSampleSize": str(self.train_sample_size), + } + + def search_param(self) -> dict: + return {"nprobe": self.nprobe} diff --git a/vectordb_bench/backend/clients/pinot/pinot.py b/vectordb_bench/backend/clients/pinot/pinot.py new file mode 100644 index 000000000..bdae58d64 --- /dev/null +++ b/vectordb_bench/backend/clients/pinot/pinot.py @@ -0,0 +1,449 @@ +"""Wrapper around Apache Pinot vector database over VectorDB""" + +import json +import logging +import tempfile +import threading +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import requests + +from ...filter import Filter, FilterOp +from ..api import DBCaseConfig, MetricType, VectorDB + +log = logging.getLogger(__name__) + +# Rows accumulated before flushing a segment to Pinot. +# Large enough to avoid thousands of tiny segments (which breaks IVF training +# and hurts query performance), yet small enough to keep memory use bounded. +# For 768-dim float32 vectors: 100K rows ≈ 300 MB in-memory. +DEFAULT_INGEST_BATCH_SIZE = 100_000 + + +class Pinot(VectorDB): + """Apache Pinot vector database client for VectorDBBench.""" + + name = "Pinot" + # thread_safe=True: each flush uses a fresh requests.Session (not a shared one), + # and each worker thread has its own row buffer via threading.local(). + # This lets the framework spawn multiple load workers that flush segments in parallel. + thread_safe: bool = True + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "VectorBenchCollection", + drop_old: bool = False, + with_scalar_labels: bool = False, + **kwargs, + ): + self.dim = dim + self.case_config = db_case_config + self.table_name = collection_name + self._primary_field = "id" + self._vector_field = "embedding" + self._label_field = "labels" + self.with_scalar_labels = with_scalar_labels + self._filter_where: str = "" # set by prepare_filter(); applied in search_embedding() + + controller_host = db_config["controller_host"] + controller_port = db_config["controller_port"] + broker_host = db_config["broker_host"] + broker_port = db_config["broker_port"] + self._controller_url = f"http://{controller_host}:{controller_port}" + self._broker_url = f"http://{broker_host}:{broker_port}" + + self._auth = None + if db_config.get("username") and db_config.get("password"): + self._auth = (db_config["username"], db_config["password"]) + + # Per-thread row buffers — each worker thread accumulates rows independently + # and flushes its own segment when the threshold is reached. + # _registered_buffers tracks all thread-local objects so init() teardown + # can flush any remaining rows from every worker thread. + self._thread_local = threading.local() + self._registered_buffers: list = [] + self._buffers_lock = threading.Lock() + self._ingest_batch_size: int = db_config.get("ingest_batch_size", DEFAULT_INGEST_BATCH_SIZE) + + self.session = None + + with requests.Session() as setup_session: + if self._auth: + setup_session.auth = self._auth + + if drop_old: + self._delete_table(setup_session) + self._delete_schema(setup_session) + + if not self._schema_exists(setup_session): + self._create_schema(setup_session) + + if not self._table_exists(setup_session): + self._create_table(setup_session) + + def _schema_exists(self, session: requests.Session) -> bool: + resp = session.get(f"{self._controller_url}/schemas/{self.table_name}") + return resp.status_code == 200 + + def _table_exists(self, session: requests.Session) -> bool: + resp = session.get(f"{self._controller_url}/tables/{self.table_name}") + if resp.status_code != 200: + return False + data = resp.json() + return bool(data.get("OFFLINE") or data.get("tables")) + + def _delete_table(self, session: requests.Session): + resp = session.delete(f"{self._controller_url}/tables/{self.table_name}?type=offline") + if resp.status_code not in (200, 404): + log.warning(f"Failed to delete Pinot table {self.table_name}: {resp.text}") + else: + log.info(f"Deleted Pinot table: {self.table_name}") + # Wait for Pinot to finish cleaning up the external view + for _ in range(30): + check = session.get(f"{self._controller_url}/tables/{self.table_name}/externalview") + if check.status_code == 404 or not check.json(): + break + log.info(f"Waiting for Pinot external view cleanup for {self.table_name}...") + time.sleep(2) + else: + log.warning(f"External view for {self.table_name} did not clear within 60s") + + def _delete_schema(self, session: requests.Session): + resp = session.delete(f"{self._controller_url}/schemas/{self.table_name}") + if resp.status_code not in (200, 404): + log.warning(f"Failed to delete Pinot schema {self.table_name}: {resp.text}") + else: + log.info(f"Deleted Pinot schema: {self.table_name}") + + def _create_schema(self, session: requests.Session): + dimension_fields = [ + {"name": self._primary_field, "dataType": "INT"}, + {"name": self._vector_field, "dataType": "FLOAT", "singleValueField": False}, + ] + if self.with_scalar_labels: + dimension_fields.append({"name": self._label_field, "dataType": "STRING"}) + + schema = { + "schemaName": self.table_name, + "dimensionFieldSpecs": dimension_fields, + } + resp = session.post( + f"{self._controller_url}/schemas", + json=schema, + headers={"Content-Type": "application/json"}, + ) + if not resp.ok: + log.error(f"Failed to create Pinot schema: {resp.text}") + resp.raise_for_status() + log.info(f"Created Pinot schema: {self.table_name}") + + def _create_table(self, session: requests.Session): + metric_str = self._get_index_metric_str() + index_params = self.case_config.index_param() + + # Pull vectorIndexType out of index_params; remaining entries are type-specific properties. + vector_index_type = index_params.pop("vectorIndexType", "HNSW") + properties: dict = { + "vectorIndexType": vector_index_type, + "vectorDimension": str(self.dim), + "vectorDistanceFunction": metric_str, + "version": "1", + } + properties.update(index_params) + + table_config = { + "tableName": self.table_name, + "tableType": "OFFLINE", + "segmentsConfig": { + "replication": "1", + "schemaName": self.table_name, + }, + "tenants": {}, + "tableIndexConfig": { + "loadMode": "MMAP", + # Inverted index on id for fast equality lookups; range index for >=/<= filters. + "invertedIndexColumns": [self._primary_field], + "rangeIndexColumns": [self._primary_field], + }, + "fieldConfigList": [ + { + "encodingType": "RAW", + "indexType": "VECTOR", + "name": self._vector_field, + "properties": properties, + } + ], + "ingestionConfig": { + "batchIngestionConfig": { + "segmentIngestionType": "APPEND", + "segmentIngestionFrequency": "DAILY", + } + }, + "metadata": {}, + } + resp = session.post( + f"{self._controller_url}/tables", + json=table_config, + headers={"Content-Type": "application/json"}, + ) + if not resp.ok: + log.error(f"Failed to create Pinot table: {resp.text}") + resp.raise_for_status() + log.info(f"Created Pinot table: {self.table_name}") + + def _get_index_metric_str(self) -> str: + if self.case_config.metric_type == MetricType.COSINE: + return "COSINE" + if self.case_config.metric_type == MetricType.IP: + return "INNER_PRODUCT" + return "L2" + + def _get_query_distance_fn(self) -> tuple[str, str]: + """Returns (sql_function_name, sort_order) for vector search.""" + if self.case_config.metric_type == MetricType.COSINE: + return "cosineDistance", "ASC" + if self.case_config.metric_type == MetricType.IP: + return "innerProduct", "DESC" + return "l2Distance", "ASC" + + def prepare_filter(self, filters: Filter): + """Pre-compute the SQL WHERE fragment for the given filter condition.""" + if filters.type == FilterOp.NonFilter: + self._filter_where = "" + elif filters.type == FilterOp.NumGE: + self._filter_where = f"{filters.int_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self._filter_where = f"{self._label_field} = '{filters.label_value}'" + + def __getstate__(self): + # threading.local and Lock cannot be pickled; the framework pickles the DB + # instance to send it to the load subprocess, so we must exclude them here + # and recreate them in __setstate__ after unpickling. + state = self.__dict__.copy() + state.pop("_thread_local", None) + state.pop("_buffers_lock", None) + state.pop("_registered_buffers", None) + return state + + def __setstate__(self, state: dict) -> None: + self.__dict__.update(state) + self._thread_local = threading.local() + self._buffers_lock = threading.Lock() + self._registered_buffers = [] + + def _get_thread_buffer(self) -> list: + """Return this thread's row buffer, creating and registering it on first access. + + The list itself (not the threading.local container) is registered so that the + teardown in init() can access buffered rows from the main thread, which has its + own (empty) thread-local slot and cannot see other threads' data through the + threading.local object. + """ + if not hasattr(self._thread_local, "pending_rows"): + self._thread_local.pending_rows = [] + with self._buffers_lock: + # Register the list itself, not self._thread_local, so teardown can + # read the contents from any thread (main thread included). + self._registered_buffers.append(self._thread_local.pending_rows) + return self._thread_local.pending_rows + + @contextmanager + def init(self): + self.session = requests.Session() + if self._auth: + self.session.auth = self._auth + # Reset buffer registry so teardown only flushes buffers from this init() scope. + self._registered_buffers = [] + try: + yield + finally: + # Flush any rows that were buffered but not yet sent to Pinot. + # Each worker thread may have its own partially-filled buffer. + # This must happen here — not in optimize() — because optimize() runs + # in a separate subprocess where the buffers are always empty. + for pending in self._registered_buffers: + if pending: + log.info(f"Pinot init teardown: flushing {len(pending)} remaining buffered rows") + _, err = self._flush_rows(pending) + if err: + log.warning(f"Pinot init teardown: flush error: {err}") + self.session.close() + self.session = None + + def _flush_rows(self, rows: list) -> tuple[int, Exception | None]: + """Flush the given row list to Pinot as one segment using a fresh HTTP session. + + Using a fresh session (not self.session) makes this method safe to call + from multiple threads concurrently. On success the list is cleared in-place. + Returns (rows_flushed, error). On error the list is left intact so the caller + can decide whether to retry. + """ + if not rows: + return 0, None + + n = len(rows) + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, prefix="pinot_ingest_") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + tmp_path = f.name + + try: + batch_config = json.dumps({"inputFormat": "json"}) + params = { + "tableNameWithType": f"{self.table_name}_OFFLINE", + "batchConfigMapStr": batch_config, + } + last_err = None + with requests.Session() as session: + if self._auth: + session.auth = self._auth + for attempt in range(3): + try: + with Path(tmp_path).open("rb") as f: + resp = session.post( + f"{self._controller_url}/ingestFromFile", + params=params, + files={"file": (Path(tmp_path).name, f, "application/json")}, + timeout=1800, # HNSW index building for 100K x 768D can take 10+ min + ) + if resp.ok: + rows.clear() + log.debug(f"Pinot: flushed segment with {n} rows") + return n, None + last_err = Exception(f"HTTP {resp.status_code}: {resp.text[:200]}") + log.warning(f"Pinot flush attempt {attempt + 1} failed: {last_err}") + time.sleep(1 + attempt) + except Exception as e: + last_err = e + log.warning(f"Pinot flush attempt {attempt + 1} error: {e}") + time.sleep(1 + attempt) + return 0, last_err + finally: + Path(tmp_path).unlink() + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs: Any, + ) -> tuple[int, Exception]: + # Each thread has its own buffer; no locking needed here. + pending = self._get_thread_buffer() + + for i, (emb, meta) in enumerate(zip(embeddings, metadata, strict=False)): + row = {self._primary_field: meta, self._vector_field: list(emb)} + if self.with_scalar_labels and labels_data is not None: + row[self._label_field] = labels_data[i] + pending.append(row) + + if len(pending) >= self._ingest_batch_size: + _flushed, err = self._flush_rows(pending) + if err: + log.warning(f"Failed to flush Pinot buffer: {err}") + return 0, err + + return len(embeddings), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + assert self.session is not None, "Session not initialized" + + query_arr = ",".join(str(v) for v in query) + dist_fn, order = self._get_query_distance_fn() + + search_params = self.case_config.search_param() + nprobe = search_params.get("nprobe") + + filter_clause = self._filter_where + + if nprobe is not None: + # IVF-based index: set probe count via session option, use ORDER BY for top-k + where = f"WHERE {filter_clause} " if filter_clause else "" + sql = ( + f"set vectorNprobe={nprobe}; " + f"SELECT {self._primary_field} " + f"FROM {self.table_name} " + f"{where}" + f"ORDER BY {dist_fn}({self._vector_field}, ARRAY[{query_arr}]) {order} " + f"LIMIT {k}" + ) + else: + # HNSW index: WHERE vectorSimilarity(..., ef) triggers Lucene HNSW graph search + # with a candidate list of size ef (defaults to k). Larger ef → better recall. + # ORDER BY dist re-ranks the ANN candidates for correct final recall. + ef = search_params.get("ef") or k + extra = f" AND {filter_clause}" if filter_clause else "" + sql = ( + f"SELECT {self._primary_field} " + f"FROM {self.table_name} " + f"WHERE vectorSimilarity({self._vector_field}, ARRAY[{query_arr}], {ef}){extra} " + f"ORDER BY {dist_fn}({self._vector_field}, ARRAY[{query_arr}]) {order} " + f"LIMIT {k}" + ) + + resp = self.session.post( + f"{self._broker_url}/query/sql", + json={"sql": sql}, + headers={"Content-Type": "application/json"}, + timeout=timeout, + ) + resp.raise_for_status() + result = resp.json() + + rows = result.get("resultTable", {}).get("rows", []) + return [row[0] for row in rows] + + def optimize(self, data_size: int | None = None): + """Wait for all ingested data to be queryable in Pinot. + + Remaining buffered rows are flushed on init() teardown (in the insert + subprocess), so by the time optimize() runs they are already in Pinot. + """ + if self.session is None: + return + + if data_size is None: + time.sleep(5) + return + + max_wait = 600 + check_interval = 10 + start = time.time() + + while time.time() - start < max_wait: + try: + resp = self.session.post( + f"{self._broker_url}/query/sql", + json={"sql": f"SELECT COUNT(*) FROM {self.table_name}"}, + headers={"Content-Type": "application/json"}, + ) + if resp.status_code == 200: + rows = resp.json().get("resultTable", {}).get("rows", []) + current_count = rows[0][0] if rows else 0 + if current_count >= data_size: + log.info(f"Pinot: all {data_size} rows are queryable") + return + log.info(f"Pinot: {current_count}/{data_size} rows queryable, waiting...") + except Exception as e: + log.warning(f"Pinot optimize check error: {e}") + + time.sleep(check_interval) + + log.warning(f"Pinot optimize timed out after {max_wait}s") diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 3e21746aa..42048d57f 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -25,6 +25,7 @@ from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn from ..backend.clients.pinecone.cli import Pinecone +from ..backend.clients.pinot.cli import Pinot from ..backend.clients.polardb.cli import ( PolarDBHNSWFlat, PolarDBHNSWPQ, @@ -90,6 +91,7 @@ cli.add_command(Pinecone) cli.add_command(VectorChordRQ) cli.add_command(VectorChordGraph) +cli.add_command(Pinot) cli.add_command(PolarDBHNSWFlat) cli.add_command(PolarDBHNSWPQ) cli.add_command(PolarDBHNSWSQ) From 0c20701725a84fbcd2a14b5d628c77cac2beb071 Mon Sep 17 00:00:00 2001 From: James <83447078+xiaofan-luan@users.noreply.github.com> Date: Sun, 19 Apr 2026 23:21:55 -0700 Subject: [PATCH 23/49] fix: support self-hosted Elasticsearch via --host/--port in elasticcloud commands (#761) * fix: support self-hosted Elasticsearch via --host/--port in elasticcloud commands ElasticCloudConfig previously required cloud_id, so the elasticcloudhnsw* subcommands could only target Elastic Cloud. Users benchmarking self-hosted stock Elasticsearch had no working path: tencentelasticsearch accepts host/port but forces Tencent's vsearch index_options type, which stock ES rejects with "Unknown vector index options type [vsearch]". Extend ElasticCloudConfig with scheme/host/port/user fields (mutually exclusive with cloud_id) and expose them on all four ElasticCloudHNSW* CLI subcommands. Existing cloud_id callers are unchanged. Refs #758 Co-Authored-By: Claude Opus 4.7 (1M context) * style: apply black formatting to elastic_cloud/config.py Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .../backend/clients/elastic_cloud/cli.py | 77 +++++++++++++++++-- .../backend/clients/elastic_cloud/config.py | 42 +++++++++- 2 files changed, 109 insertions(+), 10 deletions(-) diff --git a/vectordb_bench/backend/clients/elastic_cloud/cli.py b/vectordb_bench/backend/clients/elastic_cloud/cli.py index 55e521d8f..f277a6ff2 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/cli.py +++ b/vectordb_bench/backend/clients/elastic_cloud/cli.py @@ -17,11 +17,60 @@ class ElasticCloudTypedDict(TypedDict): cloud_id: Annotated[ str, - click.option("--cloud-id", type=str, help="Elastic Cloud ID", required=True), + click.option( + "--cloud-id", + type=str, + help="Elastic Cloud ID. Omit when connecting to a self-hosted ES via --host.", + required=False, + default="", + ), + ] + scheme: Annotated[ + str, + click.option( + "--scheme", + type=click.Choice(["http", "https"], case_sensitive=False), + help="Scheme for host-based connection.", + required=False, + default="https", + show_default=True, + ), + ] + host: Annotated[ + str, + click.option( + "--host", + type=str, + help="Elasticsearch host (for self-hosted ES; alternative to --cloud-id).", + required=False, + default="", + ), + ] + port: Annotated[ + int, + click.option( + "--port", + type=int, + help="Elasticsearch port (for host-based connection).", + required=False, + default=9200, + show_default=True, + ), + ] + user: Annotated[ + str, + click.option( + "--user", + type=str, + help="Elasticsearch user.", + required=False, + default="elastic", + show_default=True, + ), ] password: Annotated[ str, - click.option("--password", type=str, help="Elastic Cloud password", required=True), + click.option("--password", type=str, help="Elasticsearch password", required=True), ] number_of_shards: Annotated[ int, @@ -170,7 +219,11 @@ def ElasticCloudHNSW(**parameters: Unpack[ElasticCloudHNSWTypedDict]): db=DBTYPE, db_config=ElasticCloudConfig( db_label=parameters["db_label"], - cloud_id=SecretStr(parameters["cloud_id"]), + cloud_id=SecretStr(parameters["cloud_id"]) if parameters["cloud_id"] else None, + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], password=SecretStr(parameters["password"]), ), db_case_config=ElasticCloudIndexConfig( @@ -202,7 +255,11 @@ def ElasticCloudHNSWInt8(**parameters: Unpack[ElasticCloudHNSWTypedDict]): db=DBTYPE, db_config=ElasticCloudConfig( db_label=parameters["db_label"], - cloud_id=SecretStr(parameters["cloud_id"]), + cloud_id=SecretStr(parameters["cloud_id"]) if parameters["cloud_id"] else None, + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], password=SecretStr(parameters["password"]), ), db_case_config=ElasticCloudIndexConfig( @@ -234,7 +291,11 @@ def ElasticCloudHNSWInt4(**parameters: Unpack[ElasticCloudHNSWTypedDict]): db=DBTYPE, db_config=ElasticCloudConfig( db_label=parameters["db_label"], - cloud_id=SecretStr(parameters["cloud_id"]), + cloud_id=SecretStr(parameters["cloud_id"]) if parameters["cloud_id"] else None, + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], password=SecretStr(parameters["password"]), ), db_case_config=ElasticCloudIndexConfig( @@ -266,7 +327,11 @@ def ElasticCloudHNSWBBQ(**parameters: Unpack[ElasticCloudHNSWTypedDict]): db=DBTYPE, db_config=ElasticCloudConfig( db_label=parameters["db_label"], - cloud_id=SecretStr(parameters["cloud_id"]), + cloud_id=SecretStr(parameters["cloud_id"]) if parameters["cloud_id"] else None, + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], password=SecretStr(parameters["password"]), ), db_case_config=ElasticCloudIndexConfig( diff --git a/vectordb_bench/backend/clients/elastic_cloud/config.py b/vectordb_bench/backend/clients/elastic_cloud/config.py index 7ee54b49c..bdae177f3 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/config.py +++ b/vectordb_bench/backend/clients/elastic_cloud/config.py @@ -1,18 +1,52 @@ from enum import StrEnum -from pydantic import BaseModel, SecretStr +from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, IndexType, MetricType class ElasticCloudConfig(DBConfig, BaseModel): - cloud_id: SecretStr + # Elastic Cloud connection. Takes precedence when set. + cloud_id: SecretStr | None = None + # Self-hosted / host-based connection (used when cloud_id is not provided). + scheme: str = "https" + host: str = "" + port: int = 9200 + user: str = "elastic" password: SecretStr + @model_validator(mode="before") + @classmethod + def not_empty_field(cls, data: any) -> any: + if not isinstance(data, dict): + return data + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"cloud_id", "host"} + for field_name, v in data.items(): + if field_name in skip: + continue + if isinstance(v, str) and not v: + msg = "Empty string!" + raise ValueError(msg) + return data + + @model_validator(mode="after") + def _check_connection_target(self) -> "ElasticCloudConfig": + has_cloud_id = bool(self.cloud_id and self.cloud_id.get_secret_value()) + if not has_cloud_id and not self.host: + msg = "ElasticCloudConfig requires either cloud_id or host to be set." + raise ValueError(msg) + return self + def to_dict(self) -> dict: + auth = (self.user, self.password.get_secret_value()) + if self.cloud_id and self.cloud_id.get_secret_value(): + return { + "cloud_id": self.cloud_id.get_secret_value(), + "basic_auth": auth, + } return { - "cloud_id": self.cloud_id.get_secret_value(), - "basic_auth": ("elastic", self.password.get_secret_value()), + "hosts": [{"scheme": self.scheme, "host": self.host, "port": self.port}], + "basic_auth": auth, } From b3613ff6befcc5c802b77617cd880750917c1c51 Mon Sep 17 00:00:00 2001 From: B Nagaraju Reddy <107165377+NagarajuReddyBoggala@users.noreply.github.com> Date: Tue, 21 Apr 2026 09:33:17 +0530 Subject: [PATCH 24/49] Fix: Map "ivf_flat" to "ivfflat" for pgvector index access method (#763) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix: Map "ivf_flat" to "ivfflat" for pgvector index access method - IndexType.IVFFlat.value="IVF_FLAT" → .lower()="ivf_flat" caused SQL to fail with "access method 'ivf_flat' does not exist" - pgvector PostgreSQL extension expects "ivfflat" (no underscore), not "ivf_flat" - Added explicit mapping after lowercase normalization: if index_type_lower == "ivf_flat": index_type_lower = "ivfflat" * style(pgvector): fix comment wrapping and remove commented code --------- Co-authored-by: rnagaraju --- .../backend/clients/pgvector/pgvector.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 0b0750172..630758e1b 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -335,12 +335,20 @@ def _create_index(self): index_param = self.case_config.index_param() self._set_parallel_index_build_param() - # [FIX] The index access method name registered by the PostgreSQL pgvector extension is in lowercase (e.g., "hnsw", "ivfflat"), - # but the index type passed from the frontend UI is uppercase "HNSW" via IndexType.HNSW.value, causing SQL syntax "USING 'HNSW'" - # to fail with error "access method HNSW does not exist". Here we uniformly convert it to lowercase to match PostgreSQL's access method name. + # [FIX] The index access method name registered by the PostgreSQL pgvector extension is in + # lowercase (e.g., "hnsw", "ivfflat"), but the index type passed from the frontend UI is + # uppercase "HNSW" via IndexType.HNSW.value, causing SQL syntax "USING 'HNSW'" to fail + # with error "access method HNSW does not exist". Here we uniformly convert it to lowercase + # to match PostgreSQL's access method name. index_type_lower = index_param["index_type"].lower() + # [FIX] The pgvector access method name is "ivfflat" (no underscore), but IndexType.IVFFlat.value + # produces "IVF_FLAT" which becomes "ivf_flat" after lowercase conversion, causing SQL syntax + # "USING 'ivf_flat'" to fail with error "access method 'ivf_flat' does not exist". + # Here we map "ivf_flat" → "ivfflat" to match PostgreSQL pgvector's registered access method name. + if index_type_lower == "ivf_flat": + index_type_lower = "ivfflat" log.info(f"index_type (original={index_param['index_type']}, normalized={index_type_lower})") - + options = [] for option in index_param["index_creation_with_options"]: if option["val"] is not None: @@ -368,7 +376,6 @@ def _create_index(self): ), # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] index_type=sql.Identifier(index_type_lower), - # index_type=sql.Identifier(index_param["index_type"]), # This assumes that the quantization_type value matches the quantization function name quantization_type=sql.SQL(index_param["quantization_type"]), dim=self.dim, @@ -385,7 +392,6 @@ def _create_index(self): table_name=sql.Identifier(self.table_name), # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] index_type=sql.Identifier(index_type_lower), - # index_type=sql.Identifier(index_param["index_type"]), embedding_metric=sql.Identifier(index_param["metric"]), ) From 02e5d33df8b2de83a08b7fd61b66529344865781 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Tue, 21 Apr 2026 15:01:56 +0800 Subject: [PATCH 25/49] fix(pgvector): fix ConcurrentInsertRunner for non-thread-safe DBs (#764) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For non-thread-safe DBs (e.g. PgVector), ConcurrentInsertRunner clamps max_workers to 1, so there is always exactly one worker thread. There is no need to deepcopy self.db per thread — the single worker can use self.db directly via the connection already opened by task()'s `with self.db.init():`. The original code called deepcopy(self.db) inside _get_thread_db() after task() had already opened a live psycopg C-extension Connection on self.db. C-extension objects cannot be deep-copied, causing: TypeError: no default __reduce__ due to non-trivial __cinit__ Fix: remove the deepcopy branch entirely. All workers (thread-safe or not) now use self.db directly; thread-safety is guaranteed for non-thread-safe DBs by the max_workers=1 clamp. Also clean up stale comments in pgvector.py left over from #760/#763. Adds tests/test_pgvector.py with: - unit test that reproduces the bug (fails on original, passes on fix) - e2e regression test via ConcurrentInsertRunner + OpenAI 50K dataset See also: #756 Signed-off-by: yangxuan --- tests/pytest.ini | 3 + tests/test_pgvector.py | 180 ++++++++++++++++++ .../backend/clients/pgvector/pgvector.py | 15 +- .../backend/runner/concurrent_runner.py | 90 +++------ 4 files changed, 211 insertions(+), 77 deletions(-) create mode 100644 tests/test_pgvector.py diff --git a/tests/pytest.ini b/tests/pytest.ini index e5915e89e..9f5751ee3 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -3,3 +3,6 @@ filterwarnings = ignore::UserWarning ignore::DeprecationWarning + +markers = + integration: tests that require external services or network access (deselect with -m "not integration") diff --git a/tests/test_pgvector.py b/tests/test_pgvector.py new file mode 100644 index 000000000..cdd9461a9 --- /dev/null +++ b/tests/test_pgvector.py @@ -0,0 +1,180 @@ +"""Tests for PgVector client and ConcurrentInsertRunner. + +Reproduces issue #756: insert fails with + TypeError: no default __reduce__ due to non-trivial __cinit__ +when ConcurrentInsertRunner deep-copies a PgVector instance that has a live +psycopg connection open (the connection is opened by `with self.db.init():` +inside task() before the deepcopy in _get_thread_db()). + +Requires: + docker run -d --name pgvector-test \ + -e POSTGRES_USER=vectordb -e POSTGRES_PASSWORD=vectordb \ + -e POSTGRES_DB=vectordb -p 5432:5432 \ + pgvector/pgvector:pg17 + +Usage: + pytest tests/test_pgvector.py -v -s +""" + +from __future__ import annotations + +import logging +import pickle +from unittest.mock import MagicMock + +import numpy as np +import pytest + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.pgvector.config import PgVectorHNSWConfig +from vectordb_bench.backend.dataset import Dataset, DatasetSource +from vectordb_bench.backend.filter import Filter, FilterOp, non_filter +from vectordb_bench.backend.runner.concurrent_runner import ConcurrentInsertRunner + +log = logging.getLogger(__name__) + +# ── Connection config ──────────────────────────────────────────────────────── + +DB_CONFIG = { + "connect_config": { + "host": "localhost", + "port": 5432, + "dbname": "vectordb", + "user": "vectordb", + "password": "vectordb", + }, + "table_name": "test_pgvector", +} + +DIM = 128 +COUNT = 500 +RNG = np.random.default_rng(42) + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +def make_hnsw_config(**kwargs) -> PgVectorHNSWConfig: + return PgVectorHNSWConfig( + metric_type="COSINE", + m=16, + ef_construction=64, + ef_search=64, + **kwargs, + ) + + +def make_db(table_name: str = "test_pgvector", drop_old: bool = True) -> DB.PgVector.init_cls: + cfg = dict(DB_CONFIG) + cfg["table_name"] = table_name + return DB.PgVector.init_cls( + dim=DIM, + db_config=cfg, + db_case_config=make_hnsw_config(), + drop_old=drop_old, + ) + + +def random_embeddings(n: int = COUNT, d: int = DIM) -> list[list[float]]: + return RNG.random((n, d)).tolist() + + +# ── Basic client tests ──────────────────────────────────────────────────────── + + +class TestPgVectorBasic: + """Unit tests for the PgVector client (no subprocess).""" + + def test_insert_and_search(self): + db = make_db("test_basic") + embeddings = random_embeddings() + metadata = list(range(COUNT)) + + with db.init(): + count, err = db.insert_embeddings(embeddings=embeddings, metadata=metadata) + assert err is None, f"Insert error: {err}" + assert count == COUNT + + with db.init(): + db.optimize() + + with db.init(): + db.prepare_filter(Filter(type=FilterOp.NonFilter)) + results = db.search_embedding(query=embeddings[0], k=10) + assert len(results) > 0 + + def test_db_is_not_thread_safe(self): + db = make_db("test_thread_safe") + assert db.thread_safe is False + + def test_db_picklable_after_init(self): + """PgVector instance must be picklable after __init__ (conn/cursor are None). + + This is required for ConcurrentInsertRunner which spawns a subprocess + and pickles self (which includes self.db). + """ + db = make_db("test_pickle") + data = pickle.dumps(db) + db2 = pickle.loads(data) # noqa: S301 + assert db2.dim == DIM + + def test_get_thread_db_with_open_connection(self): + """Regression test for issue #756. + + ConcurrentInsertRunner.task() opens `with self.db.init()` before calling + workers. For non-thread-safe DBs the original _get_thread_db() then called + deepcopy(self.db) — but the live psycopg C-extension Connection is not + deep-copyable, causing TypeError. + + Fixed code returns self.db directly (no deepcopy), so this test must pass + without raising. + """ + db = make_db("test_get_thread_db") + runner = ConcurrentInsertRunner(db=db, dataset=MagicMock(), normalize=False) + + with db.init(): + assert db.conn is not None + result = runner._get_thread_db() # TypeError here on original code + + assert result is db + + +# ── ConcurrentInsertRunner tests ────────────────────────────────────────────── + + +class TestPgVectorConcurrentInsert: + """Tests for ConcurrentInsertRunner with PgVector (reproduces issue #756).""" + + @pytest.mark.integration + def test_concurrent_insert_e2e(self): + """E2E regression test for issue #756 using the OpenAI 50K dataset. + + Exercises the full pipeline: + ProcessPoolExecutor(spawn) → pickle runner → subprocess task() + → with self.db.init() → worker _get_thread_db() → insert batches + + FAILS on original code (TypeError: deepcopy of live psycopg connection). + PASSES on fixed code. + """ + dataset = Dataset.OPENAI.manager(50_000) + dataset.prepare(DatasetSource.AliyunOSS) + + cfg = dict(DB_CONFIG) + cfg["table_name"] = "test_e2e_insert" + db = DB.PgVector.init_cls( + dim=dataset.data.dim, + db_config=cfg, + db_case_config=PgVectorHNSWConfig( + metric_type="COSINE", + m=16, + ef_construction=64, + ef_search=64, + ), + drop_old=True, + ) + + runner = ConcurrentInsertRunner(db=db, dataset=dataset, normalize=True, filters=non_filter) + count = runner.run() + + assert count == 50_000, f"Expected 50000 rows, got {count}" + log.info(f"E2E insert completed: {count} rows") diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 630758e1b..41060af27 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -335,16 +335,9 @@ def _create_index(self): index_param = self.case_config.index_param() self._set_parallel_index_build_param() - # [FIX] The index access method name registered by the PostgreSQL pgvector extension is in - # lowercase (e.g., "hnsw", "ivfflat"), but the index type passed from the frontend UI is - # uppercase "HNSW" via IndexType.HNSW.value, causing SQL syntax "USING 'HNSW'" to fail - # with error "access method HNSW does not exist". Here we uniformly convert it to lowercase - # to match PostgreSQL's access method name. + # pgvector registers access methods in lowercase ("hnsw", "ivfflat") but + # IndexType enum values are uppercase; also IVFFlat maps to "ivfflat" (no underscore). index_type_lower = index_param["index_type"].lower() - # [FIX] The pgvector access method name is "ivfflat" (no underscore), but IndexType.IVFFlat.value - # produces "IVF_FLAT" which becomes "ivf_flat" after lowercase conversion, causing SQL syntax - # "USING 'ivf_flat'" to fail with error "access method 'ivf_flat' does not exist". - # Here we map "ivf_flat" → "ivfflat" to match PostgreSQL pgvector's registered access method name. if index_type_lower == "ivf_flat": index_type_lower = "ivfflat" log.info(f"index_type (original={index_param['index_type']}, normalized={index_type_lower})") @@ -374,9 +367,8 @@ def _create_index(self): if index_param["quantization_type"] == "bit" else sql.Identifier("embedding") ), - # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] index_type=sql.Identifier(index_type_lower), - # This assumes that the quantization_type value matches the quantization function name + # quantization_type value matches the quantization function name quantization_type=sql.SQL(index_param["quantization_type"]), dim=self.dim, embedding_metric=sql.Identifier(index_param["metric"]), @@ -390,7 +382,6 @@ def _create_index(self): ).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), - # [FIX] Use lowercase index_type_lower instead of original index_param["index_type"] index_type=sql.Identifier(index_type_lower), embedding_metric=sql.Identifier(index_param["metric"]), ) diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py index 6ed8e39fb..37201f88e 100644 --- a/vectordb_bench/backend/runner/concurrent_runner.py +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -13,7 +13,6 @@ import multiprocessing as mp import threading import time -from copy import deepcopy from enum import StrEnum from typing import TYPE_CHECKING @@ -44,7 +43,7 @@ class ConcurrentInsertRunner: """Concurrent insert runner with pluggable executor backend. Thread-safety: If db.thread_safe is False, max_workers is clamped to 1 - and each worker thread gets a deep-copied DB instance with its own connection. + so the single worker thread uses self.db directly (no deepcopy needed). Args: db: VectorDB instance. @@ -78,57 +77,31 @@ def __init__( log.info(f"DB {db.name} is not thread-safe, falling back to max_workers=1") effective_workers = 1 self.max_workers = effective_workers + assert db.thread_safe or self.max_workers == 1, ( + "Non-thread-safe DBs must use max_workers=1 — " + "_get_thread_db() relies on this to avoid concurrent access to self.db" + ) def __getstate__(self): """Exclude unpicklable thread-local state for ProcessPoolExecutor(spawn).""" state = self.__dict__.copy() - state.pop("_local", None) - state.pop("_ctx_lock", None) - state.pop("_thread_contexts", None) state.pop("_iter_lock", None) state.pop("_dataset_iter", None) return state - def __setstate__(self, state: dict): - self.__dict__.update(state) - self._local = threading.local() - self._ctx_lock = threading.Lock() - self._thread_contexts = [] - def _create_executor(self) -> TaskExecutor: if self.backend == ExecutorBackend.ASYNC: return AsyncExecutor(max_workers=self.max_workers) return ThreadExecutor(max_workers=self.max_workers) def _get_thread_db(self) -> api.VectorDB: - """Get or create a per-thread DB instance. + """Return self.db. - Thread-safe DBs reuse self.db (connection opened in task()). - Non-thread-safe DBs get a deep-copied instance with its own connection, - cached in thread-local storage so it is created once per thread. + All workers share the connection opened by task()'s `with self.db.init()`. + Thread-safe DBs share it across multiple workers. Non-thread-safe DBs are + clamped to max_workers=1, so there is never concurrent access. """ - if not hasattr(self._local, "db"): - if self.db.thread_safe: - self._local.db = self.db - else: - db = deepcopy(self.db) - # Manual __enter__/__exit__ because enter and exit happen in - # different scopes (here vs _cleanup_thread_contexts). - ctx = db.init() - ctx.__enter__() - self._local.db = db - with self._ctx_lock: - self._thread_contexts.append(ctx) - return self._local.db - - def _cleanup_thread_contexts(self) -> None: - """Close per-thread DB connections opened for non-thread-safe clients.""" - for ctx in self._thread_contexts: - try: - ctx.__exit__(None, None, None) - except Exception: - log.warning("Failed to close per-thread DB connection", exc_info=True) - self._thread_contexts.clear() + return self.db def _insert_batch_with_retry( self, @@ -160,14 +133,7 @@ def _worker_insert( metadata: list[int], labels_data: list[str] | None = None, ) -> int: - """Worker function: insert a batch with retry. - - Thread-safe DBs: reuse self.db whose connection is already open - via task()'s `with self.db.init()` — all threads share it safely. - - Non-thread-safe DBs: use a per-thread deep-copied instance with - its own connection, cached via threading.local. - """ + """Worker function: insert a batch with retry.""" db = self._get_thread_db() return self._insert_batch_with_retry(db, embeddings, metadata, labels_data) @@ -214,9 +180,6 @@ def _worker_loop(self) -> int: def task(self) -> int: """Insert entire dataset using concurrent executor. Runs in subprocess.""" count = 0 - self._local = threading.local() - self._ctx_lock = threading.Lock() - self._thread_contexts = [] self._iter_lock = threading.Lock() self._dataset_iter = iter(self.dataset) @@ -227,23 +190,20 @@ def task(self) -> int: ) start = time.perf_counter() - try: - with self._create_executor() as executor: - for _ in range(self.max_workers): - executor.submit(self._worker_loop) - - batch_results = executor.wait_all() - - # Log all errors, then raise the first one - errors = [r.error for r in batch_results if r.error is not None] - if errors: - for err in errors: - log.warning(f"Batch insert error: {err}") - raise errors[0] - - count = sum(r.value for r in batch_results) - finally: - self._cleanup_thread_contexts() + with self._create_executor() as executor: + for _ in range(self.max_workers): + executor.submit(self._worker_loop) + + batch_results = executor.wait_all() + + # Log all errors, then raise the first one + errors = [r.error for r in batch_results if r.error is not None] + if errors: + for err in errors: + log.warning(f"Batch insert error: {err}") + raise errors[0] + + count = sum(r.value for r in batch_results) log.info( f"({mp.current_process().name:16}) Finish concurrent insert, " From 63cc50a02d9e4e9e576a7fd640448fa93d9a7341 Mon Sep 17 00:00:00 2001 From: EeshaaKhan <170761203+EeshaaKhan@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:44:32 +0500 Subject: [PATCH 26/49] Feat: Add label filter support in pgdiskann client (#724) * Add label filtering support to pgdiskann client * Refactor pgdiskann filtering logic * Refactor: remove unrelated function * style: apply black formatting to pgdiskann.py * fix: remove trailing whitespace and fix import sorting * docs: add comments for label naming and vector storage optimization * Revert "docs: add comments for label naming and vector storage optimization" This reverts commit d10b296b612f6568f5c2abf043b0003d2f4ca8b4. --------- Co-authored-by: Eesha Faisal --- .../backend/clients/pgdiskann/pgdiskann.py | 204 +++++++++--------- 1 file changed, 106 insertions(+), 98 deletions(-) diff --git a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py index 5f069ace5..46e8fabd4 100644 --- a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py +++ b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py @@ -10,6 +10,8 @@ from pgvector.psycopg import register_vector from psycopg import Connection, Cursor, sql +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import VectorDB from .config import PgDiskANNConfigDict, PgDiskANNIndexConfig @@ -19,11 +21,16 @@ class PgDiskANN(VectorDB): """Use psycopg instructions""" + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + conn: psycopg.Connection[Any] | None = None - coursor: psycopg.Cursor[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None - _filtered_search: sql.Composed - _unfiltered_search: sql.Composed + _search: sql.Composed def __init__( self, @@ -32,6 +39,7 @@ def __init__( db_case_config: PgDiskANNIndexConfig, collection_name: str = "pg_diskann_collection", drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): self.name = "PgDiskANN" @@ -39,6 +47,9 @@ def __init__( self.case_config = db_case_config self.table_name = collection_name self.dim = dim + self.with_scalar_labels = with_scalar_labels + self._scalar_label_field = "label" + self.where_clause = "" self._index_name = "pgdiskann_index" self._primary_field = "id" @@ -86,83 +97,58 @@ def _create_connection(**kwargs) -> tuple[Connection, Cursor]: return conn, cursor - @contextmanager - def init(self) -> Generator[None, None, None]: - self.conn, self.cursor = self._create_connection(**self.db_config) - - session_options: dict[str, Any] = self.case_config.session_param() - - if len(session_options) > 0: - for setting_name, setting_val in session_options.items(): - command = sql.SQL("SET {setting_name} = {setting_val};").format( - setting_name=sql.Identifier(setting_name), setting_val=sql.Literal(setting_val) - ) - log.debug(command.as_string(self.cursor)) - self.cursor.execute(command) - self.conn.commit() - + def _generate_search_query(self) -> sql.Composed: + """Generate search query with where_clause placeholder""" search_params = self.case_config.search_param() if search_params.get("reranking"): - # Reranking-enabled queries - self._filtered_search = sql.SQL(""" + search_query = sql.SQL(""" SELECT i.id FROM ( SELECT id, embedding FROM public.{table_name} - WHERE id >= %s + {where_clause} ORDER BY embedding {metric_fun_op} %s::vector LIMIT {quantized_fetch_limit}::int ) i ORDER BY i.embedding {reranking_metric_fun_op} %s::vector LIMIT %s::int - """).format( + """).format( table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL(self.where_clause), metric_fun_op=sql.SQL(search_params["metric_fun_op"]), reranking_metric_fun_op=sql.SQL(search_params["reranking_metric_fun_op"]), quantized_fetch_limit=sql.Literal(search_params["quantized_fetch_limit"]), ) - - self._unfiltered_search = sql.SQL(""" - SELECT i.id - FROM ( - SELECT id, embedding - FROM public.{table_name} - ORDER BY embedding {metric_fun_op} %s::vector - LIMIT {quantized_fetch_limit}::int - ) i - ORDER BY i.embedding {reranking_metric_fun_op} %s::vector - LIMIT %s::int - """).format( - table_name=sql.Identifier(self.table_name), - metric_fun_op=sql.SQL(search_params["metric_fun_op"]), - reranking_metric_fun_op=sql.SQL(search_params["reranking_metric_fun_op"]), - quantized_fetch_limit=sql.Literal(search_params["quantized_fetch_limit"]), - ) - else: - self._filtered_search = sql.Composed( + search_query = sql.Composed( [ - sql.SQL( - "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding ", - ).format(table_name=sql.Identifier(self.table_name)), - sql.SQL(search_params["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ] - ) - - self._unfiltered_search = sql.Composed( - [ - sql.SQL("SELECT id FROM public.{table_name} ORDER BY embedding ").format( - table_name=sql.Identifier(self.table_name) + sql.SQL("SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding ").format( + table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL(self.where_clause), ), sql.SQL(search_params["metric_fun_op"]), sql.SQL(" %s::vector LIMIT %s::int"), ] ) - log.debug(f"Unfiltered search query={self._unfiltered_search.as_string(self.conn)}") - log.debug(f"Filtered search query={self._filtered_search.as_string(self.conn)}") + return search_query + + @contextmanager + def init(self) -> Generator[None, None, None]: + self.conn, self.cursor = self._create_connection(**self.db_config) + + session_options: dict[str, Any] = self.case_config.session_param() + + if len(session_options) > 0: + for setting_name, setting_val in session_options.items(): + command = sql.SQL("SET {setting_name} = {setting_val};").format( + setting_name=sql.Identifier(setting_name), + setting_val=sql.Literal(setting_val), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() try: yield @@ -281,12 +267,10 @@ def _create_index(self): with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) if any(options) else sql.Composed(()) - index_create_sql = sql.SQL( - """ + index_create_sql = sql.SQL(""" CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} USING {index_type} (embedding {embedding_metric}) - """, - ).format( + """).format( index_name=sql.Identifier(self._index_name), table_name=sql.Identifier(self.table_name), index_type=sql.Identifier(index_param["index_type"].lower()), @@ -304,11 +288,36 @@ def _create_table(self, dim: int): try: log.info(f"{self.name} client create table : {self.table_name}") + if self.with_scalar_labels: + self.cursor.execute( + sql.SQL(""" + CREATE TABLE IF NOT EXISTS public.{table_name} + ({primary_field} BIGINT PRIMARY KEY, embedding vector({dim}), {label_field} VARCHAR(64)); + """).format( + table_name=sql.Identifier(self.table_name), + dim=dim, + primary_field=sql.Identifier(self._primary_field), + label_field=sql.Identifier(self._scalar_label_field), + ), + ) + else: + self.cursor.execute( + sql.SQL(""" + CREATE TABLE IF NOT EXISTS public.{table_name} + ({primary_field} BIGINT PRIMARY KEY, embedding vector({dim})); + """).format( + table_name=sql.Identifier(self.table_name), + dim=dim, + primary_field=sql.Identifier(self._primary_field), + ), + ) + self.cursor.execute( - sql.SQL( - "CREATE TABLE IF NOT EXISTS public.{table_name} (id BIGINT PRIMARY KEY, embedding vector({dim}));", - ).format(table_name=sql.Identifier(self.table_name), dim=dim), + sql.SQL("ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;").format( + table_name=sql.Identifier(self.table_name) + ), ) + self.conn.commit() except Exception as e: log.warning(f"Failed to create pgdiskann table: {self.table_name} error: {e}") @@ -318,11 +327,15 @@ def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs: Any, ) -> tuple[int, Exception | None]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + if self.with_scalar_labels: + assert labels_data is not None, "labels_data should be provided if with_scalar_labels is set to True" + try: metadata_arr = np.array(metadata) embeddings_arr = np.array(embeddings) @@ -332,9 +345,14 @@ def insert_embeddings( table_name=sql.Identifier(self.table_name), ), ) as copy: - copy.set_types(["bigint", "vector"]) - for i, row in enumerate(metadata_arr): - copy.write_row((row, embeddings_arr[i])) + if self.with_scalar_labels: + copy.set_types(["bigint", "vector", "varchar"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i], labels_data[i])) + else: + copy.set_types(["bigint", "vector"]) + for i, row in enumerate(metadata_arr): + copy.write_row((row, embeddings_arr[i])) self.conn.commit() if kwargs.get("last_batch"): @@ -345,49 +363,39 @@ def insert_embeddings( log.warning(f"Failed to insert data into table ({self.table_name}), error: {e}") return 0, e + def prepare_filter(self, filters: Filter): + """Prepare filter - builds where_clause""" + if filters.type == FilterOp.NonFilter: + self.where_clause = "" + elif filters.type == FilterOp.NumGE: + self.where_clause = f"WHERE {self._primary_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self.where_clause = f"WHERE {self._scalar_label_field} = '{filters.label_value}'" + else: + msg = f"Not support Filter for PgDiskANN - {filters}" + raise ValueError(msg) + + self._search = self._generate_search_query() + log.debug(f"Search query={self._search.as_string(self.conn)}") + def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, timeout: int | None = None, + **kwargs: Any, ) -> list[int]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" search_params = self.case_config.search_param() - is_reranking = search_params.get("reranking", False) - q = np.asarray(query) - if filters: - gt = filters.get("id") - if is_reranking: - result = self.cursor.execute( - self._filtered_search, - (gt, q, q, k), - prepare=True, - binary=True, - ) - else: - result = self.cursor.execute( - self._filtered_search, - (gt, q, k), - prepare=True, - binary=True, - ) - elif is_reranking: - result = self.cursor.execute( - self._unfiltered_search, - (q, q, k), - prepare=True, - binary=True, - ) - else: - result = self.cursor.execute( - self._unfiltered_search, - (q, k), - prepare=True, - binary=True, - ) + + result = self.cursor.execute( + self._search, + (q, q, k) if search_params.get("reranking", False) else (q, k), + prepare=True, + binary=True, + ) return [int(i[0]) for i in result.fetchall()] From 4082eff8ff602a245abd14d915724d718bcbe2f2 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Fri, 24 Apr 2026 17:21:00 +0800 Subject: [PATCH 27/49] fix(ui): Run Test page error surfacing and streamlit upgrade (#766) - Migrate DB config validators to pydantic v2; list all empty fields instead of raising on first; consolidate via `_extra_empty_skip`. - Surface missing client modules at config render time as `{DB} needs `{module}` but it is not installed.` - Replace streamlit-autorefresh with native `@st.fragment(run_every)` so live progress does not block UI. - Bump streamlit to 1.47+ (picks up streamlit#11890 fragment fix); switch to native `st.switch_page`, drop `streamlit_extras`. - Migrate deprecated `use_container_width=True` to `width="stretch"`. - Patch tornado `write_message` to consume expected `WebSocketClosedError` on tab-close races (streamlit#9787). - Add contract test: each DB enum resolves config_cls/init_cls or raises ModuleNotFoundError. See also: #446 Signed-off-by: yangxuan --- pyproject.toml | 4 +- tests/test_db_client_resolution.py | 17 ++ vectordb_bench/backend/clients/api.py | 15 +- .../backend/clients/aws_opensearch/config.py | 18 +-- .../backend/clients/elastic_cloud/config.py | 17 +- .../backend/clients/milvus/config.py | 19 +-- .../backend/clients/oss_opensearch/config.py | 16 +- .../backend/clients/qdrant_cloud/config.py | 19 +-- vectordb_bench/backend/clients/tidb/config.py | 19 +-- .../components/check_results/charts.py | 2 +- .../frontend/components/check_results/nav.py | 9 +- .../components/check_results/priceTable.py | 2 +- .../frontend/components/concurrent/charts.py | 2 +- .../frontend/components/int_filter/charts.py | 2 +- .../components/label_filter/charts.py | 2 +- .../frontend/components/qps_recall/charts.py | 2 +- .../components/run_test/autoRefresh.py | 10 -- .../components/run_test/dbConfigSetting.py | 17 +- .../components/run_test/submitTask.py | 153 +++++++++--------- .../frontend/components/streaming/charts.py | 4 +- .../components/streaming/concurrent_detail.py | 4 +- vectordb_bench/frontend/pages/run_test.py | 4 - vectordb_bench/frontend/vdbbench.py | 23 +++ 23 files changed, 171 insertions(+), 209 deletions(-) create mode 100644 tests/test_db_client_resolution.py delete mode 100644 vectordb_bench/frontend/components/run_test/autoRefresh.py diff --git a/pyproject.toml b/pyproject.toml index 2ed18b115..8bd5de2ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,9 +27,7 @@ classifiers = [ dependencies = [ "click", "pytz", - "streamlit-autorefresh", - "streamlit<1.44,!=1.34.0", # There is a breaking change in 1.44 related to get_page https://discuss.streamlit.io/t/from-streamlit-source-util-import-get-pages-gone-in-v-1-44-0-need-urgent-help/98399 - "streamlit_extras", + "streamlit>=1.47,<2", # 1.47 fixes streamlit#11660 "tqdm", "s3fs", "oss2", diff --git a/tests/test_db_client_resolution.py b/tests/test_db_client_resolution.py new file mode 100644 index 000000000..ce0278eb0 --- /dev/null +++ b/tests/test_db_client_resolution.py @@ -0,0 +1,17 @@ +"""Every DB must resolve config_cls/init_cls or raise ModuleNotFoundError. +Anything else breaks the Run Test page's missing-optional-dep hint path. +""" + +import pytest + +from vectordb_bench.backend.clients import DB + + +@pytest.mark.parametrize("db", list(DB), ids=lambda d: d.name) +def test_db_resolves_or_missing_module(db): + for attr in ("config_cls", "init_cls"): + try: + getattr(db, attr) + except ModuleNotFoundError as e: + assert e.name, f"{db.name}.{attr}: ModuleNotFoundError has no .name" + return diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index f507abe33..118c505ff 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from contextlib import contextmanager from enum import StrEnum +from typing import ClassVar from pydantic import BaseModel, model_validator @@ -77,6 +78,9 @@ class DBConfig(ABC, BaseModel): version: str = "" note: str = "" + # Field names subclasses allow to be empty (optional creds, alt-route fields). + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset() + @staticmethod def common_short_configs() -> list[str]: """ @@ -100,12 +104,11 @@ def to_dict(self) -> dict: def not_empty_field(cls, data: any) -> any: if not isinstance(data, dict): return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") + skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | cls._extra_empty_skip + empty = [k for k, v in data.items() if k not in skip and isinstance(v, str) and not v] + if empty: + msg = f"Empty field(s): {', '.join(empty)}" + raise ValueError(msg) return data diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index 7742d421d..62c284317 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -1,7 +1,8 @@ import logging from enum import Enum +from typing import ClassVar -from pydantic import BaseModel, SecretStr, model_validator +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType @@ -9,6 +10,8 @@ class AWSOpenSearchConfig(DBConfig, BaseModel): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"user", "password", "host"}) + host: str = "" port: int = 80 user: str | None = None @@ -32,19 +35,6 @@ def to_dict(self) -> dict: "timeout": 600, } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class AWSOS_Engine(Enum): faiss = "faiss" diff --git a/vectordb_bench/backend/clients/elastic_cloud/config.py b/vectordb_bench/backend/clients/elastic_cloud/config.py index bdae177f3..be2c2dce8 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/config.py +++ b/vectordb_bench/backend/clients/elastic_cloud/config.py @@ -1,4 +1,5 @@ from enum import StrEnum +from typing import ClassVar from pydantic import BaseModel, SecretStr, model_validator @@ -6,6 +7,8 @@ class ElasticCloudConfig(DBConfig, BaseModel): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"cloud_id", "host"}) + # Elastic Cloud connection. Takes precedence when set. cloud_id: SecretStr | None = None # Self-hosted / host-based connection (used when cloud_id is not provided). @@ -15,20 +18,6 @@ class ElasticCloudConfig(DBConfig, BaseModel): user: str = "elastic" password: SecretStr - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"cloud_id", "host"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - msg = "Empty string!" - raise ValueError(msg) - return data - @model_validator(mode="after") def _check_connection_target(self) -> "ElasticCloudConfig": has_cloud_id = bool(self.cloud_id and self.cloud_id.get_secret_value()) diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 620a6b484..054a3fddb 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,9 +1,13 @@ -from pydantic import BaseModel, SecretStr, model_validator +from typing import ClassVar + +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, IndexType, MetricType, SQType class MilvusConfig(DBConfig): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"user", "password"}) + uri: SecretStr = "http://localhost:19530" user: str | None = None password: SecretStr | None = None @@ -19,19 +23,6 @@ def to_dict(self) -> dict: "replica_number": self.replica_number, } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class MilvusIndexConfig(BaseModel): """Base config for milvus""" diff --git a/vectordb_bench/backend/clients/oss_opensearch/config.py b/vectordb_bench/backend/clients/oss_opensearch/config.py index a5d69459a..7a8b1d98e 100644 --- a/vectordb_bench/backend/clients/oss_opensearch/config.py +++ b/vectordb_bench/backend/clients/oss_opensearch/config.py @@ -1,5 +1,6 @@ import logging from enum import Enum +from typing import ClassVar from pydantic import BaseModel, SecretStr, field_validator, model_validator @@ -9,6 +10,8 @@ class OSSOpenSearchConfig(DBConfig, BaseModel): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"user", "password", "host"}) + host: str = "" port: int = 80 user: str | None = None @@ -32,19 +35,6 @@ def to_dict(self) -> dict: "timeout": 600, } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"user", "password", "host"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class OSSOS_Engine(Enum): faiss = "faiss" diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index 06543aaab..c4466dc17 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,6 +1,6 @@ -from typing import TypeVar +from typing import ClassVar, TypeVar -from pydantic import BaseModel, SecretStr, model_validator +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType @@ -10,6 +10,8 @@ # Allowing `api_key` to be left empty, to ensure compatibility with the open-source Qdrant. class QdrantConfig(DBConfig): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"api_key"}) + url: SecretStr api_key: SecretStr | None = None @@ -25,19 +27,6 @@ def to_dict(self) -> dict: "url": self.url.get_secret_value(), } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"api_key"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class QdrantIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None diff --git a/vectordb_bench/backend/clients/tidb/config.py b/vectordb_bench/backend/clients/tidb/config.py index 93098ede1..5e2f032d4 100644 --- a/vectordb_bench/backend/clients/tidb/config.py +++ b/vectordb_bench/backend/clients/tidb/config.py @@ -1,6 +1,6 @@ -from typing import TypedDict +from typing import ClassVar, TypedDict -from pydantic import BaseModel, SecretStr, model_validator +from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType @@ -16,6 +16,8 @@ class TiDBConfigDict(TypedDict): class TiDBConfig(DBConfig): + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"password"}) + user_name: str = "root" password: SecretStr host: str = "127.0.0.1" @@ -35,19 +37,6 @@ def to_dict(self) -> TiDBConfigDict: "ssl_verify_identity": self.ssl, } - @model_validator(mode="before") - @classmethod - def not_empty_field(cls, data: any) -> any: - if not isinstance(data, dict): - return data - skip = set(cls.common_short_configs()) | set(cls.common_long_configs()) | {"password"} - for field_name, v in data.items(): - if field_name in skip: - continue - if isinstance(v, str) and not v: - raise ValueError("Empty string!") - return data - class TiDBIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None diff --git a/vectordb_bench/frontend/components/check_results/charts.py b/vectordb_bench/frontend/components/check_results/charts.py index 0e74d2752..d36cc5fc6 100644 --- a/vectordb_bench/frontend/components/check_results/charts.py +++ b/vectordb_bench/frontend/components/check_results/charts.py @@ -153,4 +153,4 @@ def drawMetricChart(data, metric, st, key: str): ), ) - chart.plotly_chart(fig, use_container_width=True, key=key) + chart.plotly_chart(fig, width="stretch", key=key) diff --git a/vectordb_bench/frontend/components/check_results/nav.py b/vectordb_bench/frontend/components/check_results/nav.py index ba4fa99c7..2267024bb 100644 --- a/vectordb_bench/frontend/components/check_results/nav.py +++ b/vectordb_bench/frontend/components/check_results/nav.py @@ -1,25 +1,22 @@ -from streamlit_extras.switch_page_button import switch_page - - def NavToRunTest(st): st.subheader("Run your test") st.write("You can set the configs and run your own test.") navClick = st.button("Run Your Test   >") if navClick: - switch_page("run test") + st.switch_page("pages/run_test.py") def NavToQuriesPerDollar(st): st.subheader("Compare qps with price.") navClick = st.button("QP$ (Quries per Dollar)   >") if navClick: - switch_page("quries_per_dollar") + st.switch_page("pages/quries_per_dollar.py") def NavToResults(st, key="nav-to-results"): navClick = st.button("<   Back to Results", key=key) if navClick: - switch_page("results") + st.switch_page("pages/results.py") def NavToPages(st): diff --git a/vectordb_bench/frontend/components/check_results/priceTable.py b/vectordb_bench/frontend/components/check_results/priceTable.py index f2c0ae001..f34f70872 100644 --- a/vectordb_bench/frontend/components/check_results/priceTable.py +++ b/vectordb_bench/frontend/components/check_results/priceTable.py @@ -27,7 +27,7 @@ def priceTable(container, data): expander = container.expander("Price List (Editable).") editTable = expander.data_editor( table, - use_container_width=True, + width="stretch", hide_index=True, height=height, disabled=("DB", "Label"), diff --git a/vectordb_bench/frontend/components/concurrent/charts.py b/vectordb_bench/frontend/components/concurrent/charts.py index 004fcb261..5369d5912 100644 --- a/vectordb_bench/frontend/components/concurrent/charts.py +++ b/vectordb_bench/frontend/components/concurrent/charts.py @@ -94,4 +94,4 @@ def drawChart(data, st, key: str, x_metric: str = "latency_p99", y_metric: str = fig.update_yaxes(range=yrange, title_text=gen_title(y_metric)) fig.update_traces(textposition="bottom right", texttemplate="conc-%{text:,.4~r}") - st.plotly_chart(fig, use_container_width=True, key=key) + st.plotly_chart(fig, width="stretch", key=key) diff --git a/vectordb_bench/frontend/components/int_filter/charts.py b/vectordb_bench/frontend/components/int_filter/charts.py index 881681031..5c32a089e 100644 --- a/vectordb_bench/frontend/components/int_filter/charts.py +++ b/vectordb_bench/frontend/components/int_filter/charts.py @@ -57,4 +57,4 @@ def drawChart(st, data: list[object], metric): margin=dict(l=0, r=0, t=40, b=0, pad=8), legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), ) - st.plotly_chart(fig, use_container_width=True) + st.plotly_chart(fig, width="stretch") diff --git a/vectordb_bench/frontend/components/label_filter/charts.py b/vectordb_bench/frontend/components/label_filter/charts.py index 881681031..5c32a089e 100644 --- a/vectordb_bench/frontend/components/label_filter/charts.py +++ b/vectordb_bench/frontend/components/label_filter/charts.py @@ -57,4 +57,4 @@ def drawChart(st, data: list[object], metric): margin=dict(l=0, r=0, t=40, b=0, pad=8), legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), ) - st.plotly_chart(fig, use_container_width=True) + st.plotly_chart(fig, width="stretch") diff --git a/vectordb_bench/frontend/components/qps_recall/charts.py b/vectordb_bench/frontend/components/qps_recall/charts.py index ab57dd0ce..a44c51a36 100644 --- a/vectordb_bench/frontend/components/qps_recall/charts.py +++ b/vectordb_bench/frontend/components/qps_recall/charts.py @@ -115,4 +115,4 @@ def drawlinechart(st, data: list[object], metric, key: str): margin=dict(l=0, r=0, t=40, b=0, pad=8), legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), ) - st.plotly_chart(fig, use_container_width=True, key=key) + st.plotly_chart(fig, width="stretch", key=key) diff --git a/vectordb_bench/frontend/components/run_test/autoRefresh.py b/vectordb_bench/frontend/components/run_test/autoRefresh.py deleted file mode 100644 index 034ab5017..000000000 --- a/vectordb_bench/frontend/components/run_test/autoRefresh.py +++ /dev/null @@ -1,10 +0,0 @@ -from streamlit_autorefresh import st_autorefresh -from vectordb_bench.frontend.config.styles import * - - -def autoRefresh(): - auto_refresh_count = st_autorefresh( - interval=MAX_AUTO_REFRESH_INTERVAL, - limit=MAX_AUTO_REFRESH_COUNT, - key="streamlit-auto-refresh", - ) diff --git a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py index a2d2de77f..85167fb66 100644 --- a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py +++ b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py @@ -11,19 +11,18 @@ def dbConfigSettings(st, activedDbList: list[DB]): isAllValid = True for activeDb in activedDbList: dbConfigSettingItemContainer = expander.container() - dbConfig = dbConfigSettingItem(dbConfigSettingItemContainer, activeDb) try: + dbConfig = dbConfigSettingItem(dbConfigSettingItemContainer, activeDb) dbConfigs[activeDb] = activeDb.config_cls(**dbConfig) + # Probe client module so missing optional deps surface now, not on Run. + _ = activeDb.init_cls + except ModuleNotFoundError as e: + isAllValid = False + dbConfigSettingItemContainer.error(f"{activeDb.value} needs `{e.name}` but it is not installed.") except ValidationError as e: isAllValid = False - errTexts = [] - for err in e.raw_errors: - errLocs = err.loc_tuple() - errInfo = err.exc - errText = f"{', '.join(errLocs)} - {errInfo}" - errTexts.append(errText) - - dbConfigSettingItemContainer.error(f"{'; '.join(errTexts)}") + errTexts = [f"{', '.join(str(x) for x in err['loc'])} - {err['msg']}" for err in e.errors()] + dbConfigSettingItemContainer.error("; ".join(errTexts)) return dbConfigs, isAllValid diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index e5c2a1e42..93fd280c3 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -1,77 +1,76 @@ from datetime import datetime + +import streamlit as st + from vectordb_bench import config from vectordb_bench.frontend.config import styles from vectordb_bench.interface import benchmark_runner from vectordb_bench.models import TaskConfig -def submitTask(st, tasks, isAllValid): - st.markdown( +def submitTask(container, tasks, isAllValid): + container.markdown( "
", unsafe_allow_html=True, ) - st.subheader("STEP 3: Task Label") - st.markdown( + container.subheader("STEP 3: Task Label") + container.markdown( "
This description is used to mark the result.
", unsafe_allow_html=True, ) - taskLabel = taskLabelInput(st) + taskLabel = taskLabelInput(container) - st.markdown( + container.markdown( "
", unsafe_allow_html=True, ) - controlPanelContainer = st.container() - controlPanel(controlPanelContainer, tasks, taskLabel, isAllValid) + controlPanel(container.container(), tasks, taskLabel, isAllValid) -def taskLabelInput(st): +def taskLabelInput(container): defaultTaskLabel = datetime.now().strftime("%Y%m%d%H") - columns = st.columns(styles.TASK_LABEL_INPUT_COLUMNS) - taskLabel = columns[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed") - return taskLabel + cols = container.columns(styles.TASK_LABEL_INPUT_COLUMNS) + return cols[0].text_input("task_label", defaultTaskLabel, label_visibility="collapsed") -def advancedSettings(st): - container = st.columns([1, 2]) - index_already_exists = container[0].checkbox("Index already exists", value=False) - container[1].caption("if selected, inserting and building will be skipped.") +def advancedSettings(container): + cols = container.columns([1, 2]) + index_already_exists = cols[0].checkbox("Index already exists", value=False) + cols[1].caption("if selected, inserting and building will be skipped.") - container = st.columns([1, 2]) - use_aliyun = container[0].checkbox("Dataset from Aliyun (Shanghai)", value=False) - container[1].caption( - "if selected, the dataset will be downloaded from Aliyun OSS shanghai, default AWS S3 aws-us-west." - ) + cols = container.columns([1, 2]) + use_aliyun = cols[0].checkbox("Dataset from Aliyun (Shanghai)", value=False) + cols[1].caption("if selected, the dataset will be downloaded from Aliyun OSS shanghai, default AWS S3 aws-us-west.") - container = st.columns([1, 2]) - k = container[0].number_input("k", min_value=1, value=100, label_visibility="collapsed") - container[1].caption("K value for number of nearest neighbors to search") + cols = container.columns([1, 2]) + k = cols[0].number_input("k", min_value=1, value=100, label_visibility="collapsed") + cols[1].caption("K value for number of nearest neighbors to search") - container = st.columns([1, 2]) + cols = container.columns([1, 2]) defaultconcurrentInput = ",".join(map(str, config.NUM_CONCURRENCY)) - concurrentInput = container[0].text_input( - "Concurrent Input", value=defaultconcurrentInput, label_visibility="collapsed" - ) - container[1].caption("num of concurrencies for search tests to get max-qps") + concurrentInput = cols[0].text_input("Concurrent Input", value=defaultconcurrentInput, label_visibility="collapsed") + cols[1].caption("num of concurrencies for search tests to get max-qps") - container = st.columns([1, 2]) - concurrency_duration = container[0].number_input( + cols = container.columns([1, 2]) + concurrency_duration = cols[0].number_input( "Concurrency Duration", value=config.CONCURRENCY_DURATION, label_visibility="collapsed" ) - container[1].caption("concurrency duration for each concurrency search test") + cols[1].caption("concurrency duration for each concurrency search test") - container = st.columns([1, 2]) - load_concurrency = container[0].number_input( + cols = container.columns([1, 2]) + load_concurrency = cols[0].number_input( "Load Concurrency", min_value=0, value=config.LOAD_CONCURRENCY, label_visibility="collapsed" ) - container[1].caption("number of concurrent workers for data loading in performance cases (0 = cpu_count)") + cols[1].caption("number of concurrent workers for data loading in performance cases (0 = cpu_count)") return index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency -def controlPanel(st, tasks: list[TaskConfig], taskLabel, isAllValid): - index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency = advancedSettings(st) +def controlPanel(container, tasks: list[TaskConfig], taskLabel, isAllValid): + index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration, load_concurrency = advancedSettings( + container + ) def runHandler(): benchmark_runner.set_drop_old(not index_already_exists) @@ -79,7 +78,7 @@ def runHandler(): try: concurrentInput_list = [int(item.strip()) for item in concurrentInput.split(",")] except ValueError: - st.write("please input correct number") + container.write("please input correct number") return None for task in tasks: @@ -93,41 +92,43 @@ def runHandler(): def stopHandler(): benchmark_runner.stop_running() - isRunning = benchmark_runner.has_running() - - if isRunning: - currentTaskId = benchmark_runner.get_current_task_id() - tasksCount = benchmark_runner.get_tasks_count() - text = f":running: Running Task {currentTaskId} / {tasksCount}" - - if tasksCount > 0: - st.progress(currentTaskId / tasksCount, text=text) - - columns = st.columns(6) - columns[0].button( - "Run Your Test", - disabled=True, - on_click=runHandler, - type="primary", - ) - columns[1].button( - "Stop", - on_click=stopHandler, - type="primary", - ) - - else: - errorText = benchmark_runner.latest_error or "" - if len(errorText) > 0: - st.error(errorText) - disabled = True if len(tasks) == 0 or not isAllValid else False - if not isAllValid: - st.error("Make sure all config is valid.") - elif len(tasks) == 0: - st.warning("No tests to run.") - st.button( - "Run Your Test", - disabled=disabled, - on_click=runHandler, - type="primary", - ) + @st.fragment(run_every=f"{styles.MAX_AUTO_REFRESH_INTERVAL / 1000}s") + def _renderLiveStatus(): + if benchmark_runner.has_running(): + currentTaskId = benchmark_runner.get_current_task_id() + tasksCount = benchmark_runner.get_tasks_count() + text = f":running: Running Task {currentTaskId} / {tasksCount}" + if tasksCount > 0: + st.progress(currentTaskId / tasksCount, text=text) + cols = st.columns(6) + cols[0].button( + "Run Your Test", + disabled=True, + on_click=runHandler, + type="primary", + key="run-disabled", + ) + cols[1].button( + "Stop", + on_click=stopHandler, + type="primary", + key="stop-btn", + ) + else: + errorText = benchmark_runner.latest_error or "" + if len(errorText) > 0: + st.error(errorText) + disabled = len(tasks) == 0 or not isAllValid + if not isAllValid: + st.error("Make sure all config is valid.") + elif len(tasks) == 0: + st.warning("No tests to run.") + st.button( + "Run Your Test", + disabled=disabled, + on_click=runHandler, + type="primary", + key="run-btn", + ) + + _renderLiveStatus() diff --git a/vectordb_bench/frontend/components/streaming/charts.py b/vectordb_bench/frontend/components/streaming/charts.py index a05da9b25..09357cf44 100644 --- a/vectordb_bench/frontend/components/streaming/charts.py +++ b/vectordb_bench/frontend/components/streaming/charts.py @@ -119,7 +119,7 @@ def drawLineChart( if x_metric == DisplayedMetric.search_time: x_title = "Actual Time (s)" fig.update_layout(xaxis_title=x_title) - st.plotly_chart(fig, use_container_width=True, key=key) + st.plotly_chart(fig, width="stretch", key=key) def get_normal_scatter( @@ -234,7 +234,7 @@ def drawBarChart( fig.update_layout(xaxis_title="time (s)") fig.update_layout(barmode="stack") fig.update_traces(width=0.15) - st.plotly_chart(fig, use_container_width=True, key=key) + st.plotly_chart(fig, width="stretch", key=key) def get_bar( diff --git a/vectordb_bench/frontend/components/streaming/concurrent_detail.py b/vectordb_bench/frontend/components/streaming/concurrent_detail.py index 0580bad14..37852a7ab 100644 --- a/vectordb_bench/frontend/components/streaming/concurrent_detail.py +++ b/vectordb_bench/frontend/components/streaming/concurrent_detail.py @@ -124,7 +124,7 @@ def drawQPSLatencyChart(container, qps_values, latencies_ms, stage, metric_name, showlegend=False, ) - container.plotly_chart(fig, use_container_width=True, key=f"{case_name}-chart-{stage}") + container.plotly_chart(fig, width="stretch", key=f"{case_name}-chart-{stage}") def drawMetricsTable(container, qps_values, conc_nums, p99_list, p95_list, avg_list, stage): @@ -267,7 +267,7 @@ def drawComparisonChart(container, case_data, selected_stages, metric_name, case legend=dict(yanchor="top", y=0.99, xanchor="right", x=0.99), ) - container.plotly_chart(fig, use_container_width=True, key=f"{case_name}-compare-chart") + container.plotly_chart(fig, width="stretch", key=f"{case_name}-compare-chart") # Add insight container.info("**Insight:** Compare curves across stages to understand how performance scales with data growth.") diff --git a/vectordb_bench/frontend/pages/run_test.py b/vectordb_bench/frontend/pages/run_test.py index e4472e767..64115ff17 100644 --- a/vectordb_bench/frontend/pages/run_test.py +++ b/vectordb_bench/frontend/pages/run_test.py @@ -1,5 +1,4 @@ import streamlit as st -from vectordb_bench.frontend.components.run_test.autoRefresh import autoRefresh from vectordb_bench.frontend.components.run_test.caseSelector import caseSelector from vectordb_bench.frontend.components.run_test.dbConfigSetting import dbConfigSettings from vectordb_bench.frontend.components.run_test.dbSelector import dbSelector @@ -57,9 +56,6 @@ def main(): # nav to results NavToResults(st, key="footer-nav-to-results") - # autofresh - autoRefresh() - if __name__ == "__main__": main() diff --git a/vectordb_bench/frontend/vdbbench.py b/vectordb_bench/frontend/vdbbench.py index 860467734..90769019c 100644 --- a/vectordb_bench/frontend/vdbbench.py +++ b/vectordb_bench/frontend/vdbbench.py @@ -1,10 +1,33 @@ import streamlit as st +from tornado.iostream import StreamClosedError +from tornado.websocket import WebSocketClosedError, WebSocketProtocol13 + from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon from vectordb_bench.frontend.components.custom.initStyle import initStyle from vectordb_bench.frontend.components.welcome.explainPrams import explainPrams from vectordb_bench.frontend.components.welcome.welcomePrams import welcomePrams from vectordb_bench.frontend.config.styles import FAVICON, PAGE_TITLE +# Consume expected WS-close errors on streamlit's fire-and-forget writes +# (streamlit#9787, unfixed upstream). +_orig_write_message = WebSocketProtocol13.write_message + + +def _write_message_with_consumer(self, message, binary=False): + task = _orig_write_message(self, message, binary=binary) + + def _consume(t): + exc = t.exception() + if exc is None or isinstance(exc, (WebSocketClosedError, StreamClosedError)): + return + t.get_loop().call_exception_handler({"message": "websocket write failed", "exception": exc, "task": t}) + + task.add_done_callback(_consume) + return task + + +WebSocketProtocol13.write_message = _write_message_with_consumer + def main(): st.set_page_config( From a424b025f00e246d617826b9bd793d123c927a88 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Fri, 24 Apr 2026 19:32:06 +0800 Subject: [PATCH 28/49] feat(loader): cap default insert workers to min(cpu, 4) (#767) ConcurrentInsertRunner previously defaulted to mp.cpu_count(), spawning one worker per CPU when load_concurrency was unset. On high-core hosts this opens many parallel client connections, saturating modest DBs / network paths and yielding worse load throughput than a smaller, steadier worker count. Cap the unset default to min(cpu_count, 4). Explicit load_concurrency from CLI / config / submitTask still wins. Signed-off-by: yangxuan --- vectordb_bench/backend/runner/concurrent_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py index 37201f88e..7c8aeb24f 100644 --- a/vectordb_bench/backend/runner/concurrent_runner.py +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -51,7 +51,7 @@ class ConcurrentInsertRunner: normalize: Whether to L2-normalize embeddings. filters: Filter configuration. timeout: Timeout in seconds for the overall operation. - max_workers: Number of concurrent workers (default: cpu_count). + max_workers: Number of concurrent workers (default: min(cpu_count, 4)). backend: Executor backend to use ('threading' or 'async'). """ @@ -72,7 +72,7 @@ def __init__( self.filters = filters self.backend = backend - effective_workers = max_workers or mp.cpu_count() + effective_workers = max_workers or min(mp.cpu_count(), 4) if not db.thread_safe: log.info(f"DB {db.name} is not thread-safe, falling back to max_workers=1") effective_workers = 1 From c2a6f859831d6d63ce2385d189809839b123a0ae Mon Sep 17 00:00:00 2001 From: liuhao6741 <157583880+liuhao6741@users.noreply.github.com> Date: Mon, 11 May 2026 11:45:54 +0800 Subject: [PATCH 29/49] feat(seekdb): add SeekDB backend with HNSW index support (#770) Add a new vector database backend for SeekDB, connecting via mysql-connector-python over the MySQL wire protocol. Key components: - seekdb.py: VectorDB implementation with heap-organized table, HNSW vector index, and version-aware optimize() that calls dbms_index_manager.refresh() on SeekDB >= 1.3.0 - config.py: DBConfig with host/port/user/password/database and SeekDBHNSWConfig with m/ef_construction/ef_search parameters - cli.py: Click command `SeekDBHNSW` for command-line benchmarks Registration: - Add SeekDB to the DB enum in backend/clients/__init__.py with lazy imports for init_cls, config_cls, and case_config_cls - Register SeekDBHNSW CLI command in cli/vectordbbench.py - Add seekdb optional dependency in pyproject.toml (pip install vectordb-bench[seekdb]) Filter support: - NonFilter and NumGE (id >= N) filters are supported - StrEqual (label filter) is intentionally excluded since the table schema only has id and embedding columns Thread safety: - mysql.connector is not thread-safe (thread_safe = False). ConcurrentInsertRunner uses max_workers=1 accordingly - rate_runner.py handles SeekDB specially: copies the db object, resets the connection, and calls init() per worker thread Co-authored-by: liuhao6741 Co-authored-by: Claude Opus 4.7 --- pyproject.toml | 1 + vectordb_bench/backend/clients/__init__.py | 16 + vectordb_bench/backend/clients/seekdb/cli.py | 60 ++++ .../backend/clients/seekdb/config.py | 77 +++++ .../backend/clients/seekdb/seekdb.py | 282 ++++++++++++++++++ vectordb_bench/backend/runner/rate_runner.py | 13 +- vectordb_bench/cli/vectordbbench.py | 2 + 7 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 vectordb_bench/backend/clients/seekdb/cli.py create mode 100644 vectordb_bench/backend/clients/seekdb/config.py create mode 100644 vectordb_bench/backend/clients/seekdb/seekdb.py diff --git a/pyproject.toml b/pyproject.toml index 8bd5de2ad..ee57d8339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,7 @@ turbopuffer = [ "turbopuffer" ] zvec = [ "zvec" ] endee = [ "endee==0.1.10" ] lindorm = [ "opensearch-py" ] +seekdb = [ "mysql-connector-python" ] pinot = [ "requests" ] [project.urls] diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 9029be05f..4be8d0424 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -62,6 +62,7 @@ class DB(Enum): VectorChord = "VectorChord" PolarDB = "PolarDB" Pinot = "Pinot" + SeekDB = "SeekDB" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -263,6 +264,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return Pinot + if self == DB.SeekDB: + from .seekdb.seekdb import SeekDB + + return SeekDB + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -466,6 +472,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return PinotConfig + if self == DB.SeekDB: + from .seekdb.config import SeekDBConfig + + return SeekDBConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -651,6 +662,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 IndexType.IVFPQ: PinotIVFPQConfig, }.get(index_type, PinotHNSWConfig) + if self == DB.SeekDB: + from .seekdb.config import _seekdb_case_config + + return _seekdb_case_config.get(index_type) + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/seekdb/cli.py b/vectordb_bench/backend/clients/seekdb/cli.py new file mode 100644 index 000000000..9c135220f --- /dev/null +++ b/vectordb_bench/backend/clients/seekdb/cli.py @@ -0,0 +1,60 @@ +import os +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + HNSWFlavor3, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class SeekDBTypedDict(CommonTypedDict): + host: Annotated[str, click.option("--host", type=str, help="SeekDB host", required=True)] + user: Annotated[str, click.option("--user", type=str, help="SeekDB username", required=True)] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="SeekDB password", + default=lambda: os.environ.get("SEEKDB_PASSWORD", ""), + ), + ] + database: Annotated[str, click.option("--database", type=str, help="Database name", required=True)] + port: Annotated[int, click.option("--port", type=int, help="SeekDB port", default=3306, show_default=True)] + + +class SeekDBHNSWTypedDict(SeekDBTypedDict, HNSWFlavor3): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(SeekDBHNSWTypedDict) +def SeekDBHNSW(**parameters: Unpack[SeekDBHNSWTypedDict]): + """Run VectorDBBench against SeekDB with an HNSW index.""" + from ..api import IndexType + from .config import SeekDBConfig, SeekDBHNSWConfig + + run( + DB.SeekDB, + SeekDBConfig( + db_label=parameters["db_label"], + user=SecretStr(parameters["user"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + ), + SeekDBHNSWConfig( + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + index=IndexType.HNSW, + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/seekdb/config.py b/vectordb_bench/backend/clients/seekdb/config.py new file mode 100644 index 000000000..01a43ce4b --- /dev/null +++ b/vectordb_bench/backend/clients/seekdb/config.py @@ -0,0 +1,77 @@ +from typing import TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class SeekDBConfigDict(TypedDict): + user: str + host: str + port: int + password: str + database: str + + +class SeekDBConfig(DBConfig): + user: SecretStr = SecretStr("root") + password: SecretStr + host: str + port: int = 3306 + database: str + + def to_dict(self) -> SeekDBConfigDict: + return { + "user": self.user.get_secret_value(), + "host": self.host, + "port": self.port, + "password": self.password.get_secret_value(), + "database": self.database, + } + + +class SeekDBIndexConfig(BaseModel): + index: IndexType + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2" + if self.metric_type == MetricType.IP: + return "inner_product" + return "cosine" + + def parse_metric_func_str(self) -> str: + if self.metric_type == MetricType.L2: + return "l2_distance" + if self.metric_type == MetricType.IP: + return "negative_inner_product" + return "cosine_distance" + + +class SeekDBHNSWConfig(SeekDBIndexConfig, DBCaseConfig): + m: int + ef_construction: int + ef_search: int + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "m": self.m, + "ef_construction": self.ef_construction, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric_func_str(), + "params": {"ef_search": self.ef_search}, + } + + +_seekdb_case_config = { + IndexType.HNSW: SeekDBHNSWConfig, +} diff --git a/vectordb_bench/backend/clients/seekdb/seekdb.py b/vectordb_bench/backend/clients/seekdb/seekdb.py new file mode 100644 index 000000000..f4551a577 --- /dev/null +++ b/vectordb_bench/backend/clients/seekdb/seekdb.py @@ -0,0 +1,282 @@ +import logging +import re +import struct +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any + +import mysql.connector as mysql + +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import IndexType, VectorDB +from .config import SeekDBConfigDict, SeekDBHNSWConfig + +log = logging.getLogger(__name__) + +SEEKDB_DEFAULT_LOAD_BATCH_SIZE = 256 + +# Minimum SeekDB version for dbms_index_manager.refresh() after bulk load (see VERSION()). +_SEEKDB_REFRESH_MIN_VERSION = (1, 3, 0) +_SEEKDB_VERSION_IN_VERSION_STRING = re.compile(r"seekdb-v(\d+(?:\.\d+)*)", re.IGNORECASE) + + +def _seekdb_version_tuple(version_row: str | None) -> tuple[int, ...] | None: + if not version_row: + return None + m = _SEEKDB_VERSION_IN_VERSION_STRING.search(version_row.strip()) + if not m: + return None + return tuple(int(p) for p in m.group(1).split(".") if p.isdigit()) + + +def _version_tuple_ge(parsed: tuple[int, ...], minimum: tuple[int, ...]) -> bool: + n = max(len(parsed), len(minimum)) + for i in range(n): + p = parsed[i] if i < len(parsed) else 0 + m = minimum[i] if i < len(minimum) else 0 + if p != m: + return p > m + return True + + +class SeekDB(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + # mysql.connector is not thread-safe; ConcurrentInsertRunner uses max_workers=1 when False. + # Streaming fixed-rate inserts use rate_runner with per-thread deepcopy+init() instead. + thread_safe: bool = False + + def __init__( + self, + dim: int, + db_config: SeekDBConfigDict, + db_case_config: SeekDBHNSWConfig, + collection_name: str = "items", + drop_old: bool = False, + **kwargs, + ): + self.name = "SeekDB" + self.dim = dim + self.db_config = db_config + self.db_case_config = db_case_config + self.table_name = collection_name + self.load_batch_size = SEEKDB_DEFAULT_LOAD_BATCH_SIZE + self._index_name = "vidx" + self._primary_field = "id" + self._vector_field = "embedding" + self.expr = "" + + log.info( + f"{self.name} initialized with config:\nDatabase: {self.db_config}\nCase Config: {self.db_case_config}" + ) + + self._conn = None + self._cursor = None + + try: + self._connect() + self._apply_system_settings() + if drop_old: + self._drop_table() + self._create_table() + self._create_index() + finally: + self._disconnect() + + def _connect(self): + try: + self._conn = mysql.connect( + host=self.db_config["host"], + user=self.db_config["user"], + port=self.db_config["port"], + password=self.db_config["password"], + database=self.db_config["database"], + ) + self._cursor = self._conn.cursor() + except mysql.Error: + log.exception("Failed to connect to SeekDB") + raise + + def _disconnect(self): + if self._cursor: + self._cursor.close() + self._cursor = None + if self._conn: + self._conn.close() + self._conn = None + + def _apply_system_settings(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + + self._cursor.execute('ALTER SYSTEM SET memory_limit = "0M"') + self._cursor.execute("ALTER SYSTEM SET cpu_count = 0") + + def _init_session_settings(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + + self._cursor.execute("SET autocommit=1") + if self.db_case_config.index == IndexType.HNSW: + ef_search = self.db_case_config.search_param()["params"]["ef_search"] + # SeekDB uses OceanBase-style session vars (not plain hnsw_ef_search). + self._cursor.execute(f"SET ob_hnsw_ef_search={ef_search}") + + @contextmanager + def init(self) -> Generator[None, None, None]: + try: + self._connect() + self._init_session_settings() + yield + finally: + self._disconnect() + + def _drop_table(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + log.info(f"Dropping table {self.table_name}") + self._cursor.execute(f"DROP TABLE IF EXISTS {self.table_name}") + + def _create_table(self): + """Create a heap table with a vector column. + + ORGANIZATION HEAP specifies a heap-organized table (no clustered primary + key order), which is required by SeekDB for vector workloads. + """ + if not self._cursor: + raise ValueError("Cursor is not initialized") + + log.info(f"Creating heap table {self.table_name}") + create_table_query = f""" + CREATE TABLE {self.table_name} ( + id INT, + embedding VECTOR({self.dim}) + ) ORGANIZATION HEAP; + """ + self._cursor.execute(create_table_query) + + def _create_index(self): + """Create the HNSW vector index immediately after table creation. + + Following Milvus's approach: the index is built upfront so that + streaming inserts are indexed incrementally and searches can run + concurrently with writes (StreamingPerformanceCase). + """ + if not self._cursor: + raise ValueError("Cursor is not initialized") + + index_params = self.db_case_config.index_param() + params = index_params["params"] + index_args = ", ".join(f"{k}={v}" for k, v in params.items()) + + index_query = ( + f"CREATE VECTOR INDEX {self._index_name} " + f"ON {self.table_name}({self._vector_field}) " + f"WITH (distance={index_params['metric_type']}, " + f"type={index_params['index_type']}, {index_args})" + ) + + log.info("Creating HNSW index: %s", index_query) + try: + self._cursor.execute(index_query) + log.info("HNSW index created successfully") + except mysql.Error: + log.exception("Failed to create HNSW index") + raise + + def optimize(self, data_size: int | None = None): + """Post-load hook: refresh index metadata on SeekDB >= 1.3.0 when available. + + Older releases rely on incremental HNSW indexing only. From 1.3.0 onward, + ``CALL dbms_index_manager.refresh()`` aligns on-disk index state after bulk + load (VERSION() strings look like ``... seekdb-v1.3.0.0``). + """ + if not self._cursor: + raise ValueError("Cursor is not initialized") + + self._cursor.execute("SELECT VERSION()") + row = self._cursor.fetchone() + version_str = row[0] if row else None + parsed = _seekdb_version_tuple(version_str) + + if parsed is None or not _version_tuple_ge(parsed, _SEEKDB_REFRESH_MIN_VERSION): + log.info( + "%s optimize: skip dbms_index_manager.refresh (version=%r parsed=%s; need >= %s)", + self.name, + version_str, + parsed, + ".".join(map(str, _SEEKDB_REFRESH_MIN_VERSION)), + ) + return + + log.info( + "%s optimize: SeekDB %s >= %s, calling dbms_index_manager.refresh()", + self.name, + ".".join(map(str, parsed)), + ".".join(map(str, _SEEKDB_REFRESH_MIN_VERSION)), + ) + try: + self._cursor.execute("CALL dbms_index_manager.refresh()") + except mysql.Error: + log.exception("dbms_index_manager.refresh() failed") + raise + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + if not self._cursor: + raise ValueError("Cursor is not initialized") + + insert_count = 0 + try: + for batch_start in range(0, len(embeddings), self.load_batch_size): + batch_end = min(batch_start + self.load_batch_size, len(embeddings)) + batch = [(metadata[i], embeddings[i]) for i in range(batch_start, batch_end)] + values = ", ".join(f"({item_id}, '[{','.join(map(str, embedding))}]')" for item_id, embedding in batch) + self._cursor.execute(f"INSERT INTO {self.table_name} VALUES {values}") + insert_count += len(batch) + except mysql.Error: + log.exception("Failed to insert embeddings") + raise + + return insert_count, None + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.expr = "" + elif filters.type == FilterOp.NumGE: + self.expr = f"WHERE id >= {filters.int_value}" + else: + msg = f"Unsupported filter for SeekDB: {filters}" + raise ValueError(msg) + + def search_embedding( + self, + query: list[float], + k: int = 100, + ) -> list[int]: + if not self._cursor: + raise ValueError("Cursor is not initialized") + + packed = struct.pack(f"<{len(query)}f", *query) + hex_vec = packed.hex() + + query_str = ( + f"SELECT id FROM {self.table_name} " + f"{self.expr} ORDER BY " + f"{self.db_case_config.parse_metric_func_str()}({self._vector_field}, X'{hex_vec}') " + f"APPROXIMATE LIMIT {k}" + ) + + try: + self._cursor.execute(query_str) + return [row[0] for row in self._cursor.fetchall()] + except mysql.Error: + log.exception("Failed to execute search query") + raise diff --git a/vectordb_bench/backend/runner/rate_runner.py b/vectordb_bench/backend/runner/rate_runner.py index c56c8ca61..2387abfcb 100644 --- a/vectordb_bench/backend/runner/rate_runner.py +++ b/vectordb_bench/backend/runner/rate_runner.py @@ -3,7 +3,7 @@ import multiprocessing as mp import time from concurrent.futures import ThreadPoolExecutor -from copy import deepcopy +from copy import copy, deepcopy from vectordb_bench import config from vectordb_bench.backend.clients import api @@ -65,6 +65,17 @@ def _insert_embeddings(db: api.VectorDB, emb: list[list[float]], metadata: list[ log.debug("Failed to reset Doris client or table on thread-local copy", exc_info=True) with db_copy.init(): _insert_embeddings(db_copy, emb, metadata, retry_idx=0) + elif db.name == "SeekDB": + # mysql.connector is not thread-safe; do not share one connection across workers. + # deepcopy() fails on an open _conn (socket is not picklable / not copy-safe in spawn workers). + db_copy = copy(db) + try: + db_copy._conn = None + db_copy._cursor = None + except Exception: + log.debug("Failed to reset SeekDB connection on thread-local copy", exc_info=True) + with db_copy.init(): + _insert_embeddings(db_copy, emb, metadata, retry_idx=0) else: _insert_embeddings(db, emb, metadata, retry_idx=0) diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 42048d57f..eca3dbc52 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -35,6 +35,7 @@ from ..backend.clients.qdrant_local.cli import QdrantLocal from ..backend.clients.redis.cli import Redis from ..backend.clients.s3_vectors.cli import S3Vectors +from ..backend.clients.seekdb.cli import SeekDBHNSW from ..backend.clients.tencent_elasticsearch.cli import TencentElasticsearch from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB @@ -95,6 +96,7 @@ cli.add_command(PolarDBHNSWFlat) cli.add_command(PolarDBHNSWPQ) cli.add_command(PolarDBHNSWSQ) +cli.add_command(SeekDBHNSW) if __name__ == "__main__": From aaab64324d64290e84cb42312be51a1135396805 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Fri, 15 May 2026 18:05:31 +0800 Subject: [PATCH 30/49] fix: Require pymilvus<3.0.0 and fix the overflow size (#781) Signed-off-by: yangxuan --- install/requirements_py3.11.txt | 2 +- pyproject.toml | 3 +- tests/test_milvus.py | 51 ++++++++++++++++++- .../backend/clients/milvus/milvus.py | 7 ++- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 130745816..7208439bb 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -22,7 +22,7 @@ plotly environs pydantic>=2.0,<3 scikit-learn -pymilvus +pymilvus<3.0.0 clickhouse_connect pyvespa mysql-connector-python diff --git a/pyproject.toml b/pyproject.toml index ee57d8339..cd63e5a33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "environs", "pydantic>=2.0,<3", "scikit-learn", - "pymilvus", # with pandas, numpy + "pymilvus<3.0.0", # with pandas, numpy "hdrhistogram>=0.10.1", "ujson", ] @@ -211,4 +211,3 @@ builtins-ignorelist = [ "vectordb_bench/backend/clients/*" = ["PLC0415"] "vectordb_bench/cli/batch_cli.py" = ["PLC0415"] "vectordb_bench/backend/data_source.py" = ["PLC0415"] - diff --git a/tests/test_milvus.py b/tests/test_milvus.py index 8cc391acc..1c5de7ce0 100644 --- a/tests/test_milvus.py +++ b/tests/test_milvus.py @@ -4,20 +4,67 @@ """ import logging +from types import SimpleNamespace +from unittest.mock import MagicMock +import pytest from pydantic import SecretStr +from vectordb_bench.backend.cases import CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.backend.clients.api import IndexType from vectordb_bench.backend.clients.milvus.config import MilvusConfig -from vectordb_bench.backend.cases import CaseType +from vectordb_bench.backend.clients.milvus.milvus import MILVUS_FORCE_MERGE_TARGET_SIZE_MB, Milvus from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import CaseConfig, TaskConfig - log = logging.getLogger(__name__) +class TestMilvusOptimize: + def _milvus(self, *, compact_side_effect: Exception | None = None): + milvus = Milvus.__new__(Milvus) + milvus.name = "Milvus" + milvus.collection_name = "test_collection" + milvus.case_config = SimpleNamespace(is_gpu_index=False) + milvus.client = MagicMock() + milvus.client.compact.side_effect = compact_side_effect + milvus.client.compact.return_value = 0 + milvus._wait_for_segments_sorted = MagicMock() + milvus._wait_for_index = MagicMock() + milvus._wait_for_compaction = MagicMock() + return milvus + + def test_optimize_compact_uses_safe_force_merge_target_size(self): + milvus = self._milvus() + + milvus._optimize() + + milvus.client.compact.assert_called_once_with("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) + milvus.client.refresh_load.assert_called_once_with("test_collection") + + def test_optimize_skips_property_style_permission_denied(self): + error = RuntimeError("permission denied") + error.code = SimpleNamespace(name="PERMISSION_DENIED") + milvus = self._milvus(compact_side_effect=error) + + milvus._optimize() + + milvus.client.refresh_load.assert_called_once_with("test_collection") + + def test_optimize_reraises_non_permission_error(self): + error = RuntimeError("boom") + error.code = SimpleNamespace(name="UNAVAILABLE") + milvus = self._milvus(compact_side_effect=error) + + with pytest.raises(RuntimeError, match="boom") as exc_info: + milvus._optimize() + + assert exc_info.value is error + milvus.client.refresh_load.assert_not_called() + + +@pytest.mark.integration class TestMilvus: """E2E test for Milvus using Performance1536D50K (OpenAI 50K dataset).""" diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index d36a15c24..740c89509 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -15,6 +15,7 @@ log = logging.getLogger(__name__) MILVUS_LOAD_REQS_SIZE = 1.5 * 1024 * 1024 +MILVUS_FORCE_MERGE_TARGET_SIZE_MB = ((1 << 63) - 1) // (1024**2) class Milvus(VectorDB): @@ -173,13 +174,15 @@ def _optimize(self): # wait for sort, index, compact self._wait_for_segments_sorted() self._wait_for_index() - compaction_id = self.client.compact(self.collection_name, target_size=(2**63 - 1)) + compaction_id = self.client.compact( + self.collection_name, target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB + ) if compaction_id > 0: self._wait_for_compaction(compaction_id) log.info(f"{self.name} force merge compaction completed.") except Exception as e: log.warning(f"{self.name} compact or list segments error: {e}") - if hasattr(e, "code") and e.code().name == "PERMISSION_DENIED": + if getattr(getattr(e, "code", None), "name", None) == "PERMISSION_DENIED": log.warning("Skip compact due to list segments or compact permission denied.") else: raise e from None From 191b7106a08a3e6f9f9ffe9bf5604d8f5daa8270 Mon Sep 17 00:00:00 2001 From: fan <37357096+wyfanxiao@users.noreply.github.com> Date: Fri, 15 May 2026 18:06:06 +0800 Subject: [PATCH 31/49] feat(oceanbase): configurable index params, KEY partitioning, HNSW_BQ cosine support (#776) * feat(oceanbase): configurable index params, KEY partitioning, HNSW_BQ cosine support - Add --create-index-parallel CLI option (default 16) - Add --extra-info-max-size CLI option (default 32, set 0 to omit) - Add --partitions CLI option for KEY partitioning (default 0, no partition) - HNSW_BQ: remove forced L2 for cosine, now supports cosine natively - need_normalize_cosine returns False for all index types - pyproject.toml: add pyyaml dependency, fix packages.find to include all subpackages * fix(oceanbase): declare thread_safe=False to prevent cursor sharing across threads * fix: restore seekdb dependency accidentally removed --- pyproject.toml | 3 +- .../backend/clients/oceanbase/cli.py | 38 ++++++++++++++++++- .../backend/clients/oceanbase/config.py | 11 +++--- .../backend/clients/oceanbase/oceanbase.py | 31 +++++++-------- 4 files changed, 58 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd63e5a33..3bbba8ac0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["."] -include = ["vectordb_bench", "vectordb_bench.cli"] +include = ["vectordb_bench", "vectordb_bench.*"] [project] name = "vectordb-bench" @@ -26,6 +26,7 @@ classifiers = [ ] dependencies = [ "click", + "pyyaml", "pytz", "streamlit>=1.47,<2", # 1.47 fixes streamlit#11660 "tqdm", diff --git a/vectordb_bench/backend/clients/oceanbase/cli.py b/vectordb_bench/backend/clients/oceanbase/cli.py index 61583cc82..81ccacc79 100644 --- a/vectordb_bench/backend/clients/oceanbase/cli.py +++ b/vectordb_bench/backend/clients/oceanbase/cli.py @@ -31,9 +31,40 @@ class OceanBaseTypedDict(CommonTypedDict): ] database: Annotated[str, click.option("--database", type=str, help="DataBase name", required=True)] port: Annotated[int, click.option("--port", type=int, help="OceanBase port", required=True)] + create_index_parallel: Annotated[ + int, + click.option( + "--create-index-parallel", + type=int, + default=16, + show_default=True, + help="PARALLEL hint degree for CREATE VECTOR INDEX", + ), + ] + partitions: Annotated[ + int, + click.option( + "--partitions", + type=int, + default=0, + show_default=True, + help="Number of KEY partitions for the table. 0 or 1 means no partitioning.", + ), + ] -class OceanBaseHNSWTypedDict(CommonTypedDict, OceanBaseTypedDict, HNSWFlavor4): ... +class OceanBaseHNSWTypedDict(CommonTypedDict, OceanBaseTypedDict, HNSWFlavor4): + extra_info_max_size: Annotated[ + int | None, + click.option( + "--extra-info-max-size", + type=int, + default=32, + show_default=True, + help="extra_info_max_size for HNSW index. Set to 0 to omit.", + required=False, + ), + ] @cli.command() @@ -55,6 +86,9 @@ def OceanBaseHNSW(**parameters: Unpack[OceanBaseHNSWTypedDict]): m=parameters["m"], efConstruction=parameters["ef_construction"], ef_search=parameters["ef_search"], + extra_info_max_size=parameters["extra_info_max_size"] or None, + create_index_parallel=parameters["create_index_parallel"], + partitions=parameters["partitions"], index=parameters["index_type"], ), **parameters, @@ -94,6 +128,8 @@ def OceanBaseIVF(**parameters: Unpack[OceanBaseIVFTypedDict]): nlist=parameters["nlist"], sample_per_nlist=parameters["sample_per_nlist"], nbits=parameters["nbits"], + create_index_parallel=parameters["create_index_parallel"], + partitions=parameters["partitions"], index=input_index_type, ivf_nprobes=parameters["ivf_nprobes"], ), diff --git a/vectordb_bench/backend/clients/oceanbase/config.py b/vectordb_bench/backend/clients/oceanbase/config.py index 1f37cfc75..384a7c263 100644 --- a/vectordb_bench/backend/clients/oceanbase/config.py +++ b/vectordb_bench/backend/clients/oceanbase/config.py @@ -36,20 +36,18 @@ class OceanBaseIndexConfig(BaseModel): index: IndexType metric_type: MetricType | None = None lib: str = "vsag" + create_index_parallel: int = 16 + partitions: int = 0 def parse_metric(self) -> str: - if self.metric_type == MetricType.L2 or ( - self.index == IndexType.HNSW_BQ and self.metric_type == MetricType.COSINE - ): + if self.metric_type == MetricType.L2: return "l2" if self.metric_type == MetricType.IP: return "inner_product" return "cosine" def parse_metric_func_str(self) -> str: - if self.metric_type == MetricType.L2 or ( - self.index == IndexType.HNSW_BQ and self.metric_type == MetricType.COSINE - ): + if self.metric_type == MetricType.L2: return "l2_distance" if self.metric_type == MetricType.IP: return "negative_inner_product" @@ -60,6 +58,7 @@ class OceanBaseHNSWConfig(OceanBaseIndexConfig, DBCaseConfig): m: int efConstruction: int ef_search: int | None = None + extra_info_max_size: int | None = 32 index: IndexType def index_param(self) -> dict: diff --git a/vectordb_bench/backend/clients/oceanbase/oceanbase.py b/vectordb_bench/backend/clients/oceanbase/oceanbase.py index bf615e4d0..a34161037 100644 --- a/vectordb_bench/backend/clients/oceanbase/oceanbase.py +++ b/vectordb_bench/backend/clients/oceanbase/oceanbase.py @@ -23,6 +23,8 @@ class OceanBase(VectorDB): FilterOp.NumGE, FilterOp.StrEqual, ] + # mysql-connector cursor cannot be shared across threads + thread_safe: bool = False def __init__( self, @@ -109,27 +111,28 @@ def _create_table(self): if not self._cursor: raise ValueError("Cursor is not initialized") - log.info(f"Creating table {self.table_name}") - create_table_query = f""" - CREATE TABLE {self.table_name} ( - id INT PRIMARY KEY, - embedding VECTOR({self.dim}) - ); - """ + partitions = getattr(self.db_case_config, "partitions", 0) + log.info(f"Creating table {self.table_name} (partitions={partitions})") + + create_table_query = f"CREATE TABLE {self.table_name} (id INT PRIMARY KEY, embedding VECTOR({self.dim}))" + if partitions > 1: + create_table_query += f" PARTITION BY KEY(id) PARTITIONS {partitions}" + create_table_query += ";" self._cursor.execute(create_table_query) def optimize(self, data_size: int): index_params = self.db_case_config.index_param() index_args = ", ".join(f"{k}={v}" for k, v in index_params["params"].items()) index_query = ( - f"CREATE /*+ PARALLEL(18) */ VECTOR INDEX idx1 " + f"CREATE /*+ PARALLEL({self.db_case_config.create_index_parallel}) */ VECTOR INDEX idx1 " f"ON {self.table_name}(embedding) " f"WITH (distance={self.db_case_config.parse_metric()}, " f"type={index_params['index_type']}, lib={index_params['lib']}, {index_args}" ) - if self.db_case_config.index in {IndexType.HNSW, IndexType.HNSW_SQ, IndexType.HNSW_BQ}: - index_query += ", extra_info_max_size=32" + extra_info = getattr(self.db_case_config, "extra_info_max_size", None) + if extra_info is not None: + index_query += f", extra_info_max_size={extra_info}" index_query += ")" @@ -153,10 +156,6 @@ def optimize(self, data_size: int): raise def need_normalize_cosine(self) -> bool: - if self.db_case_config.index == IndexType.HNSW_BQ: - log.info("current HNSW_BQ only supports L2, cosine dataset need normalize.") - return True - return False def _wait_for_major_compaction(self): @@ -185,9 +184,7 @@ def insert_embeddings( batch_end = min(batch_start + self.load_batch_size, len(embeddings)) batch = [(metadata[i], embeddings[i]) for i in range(batch_start, batch_end)] values = ", ".join(f"({item_id}, '[{','.join(map(str, embedding))}]')" for item_id, embedding in batch) - self._cursor.execute( - f"INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(32) */ INTO {self.table_name} VALUES {values}" - ) + self._cursor.execute(f"INSERT INTO {self.table_name} VALUES {values}") insert_count += len(batch) except mysql.Error: log.exception("Failed to insert embeddings") From c6f96f7b9829b462d392db71cccbc55202ce3c27 Mon Sep 17 00:00:00 2001 From: Yuanzhan Gao Date: Fri, 29 May 2026 16:43:57 +0800 Subject: [PATCH 32/49] feat: Add VectorDBBench Cloud Leaderboard benchmark cases and client support (#775) --- .gitignore | 1 + README.md | 11 + docs/release/2026-05-cloud-leaderboard.md | 170 +++ tests/test_case_runner_reuse.py | 171 +++ tests/test_cloud_cold_latency_case.py | 470 +++++++ tests/test_cloud_insert_case.py | 1108 +++++++++++++++++ tests/test_cloud_payload_case.py | 180 +++ tests/test_cloud_payload_search.py | 128 ++ tests/test_milvus.py | 162 +++ tests/test_milvus_zilliz_cli.py | 57 + tests/test_multitenant_case.py | 400 ++++++ tests/test_pinecone_multitenant.py | 214 ++++ tests/test_turbopuffer_cli.py | 293 +++++ vectordb_bench/__init__.py | 2 + vectordb_bench/backend/assembler.py | 7 +- vectordb_bench/backend/cases.py | 276 +++- vectordb_bench/backend/clients/api.py | 42 + vectordb_bench/backend/clients/milvus/cli.py | 257 ++-- .../backend/clients/milvus/milvus.py | 139 ++- .../backend/clients/pinecone/config.py | 2 + .../backend/clients/pinecone/pinecone.py | 236 +++- .../backend/clients/turbopuffer/cli.py | 221 +++- .../backend/clients/turbopuffer/config.py | 23 + .../clients/turbopuffer/turbopuffer.py | 285 ++++- .../backend/clients/zilliz_cloud/cli.py | 32 +- .../backend/clients/zilliz_cloud/config.py | 10 +- vectordb_bench/backend/dataset.py | 28 +- vectordb_bench/backend/payload.py | 21 + vectordb_bench/backend/runner/__init__.py | 2 + .../backend/runner/cold_warm_runner.py | 120 ++ .../backend/runner/concurrent_runner.py | 85 +- vectordb_bench/backend/runner/mp_runner.py | 27 +- .../backend/runner/serial_runner.py | 36 +- vectordb_bench/backend/task_runner.py | 263 +++- vectordb_bench/cli/cli.py | 178 ++- vectordb_bench/cli/vectordbbench.py | 3 +- vectordb_bench/interface.py | 17 +- vectordb_bench/metric.py | 9 + vectordb_bench/models.py | 112 +- 39 files changed, 5535 insertions(+), 263 deletions(-) create mode 100644 docs/release/2026-05-cloud-leaderboard.md create mode 100644 tests/test_case_runner_reuse.py create mode 100644 tests/test_cloud_cold_latency_case.py create mode 100644 tests/test_cloud_insert_case.py create mode 100644 tests/test_cloud_payload_case.py create mode 100644 tests/test_cloud_payload_search.py create mode 100644 tests/test_milvus_zilliz_cli.py create mode 100644 tests/test_multitenant_case.py create mode 100644 tests/test_pinecone_multitenant.py create mode 100644 tests/test_turbopuffer_cli.py create mode 100644 vectordb_bench/backend/payload.py create mode 100644 vectordb_bench/backend/runner/cold_warm_runner.py diff --git a/.gitignore b/.gitignore index cea1306b0..b33099105 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ venv/ .venv/ .idea/ logs/ +vectordb_bench/results/cloudleaderboard/ # Worktrees .worktrees/ diff --git a/README.md b/README.md index 685e37a47..3cdceddc0 100644 --- a/README.md +++ b/README.md @@ -708,6 +708,17 @@ vectordbbench batchcli --batch-config-file ### Introduction To facilitate the presentation of test results and provide a comprehensive performance analysis report, we offer a [leaderboard page](https://zilliz.com/benchmark). It allows us to choose from QPS, QP$, and latency metrics, and provides a comprehensive assessment of a system's performance based on the test results of various cases and a set of scoring mechanisms (to be introduced later). On this leaderboard, we can select the systems and models to be compared, and filter out cases we do not want to consider. Comprehensive scores are always ranked from best to worst, and the specific test results of each query will be presented in the list below. +### Cloud Leaderboard + +VectorDBBench now includes Cloud Leaderboard cases for production-oriented cloud vector database evaluation. These cases complement the original raw-performance leaderboard by measuring behaviors that matter for managed services: + +- `CloudInsertCase`: insert throughput plus searchable and indexed readiness delays. +- `CloudPayloadSearchCase`: search performance when responses return IDs only, scalar metadata, or vectors. +- `CloudMultiTenantSearchCase`: tenant-routed search for SaaS-shaped workloads. +- `CloudColdLatencyCase`: cold and warm serial latency for first-query and cache-sensitive serving paths. + +The May 2026 release note explains why the Cloud Leaderboard was added, what changed, which systems were tested this round, and how to run each new case: [docs/release/2026-05-cloud-leaderboard.md](docs/release/2026-05-cloud-leaderboard.md). + ### Scoring Rules 1. For each case, select a base value and score each system based on relative values. diff --git a/docs/release/2026-05-cloud-leaderboard.md b/docs/release/2026-05-cloud-leaderboard.md new file mode 100644 index 000000000..c9dc76a7b --- /dev/null +++ b/docs/release/2026-05-cloud-leaderboard.md @@ -0,0 +1,170 @@ +# VectorDBBench Cloud Leaderboard Release Note + +May 2026 + +The VectorDBBench Cloud Leaderboard moves beyond a single raw-throughput ranking. It evaluates managed vector databases around the behaviors production teams have to plan for: ingest readiness, payload-aware search, tenant-shaped workloads, cold latency, and cost at practical QPS targets. + +## Why we need a new leaderboard now + +The vector database market has moved past the "highest QPS wins" phase. Production teams choosing a managed vector database also care about budget, data freshness, tail latency, recall, metadata payloads, tenant isolation, and operational predictability. + +The existing VectorDBBench leaderboard remains useful for comparing baseline search performance across systems. But cloud buyers ask a wider set of questions: + +- When does newly inserted data become searchable? +- When is it fully indexed? +- What happens when search returns metadata or vectors instead of only IDs? +- What happens when traffic is split across many tenants? +- What does each reachable QPS tier cost? + +The Cloud Leaderboard is designed around those questions. It keeps performance visible, but puts it next to the readiness, payload, tenant, cold-start, and cost signals that determine what a customer can safely deploy. + +## What the Cloud Leaderboard changes + +The Cloud Leaderboard is a production cloud decision layer, not a replacement for the original raw-performance board. The main change is that benchmark cases now model cloud operating concerns directly instead of treating all products as simple warm search engines. + +The new cases add: + +- Insert readiness measurement, including client insert completion, searchable delay, and indexed delay. +- Explicit response payload profiles: IDs only, scalar label metadata, or vector values. +- Cloud cold-latency measurement for the first search path after idle or cache-cold conditions. +- Multi-tenant search, where data is split into deterministic tenant labels or namespaces and queries are routed by tenant. +- Cost-oriented interpretation, so raw QPS can be read together with monthly cost and readiness constraints. + +This matters because a top-line QPS table can hide important tradeoffs. A system can look strong on peak throughput while losing ground on recall, p99 latency, cold-start behavior, payload cost, or sustained cost at the same target QPS. + +## Who we tested this round + +This round focuses on three popular cloud vector databases: + +- Zilliz Cloud, including tiered and fixed-capacity configurations. +- turbopuffer, including normal, pinned, and backpressure-related configurations where applicable. +- Pinecone serverless. + +The tested matrix is intentionally cloud-oriented. It compares managed products and managed-service modes rather than only local or self-hosted engine behavior. + +## The new tests we added + +Version 2 adds four cloud-oriented cases in VectorDBBench. Each case is designed to expose a production behavior that a plain QPS benchmark can miss. + +### CloudInsertCase + +**Purpose.** CloudInsertCase measures write readiness, not just client-side insert speed. This is important for backfills, migrations, daily refreshes, and release workflows where a team needs to know when newly written vectors can safely take traffic. + +**How it works.** The case loads the dataset with `ConcurrentInsertRunner`, records insert completion time and rows per second, then polls the database until inserted data is fully searchable and fully indexed. The resulting metric separates: + +- `insert_completion_seconds` +- `insert_rows_per_second` +- `searchable_after_insert_seconds` +- `indexed_after_searchable_seconds` + +Example: run LAION 100M insert readiness on Zilliz Cloud with a 10k batch size. + +```bash +vectordbbench zillizautoindex \ + --case-type CloudInsertCase \ + --uri "$ZILLIZ_URI" \ + --token "$ZILLIZ_TOKEN" \ + --collection-name cloud_insert_laion100m_bs10k \ + --cloud-insert-batch-size 10000 \ + --load-concurrency 16 \ + --skip-search-serial \ + --skip-search-concurrent \ + --task-label cloud-insert-zilliz-12cu +``` + +### CloudPayloadSearchCase + +**Purpose.** CloudPayloadSearchCase measures search when the response body resembles production traffic. Many applications return more than vector IDs: they return scalar metadata, labels, or the vector values themselves. That response payload can change throughput, latency, and even product ranking. + +**How it works.** The case extends the normal performance case with an explicit `payload_profile`. Supported profiles are: + +- `ids_only` +- `scalar_label` +- `vector` + +The case can also run unfiltered search, integer-filter search through `--cloud-filter-rate`, or scalar-label filter search through `--cloud-label-percentage`. It records QPS, latency, recall where applicable, and estimated response payload bytes per query. + +When `payload_profile` is `scalar_label`, VectorDBBench materializes scalar label data even for unfiltered runs. This keeps the loaded schema aligned with the requested response payload instead of only loading labels for scalar-label filter runs. + +Example: run vector-payload search on Pinecone with a highly selective integer filter. + +```bash +vectordbbench pinecone \ + --case-type CloudPayloadSearchCase \ + --api-key "$PINECONE_API_KEY" \ + --index-name "$PINECONE_INDEX" \ + --payload-profile vector \ + --cloud-filter-rate 0.001 \ + --k 100 \ + --num-concurrency 60,80 \ + --concurrency-duration 30 \ + --task-label cloud-payload-pinecone-vector-int-filter-0-1p +``` + +### CloudMultiTenantSearchCase + +**Purpose.** CloudMultiTenantSearchCase models SaaS-shaped traffic. Instead of treating the dataset as one flat global collection, it splits records across many tenants and routes each query to a tenant. This highlights products whose namespace, partition-key, or tenant-filter paths behave differently from single-tenant search. + +**How it works.** The case defaults to the Cohere 10M dataset and assigns each row to a deterministic tenant by `row_id % tenant_count`. During search, queries are routed to the corresponding tenant label or namespace. The case supports the same payload profiles and optional filter modes as payload search. + +Tenant routing labels and scalar payload labels are separate concepts. A multi-tenant run can route by tenant while still storing and returning scalar-label payload data when `payload_profile` is `scalar_label`, and scalar-label filters continue to use the scalar label field rather than the tenant routing field. + +TurboPuffer tenant namespace cache warmup is explicit. By default, `CloudMultiTenantSearchCase` does not warm tenant namespaces during `optimize()`; use `--multitenant-warmup-policy all` only when the benchmark should model proactively warmed tenant namespaces. + +Example: run 1,000-tenant IDs-only search on turbopuffer. + +```bash +vectordbbench turbopuffer \ + --case-type CloudMultiTenantSearchCase \ + --dataset-with-size-type "Large Cohere (768dim, 10M)" \ + --api-key "$TURBOPUFFER_API_KEY" \ + --region aws-us-east-1 \ + --namespace vdbbench_mt_seed \ + --multitenant-namespace-prefix vdbbench_mt_ \ + --tenant-count 1000 \ + --tenant-prefix tenant_ \ + --tenant-id-width 4 \ + --payload-profile ids_only \ + --num-concurrency 40,60,80 \ + --concurrency-duration 30 \ + --task-label cloud-multitenant-turbopuffer-1000t +``` + +### CloudColdLatencyCase + +**Purpose.** CloudColdLatencyCase measures first-query and cold-path latency that warm benchmark loops can hide. This matters for serverless products, storage-tiered products, idle workloads, and customer-facing applications where the first query after an idle period is visible to users. + +**How it works.** The case is intentionally search-only and must run against an existing collection that has already become cold according to the product's cache and storage behavior. It rejects `drop_old` and `load` stages because insert-then-immediately-search runs can leave caches, indexing paths, or vendor warmup APIs in an ambiguous state. `ColdWarmSearchRunner` runs serial searches in cold and warm passes. It records cold-latency details in `additional_parameters["cold_latency"]` and also records payload profile and estimated payload bytes per query. + +Example: run a pinned turbopuffer cold-latency test with scalar-label payloads. + +```bash +vectordbbench turbopuffer \ + --case-type CloudColdLatencyCase \ + --skip-drop-old \ + --skip-load \ + --api-key "$TURBOPUFFER_API_KEY" \ + --region aws-us-east-1 \ + --namespace cloud_cold_latency_scalar_label \ + --pin-namespace \ + --pin-replicas 2 \ + --payload-profile scalar_label \ + --cloud-cold-query-count 1000 \ + --skip-search-concurrent \ + --task-label cloud-cold-latency-turbopuffer-pinned-scalar-label +``` + +## Caveats + +This release note introduces the new Cloud Leaderboard direction; it is not the full benchmark report. Detailed tables, raw JSON artifacts, pricing worksheets, and edge-case analysis should live in the benchmark report or external result artifact repository. + +Important caveats: + +- Pricing changes over time. Cost charts need a pricing date, region, and configuration assumptions. +- Managed-service configuration can materially change results, especially for serverless scaling, pinned replicas, capacity units, and storage-tiering modes. +- "Fully indexed" and "fully searchable" readiness may be exposed differently by each vendor, so the implementation must document how each status is detected or inferred. +- The current multi-tenant case uses deterministic tenant assignment and uniform tenant routing. It does not represent every SaaS tenant distribution. +- Multi-tenant routing labels or namespaces are not equivalent to scalar payload labels. Benchmark clients must keep those fields separate when a run combines tenant routing with scalar-label payload or filter behavior. +- Cold latency depends on cache state, idle window, replica pinning, storage architecture, and service warmup behavior. The idle and warmup rules must stay strict between products. +- Payload search rankings are workload-specific. IDs-only, scalar-label, vector-return, integer-filter, and label-filter runs can produce different winners. +- Cost Pareto results must be read together with recall, latency, payload profile, and readiness constraints rather than as a standalone ranking. diff --git a/tests/test_case_runner_reuse.py b/tests/test_case_runner_reuse.py new file mode 100644 index 000000000..55dfbda0b --- /dev/null +++ b/tests/test_case_runner_reuse.py @@ -0,0 +1,171 @@ +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig, MetricType +from vectordb_bench.backend.clients.doris.config import DorisCaseConfig, DorisConfig +from vectordb_bench.backend.clients.pinecone.config import PineconeConfig +from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferConfig, TurboPufferIndexConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.task_runner import CaseRunner, RunningStatus, TaskRunner +from vectordb_bench.interface import BenchMarkRunner +from vectordb_bench.metric import Metric +from vectordb_bench.models import CaseConfig, CaseType, TaskConfig, TaskStage, TestResult + + +def make_runner( + *, + case_id: CaseType = CaseType.Performance1536D50K, + custom_case: dict | None = None, + db: DB = DB.TurboPuffer, + db_config=None, + db_case_config=None, + stages: list[TaskStage] | None = None, +) -> CaseRunner: + if db_config is None: + if db == DB.TurboPuffer: + db_config = TurboPufferConfig(api_key="key", region="aws-us-east-1") + elif db == DB.Pinecone: + db_config = PineconeConfig(api_key="key", index_name="idx") + elif db == DB.Doris: + db_config = DorisConfig(password=SecretStr("")) + else: + db_config = DB.Test.config_cls() + if db_case_config is None: + if db == DB.TurboPuffer: + db_case_config = TurboPufferIndexConfig(metric_type=MetricType.COSINE) + elif db == DB.Doris: + db_case_config = DorisCaseConfig(metric_type=MetricType.COSINE) + else: + db_case_config = EmptyDBCaseConfig() + + task = TaskConfig( + db=db, + db_config=db_config, + db_case_config=db_case_config, + case_config=CaseConfig(case_id=case_id, custom_case=custom_case or {}), + stages=stages or [TaskStage.DROP_OLD, TaskStage.LOAD, TaskStage.SEARCH_SERIAL], + ) + return CaseRunner( + run_id="run-id", + config=task, + ca=task.case_config.case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + +def assert_not_reusable(left: CaseRunner, right: CaseRunner) -> None: + assert left != right + assert right != left + assert hash(left) != hash(right) + + +def test_reuse_key_distinguishes_multitenant_routing_parameters(): + base_case = { + "dataset_with_size_type": DatasetWithSizeType.CohereSmall.value, + "tenant_count": 2, + "tenant_prefix": "tenant_", + "tenant_id_width": 4, + } + + assert_not_reusable( + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case=base_case), + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case={**base_case, "tenant_count": 3}), + ) + assert_not_reusable( + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case=base_case), + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case={**base_case, "tenant_prefix": "org_"}), + ) + assert_not_reusable( + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case=base_case), + make_runner(case_id=CaseType.CloudMultiTenantSearchCase, custom_case={**base_case, "tenant_id_width": 2}), + ) + + +def test_reuse_key_distinguishes_single_layout_from_multitenant_layout(): + dataset_case = {"dataset_with_size_type": DatasetWithSizeType.CohereSmall.value} + + assert_not_reusable( + make_runner(case_id=CaseType.CloudPayloadSearchCase, custom_case=dataset_case), + make_runner( + case_id=CaseType.CloudMultiTenantSearchCase, + custom_case={**dataset_case, "tenant_count": 2}, + ), + ) + + +def test_reuse_key_preserves_safe_payload_reuse(): + dataset_case = {"dataset_with_size_type": DatasetWithSizeType.CohereSmall.value} + + ids_only = make_runner( + case_id=CaseType.CloudPayloadSearchCase, + custom_case={**dataset_case, "payload_profile": "ids_only"}, + ) + vector = make_runner( + case_id=CaseType.CloudPayloadSearchCase, + custom_case={**dataset_case, "payload_profile": "vector"}, + ) + + assert ids_only == vector + assert hash(ids_only) == hash(vector) + + +def test_reuse_key_distinguishes_physical_db_targets(): + assert_not_reusable( + make_runner(db_config=TurboPufferConfig(api_key="key", region="aws-us-east-1", namespace="namespace_a")), + make_runner(db_config=TurboPufferConfig(api_key="key", region="aws-us-east-1", namespace="namespace_b")), + ) + assert_not_reusable( + make_runner(db=DB.Pinecone, db_config=PineconeConfig(api_key="key", index_name="index_a")), + make_runner(db=DB.Pinecone, db_config=PineconeConfig(api_key="key", index_name="index_b")), + ) + + +def test_reuse_key_distinguishes_doris_case_derived_table_names(): + assert_not_reusable( + make_runner(db=DB.Doris, case_id=CaseType.Performance768D1M), + make_runner( + db=DB.Doris, + case_id=CaseType.NewIntFilterPerformanceCase, + custom_case={ + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + "filter_rate": 0.01, + }, + ), + ) + + +def test_search_only_runner_does_not_suppress_later_full_load(monkeypatch): + calls: list[bool] = [] + search_only = make_runner( + stages=[TaskStage.SEARCH_SERIAL], + ) + full_load = make_runner( + stages=[TaskStage.DROP_OLD, TaskStage.LOAD, TaskStage.SEARCH_SERIAL], + ) + + def fake_run(self: CaseRunner, drop_old: bool = True) -> Metric: + calls.append(drop_old) + return Metric() + + class SendConn: + def __init__(self): + self.sent = [] + + def send(self, value): + self.sent.append(value) + + def close(self): + return None + + monkeypatch.setattr(CaseRunner, "run", fake_run) + monkeypatch.setattr(TestResult, "display", lambda self: None) + monkeypatch.setattr(TestResult, "flush", lambda self: None) + + BenchMarkRunner()._async_task_v2( + TaskRunner(run_id="run-id", task_label="task", case_runners=[search_only, full_load]), + SendConn(), + ) + + assert calls == [False, True] diff --git a/tests/test_cloud_cold_latency_case.py b/tests/test_cloud_cold_latency_case.py new file mode 100644 index 000000000..55f610b38 --- /dev/null +++ b/tests/test_cloud_cold_latency_case.py @@ -0,0 +1,470 @@ +import json +from contextlib import contextmanager +from pathlib import Path + +import numpy as np +import pytest + +from vectordb_bench.backend.assembler import Assembler +from vectordb_bench.backend.cases import CaseLabel, CaseType, CloudColdLatencyCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig +from vectordb_bench.backend.clients.pinecone.config import PineconeConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.result_collector import ResultCollector +from vectordb_bench.backend.runner.cold_warm_runner import ColdWarmSearchRunner +from vectordb_bench.backend.task_runner import CaseRunner, RunningStatus +from vectordb_bench.cli.cli import get_custom_case_config +from vectordb_bench.metric import Metric +from vectordb_bench.models import CaseConfig, CaseResult, TaskConfig, TaskStage, TestResult + + +def test_cloud_cold_latency_case_defaults_to_laion_100m(): + case = CloudColdLatencyCase() + + assert case.case_id == CaseType.CloudColdLatencyCase + assert case.label == CaseLabel.CloudColdLatency + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + assert case.dataset.data.dim == 768 + assert case.payload_profile == PayloadProfile.IDS_ONLY + assert case.query_count == 1000 + assert case.filters.type == FilterOp.NonFilter + + +def test_cloud_cold_latency_case_accepts_payload_dataset_and_int_filter(): + case = CloudColdLatencyCase( + dataset_with_size_type=DatasetWithSizeType.CohereSmall.value, + payload_profile="vector", + filter_rate=0.9, + query_count=10, + ) + + assert case.dataset_with_size_type == DatasetWithSizeType.CohereSmall + assert case.dataset.data.name == "Cohere" + assert case.payload_profile == PayloadProfile.VECTOR + assert case.filter_rate == 0.9 + assert case.query_count == 10 + assert case.filters.type == FilterOp.NumGE + + +def test_cloud_cold_latency_case_accepts_label_filter(): + case = CloudColdLatencyCase(label_percentage=0.9) + + assert case.label_percentage == 0.9 + assert case.filters.type == FilterOp.StrEqual + + +def test_cloud_cold_latency_case_rejects_two_filter_types(): + with pytest.raises(ValueError, match="supports only one filter type"): + CloudColdLatencyCase(filter_rate=0.9, label_percentage=0.9) + + +def test_cloud_cold_latency_case_rejects_invalid_query_count(): + with pytest.raises(ValueError, match="query_count must be positive"): + CloudColdLatencyCase(query_count=0) + + +def test_case_config_builds_cloud_cold_latency_case_from_custom_case(): + case = CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={ + "payload_profile": "scalar_label", + "label_percentage": 0.9, + "query_count": 12, + }, + ).case + + assert isinstance(case, CloudColdLatencyCase) + assert case.payload_profile == PayloadProfile.SCALAR_LABEL + assert case.label_percentage == 0.9 + assert case.query_count == 12 + + +def test_cli_builds_cloud_cold_latency_custom_case_config(): + params = { + "case_type": "CloudColdLatencyCase", + "payload_profile": "vector", + "cloud_filter_rate": 0.9, + "cloud_label_percentage": None, + "cloud_cold_query_count": 1000, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + assert get_custom_case_config(params) == { + "payload_profile": "vector", + "filter_rate": 0.9, + "query_count": 1000, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + +def test_cli_keeps_cloud_cold_latency_default_dataset_as_laion(): + params = { + "case_type": "CloudColdLatencyCase", + "payload_profile": "ids_only", + "cloud_filter_rate": None, + "cloud_label_percentage": None, + "cloud_cold_query_count": 1000, + "dataset_with_size_type": None, + } + + custom_case = get_custom_case_config(params) + case = CaseConfig(case_id=CaseType.CloudColdLatencyCase, custom_case=custom_case).case + + assert custom_case == { + "payload_profile": "ids_only", + "query_count": 1000, + } + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + + +def test_cloud_cold_latency_result_file_uses_cold_latency_metrics(tmp_path: Path): + cold_latency = { + "cold_stats": { + "first_query_latency": 0.2, + "p99_latency": 0.3, + "p95_latency": 0.25, + "avg_latency": 0.21, + }, + "warm_stats": { + "first_query_latency": 0.1, + "p99_latency": 0.15, + "p95_latency": 0.12, + "avg_latency": 0.11, + }, + "ratios": { + "first_query_latency": 2.0, + "p99_latency": 2.0, + "p95_latency": 2.0833, + "avg_latency": 1.9091, + }, + } + result = CaseResult( + task_config=TaskConfig( + db=DB.Pinecone, + db_config=PineconeConfig( + db_label="pinecone_cloud_cold_latency", + api_key="secret-key", + index_name="laion100m", + ), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"payload_profile": "vector", "query_count": 1000}, + ), + stages=[TaskStage.SEARCH_SERIAL], + load_concurrency=0, + ), + metrics=Metric( + insert_duration=0.0, + optimize_duration=0.0, + load_duration=0.0, + payload_profile="vector", + payload_estimated_bytes_per_query=309200, + additional_parameters={"cold_latency": cold_latency}, + ), + ) + test_result = TestResult(run_id="run-id", task_label="cloud_cold_latency_pinecone", results=[result]) + + test_result.write_db_file(tmp_path, test_result, "pinecone") + + result_file = next(tmp_path.glob("result_*_pinecone.json")) + raw_output = result_file.read_text() + assert raw_output.startswith('{\n "run_id"') + written = json.loads(raw_output) + assert written["results"][0]["metrics"] == { + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "payload_profile": "vector", + "payload_estimated_bytes_per_query": 309200, + "cold_latency": cold_latency, + } + assert written["results"][0]["task_config"]["db_config"]["api_key"] == "**********" + assert written["results"][0]["task_config"]["db_config"]["index_name"] == "laion100m" + assert written["results"][0]["task_config"]["case_config"] == { + "case_id": 700, + "custom_case": {"payload_profile": "vector", "query_count": 1000}, + } + + read_back = TestResult.read_file(result_file) + assert read_back.results[0].task_config.case_config.case_id == CaseType.CloudColdLatencyCase + assert read_back.results[0].task_config.case_config.custom_case == { + "payload_profile": "vector", + "query_count": 1000, + } + assert read_back.results[0].metrics.additional_parameters["cold_latency"] == cold_latency + + collected = ResultCollector.collect(tmp_path) + assert len(collected) == 1 + assert collected[0].results[0].metrics.additional_parameters["cold_latency"] == cold_latency + + +class FakeColdWarmDB: + name = "FakeColdWarmDB" + + def __init__(self, supported_payload_profiles: set[PayloadProfile] | None = None): + self.supported_payload_profiles = supported_payload_profiles or {PayloadProfile.IDS_ONLY} + self.calls = [] + self.prepare_filter_calls = [] + self.init_enter_count = 0 + + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in self.supported_payload_profiles + + def need_normalize_cosine(self) -> bool: + return False + + @contextmanager + def init(self): + self.init_enter_count += 1 + yield + + def prepare_filter(self, filters: Filter): + self.prepare_filter_calls.append(filters) + + def search_embedding(self, query: list[float], k: int = 100, **kwargs) -> list[int]: + self.calls.append((query, k, kwargs)) + return list(range(k)) + + +def test_cold_warm_runner_computes_stats_and_ratios(monkeypatch: pytest.MonkeyPatch): + db = FakeColdWarmDB() + # Cold latencies: 0.2, 0.2, 0.2. Warm latencies: 0.1, 0.1, 0.1. + perf_values = iter([0.0, 0.2, 0.2, 0.4, 0.4, 0.6, 0.6, 0.7, 0.7, 0.8, 0.8, 0.9]) + monkeypatch.setattr("vectordb_bench.backend.runner.cold_warm_runner.time.perf_counter", lambda: next(perf_values)) + + runner = ColdWarmSearchRunner( + db=db, + test_data=[[0.1], [0.2], [0.3]], + k=3, + query_count=3, + ) + + result = runner.run() + + assert result == { + "cold_stats": { + "first_query_latency": 0.2, + "p99_latency": 0.2, + "p95_latency": 0.2, + "avg_latency": 0.2, + }, + "warm_stats": { + "first_query_latency": 0.1, + "p99_latency": 0.1, + "p95_latency": 0.1, + "avg_latency": 0.1, + }, + "cold_warm_ratio": { + "first_query_latency_ratio": 2.0, + "p99_latency_ratio": 2.0, + "p95_latency_ratio": 2.0, + "avg_latency_ratio": 2.0, + }, + } + assert db.init_enter_count == 1 + assert len(db.prepare_filter_calls) == 1 + assert [call[0] for call in db.calls] == [[0.1], [0.2], [0.3], [0.1], [0.2], [0.3]] + + +def test_cold_warm_runner_passes_payload_profile_in_both_passes(monkeypatch: pytest.MonkeyPatch): + db = FakeColdWarmDB(supported_payload_profiles={PayloadProfile.IDS_ONLY, PayloadProfile.VECTOR}) + perf_values = iter([0.0, 0.1, 0.1, 0.2]) + monkeypatch.setattr("vectordb_bench.backend.runner.cold_warm_runner.time.perf_counter", lambda: next(perf_values)) + + runner = ColdWarmSearchRunner( + db=db, + test_data=[np.array([0.1])], + k=3, + payload_profile=PayloadProfile.VECTOR, + query_count=1, + ) + + runner.run() + + assert db.calls == [ + ([0.1], 3, {"payload_profile": PayloadProfile.VECTOR}), + ([0.1], 3, {"payload_profile": PayloadProfile.VECTOR}), + ] + + +def test_cold_warm_runner_omits_payload_profile_for_ids_only(monkeypatch: pytest.MonkeyPatch): + db = FakeColdWarmDB() + perf_values = iter([0.0, 0.1, 0.1, 0.2]) + monkeypatch.setattr("vectordb_bench.backend.runner.cold_warm_runner.time.perf_counter", lambda: next(perf_values)) + + runner = ColdWarmSearchRunner( + db=db, + test_data=[[0.1]], + k=3, + payload_profile=PayloadProfile.IDS_ONLY, + query_count=1, + ) + + runner.run() + + assert db.calls == [ + ([0.1], 3, {}), + ([0.1], 3, {}), + ] + + +def test_cold_warm_runner_fails_for_unsupported_payload_profile(): + db = FakeColdWarmDB() + + with pytest.raises(NotImplementedError, match="payload_profile=vector"): + ColdWarmSearchRunner( + db=db, + test_data=[[0.1]], + payload_profile=PayloadProfile.VECTOR, + query_count=1, + ) + + +def test_cold_warm_runner_rejects_too_few_queries(): + db = FakeColdWarmDB() + + with pytest.raises(ValueError, match="query_count=2 exceeds test_data size=1"): + ColdWarmSearchRunner(db=db, test_data=[[0.1]], query_count=2) + + +def test_assembler_schedules_cloud_cold_latency_case(): + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"query_count": 1}, + ), + stages=[TaskStage.SEARCH_SERIAL], + ) + + runner = Assembler.assemble_all("run-id", "task-label", [task], DatasetSource.S3) + + assert len(runner.case_runners) == 1 + assert runner.case_runners[0].ca.label == CaseLabel.CloudColdLatency + + +def test_case_runner_stores_cloud_cold_latency_metric(monkeypatch: pytest.MonkeyPatch): + case = CloudColdLatencyCase(query_count=1) + case.dataset.test_data = [[0.1]] + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"query_count": 1}, + ), + stages=[TaskStage.SEARCH_SERIAL], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + runner.db = FakeColdWarmDB() + + expected = { + "cold_stats": { + "first_query_latency": 0.2, + "p99_latency": 0.2, + "p95_latency": 0.2, + "avg_latency": 0.2, + }, + "warm_stats": { + "first_query_latency": 0.1, + "p99_latency": 0.1, + "p95_latency": 0.1, + "avg_latency": 0.1, + }, + "cold_warm_ratio": { + "first_query_latency_ratio": 2.0, + "p99_latency_ratio": 2.0, + "p95_latency_ratio": 2.0, + "avg_latency_ratio": 2.0, + }, + } + captured_kwargs = {} + + class FakeRunner: + def __init__(self, **kwargs): + captured_kwargs.update(kwargs) + + def run(self): + return expected + + monkeypatch.setattr("vectordb_bench.backend.task_runner.ColdWarmSearchRunner", FakeRunner) + + metric = runner._run_cloud_cold_latency_case(drop_old=False) + + assert metric.additional_parameters["cold_latency"] == expected + assert metric.payload_profile == "ids_only" + assert metric.payload_estimated_bytes_per_query == case.estimated_payload_bytes_per_query(task.case_config.k) + assert captured_kwargs == { + "db": runner.db, + "test_data": [[0.1]], + "filters": case.filters, + "k": task.case_config.k, + "payload_profile": case.payload_profile, + "query_count": case.query_count, + } + + +def test_cloud_cold_latency_case_rejects_drop_old(): + case = CloudColdLatencyCase(query_count=1) + case.dataset.test_data = [[0.1]] + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"query_count": 1}, + ), + stages=[TaskStage.SEARCH_SERIAL], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + with pytest.raises(ValueError, match="requires an existing cold collection"): + runner._run_cloud_cold_latency_case(drop_old=True) + + +def test_cloud_cold_latency_case_rejects_load_stage(): + case = CloudColdLatencyCase(query_count=1) + case.dataset.test_data = [[0.1]] + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudColdLatencyCase, + custom_case={"query_count": 1}, + ), + stages=[TaskStage.LOAD, TaskStage.SEARCH_SERIAL], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + with pytest.raises(ValueError, match="search-only"): + runner._run_cloud_cold_latency_case(drop_old=False) diff --git a/tests/test_cloud_insert_case.py b/tests/test_cloud_insert_case.py new file mode 100644 index 000000000..28e70f7bd --- /dev/null +++ b/tests/test_cloud_insert_case.py @@ -0,0 +1,1108 @@ +import json +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import MagicMock + +import numpy as np +import pandas as pd +import pytest + +from vectordb_bench import config +from vectordb_bench.backend.assembler import Assembler +from vectordb_bench.backend.cases import CaseLabel, CaseType, CloudInsertCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig, VectorDB +from vectordb_bench.backend.clients.milvus.milvus import Milvus +from vectordb_bench.backend.clients.pinecone.config import PineconeConfig +from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone +from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig +from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer +from vectordb_bench.backend.clients.zilliz_cloud.config import AutoIndexConfig, ZillizCloudConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.result_collector import ResultCollector +from vectordb_bench.backend.runner.concurrent_runner import ConcurrentInsertRunner +from vectordb_bench.backend.runner.executor import TaskResult +from vectordb_bench.backend.task_runner import CaseRunner +from vectordb_bench.cli.cli import get_custom_case_config +from vectordb_bench.metric import Metric +from vectordb_bench.models import CaseConfig, CaseResult, TaskConfig, TaskStage, TestResult + + +class _SerialTaskExecutor: + def __init__(self): + self.submitted = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return False + + def submit(self, fn): + self.submitted.append(fn) + + def wait_all(self): + results = [] + for fn in self.submitted: + try: + results.append(TaskResult(value=fn())) + except Exception as e: + results.append(TaskResult(error=e)) + return results + + +class _ConcurrentRunnerData: + train_id_field = "id" + train_vector_field = "vector" + + +class _ConcurrentRunnerDataset: + data = _ConcurrentRunnerData() + + def __init__(self, ids): + self.ids = ids + + def iter_batches(self, batch_size): + return iter([pd.DataFrame({"id": [row_id], "vector": [np.array([row_id / 10 + 0.1])]}) for row_id in self.ids]) + + +def test_cloud_insert_case_defaults_to_laion_100m(): + case = CloudInsertCase(batch_size=1000) + + assert case.case_id == CaseType.CloudInsertCase + assert case.label == CaseLabel.CloudInsert + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + assert case.batch_size == 1000 + assert case.duration is None + assert case.readiness_timeout is None + + +def test_case_config_builds_cloud_insert_case_from_custom_case(): + case = CaseConfig( + case_id=CaseType.CloudInsertCase, + custom_case={ + "batch_size": 5000, + "duration": 1800, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + }, + ).case + + assert isinstance(case, CloudInsertCase) + assert case.batch_size == 5000 + assert case.duration == 1800 + assert case.dataset.data.name == "Cohere" + assert case.dataset.data.size == 1_000_000 + + +def test_case_config_builds_cloud_insert_case_from_laion_100m_dataset_option(): + case = CaseConfig( + case_id=CaseType.CloudInsertCase, + custom_case={ + "batch_size": 10_000, + "dataset_with_size_type": "Large LAION (768dim, 100M)", + }, + ).case + + assert isinstance(case, CloudInsertCase) + assert case.batch_size == 10_000 + assert case.dataset_with_size_type == DatasetWithSizeType.LAIONLarge + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + assert case.dataset.data.dim == 768 + + +def test_laion_100m_dataset_option_uses_100m_timeouts(): + assert DatasetWithSizeType.LAIONLarge.get_load_timeout() == config.LOAD_TIMEOUT_768D_100M + assert DatasetWithSizeType.LAIONLarge.get_optimize_timeout() == config.OPTIMIZE_TIMEOUT_768D_100M + + +def test_cli_builds_cloud_insert_custom_case_config(): + params = { + "case_type": "CloudInsertCase", + "cloud_insert_batch_size": 10_000, + "cloud_insert_duration": 1800, + "cloud_insert_readiness_timeout": 7200, + "cloud_insert_readiness_poll_interval": 10, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + assert get_custom_case_config(params) == { + "batch_size": 10_000, + "duration": 1800, + "readiness_timeout": 7200, + "readiness_poll_interval": 10, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + +def test_cli_builds_cloud_insert_custom_case_config_with_laion_100m_dataset(): + cfg = get_custom_case_config( + { + "case_type": "CloudInsertCase", + "cloud_insert_batch_size": 10_000, + "cloud_insert_duration": None, + "cloud_insert_readiness_timeout": None, + "cloud_insert_readiness_poll_interval": None, + "dataset_with_size_type": DatasetWithSizeType.LAIONLarge.value, + } + ) + + assert cfg == { + "batch_size": 10_000, + "duration": None, + "dataset_with_size_type": DatasetWithSizeType.LAIONLarge.value, + } + + case = CaseConfig(case_id=CaseType.CloudInsertCase, custom_case=cfg).case + assert case.dataset_with_size_type == DatasetWithSizeType.LAIONLarge + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + + +def test_cli_builds_cloud_insert_custom_case_config_with_default_dataset(): + cfg = get_custom_case_config( + { + "case_type": "CloudInsertCase", + "cloud_insert_batch_size": 10_000, + "cloud_insert_duration": None, + "cloud_insert_readiness_timeout": None, + "cloud_insert_readiness_poll_interval": None, + "dataset_with_size_type": None, + } + ) + + assert cfg == { + "batch_size": 10_000, + "duration": None, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + } + + case = CaseConfig(case_id=CaseType.CloudInsertCase, custom_case=cfg).case + assert case.dataset_with_size_type == DatasetWithSizeType.CohereMedium + assert case.dataset.data.size == 1_000_000 + + +def test_cli_builds_multitenant_custom_case_config(): + cfg = get_custom_case_config( + { + "case_type": "CloudMultiTenantSearchCase", + "dataset_with_size_type": "Small Cohere (768dim, 100K)", + "tenant_count": 13, + "tenant_prefix": "acct_", + "tenant_id_width": 3, + "payload_profile": "vector", + "cloud_filter_rate": 0.01, + "cloud_label_percentage": None, + } + ) + + assert cfg == { + "dataset_with_size_type": "Small Cohere (768dim, 100K)", + "tenant_count": 13, + "tenant_prefix": "acct_", + "tenant_id_width": 3, + "payload_profile": "vector", + "filter_rate": 0.01, + } + + +def test_cli_omits_multitenant_dataset_when_not_selected(): + cfg = get_custom_case_config( + { + "case_type": "CloudMultiTenantSearchCase", + "dataset_with_size_type": None, + "tenant_count": 13, + "tenant_prefix": "acct_", + "tenant_id_width": 3, + "payload_profile": "ids_only", + "cloud_filter_rate": None, + "cloud_label_percentage": None, + } + ) + + assert cfg == { + "tenant_count": 13, + "tenant_prefix": "acct_", + "tenant_id_width": 3, + "payload_profile": "ids_only", + } + + case = CaseConfig(case_id=CaseType.CloudMultiTenantSearchCase, custom_case=cfg).case + assert case.dataset_with_size_type == DatasetWithSizeType.CohereLarge + assert case.dataset.data.size == 10_000_000 + + +def test_assembler_schedules_cloud_insert_case(): + task = TaskConfig( + db=DB.ZillizCloud, + db_config=ZillizCloudConfig(uri="https://example.com", user="db_admin"), + db_case_config=AutoIndexConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudInsertCase, + custom_case={ + "batch_size": 1000, + "dataset_with_size_type": DatasetWithSizeType.CohereMedium.value, + }, + ), + stages=[TaskStage.DROP_OLD, TaskStage.LOAD], + ) + + runner = Assembler.assemble_all("run-id", "task-label", [task], DatasetSource.S3) + + assert len(runner.case_runners) == 1 + assert runner.case_runners[0].ca.label == CaseLabel.CloudInsert + + +def test_default_insert_readiness_is_immediately_ready(): + class FakeVectorDB(VectorDB): + def __init__(self, dim, db_config, db_case_config, collection_name="", drop_old=False): + pass + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None, **kwargs): + return len(embeddings), None + + def search_embedding(self, query, k=100, payload_profile=None): + return [] + + def optimize(self, data_size=None): + pass + + assert FakeVectorDB(1, {}, EmptyDBCaseConfig()).poll_insert_readiness(10) == { + "fully_searchable": True, + "fully_indexed": True, + "additional_parameters": {}, + } + + +def test_metric_contains_cloud_insert_output_fields(): + metric = Metric( + inserted_count=100, + insert_rows_per_second=83.33, + insert_completion_seconds=1.2, + searchable_after_insert_seconds=3.4, + indexed_after_searchable_seconds=5.6, + additional_parameters={"disable_backpressure": True}, + ) + + assert metric.inserted_count == 100 + assert metric.insert_rows_per_second == 83.33 + assert metric.insert_completion_seconds == 1.2 + assert metric.searchable_after_insert_seconds == 3.4 + assert metric.indexed_after_searchable_seconds == 5.6 + assert metric.additional_parameters == {"disable_backpressure": True} + + +def test_cloud_insert_result_file_uses_insert_only_metrics(tmp_path: Path): + result = CaseResult( + task_config=TaskConfig( + db=DB.Pinecone, + db_config=PineconeConfig( + db_label="pinecone_cloud_insert_laion100m_bs1k", + api_key="secret-key", + index_name="laion100m", + ), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig( + case_id=CaseType.CloudInsertCase, + custom_case={"batch_size": 1000, "duration": None}, + ), + stages=[TaskStage.DROP_OLD, TaskStage.LOAD], + load_concurrency=0, + ), + metrics=Metric( + inserted_count=100_000_000, + insert_rows_per_second=3919.9296, + insert_completion_seconds=255.1066, + searchable_after_insert_seconds=0.0, + indexed_after_searchable_seconds=28.5956, + additional_parameters={}, + ), + ) + test_result = TestResult(run_id="run-id", task_label="cloud_insert_pinecone_laion100m_bs1k", results=[result]) + + test_result.write_db_file(tmp_path, test_result, "pinecone") + + result_file = next(tmp_path.glob("result_*_pinecone.json")) + raw_output = result_file.read_text() + assert raw_output.startswith('{\n "run_id"') + written = json.loads(raw_output) + assert written["results"][0]["metrics"] == { + "inserted_count": 100_000_000, + "insert_rows_per_second": 3919.9296, + "insert_completion_seconds": 255.1066, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 28.5956, + "additional_parameters": {}, + } + assert written["results"][0]["task_config"]["db_config"]["api_key"] == "**********" + assert written["results"][0]["task_config"]["db_config"]["index_name"] == "laion100m" + assert written["results"][0]["task_config"]["case_config"] == { + "case_id": 600, + "custom_case": {"batch_size": 1000, "duration": None}, + } + + read_back = TestResult.read_file(result_file) + assert read_back.results[0].task_config.case_config.case_id == CaseType.CloudInsertCase + assert read_back.results[0].task_config.case_config.custom_case == {"batch_size": 1000, "duration": None} + + collected = ResultCollector.collect(tmp_path) + assert len(collected) == 1 + assert collected[0].results[0].metrics.inserted_count == 100_000_000 + + +def test_turbopuffer_insert_can_disable_backpressure(): + db = TurboPuffer.__new__(TurboPuffer) + db.with_scalar_labels = False + db._scalar_id_field = "id" + db._vector_field = "vector" + db.metric = "cosine_distance" + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=True) + + class Namespace: + kwargs = None + + def write(self, **kwargs): + self.kwargs = kwargs + + db.ns = Namespace() + + assert db.insert_embeddings([[0.1]], [1]) == (1, None) + assert db.ns.kwargs["disable_backpressure"] is True + + +def test_turbopuffer_insert_serializes_numpy_vectors(): + db = TurboPuffer.__new__(TurboPuffer) + db.with_scalar_labels = False + db._scalar_id_field = "id" + db._vector_field = "vector" + db.metric = "cosine_distance" + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=False) + + class Namespace: + kwargs = None + + def write(self, **kwargs): + self.kwargs = kwargs + + db.ns = Namespace() + + insert_count, error = db.insert_embeddings([np.array([0.1, 0.2])], [1]) + + assert error is None + assert insert_count == 1 + assert db.ns.kwargs["upsert_columns"]["vector"] == [[0.1, 0.2]] + + +def test_turbopuffer_insert_returns_write_error(): + db = TurboPuffer.__new__(TurboPuffer) + db.with_scalar_labels = False + db._scalar_id_field = "id" + db._vector_field = "vector" + db.metric = "cosine_distance" + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=False) + + class Namespace: + def write(self, **kwargs): + raise RuntimeError("write failed") + + db.ns = Namespace() + + insert_count, error = db.insert_embeddings([[0.1]], [1]) + + assert insert_count == 0 + assert isinstance(error, RuntimeError) + + +def test_milvus_insert_readiness_uses_entity_count_and_index_progress(): + db = Milvus.__new__(Milvus) + db.collection_name = "c" + db._vector_index_name = "vector_idx" + db.client = type( + "Client", + (), + { + "flush": lambda self, collection_name: None, + "get_collection_stats": lambda self, collection_name: {"row_count": "10"}, + "describe_index": lambda self, collection_name, index_name: {"pending_index_rows": 0}, + }, + )() + + assert db.poll_insert_readiness(10) == { + "fully_searchable": True, + "fully_indexed": True, + "additional_parameters": {}, + } + + +def test_pinecone_insert_readiness_uses_vector_count(): + db = Pinecone.__new__(Pinecone) + db.index = type("Index", (), {"describe_index_stats": lambda self: {"total_vector_count": 9}})() + + assert db.poll_insert_readiness(10)["fully_searchable"] is False + assert db.poll_insert_readiness(9)["fully_indexed"] is True + + +def test_pinecone_supports_cloud_payload_profiles(): + db = Pinecone.__new__(Pinecone) + + assert db.supports_payload_profile(PayloadProfile.IDS_ONLY) + assert db.supports_payload_profile(PayloadProfile.SCALAR_LABEL) + assert db.supports_payload_profile(PayloadProfile.VECTOR) + + +def test_pinecone_search_requests_metadata_for_scalar_label_payload(): + db = Pinecone.__new__(Pinecone) + db.expr = None + + class Index: + kwargs = None + + def query(self, **kwargs): + self.kwargs = kwargs + return {"matches": [{"id": "1"}]} + + db.index = Index() + + assert db.search_embedding([0.1], payload_profile=PayloadProfile.SCALAR_LABEL) == [1] + assert db.index.kwargs["include_metadata"] is True + assert db.index.kwargs["include_values"] is False + + +def test_pinecone_search_requests_values_for_vector_payload(): + db = Pinecone.__new__(Pinecone) + db.expr = {"meta": {"$gte": 10}} + + class Index: + kwargs = None + + def query(self, **kwargs): + self.kwargs = kwargs + return {"matches": [{"id": "2"}]} + + db.index = Index() + + assert db.search_embedding([0.1], payload_profile=PayloadProfile.VECTOR) == [2] + assert db.index.kwargs["include_metadata"] is False + assert db.index.kwargs["include_values"] is True + assert db.index.kwargs["filter"] == {"meta": {"$gte": 10}} + + +def test_pinecone_search_retries_rate_limited_queries(monkeypatch): + db = Pinecone.__new__(Pinecone) + db.expr = None + monkeypatch.setenv("PINECONE_QUERY_RETRY_SLEEP_SECONDS", "0.25") + sleeps = [] + monkeypatch.setattr("vectordb_bench.backend.clients.pinecone.pinecone.time.sleep", sleeps.append) + + class RateLimitError(Exception): + status = 429 + + class Index: + calls = 0 + + def query(self, **kwargs): + self.calls += 1 + if self.calls < 3: + raise RateLimitError("too many requests") + return {"matches": [{"id": "3"}]} + + db.index = Index() + + assert db.search_embedding([0.1]) == [3] + assert db.index.calls == 3 + assert sleeps == [0.25, 0.25] + + +def test_pinecone_search_stops_after_rate_limit_retry_budget(monkeypatch): + db = Pinecone.__new__(Pinecone) + db.expr = None + monkeypatch.setenv("PINECONE_QUERY_MAX_RETRIES", "1") + monkeypatch.setattr("vectordb_bench.backend.clients.pinecone.pinecone.time.sleep", lambda _: None) + + class RateLimitError(Exception): + status = 429 + + class Index: + def query(self, **kwargs): + raise RateLimitError("too many requests") + + db.index = Index() + + try: + db.search_embedding([0.1]) + except RateLimitError: + pass + else: + raise AssertionError("expected Pinecone rate limit error") + + +def test_pinecone_insert_tracks_last_write_lsn(): + db = Pinecone.__new__(Pinecone) + db.batch_size = 1000 + db.with_scalar_labels = False + db._scalar_id_field = "meta" + + class UpsertResponse: + _response_info = {"raw_headers": {"x-pinecone-request-lsn": "123"}} + + class Index: + def upsert(self, records): + return UpsertResponse() + + db.index = Index() + + insert_count, error = db.insert_embeddings([[0.1]], [1]) + + assert error is None + assert insert_count == 1 + assert db._last_write_lsn == 123 + + +def test_pinecone_insert_keeps_highest_write_lsn(): + db = Pinecone.__new__(Pinecone) + db.batch_size = 1 + db.with_scalar_labels = False + db._scalar_id_field = "meta" + responses = iter(["123", "122"]) + + class UpsertResponse: + def __init__(self, lsn): + self._response_info = {"raw_headers": {"x-pinecone-request-lsn": lsn}} + + class Index: + def upsert(self, records): + return UpsertResponse(next(responses)) + + db.index = Index() + + insert_count, error = db.insert_embeddings([[0.1], [0.2]], [1, 2]) + + assert error is None + assert insert_count == 2 + assert db._last_write_lsn == 123 + + +def test_pinecone_record_write_lsn_keeps_highest_value(): + db = Pinecone.__new__(Pinecone) + + db._record_write_lsn(123) + db._record_write_lsn(122) + + assert db._last_write_lsn == 123 + + +def test_pinecone_insert_readiness_uses_lsn_when_available(): + db = Pinecone.__new__(Pinecone) + db._last_write_lsn = 123 + db._readiness_probe_vector = [0.0] + + class QueryResponse(dict): + def __init__(self, indexed_lsn): + super().__init__({"matches": []}) + self._response_info = {"raw_headers": {"x-pinecone-max-indexed-lsn": str(indexed_lsn)}} + + class Index: + indexed_lsn = 122 + + def describe_index_stats(self): + return {"total_vector_count": 10} + + def query(self, **kwargs): + return QueryResponse(self.indexed_lsn) + + db.index = Index() + + assert db.poll_insert_readiness(10)["fully_searchable"] is False + db.index.indexed_lsn = 123 + assert db.poll_insert_readiness(10)["fully_indexed"] is True + + +def test_turbopuffer_insert_readiness_uses_unindexed_bytes(): + db = TurboPuffer.__new__(TurboPuffer) + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=True) + db.ns = type("Namespace", (), {"metadata": lambda self: {"unindexed_bytes": 1}})() + + status = db.poll_insert_readiness(10) + + assert status["fully_searchable"] is True + assert status["fully_indexed"] is False + assert status["additional_parameters"] == {"disable_backpressure": True} + + +def test_turbopuffer_insert_readiness_uses_nested_unindexed_bytes(): + db = TurboPuffer.__new__(TurboPuffer) + db.db_case_config = TurboPufferIndexConfig(disable_backpressure=False) + db.ns = type( + "Namespace", + (), + {"metadata": lambda self: {"index": {"status": "updating", "unindexed_bytes": 1}}}, + )() + + status = db.poll_insert_readiness(10) + + assert status["fully_searchable"] is True + assert status["fully_indexed"] is False + assert status["additional_parameters"] == {"disable_backpressure": False} + + +def test_cloud_insert_runner_records_insert_and_readiness_metrics(monkeypatch): + class Data: + train_id_field = "id" + train_vector_field = "vector" + metric_type = "L2" + + class Dataset: + data = Data() + + def iter_batches(self, batch_size): + assert batch_size == 2 + return iter( + [ + pd.DataFrame({"id": [1, 2], "vector": [np.array([0.1]), np.array([0.2])]}), + pd.DataFrame({"id": [3], "vector": [np.array([0.3])]}), + ] + ) + + class DB: + thread_safe = True + name = "fake" + inserts = [] + readiness_calls = 0 + + @contextmanager + def init(self): + yield + + def need_normalize_cosine(self): + return False + + def insert_embeddings(self, embeddings, metadata, labels_data=None): + self.inserts.append((embeddings, metadata)) + return len(metadata), None + + def poll_insert_readiness(self, expected_count): + self.readiness_calls += 1 + return { + "fully_searchable": self.readiness_calls >= 2, + "fully_indexed": self.readiness_calls >= 3, + "additional_parameters": {"example": "value"}, + } + + db = DB() + monkeypatch.setattr("vectordb_bench.backend.task_runner.time.sleep", lambda _: None) + case = CloudInsertCase(batch_size=2) + case.dataset = Dataset() + config = type("Config", (), {"load_concurrency": 1})() + runner = CaseRunner.construct(ca=case, db=db, config=config) + + metric = runner._run_cloud_insert_case() + + assert [metadata for _, metadata in db.inserts] == [[1, 2], [3]] + assert metric.inserted_count == 3 + assert metric.insert_rows_per_second > 0 + assert metric.insert_completion_seconds >= 0 + assert metric.searchable_after_insert_seconds >= 0 + assert metric.indexed_after_searchable_seconds >= 0 + assert metric.additional_parameters == {"example": "value"} + + +def test_cloud_insert_runner_times_out_when_readiness_never_completes(monkeypatch): + class Data: + train_id_field = "id" + train_vector_field = "vector" + metric_type = "L2" + + class Dataset: + data = Data() + + def iter_batches(self, batch_size): + return iter([pd.DataFrame({"id": [1], "vector": [np.array([0.1])]})]) + + class DB: + @contextmanager + def init(self): + yield + + def need_normalize_cosine(self): + return False + + def poll_insert_readiness(self, expected_count): + return { + "fully_searchable": False, + "fully_indexed": False, + "additional_parameters": {"reason": "stalled"}, + } + + class FakeConcurrentInsertRunner: + def __init__(self, *args, **kwargs): + pass + + def task(self): + return 1 + + def fail_on_sleep(_seconds): + raise AssertionError("readiness polling did not time out before sleeping") + + monkeypatch.setattr("vectordb_bench.backend.task_runner.ConcurrentInsertRunner", FakeConcurrentInsertRunner) + monkeypatch.setattr("vectordb_bench.backend.task_runner.time.sleep", fail_on_sleep) + case = CloudInsertCase(batch_size=1, readiness_timeout=0, readiness_poll_interval=0) + case.dataset = Dataset() + runner = CaseRunner.construct(ca=case, db=DB(), config=type("Config", (), {"load_concurrency": 1})()) + + with pytest.raises(TimeoutError, match="fully_searchable.*last_status.*stalled"): + runner._run_cloud_insert_case() + + +def test_cloud_insert_runner_uses_concurrent_insert_runner(monkeypatch): + created = {} + + class Data: + metric_type = "L2" + + class Dataset: + data = Data() + + class FakeConcurrentInsertRunner: + def __init__(self, db, dataset, normalize, filters, max_workers, batch_size, duration): + created.update( + { + "db": db, + "dataset": dataset, + "normalize": normalize, + "filters": filters, + "max_workers": max_workers, + "batch_size": batch_size, + "duration": duration, + } + ) + + def task(self): + return 3 + + class DB: + @contextmanager + def init(self): + yield + + def need_normalize_cosine(self): + return False + + def poll_insert_readiness(self, expected_count): + return {"fully_searchable": True, "fully_indexed": True, "additional_parameters": {}} + + monkeypatch.setattr("vectordb_bench.backend.task_runner.ConcurrentInsertRunner", FakeConcurrentInsertRunner) + case = CloudInsertCase(batch_size=1000, duration=60) + case.dataset = Dataset() + config = type("Config", (), {"load_concurrency": 7})() + runner = CaseRunner.construct(ca=case, db=DB(), config=config) + + metric = runner._run_cloud_insert_case() + + assert metric.inserted_count == 3 + assert created["batch_size"] == 1000 + assert created["duration"] == 60 + assert created["max_workers"] == 7 + assert created["dataset"] is case.dataset + + +def test_concurrent_insert_runner_does_not_retry_non_retryable_insert_errors(monkeypatch): + from vectordb_bench.backend.runner import concurrent_runner as concurrent_runner_module + + class NonRetryableInsertError(RuntimeError): + non_retryable = True + + class FakeDB: + def __init__(self): + self.calls = 0 + + def insert_embeddings(self, **kwargs): + self.calls += 1 + return 2, NonRetryableInsertError("partial tenant insert") + + monkeypatch.setattr(concurrent_runner_module.time, "sleep", lambda _seconds: None) + + runner = ConcurrentInsertRunner.__new__(ConcurrentInsertRunner) + db = FakeDB() + + with pytest.raises(RuntimeError, match="Non-retryable insert failure"): + runner._insert_batch_with_retry( + db, + embeddings=[[1.0], [2.0], [3.0]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + + assert db.calls == 1 + + +def test_concurrent_insert_runner_stops_handing_out_batches_after_non_retryable_error(): + class NonRetryableInsertError(RuntimeError): + non_retryable = True + + class DB: + thread_safe = True + name = "fake" + + def __init__(self): + self.calls = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None): + self.calls.append(metadata) + if len(self.calls) == 1: + return 0, NonRetryableInsertError("partial tenant insert") + return len(metadata), None + + db = DB() + runner = ConcurrentInsertRunner( + db, _ConcurrentRunnerDataset([0, 1, 2]), normalize=False, max_workers=2, batch_size=1 + ) + runner._create_executor = lambda: _SerialTaskExecutor() + + with pytest.raises(RuntimeError, match="Non-retryable insert failure"): + runner.task() + + assert db.calls == [[0]] + + +def test_concurrent_insert_runner_stops_handing_out_batches_after_retry_exhaustion(monkeypatch): + from vectordb_bench.backend.runner import concurrent_runner as concurrent_runner_module + + class DB: + thread_safe = True + name = "fake" + + def __init__(self): + self.calls = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None): + self.calls.append(metadata) + return 0, RuntimeError("insert failed") + + monkeypatch.setattr(concurrent_runner_module.config, "MAX_INSERT_RETRY", 0) + + db = DB() + runner = ConcurrentInsertRunner(db, _ConcurrentRunnerDataset([0, 1]), normalize=False, max_workers=2, batch_size=1) + runner._create_executor = lambda: _SerialTaskExecutor() + + with pytest.raises(RuntimeError, match="Insert failed and retried more than 0 times"): + runner.task() + + assert db.calls == [[0]] + + +def test_concurrent_insert_runner_uses_custom_batch_size_iterator(): + class Data: + train_id_field = "id" + train_vector_field = "vector" + + class Dataset: + data = Data() + requested_batch_size = None + + def iter_batches(self, batch_size): + self.requested_batch_size = batch_size + return iter( + [ + pd.DataFrame( + { + "id": [1, 2], + "vector": [np.array([0.1]), np.array([0.2])], + } + ) + ] + ) + + def __iter__(self): + raise AssertionError("ConcurrentInsertRunner should request an explicit batch size") + + class DB: + thread_safe = True + name = "fake" + + def __init__(self): + self.inserts = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None): + self.inserts.append((embeddings, metadata)) + return len(metadata), None + + dataset = Dataset() + db = DB() + runner = ConcurrentInsertRunner(db, dataset, normalize=False, max_workers=1, batch_size=1000) + + assert runner.task() == 2 + assert dataset.requested_batch_size == 1000 + assert db.inserts == [([[0.1], [0.2]], [1, 2])] + + +class TenantInsertProbeDB: + name = "TenantInsertProbeDB" + thread_safe = True + + def __init__(self): + self.calls = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None, tenant_labels_data=None): + self.calls.append( + { + "embeddings": embeddings, + "metadata": metadata, + "labels_data": labels_data, + "tenant_labels_data": tenant_labels_data, + } + ) + return len(embeddings), None + + +class TenantAwareCase: + is_multitenant = True + + def tenant_labels_for_ids(self, row_ids): + return [f"tenant_{int(row_id) % 3:04d}" for row_id in row_ids] + + +def test_concurrent_insert_runner_passes_tenant_labels(): + db = TenantInsertProbeDB() + dataset = MagicMock() + dataset.data.train_id_field = "id" + dataset.data.train_vector_field = "emb" + dataset.iter_batches.return_value = iter( + [ + pd.DataFrame( + { + "id": [0, 1, 5], + "emb": [[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + } + ) + ] + ) + + runner = ConcurrentInsertRunner( + db=db, + dataset=dataset, + normalize=False, + max_workers=1, + batch_size=10, + tenant_case=TenantAwareCase(), + ) + + count = runner.task() + + assert count == 3 + assert db.calls[0]["metadata"] == [0, 1, 5] + assert db.calls[0]["tenant_labels_data"] == ["tenant_0000", "tenant_0001", "tenant_0002"] + + +def test_concurrent_insert_runner_passes_scalar_labels_for_scalar_payload_without_filter(): + db = TenantInsertProbeDB() + dataset = MagicMock() + dataset.data.train_id_field = "id" + dataset.data.train_vector_field = "emb" + dataset.data.scalar_labels_file_separated = False + dataset.iter_batches.return_value = iter( + [ + pd.DataFrame( + { + "id": [0, 1, 5], + "emb": [[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + "labels": ["label_a", "label_b", "label_c"], + } + ) + ] + ) + + runner = ConcurrentInsertRunner( + db=db, + dataset=dataset, + normalize=False, + max_workers=1, + batch_size=10, + with_scalar_labels=True, + ) + + count = runner.task() + + assert count == 3 + assert db.calls[0]["metadata"] == [0, 1, 5] + assert db.calls[0]["labels_data"] == ["label_a", "label_b", "label_c"] + assert db.calls[0]["tenant_labels_data"] is None + + +def test_pre_run_prepares_separated_scalar_labels_for_scalar_payload(): + class Dataset: + def __init__(self): + self.prepare_kwargs = None + self.data = type("Data", (), {"dim": 2, "metric_type": "L2"})() + + def prepare(self, source, filters, with_train_files, with_scalar_labels=False): + self.prepare_kwargs = { + "source": source, + "filters": filters, + "with_train_files": with_train_files, + "with_scalar_labels": with_scalar_labels, + } + return True + + class Case: + is_multitenant = False + with_scalar_labels = True + filters = MagicMock() + dataset = Dataset() + + class DB: + init_args = None + + @classmethod + def init_cls(cls, **kwargs): + cls.init_args = kwargs + return cls() + + def need_normalize_cosine(self): + return False + + class DBConfig: + def to_dict(self): + return {} + + config = type( + "Config", + (), + { + "db": DB, + "db_config": DBConfig(), + "db_case_config": EmptyDBCaseConfig(), + "stages": [TaskStage.LOAD], + }, + )() + runner = CaseRunner.construct( + ca=Case(), + config=config, + dataset_source=DatasetSource.S3, + ) + + runner._pre_run() + + assert Case.dataset.prepare_kwargs["with_scalar_labels"] is True diff --git a/tests/test_cloud_payload_case.py b/tests/test_cloud_payload_case.py new file mode 100644 index 000000000..d227dd176 --- /dev/null +++ b/tests/test_cloud_payload_case.py @@ -0,0 +1,180 @@ +import pytest + +from vectordb_bench import config +from vectordb_bench.backend.cases import CaseType, CloudPayloadSearchCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.runner.mp_runner import MultiProcessingSearchRunner +from vectordb_bench.backend.runner.serial_runner import SerialSearchRunner +from vectordb_bench.backend.task_runner import CaseRunner, RunningStatus +from vectordb_bench.cli.cli import get_custom_case_config +from vectordb_bench.models import CaseConfig, TaskConfig + + +class FakeDB: + name = "FakeDB" + + def __init__(self, supported_payload_profiles=None): + self.supported_payload_profiles = supported_payload_profiles or {PayloadProfile.IDS_ONLY} + self.calls = [] + + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in self.supported_payload_profiles + + def search_embedding(self, query: list[float], k: int = 100, **kwargs) -> list[int]: + self.calls.append((query, k, kwargs)) + return list(range(k)) + + +def test_payload_profile_estimates_response_bytes(): + assert PayloadProfile.IDS_ONLY.estimated_bytes_per_query(k=10, dim=768) == 200 + assert PayloadProfile.VECTOR.estimated_bytes_per_query(k=10, dim=768) == 30_920 + + +def test_cloud_payload_case_defaults_to_laion_100m(): + case = CloudPayloadSearchCase(payload_profile="vector") + + assert case.case_id == CaseType.CloudPayloadSearchCase + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + assert case.dataset.data.dim == 768 + assert case.payload_profile == PayloadProfile.VECTOR + assert case.estimated_payload_bytes_per_query(config.K_DEFAULT) == 309_200 + + +def test_cloud_payload_case_accepts_sized_dataset(): + case = CloudPayloadSearchCase(dataset_with_size_type=DatasetWithSizeType.CohereSmall.value) + + assert case.dataset_with_size_type == DatasetWithSizeType.CohereSmall + assert case.dataset.data.name == "Cohere" + assert case.dataset.data.size == 100_000 + + +def test_case_config_builds_cloud_payload_case_from_custom_case(): + case = CaseConfig( + case_id=CaseType.CloudPayloadSearchCase, + custom_case={"payload_profile": "vector"}, + ).case + + assert isinstance(case, CloudPayloadSearchCase) + assert case.payload_profile == PayloadProfile.VECTOR + + +def test_case_runner_reuse_key_distinguishes_scalar_label_schema_requirement(): + ids_only_case = CloudPayloadSearchCase( + dataset_with_size_type=DatasetWithSizeType.CohereSmall.value, + payload_profile=PayloadProfile.IDS_ONLY, + ) + scalar_label_case = CloudPayloadSearchCase( + dataset_with_size_type=DatasetWithSizeType.CohereSmall.value, + payload_profile=PayloadProfile.SCALAR_LABEL, + ) + task = TaskConfig( + db=DB.Test, + db_config=DB.Test.config_cls(), + db_case_config=EmptyDBCaseConfig(), + case_config=CaseConfig(case_id=CaseType.CloudPayloadSearchCase), + ) + + ids_only_runner = CaseRunner( + run_id="run-id", + config=task, + ca=ids_only_case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + scalar_label_runner = CaseRunner( + run_id="run-id", + config=task, + ca=scalar_label_case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + assert ids_only_case.with_scalar_labels is False + assert scalar_label_case.with_scalar_labels is True + assert ids_only_runner != scalar_label_runner + assert hash(ids_only_runner) != hash(scalar_label_runner) + + +def test_cli_propagates_cloud_payload_dataset_selection(): + custom_case = get_custom_case_config( + { + "case_type": "CloudPayloadSearchCase", + "dataset_with_size_type": DatasetWithSizeType.CohereSmall.value, + "payload_profile": "vector", + "cloud_filter_rate": None, + "cloud_label_percentage": None, + } + ) + + assert custom_case["dataset_with_size_type"] == DatasetWithSizeType.CohereSmall.value + + case = CaseConfig(case_id=CaseType.CloudPayloadSearchCase, custom_case=custom_case).case + assert case.dataset_with_size_type == DatasetWithSizeType.CohereSmall + assert case.dataset.data.size == 100_000 + + +def test_cli_omits_cloud_payload_dataset_when_not_selected(): + custom_case = get_custom_case_config( + { + "case_type": "CloudPayloadSearchCase", + "dataset_with_size_type": None, + "payload_profile": "ids_only", + "cloud_filter_rate": None, + "cloud_label_percentage": None, + } + ) + + assert "dataset_with_size_type" not in custom_case + + case = CaseConfig(case_id=CaseType.CloudPayloadSearchCase, custom_case=custom_case).case + assert case.dataset_with_size_type is None + assert case.dataset.data.name == "LAION" + assert case.dataset.data.size == 100_000_000 + + +def test_serial_runner_omits_payload_argument_for_ids_only(): + db = FakeDB() + runner = SerialSearchRunner(db=db, test_data=[[0.1]], ground_truth=[[0]], k=3) + + assert runner._get_db_search_res([0.1]) == [0, 1, 2] + assert db.calls == [([0.1], 3, {})] + + +def test_serial_runner_passes_payload_argument_for_vector_profile(): + db = FakeDB(supported_payload_profiles={PayloadProfile.IDS_ONLY, PayloadProfile.VECTOR}) + runner = SerialSearchRunner( + db=db, + test_data=[[0.1]], + ground_truth=[[0]], + k=3, + payload_profile=PayloadProfile.VECTOR, + ) + + assert runner._get_db_search_res([0.1]) == [0, 1, 2] + assert db.calls == [([0.1], 3, {"payload_profile": PayloadProfile.VECTOR})] + + +def test_search_runners_fail_fast_for_unsupported_payload_profile(): + db = FakeDB() + + with pytest.raises(NotImplementedError, match="payload_profile=vector"): + SerialSearchRunner( + db=db, + test_data=[[0.1]], + ground_truth=[[0]], + k=3, + payload_profile=PayloadProfile.VECTOR, + ) + + with pytest.raises(NotImplementedError, match="payload_profile=vector"): + MultiProcessingSearchRunner( + db=db, + test_data=[[0.1]], + k=3, + payload_profile=PayloadProfile.VECTOR, + ) diff --git a/tests/test_cloud_payload_search.py b/tests/test_cloud_payload_search.py new file mode 100644 index 000000000..cda8f63ba --- /dev/null +++ b/tests/test_cloud_payload_search.py @@ -0,0 +1,128 @@ +from contextlib import contextmanager + +import polars as pl + +from vectordb_bench.backend.cases import CloudPayloadSearchCase +from vectordb_bench.backend.clients.milvus.milvus import Milvus +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import Dataset +from vectordb_bench.backend.filter import FilterOp, non_filter +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.runner.serial_runner import SerialSearchRunner + + +class _Client: + def __init__(self): + self.search_kwargs = None + + def search(self, **kwargs): + self.search_kwargs = kwargs + return [[{"pk": 1}]] + + +def test_laion_100m_declares_scalar_label_assets(): + laion = Dataset.LAION.get(100_000_000) + + assert laion.with_scalar_labels is True + assert laion.scalar_labels_file == "scalar_labels.parquet" + assert 0.01 in laion.scalar_label_percentages + + +def test_scalar_label_payload_profile_estimates_small_string_payload(): + assert PayloadProfile.SCALAR_LABEL.estimated_bytes_per_query(k=100, dim=768) == 3600 + + +def test_scalar_label_payload_profile_requires_scalar_label_materialization_without_filter(): + case = CloudPayloadSearchCase(payload_profile="scalar_label") + + assert case.filters.type == FilterOp.NonFilter + assert case.with_scalar_labels is True + + +def test_dataset_prepare_loads_separated_scalar_labels_for_scalar_payload(monkeypatch): + dataset = Dataset.LAION.manager(100_000_000) + dataset.data.with_remote_resource = False + loaded_scalar_labels = object() + + def fake_read_file(file_name): + if file_name == dataset.data.scalar_labels_file: + return loaded_scalar_labels + return pl.DataFrame({dataset.data.test_vector_field: [], dataset.data.gt_neighbors_field: []}) + + monkeypatch.setattr(dataset, "_read_file", fake_read_file) + + dataset.prepare( + source=DatasetSource.S3, + filters=non_filter, + with_train_files=False, + with_scalar_labels=True, + ) + + assert dataset.scalar_labels is loaded_scalar_labels + + +def test_cloud_payload_case_can_combine_label_filter_with_scalar_label_payload(): + case = CloudPayloadSearchCase( + payload_profile="scalar_label", + label_percentage=0.01, + ) + + assert case.payload_profile == PayloadProfile.SCALAR_LABEL + assert case.filters.type == FilterOp.StrEqual + assert case.filters.label_value == "label_1p" + + +def test_milvus_scalar_label_payload_requests_label_output_field(): + db = Milvus.__new__(Milvus) + db.client = _Client() + db.case_config = type("CaseConfig", (), {"search_param": lambda self: {}})() + db.collection_name = "collection" + db.expr = "label == 'label_1p'" + db._primary_field = "pk" + db._vector_field = "vector" + db._scalar_label_field = "label" + + assert db.supports_payload_profile(PayloadProfile.SCALAR_LABEL) + assert db.search_embedding([0.1, 0.2], payload_profile=PayloadProfile.SCALAR_LABEL) == [1] + assert db.client.search_kwargs["output_fields"] == ["label"] + + +class TenantSearchProbeDB: + name = "TenantSearchProbeDB" + + def __init__(self): + self.tenants = [] + + def supports_payload_profile(self, payload_profile): + return True + + @contextmanager + def init(self): + yield + + def prepare_filter(self, filters): + return None + + def search_embedding(self, query, k=100, payload_profile=None, tenant=None): + self.tenants.append(tenant) + return [] + + +def test_serial_search_runner_passes_tenant_and_skips_recall(): + db = TenantSearchProbeDB() + runner = SerialSearchRunner( + db=db, + test_data=[[1.0, 0.0], [0.0, 1.0]], + ground_truth=None, + tenant_labels=["tenant_0000", "tenant_0001"], + measure_recall=False, + ) + + recall, ndcg, p99, p95 = runner.search((runner.test_data, runner.ground_truth)) + + assert recall == 0 + assert ndcg == 0 + assert p99 >= 0 + assert p95 >= 0 + assert set(db.tenants).issubset({"tenant_0000", "tenant_0001"}) + assert db.tenants diff --git a/tests/test_milvus.py b/tests/test_milvus.py index 1c5de7ce0..dfc88cad8 100644 --- a/tests/test_milvus.py +++ b/tests/test_milvus.py @@ -15,6 +15,7 @@ from vectordb_bench.backend.clients.api import IndexType from vectordb_bench.backend.clients.milvus.config import MilvusConfig from vectordb_bench.backend.clients.milvus.milvus import MILVUS_FORCE_MERGE_TARGET_SIZE_MB, Milvus +from vectordb_bench.backend.payload import PayloadProfile from vectordb_bench.interface import BenchMarkRunner from vectordb_bench.models import CaseConfig, TaskConfig @@ -84,3 +85,164 @@ def test_performance_1536d_50k(self): result = runner.get_results() log.info(f"test result: {result}") assert len(result) > 0 + + +def test_milvus_multitenant_search_uses_tenant_label_filter(): + captured = {} + + def search(**kwargs): + captured.update(kwargs) + return [[{"pk": 1}]] + + db = object.__new__(Milvus) + db.client = SimpleNamespace(search=search) + db.collection_name = "test_collection" + db._vector_field = "vector" + db._primary_field = "pk" + db._scalar_label_field = "label" + db.case_config = SimpleNamespace(search_param=lambda: {"metric_type": "COSINE"}) + db.expr = "" + + result = db.search_embedding([0.1, 0.2], k=3, payload_profile=PayloadProfile.IDS_ONLY, tenant="tenant_0003") + + assert result == [1] + assert captured["filter"] == "label == 'tenant_0003'" + + +def test_milvus_validate_multitenant_schema_accepts_partition_key_label( + monkeypatch: pytest.MonkeyPatch, +) -> None: + closed = [] + + class FakeMilvusClient: + def __init__(self, **_kwargs: object) -> None: + pass + + def describe_collection(self, _collection_name: str) -> dict: + return { + "fields": [ + {"name": "pk", "is_primary": True}, + {"name": "label", "is_partition_key": True}, + ] + } + + def close(self) -> None: + closed.append(True) + + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.MilvusClient", FakeMilvusClient) + + db = object.__new__(Milvus) + db.name = "Milvus" + db.db_config = {"uri": "http://example.invalid", "user": None, "password": None, "token": ""} + db.collection_name = "existing" + db._scalar_label_field = "label" + + db.validate_multitenant_schema() + + assert closed == [True] + + +def test_milvus_validate_multitenant_schema_rejects_non_partition_key_label( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeMilvusClient: + def __init__(self, **_kwargs: object) -> None: + pass + + def describe_collection(self, _collection_name: str) -> dict: + return {"fields": [{"name": "label", "is_partition_key": False}]} + + def close(self) -> None: + pass + + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.MilvusClient", FakeMilvusClient) + + db = object.__new__(Milvus) + db.name = "Milvus" + db.db_config = {"uri": "http://example.invalid", "user": None, "password": None, "token": ""} + db.collection_name = "existing" + db._scalar_label_field = "label" + + with pytest.raises(ValueError, match="label field is not a partition key"): + db.validate_multitenant_schema() + + +def test_milvus_validate_multitenant_schema_uses_existing_labels_partition_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured = {} + + class FakeMilvusClient: + def __init__(self, **_kwargs: object) -> None: + pass + + def describe_collection(self, _collection_name: str) -> dict: + return { + "fields": [ + {"name": "pk", "is_primary": True}, + {"name": "labels", "is_partition_key": True}, + {"name": "scalar_label", "nullable": True}, + ] + } + + def close(self) -> None: + pass + + def search(**kwargs): + captured.update(kwargs) + return [[{"pk": 1}]] + + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.MilvusClient", FakeMilvusClient) + + db = object.__new__(Milvus) + db.name = "Milvus" + db.db_config = {"uri": "http://example.invalid", "user": None, "password": None, "token": ""} + db.collection_name = "existing" + db._vector_field = "vector" + db._primary_field = "pk" + db._scalar_label_field = "label" + db.case_config = SimpleNamespace(search_param=lambda: {"metric_type": "COSINE"}) + db.expr = "" + + db.validate_multitenant_schema() + db.client = SimpleNamespace(search=search) + + db.search_embedding([0.1, 0.2], payload_profile=PayloadProfile.SCALAR_LABEL, tenant="tenant_0003") + + assert captured["filter"] == "labels == 'tenant_0003'" + assert captured["output_fields"] == ["scalar_label"] + + +def test_milvus_multitenant_insert_writes_tenant_and_scalar_payload_labels() -> None: + inserted = {} + + def insert(collection_name, batch_data): + inserted["collection_name"] = collection_name + inserted["batch_data"] = batch_data + return {"insert_count": len(batch_data)} + + db = object.__new__(Milvus) + db.client = SimpleNamespace(insert=insert) + db.collection_name = "test_collection" + db.batch_size = 100 + db._primary_field = "pk" + db._scalar_id_field = "id" + db._vector_field = "vector" + db._scalar_label_field = "label" + db._scalar_payload_label_field = "scalar_label" + db._multitenant_partition_key_field = "labels" + db.with_scalar_labels = True + + count, err = db.insert_embeddings( + embeddings=[[0.1, 0.2], [0.3, 0.4]], + metadata=[1, 2], + labels_data=["label_a", "label_b"], + tenant_labels_data=["tenant_0001", "tenant_0002"], + ) + + assert count == 2 + assert err is None + assert inserted["batch_data"] == [ + {"pk": 1, "id": 1, "vector": [0.1, 0.2], "labels": "tenant_0001", "scalar_label": "label_a"}, + {"pk": 2, "id": 2, "vector": [0.3, 0.4], "labels": "tenant_0002", "scalar_label": "label_b"}, + ] diff --git a/tests/test_milvus_zilliz_cli.py b/tests/test_milvus_zilliz_cli.py new file mode 100644 index 000000000..a028c535b --- /dev/null +++ b/tests/test_milvus_zilliz_cli.py @@ -0,0 +1,57 @@ +from click.testing import CliRunner +from pytest import MonkeyPatch + +from vectordb_bench.backend.clients.milvus import cli as milvus_cli +from vectordb_bench.backend.clients.zilliz_cloud import cli as zilliz_cli + + +def test_milvus_autoindex_cli_enables_partition_key_for_multitenant_case( + monkeypatch: MonkeyPatch, +) -> None: + captured = {} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(milvus_cli, "run", fake_run) + + result = CliRunner().invoke( + milvus_cli.MilvusAutoIndex, + [ + "--case-type", + "CloudMultiTenantSearchCase", + "--uri", + "http://localhost:19530", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["db_case_config"].use_partition_key is True + + +def test_zilliz_autoindex_cli_enables_partition_key_for_multitenant_case( + monkeypatch: MonkeyPatch, +) -> None: + captured = {} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(zilliz_cli, "run", fake_run) + + result = CliRunner().invoke( + zilliz_cli.ZillizAutoIndex, + [ + "--case-type", + "CloudMultiTenantSearchCase", + "--uri", + "https://example.api.gcp-us-west1.zillizcloud.com", + "--token", + "secret", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["db_case_config"].use_partition_key is True diff --git a/tests/test_multitenant_case.py b/tests/test_multitenant_case.py new file mode 100644 index 000000000..45939c32a --- /dev/null +++ b/tests/test_multitenant_case.py @@ -0,0 +1,400 @@ +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest + +from vectordb_bench.backend.cases import CaseType, CloudMultiTenantSearchCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import EmptyDBCaseConfig, VectorDB +from vectordb_bench.backend.clients.zilliz_cloud.config import AutoIndexConfig, ZillizCloudConfig +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import DatasetManager, DatasetWithSizeType +from vectordb_bench.backend.filter import FilterOp +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.task_runner import CaseRunner, RunningStatus +from vectordb_bench.models import CaseConfig, TaskConfig, TaskStage + + +def test_multitenant_case_defaults_to_cohere_large_1000_tenants(): + case = CloudMultiTenantSearchCase() + + assert case.case_id == CaseType.CloudMultiTenantSearchCase + assert case.dataset_with_size_type == DatasetWithSizeType.CohereLarge + assert case.dataset.data.size == 10_000_000 + assert case.tenant_count == 1000 + assert case.tenant_prefix == "tenant_" + assert case.tenant_id_width == 4 + assert case.measure_recall is False + assert case.is_multitenant is True + assert case.filters.type == FilterOp.NonFilter + + +def test_multitenant_case_accepts_dataset_and_tenant_count(): + case = CloudMultiTenantSearchCase( + dataset_with_size_type=DatasetWithSizeType.CohereSmall.value, + tenant_count=7, + tenant_prefix="acct_", + tenant_id_width=2, + payload_profile=PayloadProfile.VECTOR.value, + filter_rate=0.01, + ) + + assert case.dataset_with_size_type == DatasetWithSizeType.CohereSmall + assert case.dataset.data.size == 100_000 + assert case.payload_profile == PayloadProfile.VECTOR + assert case.estimated_payload_bytes_per_query(k=10) == PayloadProfile.VECTOR.estimated_bytes_per_query( + k=10, + dim=case.dataset.data.dim, + ) + assert case.filters.type == FilterOp.NumGE + assert case.tenant_count == 7 + assert case.tenant_for_id(0) == "acct_00" + assert case.tenant_for_id(8) == "acct_01" + assert case.tenant_labels_for_ids([0, 1, 8, 13]) == ["acct_00", "acct_01", "acct_01", "acct_06"] + + +def test_case_config_constructs_multitenant_case(): + case = CaseType.CloudMultiTenantSearchCase.case_cls( + { + "dataset_with_size_type": DatasetWithSizeType.CohereSmall.value, + "tenant_count": 5, + "payload_profile": PayloadProfile.SCALAR_LABEL.value, + "label_percentage": 0.01, + } + ) + + assert isinstance(case, CloudMultiTenantSearchCase) + assert case.payload_profile == PayloadProfile.SCALAR_LABEL + assert case.filters.type == FilterOp.StrEqual + assert case.tenant_labels() == ["tenant_0000", "tenant_0001", "tenant_0002", "tenant_0003", "tenant_0004"] + + +class TenantApiProbeDB(VectorDB): + name = "TenantApiProbeDB" + + def __init__(self, dim=2, db_config=None, db_case_config=None, collection_name="test", drop_old=False, **kwargs): + self.insert_calls = [] + self.search_calls = [] + + @contextmanager + def init(self): + yield + + def insert_embeddings(self, embeddings, metadata, labels_data=None, tenant_labels_data=None, **kwargs): + self.insert_calls.append((embeddings, metadata, labels_data, tenant_labels_data)) + return len(embeddings), None + + def search_embedding(self, query, k=100, payload_profile=None, tenant=None): + self.search_calls.append((query, k, payload_profile, tenant)) + return [] + + def optimize(self, data_size=None): + return None + + +def test_vector_db_accepts_optional_tenant_context(): + db = TenantApiProbeDB(db_case_config=EmptyDBCaseConfig()) + + count, err = db.insert_embeddings([[0.1, 0.2]], [42], tenant_labels_data=["tenant_0002"]) + result = db.search_embedding([0.1, 0.2], tenant="tenant_0002") + + assert count == 1 + assert err is None + assert result == [] + assert db.insert_calls[0][3] == ["tenant_0002"] + assert db.search_calls[0][3] == "tenant_0002" + + +def test_search_only_zilliz_multitenant_validates_existing_partition_key_schema( + monkeypatch: pytest.MonkeyPatch, +) -> None: + case = CloudMultiTenantSearchCase() + task = TaskConfig( + db=DB.ZillizCloud, + db_config=ZillizCloudConfig(uri="http://example.invalid", collection_name="existing"), + db_case_config=AutoIndexConfig(use_partition_key=False), + case_config=CaseConfig(case_id=CaseType.CloudMultiTenantSearchCase), + stages=[TaskStage.SEARCH_CONCURRENT], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + calls: list[tuple[str, object]] = [] + + class ExistingCollectionDB: + def supports_multitenant(self) -> bool: + return True + + def set_multitenant_context(self, tenant_labels: list[str]) -> None: + calls.append(("set_context", tenant_labels)) + + def validate_multitenant_schema(self) -> None: + calls.append(("validate_schema", None)) + + def fake_init_db(self: CaseRunner, _drop_old: bool = True) -> None: + self.db = ExistingCollectionDB() + + def fake_prepare(*_args: object, **_kwargs: object) -> None: + calls.append(("prepare", None)) + + monkeypatch.setattr(CaseRunner, "init_db", fake_init_db) + monkeypatch.setattr(DatasetManager, "prepare", fake_prepare) + + runner._pre_run(drop_old=False) + + assert ("validate_schema", None) in calls + assert calls[0][0] == "set_context" + + +def test_zilliz_multitenant_create_still_requires_partition_key() -> None: + case = CloudMultiTenantSearchCase() + task = TaskConfig( + db=DB.ZillizCloud, + db_config=ZillizCloudConfig(uri="http://example.invalid", collection_name="new_collection"), + db_case_config=AutoIndexConfig(use_partition_key=False), + case_config=CaseConfig(case_id=CaseType.CloudMultiTenantSearchCase), + stages=[TaskStage.DROP_OLD, TaskStage.LOAD], + ) + runner = CaseRunner( + run_id="run-id", + config=task, + ca=case, + status=RunningStatus.PENDING, + dataset_source=DatasetSource.S3, + ) + + with pytest.raises(ValueError, match="requires use_partition_key=True"): + runner._pre_run(drop_old=True) + + +class FakeTurboNamespace: + def __init__(self): + self.write_calls = [] + self.query_calls = [] + + def write(self, **kwargs): + self.write_calls.append(kwargs) + + def query(self, **kwargs): + self.query_calls.append(kwargs) + return SimpleNamespace(rows=[SimpleNamespace(id="10")]) + + def metadata(self): + return {"index": {"unindexed_bytes": 0}} + + +class FakeTurboClient: + def __init__(self): + self.namespaces = {} + + def namespace(self, name): + self.namespaces.setdefault(name, FakeTurboNamespace()) + return self.namespaces[name] + + +def test_turbopuffer_groups_multitenant_insert_and_search(monkeypatch): + from vectordb_bench.backend.clients.api import MetricType + from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + fake_client = FakeTurboClient() + monkeypatch.setattr(TurboPuffer, "_create_client", lambda self: fake_client) + + db = TurboPuffer( + dim=2, + db_config={ + "api_key": "k", + "region": "r", + "api_base_url": None, + "namespace": "single", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=TurboPufferIndexConfig(metric_type=MetricType.COSINE), + drop_old=False, + ) + db.set_multitenant_context(["tenant_0000", "tenant_0001"]) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + result = db.search_embedding([1.0, 0.0], k=1, tenant="tenant_0001") + + assert count == 3 + assert err is None + assert result == [10] + assert fake_client.namespaces["mt_tenant_0000"].write_calls[0]["upsert_columns"]["id"] == [0, 2] + assert fake_client.namespaces["mt_tenant_0001"].write_calls[0]["upsert_columns"]["id"] == [1] + assert fake_client.namespaces["mt_tenant_0001"].query_calls + + +def test_turbopuffer_multitenant_insert_preserves_scalar_payload_labels(monkeypatch): + from vectordb_bench.backend.clients.api import MetricType + from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + fake_client = FakeTurboClient() + monkeypatch.setattr(TurboPuffer, "_create_client", lambda self: fake_client) + + db = TurboPuffer( + dim=2, + db_config={ + "api_key": "k", + "region": "r", + "api_base_url": None, + "namespace": "single", + "multitenant_namespace_prefix": "mt_", + "scalar_payload_label_field": "scalar_label", + }, + db_case_config=TurboPufferIndexConfig(metric_type=MetricType.COSINE), + drop_old=False, + with_scalar_labels=True, + ) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0]], + metadata=[0, 1], + labels_data=["label_a", "label_b"], + tenant_labels_data=["tenant_0000", "tenant_0001"], + ) + + assert count == 2 + assert err is None + assert fake_client.namespaces["mt_tenant_0000"].write_calls[0]["upsert_columns"]["scalar_label"] == ["label_a"] + assert fake_client.namespaces["mt_tenant_0001"].write_calls[0]["upsert_columns"]["scalar_label"] == ["label_b"] + + +def test_turbopuffer_multitenant_partial_insert_failure_is_explicit(monkeypatch): + from vectordb_bench.backend.clients.api import MetricType + from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + class FailingNamespace(FakeTurboNamespace): + def __init__(self, fail_write: bool = False): + super().__init__() + self.fail_write = fail_write + + def write(self, **kwargs): + self.write_calls.append(kwargs) + if self.fail_write: + raise RuntimeError("tenant write failed") + + class FailingTurboClient: + def __init__(self): + self.namespaces = {} + + def namespace(self, name): + self.namespaces.setdefault(name, FailingNamespace(fail_write=name == "mt_tenant_0001")) + return self.namespaces[name] + + fake_client = FailingTurboClient() + monkeypatch.setattr(TurboPuffer, "_create_client", lambda self: fake_client) + + db = TurboPuffer( + dim=2, + db_config={ + "api_key": "k", + "region": "r", + "api_base_url": None, + "namespace": "single", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=TurboPufferIndexConfig(metric_type=MetricType.COSINE), + drop_old=False, + ) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + + assert count == 2 + assert getattr(err, "non_retryable", False) is True + assert getattr(err, "inserted_count") == 2 + assert getattr(err, "successful_tenants") == {"tenant_0000": 2} + assert getattr(err, "failed_tenant") == "tenant_0001" + assert "tenant_0001" in str(err) + assert fake_client.namespaces["mt_tenant_0000"].write_calls + assert fake_client.namespaces["mt_tenant_0001"].write_calls + + +def test_turbopuffer_pins_multitenant_namespaces_on_init(monkeypatch): + from vectordb_bench.backend.clients.api import MetricType + from vectordb_bench.backend.clients.turbopuffer import turbopuffer as turbopuffer_module + from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + fake_client = FakeTurboClient() + calls = [] + + def fake_metadata_request(api_key, region, namespace, method, payload=None, api_base_url=None): + calls.append((method, namespace, payload, api_base_url)) + return {} + + def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=None, timeout=None): + calls.append(("WAIT", namespace, replicas, timeout)) + return {"pinning": {"replicas": replicas, "status": {"ready_replicas": replicas}}} + + monkeypatch.setattr(TurboPuffer, "_create_client", lambda self: fake_client) + monkeypatch.setattr(turbopuffer_module, "namespace_metadata_request", fake_metadata_request) + monkeypatch.setattr(turbopuffer_module, "wait_for_namespace_pinning", fake_wait_for_pinning) + + db = TurboPuffer( + dim=2, + db_config={ + "api_key": "k", + "region": "r", + "api_base_url": "https://tpuf.example", + "namespace": "single", + "multitenant_namespace_prefix": "mt_", + "pin_namespace": True, + "pin_replicas": 2, + "pin_timeout": 30, + }, + db_case_config=TurboPufferIndexConfig(metric_type=MetricType.COSINE), + drop_old=False, + ) + db.set_multitenant_context(["tenant_0000", "tenant_0001"]) + + with db.init(): + pass + + assert calls == [ + ("PATCH", "mt_tenant_0000", {"pinning": {"replicas": 2}}, "https://tpuf.example"), + ("WAIT", "mt_tenant_0000", 2, 30), + ("PATCH", "mt_tenant_0001", {"pinning": {"replicas": 2}}, "https://tpuf.example"), + ("WAIT", "mt_tenant_0001", 2, 30), + ] + + +def test_turbopuffer_supports_scalar_label_payload_for_multitenant_search() -> None: + from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + fake_client = FakeTurboClient() + db = TurboPuffer.__new__(TurboPuffer) + db.client = fake_client + db.namespace = "single" + db.multitenant_namespace_prefix = "mt_" + db._ns_cache = {} + db._vector_field = "vector" + db._scalar_label_field = "label" + db._scalar_payload_label_field = "scalar_label" + db.expr = None + + assert db.supports_payload_profile(PayloadProfile.SCALAR_LABEL) + assert db.search_embedding( + [1.0, 0.0], + k=50, + payload_profile=PayloadProfile.SCALAR_LABEL, + tenant="tenant_0001", + ) == [10] + assert fake_client.namespaces["mt_tenant_0001"].query_calls[0]["include_attributes"] == ["scalar_label"] diff --git a/tests/test_pinecone_multitenant.py b/tests/test_pinecone_multitenant.py new file mode 100644 index 000000000..1a81d55ff --- /dev/null +++ b/tests/test_pinecone_multitenant.py @@ -0,0 +1,214 @@ +import threading +from types import SimpleNamespace + + +class FakePineconeIndex: + def __init__(self): + self.upserts = [] + self.queries = [] + self.deletes = [] + + def describe_index_stats(self): + return {"dimension": 2, "total_vector_count": 3, "namespaces": {"mt_tenant_0000": {}, "mt_tenant_0001": {}}} + + def upsert(self, vectors, namespace=None): + self.upserts.append((vectors, namespace)) + return SimpleNamespace(_response_info={"raw_headers": {"x-pinecone-request-lsn": "7"}}) + + def query(self, **kwargs): + self.queries.append(kwargs) + return SimpleNamespace( + matches=[{"id": "11"}], + _response_info={"raw_headers": {"x-pinecone-max-indexed-lsn": "7"}}, + ) + + def delete(self, delete_all=False, namespace=None): + self.deletes.append((delete_all, namespace)) + + +class FailingPineconeIndex(FakePineconeIndex): + def upsert(self, vectors, namespace=None): + self.upserts.append((vectors, namespace)) + if namespace == "mt_tenant_0001": + raise RuntimeError("tenant upsert failed") + return SimpleNamespace(_response_info={"raw_headers": {"x-pinecone-request-lsn": "7"}}) + + +class FakePineconeClient: + def __init__(self, index): + self.index = index + + def Index(self, name): + return self.index + + +def test_pinecone_groups_multitenant_upsert_and_query(monkeypatch): + from vectordb_bench.backend.clients.pinecone import pinecone as pinecone_module + from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone + + fake_index = FakePineconeIndex() + monkeypatch.setattr( + pinecone_module.pinecone, + "Pinecone", + lambda api_key: FakePineconeClient(fake_index), + ) + + db = Pinecone( + dim=2, + db_config={ + "api_key": "k", + "index_name": "idx", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=None, + drop_old=False, + ) + db.set_multitenant_context(["tenant_0000", "tenant_0001"]) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + result = db.search_embedding([1.0, 0.0], k=1, tenant="tenant_0001") + + assert count == 3 + assert err is None + assert result == [11] + assert fake_index.upserts[0][1] == "mt_tenant_0000" + assert fake_index.upserts[1][1] == "mt_tenant_0001" + assert fake_index.queries[-1]["namespace"] == "mt_tenant_0001" + + +def test_pinecone_multitenant_upsert_preserves_scalar_payload_labels(monkeypatch): + from vectordb_bench.backend.clients.pinecone import pinecone as pinecone_module + from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone + + fake_index = FakePineconeIndex() + monkeypatch.setattr( + pinecone_module.pinecone, + "Pinecone", + lambda api_key: FakePineconeClient(fake_index), + ) + + db = Pinecone( + dim=2, + db_config={ + "api_key": "k", + "index_name": "idx", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=None, + drop_old=False, + with_scalar_labels=True, + ) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0]], + metadata=[0, 1], + labels_data=["label_a", "label_b"], + tenant_labels_data=["tenant_0000", "tenant_0001"], + ) + + assert count == 2 + assert err is None + assert fake_index.upserts[0][0][0][2]["label"] == "label_a" + assert fake_index.upserts[1][0][0][2]["label"] == "label_b" + + +def test_pinecone_multitenant_partial_insert_failure_is_explicit(monkeypatch): + from vectordb_bench.backend.clients.pinecone import pinecone as pinecone_module + from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone + + fake_index = FailingPineconeIndex() + monkeypatch.setattr( + pinecone_module.pinecone, + "Pinecone", + lambda api_key: FakePineconeClient(fake_index), + ) + + db = Pinecone( + dim=2, + db_config={ + "api_key": "k", + "index_name": "idx", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=None, + drop_old=False, + ) + + with db.init(): + count, err = db.insert_embeddings( + embeddings=[[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]], + metadata=[0, 1, 2], + tenant_labels_data=["tenant_0000", "tenant_0001", "tenant_0000"], + ) + + assert count == 2 + assert getattr(err, "non_retryable", False) is True + assert getattr(err, "inserted_count") == 2 + assert getattr(err, "successful_tenants") == {"tenant_0000": 2} + assert getattr(err, "failed_tenant") == "tenant_0001" + assert getattr(err, "failed_tenant_count") == 1 + assert db._multitenant_insert_counts == {"tenant_0000": 2} + + +def test_pinecone_multitenant_insert_counts_are_thread_safe(monkeypatch): + from vectordb_bench.backend.clients.pinecone import pinecone as pinecone_module + from vectordb_bench.backend.clients.pinecone.pinecone import Pinecone + + class RacingCounts(dict): + def __init__(self, lock_getter): + super().__init__() + self.barrier = threading.Barrier(2) + self.lock_getter = lock_getter + + def get(self, key, default=None): + value = super().get(key, default) + lock = self.lock_getter() + if key == "tenant_0000" and (lock is None or not lock.locked()): + self.barrier.wait(timeout=5) + return value + + fake_index = FakePineconeIndex() + monkeypatch.setattr( + pinecone_module.pinecone, + "Pinecone", + lambda api_key: FakePineconeClient(fake_index), + ) + + db = Pinecone( + dim=2, + db_config={ + "api_key": "k", + "index_name": "idx", + "multitenant_namespace_prefix": "mt_", + }, + db_case_config=None, + drop_old=False, + ) + db._multitenant_insert_counts = RacingCounts(lambda: getattr(db, "_multitenant_insert_counts_lock", None)) + results = [] + + def insert_one(row_id): + results.append( + db.insert_embeddings( + embeddings=[[float(row_id), 0.0]], + metadata=[row_id], + tenant_labels_data=["tenant_0000"], + ) + ) + + with db.init(): + threads = [threading.Thread(target=insert_one, args=(row_id,)) for row_id in (1, 2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(fake_index.upserts) == 2 + assert sorted(results) == [(1, None), (1, None)] + assert dict(db._multitenant_insert_counts) == {"tenant_0000": 2} diff --git a/tests/test_turbopuffer_cli.py b/tests/test_turbopuffer_cli.py new file mode 100644 index 000000000..b167a319e --- /dev/null +++ b/tests/test_turbopuffer_cli.py @@ -0,0 +1,293 @@ +from types import SimpleNamespace + +from click.testing import CliRunner +from pytest import MonkeyPatch + +from vectordb_bench.backend.clients.api import MetricType +from vectordb_bench.backend.clients.turbopuffer import cli as turbopuffer_cli +from vectordb_bench.backend.clients.turbopuffer import turbopuffer as turbopuffer_client +from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer + + +def test_turbopuffer_cli_accepts_multitenant_namespace_prefix_and_metric_type( + monkeypatch: MonkeyPatch, +) -> None: + captured = {} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(turbopuffer_cli, "run", fake_run) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPuffer, + [ + "--skip-drop-old", + "--skip-load", + "--skip-search-serial", + "--search-concurrent", + "--case-type", + "CloudMultiTenantSearchCase", + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--namespace", + "cohere10m_multitenant", + "--multitenant-namespace-prefix", + "cohere10m_", + "--scalar-payload-label-field", + "scalar_label", + "--metric-type", + "COSINE", + "--dry-run", + ], + ) + + assert result.exit_code == 0 + assert captured["db_config"].multitenant_namespace_prefix == "cohere10m_" + assert captured["db_config"].scalar_payload_label_field == "scalar_label" + assert captured["db_case_config"].metric_type == MetricType.COSINE + assert captured["db_case_config"].multitenant_warmup_policy == "none" + + +def test_turbopuffer_cli_skips_pin_namespace_during_dry_run(monkeypatch: MonkeyPatch) -> None: + calls = [] + captured = {} + + def fake_metadata_request(api_key, region, namespace, method, payload=None, api_base_url=None): + calls.append((method, payload, api_key, region, namespace, api_base_url)) + return {} + + def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=None, timeout=None): + calls.append(("WAIT", replicas, api_key, region, namespace, api_base_url, timeout)) + return {"pinning": {"replicas": replicas, "status": {"ready_replicas": replicas}}} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(turbopuffer_client, "namespace_metadata_request", fake_metadata_request) + monkeypatch.setattr(turbopuffer_client, "wait_for_namespace_pinning", fake_wait_for_pinning) + monkeypatch.setattr(turbopuffer_cli, "run", fake_run) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPuffer, + [ + "--skip-drop-old", + "--skip-load", + "--skip-search-serial", + "--search-concurrent", + "--case-type", + "CloudPayloadSearchCase", + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--namespace", + "laion100m", + "--pin-namespace", + "--pin-replicas", + "2", + "--pin-timeout", + "7200", + "--metric-type", + "COSINE", + "--disable-backpressure", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert calls == [] + assert captured["db_config"].pin_namespace is False + assert captured["db_config"].pin_namespace_requested is True + assert captured["db_config"].pin_replicas == 2 + assert captured["db_config"].pin_timeout == 7200 + assert captured["db_config"].pin_target_namespace_count == 1 + assert captured["db_case_config"].metric_type == MetricType.COSINE + assert captured["db_case_config"].disable_backpressure is True + + +def test_turbopuffer_cli_accepts_multitenant_warmup_policy( + monkeypatch: MonkeyPatch, +) -> None: + captured = {} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(turbopuffer_cli, "run", fake_run) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPuffer, + [ + "--case-type", + "CloudMultiTenantSearchCase", + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--multitenant-warmup-policy", + "all", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["db_case_config"].multitenant_warmup_policy == "all" + + +def test_turbopuffer_multitenant_optimize_skips_base_namespace_by_default( + monkeypatch: MonkeyPatch, +) -> None: + warmed = [] + + class FakeNamespace: + def __init__(self, name: str): + self.name = name + + def hint_cache_warm(self): + warmed.append(self.name) + + class FakeClient: + def namespace(self, name: str): + return FakeNamespace(name) + + monkeypatch.setattr(turbopuffer_client.time, "sleep", lambda _seconds: None) + db = object.__new__(TurboPuffer) + db.client = FakeClient() + db.ns = FakeNamespace("base") + db.namespace = "base" + db.multitenant_namespace_prefix = "mt_" + db.multitenant_tenant_labels = ["tenant_0000", "tenant_0001"] + db._ns_cache = {} + db.db_case_config = SimpleNamespace(time_wait_warmup=1, multitenant_warmup_policy="none") + + db.optimize() + + assert warmed == [] + + +def test_turbopuffer_multitenant_optimize_can_warm_all_tenant_namespaces( + monkeypatch: MonkeyPatch, +) -> None: + warmed = [] + + class FakeNamespace: + def __init__(self, name: str): + self.name = name + + def hint_cache_warm(self): + warmed.append(self.name) + + class FakeClient: + def namespace(self, name: str): + return FakeNamespace(name) + + monkeypatch.setattr(turbopuffer_client.time, "sleep", lambda _seconds: None) + db = object.__new__(TurboPuffer) + db.client = FakeClient() + db.ns = FakeNamespace("base") + db.namespace = "base" + db.multitenant_namespace_prefix = "mt_" + db.multitenant_tenant_labels = ["tenant_0000", "tenant_0001"] + db._ns_cache = {} + db.db_case_config = SimpleNamespace(time_wait_warmup=1, multitenant_warmup_policy="all") + + db.optimize() + + assert warmed == ["mt_tenant_0000", "mt_tenant_0001"] + + +def test_turbopuffer_cli_skips_multitenant_pin_namespaces_during_dry_run( + monkeypatch: MonkeyPatch, +) -> None: + calls = [] + captured = {} + + def fake_metadata_request(api_key, region, namespace, method, payload=None, api_base_url=None): + calls.append((method, namespace, payload)) + return {} + + def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=None, timeout=None): + calls.append(("WAIT", namespace, replicas)) + return {"pinning": {"replicas": replicas, "status": {"ready_replicas": replicas}}} + + def fake_run(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(turbopuffer_client, "namespace_metadata_request", fake_metadata_request) + monkeypatch.setattr(turbopuffer_client, "wait_for_namespace_pinning", fake_wait_for_pinning) + monkeypatch.setattr(turbopuffer_cli, "run", fake_run) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPuffer, + [ + "--skip-drop-old", + "--skip-load", + "--skip-search-serial", + "--search-concurrent", + "--case-type", + "CloudMultiTenantSearchCase", + "--tenant-count", + "2", + "--tenant-prefix", + "tenant_", + "--tenant-id-width", + "4", + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--namespace", + "unused_single_namespace", + "--multitenant-namespace-prefix", + "cohere10m_", + "--pin-namespace", + "--pin-replicas", + "1", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert calls == [] + assert captured["db_config"].pin_namespace is False + assert captured["db_config"].pin_namespace_requested is True + assert captured["db_config"].pin_target_namespace_count == 2 + assert captured["db_config"].multitenant_namespace_prefix == "cohere10m_" + + +def test_turbopuffer_unpin_namespace_uses_pin_timeout(monkeypatch: MonkeyPatch) -> None: + calls = [] + + def fake_metadata_request(api_key, region, namespace, method, payload=None, api_base_url=None): + calls.append((method, payload, api_key, region, namespace, api_base_url)) + return {} + + def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=None, timeout=None): + calls.append(("WAIT", replicas, api_key, region, namespace, api_base_url, timeout)) + return {"pinning": None} + + monkeypatch.setattr(turbopuffer_client, "namespace_metadata_request", fake_metadata_request) + monkeypatch.setattr(turbopuffer_client, "wait_for_namespace_pinning", fake_wait_for_pinning) + + result = CliRunner().invoke( + turbopuffer_cli.TurboPufferUnpin, + [ + "--api-key", + "secret", + "--region", + "aws-us-west-2", + "--namespace", + "laion100m", + "--pin-timeout", + "7200", + ], + ) + + assert result.exit_code == 0, result.output + assert calls == [ + ("PATCH", {"pinning": None}, "secret", "aws-us-west-2", "laion100m", None), + ("WAIT", None, "secret", "aws-us-west-2", "laion100m", None, 7200), + ] diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index fc1813b38..9b3cbdff8 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -35,6 +35,8 @@ class config: CONCURRENCY_DURATION = 30 CONCURRENCY_TIMEOUT = 3600 + CLOUD_INSERT_READINESS_TIMEOUT = env.float("CLOUD_INSERT_READINESS_TIMEOUT", None) + CLOUD_INSERT_READINESS_POLL_INTERVAL = env.float("CLOUD_INSERT_READINESS_POLL_INTERVAL", 5.0) RESULTS_LOCAL_DIR = env.path( "RESULTS_LOCAL_DIR", diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index 3268cde35..b1177f7f4 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -48,10 +48,14 @@ def assemble_all( load_runners = [r for r in runners if r.ca.label == CaseLabel.Load] perf_runners = [r for r in runners if r.ca.label == CaseLabel.Performance] streaming_runners = [r for r in runners if r.ca.label == CaseLabel.Streaming] + cloud_insert_runners = [r for r in runners if r.ca.label == CaseLabel.CloudInsert] + cloud_cold_latency_runners = [r for r in runners if r.ca.label == CaseLabel.CloudColdLatency] + + search_filter_runners = [*perf_runners, *cloud_cold_latency_runners] # group by db db2runner: dict[DB, list[CaseRunner]] = {} - for r in perf_runners: + for r in search_filter_runners: db = r.config.db if db not in db2runner: db2runner[db] = [] @@ -71,6 +75,7 @@ def assemble_all( all_runners = [] all_runners.extend(load_runners) all_runners.extend(streaming_runners) + all_runners.extend(cloud_insert_runners) for v in db2runner.values(): all_runners.extend(v) diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index edfcdea19..b93bd04c3 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -5,6 +5,7 @@ from vectordb_bench import config from vectordb_bench.backend.clients.api import MetricType from vectordb_bench.backend.filter import Filter, FilterOp, IntFilter, LabelFilter, NewIntFilter, NonFilter, non_filter +from vectordb_bench.backend.payload import PayloadProfile from vectordb_bench.base import BaseModel from vectordb_bench.frontend.components.custom.getCustomConfig import CustomDatasetConfig @@ -56,6 +57,10 @@ class CaseType(Enum): LabelFilterPerformanceCase = 300 NewIntFilterPerformanceCase = 400 + CloudPayloadSearchCase = 500 + CloudInsertCase = 600 + CloudColdLatencyCase = 700 + CloudMultiTenantSearchCase = 800 def case_cls(self, custom_configs: dict | None = None) -> type["Case"]: if custom_configs is None: @@ -79,6 +84,8 @@ class CaseLabel(Enum): Load = auto() Performance = auto() Streaming = auto() + CloudInsert = auto() + CloudColdLatency = auto() class Case(BaseModel): @@ -102,14 +109,24 @@ class Case(BaseModel): optimize_timeout: float | int | None = None filter_rate: float | None = None + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY @property def filters(self) -> Filter: return non_filter + def estimated_payload_bytes_per_query(self, k: int | None) -> int: + if k is None: + k = config.K_DEFAULT + return self.payload_profile.estimated_bytes_per_query(k=k, dim=self.dataset.data.dim) + + @property + def is_multitenant(self) -> bool: + return False + @property def with_scalar_labels(self) -> bool: - return self.filters.type == FilterOp.StrEqual + return self.filters.type == FilterOp.StrEqual or self.payload_profile == PayloadProfile.SCALAR_LABEL def check_scalar_labels(self) -> None: if self.with_scalar_labels and not self.dataset.data.with_scalar_labels: @@ -594,6 +611,259 @@ def filters(self) -> Filter: return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) +class CloudPayloadSearchCase(PerformanceCase): + case_id: CaseType = CaseType.CloudPayloadSearchCase + dataset_with_size_type: DatasetWithSizeType | None = None + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY + filter_rate: float | None = None + label_percentage: float | None = None + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str | None = None, + payload_profile: PayloadProfile | str = PayloadProfile.IDS_ONLY, + filter_rate: float | None = None, + label_percentage: float | None = None, + **kwargs, + ): + if filter_rate is not None and label_percentage is not None: + msg = "CloudPayloadSearchCase supports only one filter type per run" + raise ValueError(msg) + if dataset_with_size_type is not None and not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + if not isinstance(payload_profile, PayloadProfile): + payload_profile = PayloadProfile(payload_profile) + + if dataset_with_size_type is None: + dataset = Dataset.LAION.manager(100_000_000) + load_timeout = config.LOAD_TIMEOUT_768D_100M + optimize_timeout = config.OPTIMIZE_TIMEOUT_768D_100M + dataset_name = "LAION 100M (768dim)" + else: + dataset = dataset_with_size_type.get_manager() + load_timeout = dataset_with_size_type.get_load_timeout() + optimize_timeout = dataset_with_size_type.get_optimize_timeout() + dataset_name = dataset_with_size_type.value + + name = f"Cloud Payload Search - {payload_profile.value} - {dataset_name}" + description = ( + "Cloud leaderboard search envelope case with explicit response payload profile. " + f"Payload profile: {payload_profile.value}; dataset: {dataset_name}." + ) + super().__init__( + name=name, + description=description, + dataset=dataset, + load_timeout=load_timeout, + optimize_timeout=optimize_timeout, + dataset_with_size_type=dataset_with_size_type, + payload_profile=payload_profile, + filter_rate=filter_rate, + label_percentage=label_percentage, + **kwargs, + ) + + @property + def filters(self) -> Filter: + if self.label_percentage is not None: + return LabelFilter(label_percentage=self.label_percentage) + if self.filter_rate is None: + return non_filter + int_field = self.dataset.data.train_id_field + int_value = int(self.dataset.data.size * self.filter_rate) + return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) + + +class CloudColdLatencyCase(Case): + case_id: CaseType = CaseType.CloudColdLatencyCase + label: CaseLabel = CaseLabel.CloudColdLatency + dataset_with_size_type: DatasetWithSizeType | None = None + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY + filter_rate: float | None = None + label_percentage: float | None = None + query_count: int = 1000 + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str | None = None, + payload_profile: PayloadProfile | str = PayloadProfile.IDS_ONLY, + filter_rate: float | None = None, + label_percentage: float | None = None, + query_count: int = 1000, + **kwargs, + ): + if filter_rate is not None and label_percentage is not None: + msg = "CloudColdLatencyCase supports only one filter type per run" + raise ValueError(msg) + if query_count <= 0: + msg = "query_count must be positive" + raise ValueError(msg) + if dataset_with_size_type is not None and not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + if not isinstance(payload_profile, PayloadProfile): + payload_profile = PayloadProfile(payload_profile) + + if dataset_with_size_type is None: + dataset = Dataset.LAION.manager(100_000_000) + load_timeout = config.LOAD_TIMEOUT_768D_100M + optimize_timeout = config.OPTIMIZE_TIMEOUT_768D_100M + dataset_name = "LAION 100M (768dim)" + else: + dataset = dataset_with_size_type.get_manager() + load_timeout = dataset_with_size_type.get_load_timeout() + optimize_timeout = dataset_with_size_type.get_optimize_timeout() + dataset_name = dataset_with_size_type.value + + name = f"Cloud Cold Latency - {payload_profile.value} - {dataset_name}" + description = ( + "Cloud leaderboard cold/warm serial latency case with explicit response payload profile. " + f"Payload profile: {payload_profile.value}; dataset: {dataset_name}; query count: {query_count}." + ) + super().__init__( + name=name, + description=description, + dataset=dataset, + load_timeout=load_timeout, + optimize_timeout=optimize_timeout, + dataset_with_size_type=dataset_with_size_type, + payload_profile=payload_profile, + filter_rate=filter_rate, + label_percentage=label_percentage, + query_count=query_count, + **kwargs, + ) + + @property + def filters(self) -> Filter: + if self.label_percentage is not None: + return LabelFilter(label_percentage=self.label_percentage) + if self.filter_rate is None: + return non_filter + int_field = self.dataset.data.train_id_field + int_value = int(self.dataset.data.size * self.filter_rate) + return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) + + +class CloudInsertCase(Case): + case_id: CaseType = CaseType.CloudInsertCase + label: CaseLabel = CaseLabel.CloudInsert + batch_size: int + duration: float | None = None + readiness_timeout: float | None = config.CLOUD_INSERT_READINESS_TIMEOUT + readiness_poll_interval: float = config.CLOUD_INSERT_READINESS_POLL_INTERVAL + dataset_with_size_type: DatasetWithSizeType | None = None + + def __init__( + self, + batch_size: int, + duration: float | None = None, + readiness_timeout: float | None = config.CLOUD_INSERT_READINESS_TIMEOUT, + readiness_poll_interval: float = config.CLOUD_INSERT_READINESS_POLL_INTERVAL, + dataset_with_size_type: DatasetWithSizeType | str | None = None, + **kwargs, + ): + if dataset_with_size_type is not None and not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + dataset = ( + Dataset.LAION.manager(100_000_000) + if dataset_with_size_type is None + else dataset_with_size_type.get_manager() + ) + super().__init__( + name=f"Cloud Insert - batch {batch_size}", + description="Cloud leaderboard insert-only case with readiness polling.", + dataset=dataset, + batch_size=batch_size, + duration=duration, + readiness_timeout=readiness_timeout, + readiness_poll_interval=readiness_poll_interval, + dataset_with_size_type=dataset_with_size_type, + **kwargs, + ) + + +class CloudMultiTenantSearchCase(PerformanceCase): + case_id: CaseType = CaseType.CloudMultiTenantSearchCase + dataset_with_size_type: DatasetWithSizeType = DatasetWithSizeType.CohereLarge + tenant_count: int = 1000 + tenant_prefix: str = "tenant_" + tenant_id_width: int = 4 + tenant_distribution: str = "uniform_by_id_mod" + measure_recall: bool = False + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY + filter_rate: float | None = None + label_percentage: float | None = None + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str = DatasetWithSizeType.CohereLarge, + tenant_count: int = 1000, + tenant_prefix: str = "tenant_", + tenant_id_width: int = 4, + payload_profile: PayloadProfile | str = PayloadProfile.IDS_ONLY, + filter_rate: float | None = None, + label_percentage: float | None = None, + **kwargs, + ): + if filter_rate is not None and label_percentage is not None: + msg = "CloudMultiTenantSearchCase supports only one filter type per run" + raise ValueError(msg) + if not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + if not isinstance(payload_profile, PayloadProfile): + payload_profile = PayloadProfile(payload_profile) + if tenant_count <= 0: + msg = "tenant_count must be greater than 0" + raise ValueError(msg) + if tenant_id_width <= 0: + msg = "tenant_id_width must be greater than 0" + raise ValueError(msg) + + dataset = dataset_with_size_type.get_manager() + super().__init__( + name=f"Cloud Multi-Tenant Search - {dataset_with_size_type.value}, {tenant_count} tenants", + description=( + "Multi-tenant QPS/latency benchmark with deterministic tenant routing " + f"({dataset_with_size_type.value}, {tenant_count} tenants)." + ), + dataset=dataset, + load_timeout=dataset_with_size_type.get_load_timeout(), + optimize_timeout=dataset_with_size_type.get_optimize_timeout(), + dataset_with_size_type=dataset_with_size_type, + tenant_count=tenant_count, + tenant_prefix=tenant_prefix, + tenant_id_width=tenant_id_width, + payload_profile=payload_profile, + filter_rate=filter_rate, + label_percentage=label_percentage, + **kwargs, + ) + + @property + def is_multitenant(self) -> bool: + return True + + def tenant_for_id(self, row_id: int) -> str: + tenant_id = int(row_id) % self.tenant_count + return f"{self.tenant_prefix}{tenant_id:0{self.tenant_id_width}d}" + + def tenant_labels_for_ids(self, row_ids: list[int]) -> list[str]: + return [self.tenant_for_id(row_id) for row_id in row_ids] + + def tenant_labels(self) -> list[str]: + return [f"{self.tenant_prefix}{tenant_id:0{self.tenant_id_width}d}" for tenant_id in range(self.tenant_count)] + + @property + def filters(self) -> Filter: + if self.label_percentage is not None: + return LabelFilter(label_percentage=self.label_percentage) + if self.filter_rate is None: + return non_filter + int_field = self.dataset.data.train_id_field + int_value = int(self.dataset.data.size * self.filter_rate) + return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) + + class LabelFilterPerformanceCase(PerformanceCase): case_id: CaseType = CaseType.LabelFilterPerformanceCase dataset_with_size_type: DatasetWithSizeType @@ -655,4 +925,8 @@ def filters(self) -> Filter: CaseType.StreamingCustomDataset: StreamingCustomDataset, CaseType.NewIntFilterPerformanceCase: NewIntFilterPerformanceCase, CaseType.LabelFilterPerformanceCase: LabelFilterPerformanceCase, + CaseType.CloudPayloadSearchCase: CloudPayloadSearchCase, + CaseType.CloudInsertCase: CloudInsertCase, + CaseType.CloudColdLatencyCase: CloudColdLatencyCase, + CaseType.CloudMultiTenantSearchCase: CloudMultiTenantSearchCase, } diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 118c505ff..37a5c71dc 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, model_validator from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile class MetricType(StrEnum): @@ -63,6 +64,29 @@ class SQType(StrEnum): FP32 = "FP32" +class NonRetryableInsertError(RuntimeError): + non_retryable = True + + +class PartialInsertError(NonRetryableInsertError): + def __init__( + self, + message: str, + *, + inserted_count: int, + successful_tenants: dict[str, int] | None = None, + failed_tenant: str | None = None, + failed_tenant_count: int | None = None, + cause: Exception | None = None, + ): + super().__init__(message) + self.inserted_count = inserted_count + self.successful_tenants = successful_tenants or {} + self.failed_tenant = failed_tenant + self.failed_tenant_count = failed_tenant_count + self.__cause__ = cause + + class DBConfig(ABC, BaseModel): """DBConfig contains the connection info of vector database @@ -216,12 +240,28 @@ def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" return False + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile == PayloadProfile.IDS_ONLY + + def poll_insert_readiness(self, expected_count: int) -> dict: + return {"fully_searchable": True, "fully_indexed": True, "additional_parameters": {}} + + def set_multitenant_context(self, tenant_labels: list[str]) -> None: + self.multitenant_tenant_labels = tenant_labels + + def supports_multitenant(self) -> bool: + return False + + def validate_multitenant_schema(self) -> None: + return None + @abstractmethod def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert the embeddings to the vector database. The default number of embeddings for @@ -242,6 +282,8 @@ def search_embedding( self, query: list[float], k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant: str | None = None, ) -> list[int]: """Get k most similar embeddings to query vector. diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index ae7269801..146d23aed 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -1,7 +1,7 @@ from typing import Annotated, TypedDict, Unpack import click -from pydantic import SecretStr +from pydantic import BaseModel, SecretStr from vectordb_bench.backend.clients import DB from vectordb_bench.cli.cli import ( @@ -16,6 +16,17 @@ DBTYPE = DB.Milvus +def _use_partition_key(parameters: dict) -> bool: + explicit = parameters.get("use_partition_key") + if explicit is not None: + return explicit + return parameters.get("case_type") == "CloudMultiTenantSearchCase" + + +def _with_partition_key(db_case_config: BaseModel, parameters: dict) -> BaseModel: + return db_case_config.model_copy(update={"use_partition_key": _use_partition_key(parameters)}) + + class MilvusTypedDict(TypedDict): uri: Annotated[ str, @@ -51,6 +62,17 @@ class MilvusTypedDict(TypedDict): show_default=True, ), ] + use_partition_key: Annotated[ + bool | None, + click.option( + "--use-partition-key/--no-use-partition-key", + default=None, + help=( + "Use the Milvus partition key on the label field. " + "Defaults to enabled for CloudMultiTenantSearchCase and disabled otherwise." + ), + ), + ] class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): ... @@ -71,7 +93,7 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=AutoIndexConfig(), + db_case_config=_with_partition_key(AutoIndexConfig(), parameters), **parameters, ) @@ -91,7 +113,7 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=FLATConfig(), + db_case_config=_with_partition_key(FLATConfig(), parameters), **parameters, ) @@ -114,10 +136,13 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=HNSWConfig( - M=parameters["m"], - efConstruction=parameters["ef_construction"], - ef=parameters["ef_search"], + db_case_config=_with_partition_key( + HNSWConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + ), + parameters, ), **parameters, ) @@ -179,14 +204,17 @@ def MilvusHNSWPQ(**parameters: Unpack[MilvusHNSWPQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=HNSWPQConfig( - M=parameters["m"], - efConstruction=parameters["ef_construction"], - ef=parameters["ef_search"], - nbits=parameters["nbits"], - refine=parameters["refine"], - refine_type=parameters["refine_type"], - refine_k=parameters["refine_k"], + db_case_config=_with_partition_key( + HNSWPQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + nbits=parameters["nbits"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + parameters, ), **parameters, ) @@ -223,15 +251,18 @@ def MilvusHNSWPRQ(**parameters: Unpack[MilvusHNSWPRQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=HNSWPRQConfig( - M=parameters["m"], - efConstruction=parameters["ef_construction"], - ef=parameters["ef_search"], - nbits=parameters["nbits"], - refine=parameters["refine"], - refine_type=parameters["refine_type"], - refine_k=parameters["refine_k"], - nrq=parameters["nrq"], + db_case_config=_with_partition_key( + HNSWPRQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + nbits=parameters["nbits"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + nrq=parameters["nrq"], + ), + parameters, ), **parameters, ) @@ -264,14 +295,17 @@ def MilvusHNSWSQ(**parameters: Unpack[MilvusHNSWSQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=HNSWSQConfig( - M=parameters["m"], - efConstruction=parameters["ef_construction"], - ef=parameters["ef_search"], - sq_type=parameters["sq_type"], - refine=parameters["refine"], - refine_type=parameters["refine_type"], - refine_k=parameters["refine_k"], + db_case_config=_with_partition_key( + HNSWSQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + sq_type=parameters["sq_type"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + parameters, ), **parameters, ) @@ -295,9 +329,12 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=IVFFlatConfig( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], + db_case_config=_with_partition_key( + IVFFlatConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + ), + parameters, ), **parameters, ) @@ -318,9 +355,12 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=IVFSQ8Config( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], + db_case_config=_with_partition_key( + IVFSQ8Config( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + ), + parameters, ), **parameters, ) @@ -380,13 +420,16 @@ def MilvusIVFRabitQ(**parameters: Unpack[MilvusIVFRABITQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=IVFRABITQConfig( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], - rbq_bits_query=parameters["rbq_bits_query"], - refine=parameters["refine"], - refine_type=parameters["refine_type"], - refine_k=parameters["refine_k"], + db_case_config=_with_partition_key( + IVFRABITQConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + rbq_bits_query=parameters["rbq_bits_query"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + parameters, ), **parameters, ) @@ -411,8 +454,11 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=DISKANNConfig( - search_list=parameters["search_list"], + db_case_config=_with_partition_key( + DISKANNConfig( + search_list=parameters["search_list"], + ), + parameters, ), **parameters, ) @@ -441,11 +487,14 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=GPUIVFFlatConfig( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], - cache_dataset_on_device=parameters["cache_dataset_on_device"], - refine_ratio=parameters.get("refine_ratio"), + db_case_config=_with_partition_key( + GPUIVFFlatConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters.get("refine_ratio"), + ), + parameters, ), **parameters, ) @@ -477,9 +526,12 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=GPUBruteForceConfig( - metric_type=parameters["metric_type"], - limit=parameters["limit"], # top-k for search + db_case_config=_with_partition_key( + GPUBruteForceConfig( + metric_type=parameters["metric_type"], + limit=parameters["limit"], # top-k for search + ), + parameters, ), **parameters, ) @@ -567,13 +619,16 @@ def MilvusSVSVamana(**parameters: Unpack[MilvusSVSVamanaTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=SVSVamanaConfig( - svs_graph_max_degree=parameters["svs_graph_max_degree"], - svs_construction_window_size=parameters["svs_construction_window_size"], - svs_alpha=parameters["svs_alpha"], - svs_storage_kind=parameters["svs_storage_kind"], - svs_search_window_size=parameters["svs_search_window_size"], - svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + db_case_config=_with_partition_key( + SVSVamanaConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + parameters, ), **parameters, ) @@ -594,13 +649,16 @@ def MilvusSVSVamanaLVQ(**parameters: Unpack[MilvusSVSVamanaTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=SVSVamanaLVQConfig( - svs_graph_max_degree=parameters["svs_graph_max_degree"], - svs_construction_window_size=parameters["svs_construction_window_size"], - svs_alpha=parameters["svs_alpha"], - svs_storage_kind=parameters["svs_storage_kind"], - svs_search_window_size=parameters["svs_search_window_size"], - svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + db_case_config=_with_partition_key( + SVSVamanaLVQConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + ), + parameters, ), **parameters, ) @@ -635,14 +693,17 @@ def MilvusSVSVamanaLeanVec(**parameters: Unpack[MilvusSVSVamanaLeanVecTypedDict] num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=SVSVamanaLeanVecConfig( - svs_graph_max_degree=parameters["svs_graph_max_degree"], - svs_construction_window_size=parameters["svs_construction_window_size"], - svs_alpha=parameters["svs_alpha"], - svs_storage_kind=parameters["svs_storage_kind"], - svs_search_window_size=parameters["svs_search_window_size"], - svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], - svs_leanvec_dim=parameters["svs_leanvec_dim"], + db_case_config=_with_partition_key( + SVSVamanaLeanVecConfig( + svs_graph_max_degree=parameters["svs_graph_max_degree"], + svs_construction_window_size=parameters["svs_construction_window_size"], + svs_alpha=parameters["svs_alpha"], + svs_storage_kind=parameters["svs_storage_kind"], + svs_search_window_size=parameters["svs_search_window_size"], + svs_search_buffer_capacity=parameters["svs_search_buffer_capacity"], + svs_leanvec_dim=parameters["svs_leanvec_dim"], + ), + parameters, ), **parameters, ) @@ -673,13 +734,16 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=GPUIVFPQConfig( - nlist=parameters["nlist"], - nprobe=parameters["nprobe"], - m=parameters["m"], - nbits=parameters["nbits"], - cache_dataset_on_device=parameters["cache_dataset_on_device"], - refine_ratio=parameters["refine_ratio"], + db_case_config=_with_partition_key( + GPUIVFPQConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + m=parameters["m"], + nbits=parameters["nbits"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters["refine_ratio"], + ), + parameters, ), **parameters, ) @@ -714,17 +778,20 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): num_shards=int(parameters["num_shards"]), replica_number=int(parameters["replica_number"]), ), - db_case_config=GPUCAGRAConfig( - intermediate_graph_degree=parameters["intermediate_graph_degree"], - graph_degree=parameters["graph_degree"], - itopk_size=parameters["itopk_size"], - team_size=parameters["team_size"], - search_width=parameters["search_width"], - min_iterations=parameters["min_iterations"], - max_iterations=parameters["max_iterations"], - build_algo=parameters["build_algo"], - cache_dataset_on_device=parameters["cache_dataset_on_device"], - refine_ratio=parameters["refine_ratio"], + db_case_config=_with_partition_key( + GPUCAGRAConfig( + intermediate_graph_degree=parameters["intermediate_graph_degree"], + graph_degree=parameters["graph_degree"], + itopk_size=parameters["itopk_size"], + team_size=parameters["team_size"], + search_width=parameters["search_width"], + min_iterations=parameters["min_iterations"], + max_iterations=parameters["max_iterations"], + build_algo=parameters["build_algo"], + cache_dataset_on_device=parameters["cache_dataset_on_device"], + refine_ratio=parameters["refine_ratio"], + ), + parameters, ), **parameters, ) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 740c89509..063faf3bd 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -4,10 +4,12 @@ import time from collections.abc import Iterable from contextlib import contextmanager +from typing import Any from pymilvus import DataType, MilvusClient, MilvusException from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile from ..api import VectorDB from .config import MilvusIndexConfig @@ -47,6 +49,13 @@ def __init__( self._primary_field = "pk" self._scalar_id_field = "id" self._scalar_label_field = "label" + self._scalar_payload_label_field = self._scalar_label_field + self._multitenant_partition_key_field = self._scalar_label_field + self.multitenant_tenant_labels: list[str] = kwargs.get("multitenant_tenant_labels", []) + if self.multitenant_tenant_labels: + self._multitenant_partition_key_field = "labels" + if self.with_scalar_labels: + self._scalar_payload_label_field = "scalar_label" self._vector_field = "vector" self._vector_index_name = "vector_idx" self._scalar_id_index_name = "id_sort_idx" @@ -56,6 +65,7 @@ def __init__( uri=self.db_config.get("uri"), user=self.db_config.get("user"), password=self.db_config.get("password"), + token=self.db_config.get("token", ""), timeout=30, ) @@ -69,16 +79,27 @@ def __init__( schema.add_field(self._scalar_id_field, DataType.INT64) schema.add_field(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) - if self.with_scalar_labels: - is_partition_key = db_case_config.use_partition_key - log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") + if self.multitenant_tenant_labels: schema.add_field( - self._scalar_label_field, + self._multitenant_partition_key_field, DataType.VARCHAR, max_length=256, - is_partition_key=is_partition_key, + is_partition_key=True, ) + if self.with_scalar_labels: + is_partition_key = db_case_config.use_partition_key + log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") + if not self.multitenant_tenant_labels or ( + self._scalar_payload_label_field != self._multitenant_partition_key_field + ): + schema.add_field( + self._scalar_payload_label_field, + DataType.VARCHAR, + max_length=256, + is_partition_key=is_partition_key and not self.multitenant_tenant_labels, + ) + log.info(f"{self.name} create collection: {self.collection_name}") index_params = self._build_index_params() @@ -113,12 +134,59 @@ def _build_index_params(self): ) if self.with_scalar_labels: index_params.add_index( - field_name=self._scalar_label_field, + field_name=self._scalar_payload_label_field, index_name=self._scalar_labels_index_name, index_type="BITMAP", ) return index_params + def supports_multitenant(self) -> bool: + return True + + def validate_multitenant_schema(self) -> None: + client = MilvusClient( + uri=self.db_config.get("uri"), + user=self.db_config.get("user"), + password=self.db_config.get("password"), + token=self.db_config.get("token", ""), + timeout=30, + ) + try: + desc = client.describe_collection(self.collection_name) + fields = desc.get("fields", []) if isinstance(desc, dict) else [] + fields_by_name = {self._field_property(field, "name"): field for field in fields} + partition_key_field = self._find_multitenant_partition_key_field(fields_by_name) + if partition_key_field is None: + label_field = fields_by_name.get(self._scalar_label_field) + if label_field is None: + msg = f"{self.name} multitenant collection {self.collection_name} is missing tenant label field" + raise ValueError(msg) + msg = f"{self.name} multitenant collection {self.collection_name} label field is not a partition key" + raise ValueError(msg) + self._multitenant_partition_key_field = partition_key_field + if "scalar_label" in fields_by_name: + self._scalar_payload_label_field = "scalar_label" + finally: + client.close() + + def _find_multitenant_partition_key_field(self, fields_by_name: dict[str, dict | object]) -> str | None: + for field_name in [self._scalar_label_field, "labels"]: + field = fields_by_name.get(field_name) + if field is not None and self._field_property(field, "is_partition_key", False): + return field_name + return None + + @staticmethod + def _field_property(field: dict | object, name: str, default: Any = None): + if isinstance(field, dict): + if name in field: + return field[name] + params = field.get("params") + if isinstance(params, dict) and name in params: + return params[name] + return default + return getattr(field, name, default) + @contextmanager def init(self): """ @@ -132,6 +200,7 @@ def init(self): uri=self.db_config.get("uri"), user=self.db_config.get("user"), password=self.db_config.get("password"), + token=self.db_config.get("token", ""), timeout=60, ) yield @@ -211,6 +280,7 @@ def insert_embeddings( embeddings: Iterable[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert embeddings into Milvus. should call self.init() first""" @@ -227,8 +297,10 @@ def insert_embeddings( self._scalar_id_field: metadata[i], self._vector_field: embeddings[i], } + if tenant_labels_data is not None: + row[self._multitenant_partition_key_field] = tenant_labels_data[i] if self.with_scalar_labels: - row[self._scalar_label_field] = labels_data[i] + row[self._scalar_payload_label_field] = labels_data[i] batch_data.append(row) res = self.client.insert(self.collection_name, batch_data) insert_count += res["insert_count"] @@ -243,27 +315,62 @@ def prepare_filter(self, filters: Filter): elif filters.type == FilterOp.NumGE: self.expr = f"{self._scalar_id_field} >= {filters.int_value}" elif filters.type == FilterOp.StrEqual: - self.expr = f"{self._scalar_label_field} == '{filters.label_value}'" + self.expr = f"{self._scalar_payload_label_field} == '{filters.label_value}'" else: msg = f"Not support Filter for Milvus - {filters}" raise ValueError(msg) + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in { + PayloadProfile.IDS_ONLY, + PayloadProfile.VECTOR, + PayloadProfile.SCALAR_LABEL, + } + + def poll_insert_readiness(self, expected_count: int) -> dict: + assert self.client is not None + self.client.flush(self.collection_name) + stats = self.client.get_collection_stats(self.collection_name) + count = int(stats.get("row_count", stats.get("num_entities", 0))) + progress = self.client.describe_index(self.collection_name, self._vector_index_name) + return { + "fully_searchable": count >= expected_count, + "fully_indexed": progress.get("pending_index_rows", -1) == 0, + "additional_parameters": {}, + } + def search_embedding( self, query: list[float], k: int = 100, timeout: int | None = None, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant: str | None = None, ) -> list[int]: """Perform a search on a query embedding and return results.""" assert self.client is not None - res = self.client.search( - collection_name=self.collection_name, - data=[query], - anns_field=self._vector_field, - search_params=self.case_config.search_param(), - limit=k, - filter=self.expr, - ) + output_fields = None + if payload_profile == PayloadProfile.VECTOR: + output_fields = [self._vector_field] + elif payload_profile == PayloadProfile.SCALAR_LABEL: + output_fields = [getattr(self, "_scalar_payload_label_field", self._scalar_label_field)] + + expr = self.expr + if tenant is not None: + tenant_field = getattr(self, "_multitenant_partition_key_field", self._scalar_label_field) + tenant_expr = f"{tenant_field} == '{tenant}'" + expr = tenant_expr if not expr else f"({expr}) and ({tenant_expr})" + + search_kwargs = { + "collection_name": self.collection_name, + "data": [query], + "anns_field": self._vector_field, + "search_params": self.case_config.search_param(), + "limit": k, + "filter": expr, + "output_fields": output_fields, + } + res = self.client.search(**search_kwargs) return [result[self._primary_field] for result in res[0]] diff --git a/vectordb_bench/backend/clients/pinecone/config.py b/vectordb_bench/backend/clients/pinecone/config.py index fe1a039ed..c42c876ff 100644 --- a/vectordb_bench/backend/clients/pinecone/config.py +++ b/vectordb_bench/backend/clients/pinecone/config.py @@ -6,9 +6,11 @@ class PineconeConfig(DBConfig): api_key: SecretStr index_name: str + multitenant_namespace_prefix: str = "vdbbench_mt_" def to_dict(self) -> dict: return { "api_key": self.api_key.get_secret_value(), "index_name": self.index_name, + "multitenant_namespace_prefix": self.multitenant_namespace_prefix, } diff --git a/vectordb_bench/backend/clients/pinecone/pinecone.py b/vectordb_bench/backend/clients/pinecone/pinecone.py index 9c2b38888..f123ae231 100644 --- a/vectordb_bench/backend/clients/pinecone/pinecone.py +++ b/vectordb_bench/backend/clients/pinecone/pinecone.py @@ -1,18 +1,27 @@ """Wrapper around the Pinecone vector database over VectorDB""" import logging +import os +import threading +import time from contextlib import contextmanager +from typing import Any import pinecone from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile -from ..api import DBCaseConfig, VectorDB +from ..api import DBCaseConfig, PartialInsertError, VectorDB log = logging.getLogger(__name__) PINECONE_MAX_NUM_PER_BATCH = 1000 PINECONE_MAX_SIZE_PER_BATCH = 2 * 1024 * 1024 # 2MB +PINECONE_QUERY_MAX_RETRIES_ENV = "PINECONE_QUERY_MAX_RETRIES" +PINECONE_QUERY_RETRY_SLEEP_ENV = "PINECONE_QUERY_RETRY_SLEEP_SECONDS" +PINECONE_QUERY_DEFAULT_MAX_RETRIES = 10 +PINECONE_QUERY_DEFAULT_RETRY_SLEEP_SECONDS = 0.5 class Pinecone(VectorDB): @@ -34,15 +43,28 @@ def __init__( """Initialize wrapper around the milvus vector database.""" self.index_name = db_config.get("index_name", "") self.api_key = db_config.get("api_key", "") + self.multitenant_namespace_prefix = db_config.get("multitenant_namespace_prefix", "vdbbench_mt_") + self.multitenant_tenant_labels: list[str] = kwargs.get("multitenant_tenant_labels", []) + self._multitenant_insert_counts: dict[str, int] = {} + self._multitenant_insert_counts_lock = threading.Lock() self.batch_size = int( min(PINECONE_MAX_SIZE_PER_BATCH / (dim * 5), PINECONE_MAX_NUM_PER_BATCH), ) + self._last_write_lsn: int | None = None + self._last_write_lsn_lock = threading.Lock() + self._readiness_probe_vector = [0.0] * dim pc = pinecone.Pinecone(api_key=self.api_key) index = pc.Index(self.index_name) self.with_scalar_labels = with_scalar_labels - if drop_old: + self.expr = None + if drop_old and self.multitenant_tenant_labels: + for tenant in self.multitenant_tenant_labels: + namespace = self._namespace_for_tenant(tenant) + log.info(f"Pinecone index delete multitenant namespace: {namespace}") + index.delete(delete_all=True, namespace=namespace) + elif drop_old: index_stats = index.describe_index_stats() index_dim = index_stats["dimension"] if index_dim != dim: @@ -61,18 +83,154 @@ def init(self): self.index = pc.Index(self.index_name) yield + def __getstate__(self): + state = self.__dict__.copy() + state.pop("_last_write_lsn_lock", None) + state.pop("_multitenant_insert_counts_lock", None) + return state + def optimize(self, data_size: int | None = None): pass + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in { + PayloadProfile.IDS_ONLY, + PayloadProfile.SCALAR_LABEL, + PayloadProfile.VECTOR, + } + + def supports_multitenant(self) -> bool: + return True + + def set_multitenant_context(self, tenant_labels: list[str]) -> None: + self.multitenant_tenant_labels = tenant_labels + + def _namespace_for_tenant(self, tenant: str | None) -> str | None: + if tenant is None: + return None + return f"{self.multitenant_namespace_prefix}{tenant}" + + def poll_insert_readiness(self, expected_count: int) -> dict: + stats = self.index.describe_index_stats() + if getattr(self, "multitenant_tenant_labels", []): + namespaces = stats.get("namespaces", {}) + expected_by_tenant = self._expected_multitenant_counts(expected_count) + count_ready = True + for tenant, expected_tenant_count in expected_by_tenant.items(): + namespace = self._namespace_for_tenant(tenant) + namespace_stats = namespaces.get(namespace, {}) + namespace_count = namespace_stats.get("vector_count", 0) + count_ready = count_ready and namespace_count >= expected_tenant_count + fresh = self._multitenant_lsn_ready(expected_by_tenant) + return { + "fully_searchable": count_ready and fresh, + "fully_indexed": count_ready and fresh, + "additional_parameters": {}, + } + + count = stats.get("total_vector_count", 0) + count_ready = count >= expected_count + last_write_lsn = getattr(self, "_last_write_lsn", None) + if last_write_lsn is None: + return { + "fully_searchable": count_ready, + "fully_indexed": count_ready, + "additional_parameters": {}, + } + query_res = self.index.query(vector=self._readiness_probe_vector, top_k=1) + indexed_lsn = self._extract_lsn(query_res, "x-pinecone-max-indexed-lsn") + if indexed_lsn is None: + return { + "fully_searchable": count_ready, + "fully_indexed": count_ready, + "additional_parameters": {}, + } + fresh = indexed_lsn >= last_write_lsn + return { + "fully_searchable": count_ready and fresh, + "fully_indexed": count_ready and fresh, + "additional_parameters": {}, + } + + def _expected_multitenant_counts(self, expected_count: int) -> dict[str, int]: + insert_counts = self._multitenant_insert_count_snapshot() + if insert_counts: + return insert_counts + tenant_labels = self.multitenant_tenant_labels + tenant_count = len(tenant_labels) + base_count = expected_count // tenant_count if tenant_count else 0 + remainder = expected_count % tenant_count if tenant_count else 0 + return {tenant: base_count + (1 if idx < remainder else 0) for idx, tenant in enumerate(tenant_labels)} + + def _multitenant_insert_count_snapshot(self) -> dict[str, int]: + if not hasattr(self, "_multitenant_insert_counts_lock"): + self._multitenant_insert_counts_lock = threading.Lock() + with self._multitenant_insert_counts_lock: + return dict(getattr(self, "_multitenant_insert_counts", {})) + + def _record_multitenant_insert_count(self, tenant: str, count: int) -> None: + if not hasattr(self, "_multitenant_insert_counts_lock"): + self._multitenant_insert_counts_lock = threading.Lock() + with self._multitenant_insert_counts_lock: + # Pinecone readiness is count-based per tenant namespace. Unlike + # providers that only track touched tenants, this counter must keep + # every successful write from concurrent insert workers. + self._multitenant_insert_counts[tenant] = self._multitenant_insert_counts.get(tenant, 0) + count + + def _multitenant_lsn_ready(self, expected_by_tenant: dict[str, int]) -> bool: + last_write_lsn = getattr(self, "_last_write_lsn", None) + if last_write_lsn is None: + return True + for tenant, expected_tenant_count in expected_by_tenant.items(): + if expected_tenant_count <= 0: + continue + query_res = self.index.query( + vector=self._readiness_probe_vector, + top_k=1, + namespace=self._namespace_for_tenant(tenant), + ) + indexed_lsn = self._extract_lsn(query_res, "x-pinecone-max-indexed-lsn") + if indexed_lsn is not None and indexed_lsn < last_write_lsn: + return False + return True + + @staticmethod + def _extract_lsn(response: Any, header_name: str) -> int | None: + response_info = getattr(response, "_response_info", None) + if not response_info: + return None + raw_headers = response_info.get("raw_headers", {}) + value = raw_headers.get(header_name.lower()) or raw_headers.get(header_name) + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + def _record_write_lsn(self, write_lsn: int) -> None: + if not hasattr(self, "_last_write_lsn_lock"): + self._last_write_lsn_lock = threading.Lock() + with self._last_write_lsn_lock: + self._last_write_lsn = max(getattr(self, "_last_write_lsn", 0) or 0, write_lsn) + + @staticmethod + def _matches(response: Any) -> list: + if isinstance(response, dict): + return response["matches"] + return response.matches + def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: assert len(embeddings) == len(metadata) insert_count = 0 + successful_tenants: dict[str, int] = {} try: for batch_start_offset in range(0, len(embeddings), self.batch_size): batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) @@ -87,7 +245,46 @@ def insert_embeddings( metadata_dict, ) insert_datas.append(insert_data) - self.index.upsert(insert_datas) + if tenant_labels_data is None: + upsert_res = self.index.upsert(insert_datas) + write_lsn = self._extract_lsn(upsert_res, "x-pinecone-request-lsn") + if write_lsn is not None: + self._record_write_lsn(write_lsn) + else: + batch_tenant_labels = tenant_labels_data[batch_start_offset:batch_end_offset] + for tenant in sorted(set(batch_tenant_labels)): + tenant_insert_datas = [ + insert_data + for insert_data, tenant_label in zip(insert_datas, batch_tenant_labels, strict=True) + if tenant_label == tenant + ] + try: + upsert_res = self.index.upsert( + tenant_insert_datas, + namespace=self._namespace_for_tenant(tenant), + ) + except Exception as e: + msg = ( + "Pinecone multitenant insert failed for " + f"tenant={tenant} after writing {insert_count} rows; " + f"successful_tenants={successful_tenants}; " + f"failed_tenant_count={len(tenant_insert_datas)}" + ) + return insert_count, PartialInsertError( + msg, + inserted_count=insert_count, + successful_tenants=successful_tenants, + failed_tenant=tenant, + failed_tenant_count=len(tenant_insert_datas), + cause=e, + ) + write_lsn = self._extract_lsn(upsert_res, "x-pinecone-request-lsn") + if write_lsn is not None: + self._record_write_lsn(write_lsn) + insert_count += len(tenant_insert_datas) + successful_tenants[tenant] = successful_tenants.get(tenant, 0) + len(tenant_insert_datas) + self._record_multitenant_insert_count(tenant, len(tenant_insert_datas)) + continue insert_count += batch_end_offset - batch_start_offset except Exception as e: return insert_count, e @@ -98,15 +295,38 @@ def search_embedding( query: list[float], k: int = 100, timeout: int | None = None, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant: str | None = None, ) -> list[int]: pinecone_filters = self.expr - res = self.index.query( - top_k=k, - vector=query, - filter=pinecone_filters, - )["matches"] + include_metadata = payload_profile == PayloadProfile.SCALAR_LABEL + include_values = payload_profile == PayloadProfile.VECTOR + max_retries = int(os.getenv(PINECONE_QUERY_MAX_RETRIES_ENV, PINECONE_QUERY_DEFAULT_MAX_RETRIES)) + retry_sleep = float( + os.getenv(PINECONE_QUERY_RETRY_SLEEP_ENV, PINECONE_QUERY_DEFAULT_RETRY_SLEEP_SECONDS), + ) + for retry_idx in range(max_retries + 1): + try: + query_res = self.index.query( + top_k=k, + vector=query, + filter=pinecone_filters, + include_metadata=include_metadata, + include_values=include_values, + namespace=self._namespace_for_tenant(tenant), + ) + res = self._matches(query_res) + break + except Exception as exc: + if not self._is_rate_limited(exc) or retry_idx >= max_retries: + raise + time.sleep(retry_sleep) return [int(one_res["id"]) for one_res in res] + @staticmethod + def _is_rate_limited(exc: Exception) -> bool: + return getattr(exc, "status", None) == 429 + def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: self.expr = None diff --git a/vectordb_bench/backend/clients/turbopuffer/cli.py b/vectordb_bench/backend/clients/turbopuffer/cli.py index d510889a0..def442d69 100644 --- a/vectordb_bench/backend/clients/turbopuffer/cli.py +++ b/vectordb_bench/backend/clients/turbopuffer/cli.py @@ -10,42 +10,134 @@ run, ) from .. import DB +from ..api import MetricType +from .config import TurboPufferMultitenantWarmupPolicy + +DEFAULT_PIN_TIMEOUT = 45 * 60 + +ApiKeyOption = Annotated[ + str, + click.option("--api-key", type=str, help="TurboPuffer API key", required=True), +] +RegionOption = Annotated[ + str, + click.option( + "--region", + type=str, + help="TurboPuffer region (e.g. aws-us-east-1, gcp-us-central1)", + required=True, + ), +] +ApiBaseUrlOption = Annotated[ + str, + click.option( + "--api-base-url", + type=str, + help="Override the region-based API URL", + required=False, + default="", + show_default=False, + ), +] +NamespaceOption = Annotated[ + str, + click.option( + "--namespace", + type=str, + help="TurboPuffer namespace", + required=False, + default="vdbbench_test", + show_default=True, + ), +] +PinTimeoutOption = Annotated[ + int, + click.option( + "--pin-timeout", + type=click.IntRange(min=1), + default=DEFAULT_PIN_TIMEOUT, + show_default=True, + help="Seconds to wait for TurboPuffer namespace pinning or unpinning to complete", + ), +] class TurboPufferTypedDict(TypedDict): - api_key: Annotated[ - str, - click.option("--api-key", type=str, help="TurboPuffer API key", required=True), - ] - region: Annotated[ + api_key: ApiKeyOption + region: RegionOption + api_base_url: ApiBaseUrlOption + namespace: NamespaceOption + multitenant_namespace_prefix: Annotated[ str, click.option( - "--region", + "--multitenant-namespace-prefix", type=str, - help="TurboPuffer region (e.g. aws-us-east-1, gcp-us-central1)", - required=True, + help="Namespace prefix for CloudMultiTenantSearchCase tenant namespaces", + required=False, + default="vdbbench_mt_", + show_default=True, ), ] - api_base_url: Annotated[ + scalar_payload_label_field: Annotated[ str, click.option( - "--api-base-url", + "--scalar-payload-label-field", type=str, - help="Override the region-based API URL", + help="TurboPuffer attribute used for scalar_label payload and label filtering", required=False, - default="", - show_default=False, + default="label", + show_default=True, ), ] - namespace: Annotated[ + metric_type: Annotated[ str, click.option( - "--namespace", - type=str, - help="TurboPuffer namespace", + "--metric-type", + type=click.Choice([MetricType.COSINE.value, MetricType.L2.value]), + help="TurboPuffer distance metric type", required=False, - default="vdbbench_test", + default=MetricType.COSINE.value, + show_default=True, + ), + ] + disable_backpressure: Annotated[ + bool, + click.option( + "--disable-backpressure/--enable-backpressure", + type=bool, + default=False, + show_default=True, + help="Disable Turbopuffer write backpressure", + ), + ] + pin_namespace: Annotated[ + bool, + click.option( + "--pin-namespace/--no-pin-namespace", + default=False, + show_default=True, + help="Pin TurboPuffer namespace(s) before benchmark workers run", + ), + ] + pin_replicas: Annotated[ + int, + click.option( + "--pin-replicas", + type=click.IntRange(min=1), + default=1, + show_default=True, + help="Number of TurboPuffer pinning replicas to request", + ), + ] + pin_timeout: PinTimeoutOption + multitenant_warmup_policy: Annotated[ + str, + click.option( + "--multitenant-warmup-policy", + type=click.Choice([policy.value for policy in TurboPufferMultitenantWarmupPolicy]), + default=TurboPufferMultitenantWarmupPolicy.NONE.value, show_default=True, + help="TurboPuffer cache warmup policy for CloudMultiTenantSearchCase tenant namespaces", ), ] @@ -53,11 +145,67 @@ class TurboPufferTypedDict(TypedDict): class TurboPufferIndexTypedDict(CommonTypedDict, TurboPufferTypedDict): ... +class TurboPufferUnpinTypedDict(TypedDict): + """Options for explicit TurboPuffer namespace pinning cleanup. + + Namespace pinning is persistent service state: enabling it before a benchmark + reserves replicas for the namespace until it is cleared. Unpinning is kept + as a separate command so failed or interrupted benchmark runs can be cleaned + up later, and so a billing-affecting teardown action is not hidden behind + ordinary benchmark flags. + """ + + api_key: ApiKeyOption + region: RegionOption + api_base_url: ApiBaseUrlOption + namespace: NamespaceOption + pin_timeout: PinTimeoutOption + + +def target_namespaces_for_pinning(parameters: TurboPufferIndexTypedDict) -> list[str]: + if parameters.get("case_type") != "CloudMultiTenantSearchCase": + return [parameters["namespace"]] + + namespace_prefix = parameters["multitenant_namespace_prefix"] + tenant_prefix = parameters["tenant_prefix"] + tenant_id_width = parameters["tenant_id_width"] + return [ + f"{namespace_prefix}{tenant_prefix}{tenant_id:0{tenant_id_width}d}" + for tenant_id in range(parameters["tenant_count"]) + ] + + +def pin_namespaces_once(parameters: TurboPufferIndexTypedDict) -> None: + from .turbopuffer import namespace_metadata_request, wait_for_namespace_pinning + + for namespace in target_namespaces_for_pinning(parameters): + namespace_metadata_request( + parameters["api_key"], + parameters["region"], + namespace, + "PATCH", + {"pinning": {"replicas": parameters["pin_replicas"]}}, + parameters["api_base_url"] or None, + ) + wait_for_namespace_pinning( + parameters["api_key"], + parameters["region"], + namespace, + parameters["pin_replicas"], + parameters["api_base_url"] or None, + parameters["pin_timeout"], + ) + + @cli.command() @click_parameter_decorators_from_typed_dict(TurboPufferIndexTypedDict) def TurboPuffer(**parameters: Unpack[TurboPufferIndexTypedDict]): from .config import TurboPufferConfig, TurboPufferIndexConfig + pin_target_namespace_count = len(target_namespaces_for_pinning(parameters)) if parameters["pin_namespace"] else 0 + if parameters["pin_namespace"] and not parameters["dry_run"]: + pin_namespaces_once(parameters) + run( db=DB.TurboPuffer, db_config=TurboPufferConfig( @@ -66,7 +214,42 @@ def TurboPuffer(**parameters: Unpack[TurboPufferIndexTypedDict]): region=parameters["region"], api_base_url=parameters["api_base_url"] or None, namespace=parameters["namespace"], + multitenant_namespace_prefix=parameters["multitenant_namespace_prefix"], + scalar_payload_label_field=parameters["scalar_payload_label_field"], + pin_namespace=False, + pin_namespace_requested=parameters["pin_namespace"], + pin_replicas=parameters["pin_replicas"], + pin_timeout=parameters["pin_timeout"], + pin_target_namespace_count=pin_target_namespace_count, + ), + db_case_config=TurboPufferIndexConfig( + metric_type=MetricType(parameters["metric_type"]), + disable_backpressure=parameters["disable_backpressure"], + multitenant_warmup_policy=TurboPufferMultitenantWarmupPolicy(parameters["multitenant_warmup_policy"]), ), - db_case_config=TurboPufferIndexConfig(), **parameters, ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(TurboPufferUnpinTypedDict) +def TurboPufferUnpin(**parameters: Unpack[TurboPufferUnpinTypedDict]): + from .turbopuffer import namespace_metadata_request, wait_for_namespace_pinning + + namespace_metadata_request( + parameters["api_key"], + parameters["region"], + parameters["namespace"], + "PATCH", + {"pinning": None}, + parameters["api_base_url"] or None, + ) + meta = wait_for_namespace_pinning( + parameters["api_key"], + parameters["region"], + parameters["namespace"], + None, + parameters["api_base_url"] or None, + parameters["pin_timeout"], + ) + click.echo(f"TurboPuffer namespace unpinned: {parameters['namespace']} pinning={meta.get('pinning')}") diff --git a/vectordb_bench/backend/clients/turbopuffer/config.py b/vectordb_bench/backend/clients/turbopuffer/config.py index 88e797351..ed28675f8 100644 --- a/vectordb_bench/backend/clients/turbopuffer/config.py +++ b/vectordb_bench/backend/clients/turbopuffer/config.py @@ -1,13 +1,27 @@ +from enum import StrEnum + from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, MetricType +class TurboPufferMultitenantWarmupPolicy(StrEnum): + NONE = "none" + ALL = "all" + + class TurboPufferConfig(DBConfig): api_key: SecretStr region: str api_base_url: str | None = None namespace: str = "vdbbench_test" + multitenant_namespace_prefix: str = "vdbbench_mt_" + scalar_payload_label_field: str = "label" + pin_namespace: bool = False + pin_namespace_requested: bool = False + pin_replicas: int = 1 + pin_timeout: int = 45 * 60 + pin_target_namespace_count: int = 0 def to_dict(self) -> dict: return { @@ -15,6 +29,13 @@ def to_dict(self) -> dict: "region": self.region, "api_base_url": self.api_base_url, "namespace": self.namespace, + "multitenant_namespace_prefix": self.multitenant_namespace_prefix, + "scalar_payload_label_field": self.scalar_payload_label_field, + "pin_namespace": self.pin_namespace, + "pin_namespace_requested": self.pin_namespace_requested, + "pin_replicas": self.pin_replicas, + "pin_timeout": self.pin_timeout, + "pin_target_namespace_count": self.pin_target_namespace_count, } @@ -22,6 +43,8 @@ class TurboPufferIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None use_multi_ns_for_filter: bool = False time_wait_warmup: int = 60 * 1 # 1min + disable_backpressure: bool = False + multitenant_warmup_policy: TurboPufferMultitenantWarmupPolicy = TurboPufferMultitenantWarmupPolicy.NONE def parse_metric(self) -> str: if self.metric_type == MetricType.COSINE: diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py index 6de0df21d..e28e38ae3 100644 --- a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -3,15 +3,81 @@ import logging import time from contextlib import contextmanager +from json import dumps, loads +from typing import Any +from urllib.error import HTTPError +from urllib.parse import quote +from urllib.request import Request, urlopen import turbopuffer as tpuf -from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig +from vectordb_bench.backend.clients.turbopuffer.config import ( + TurboPufferIndexConfig, + TurboPufferMultitenantWarmupPolicy, +) from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile -from ..api import VectorDB +from ..api import PartialInsertError, VectorDB log = logging.getLogger(__name__) +TURBOPUFFER_SEARCHABLE_UNINDEXED_BYTES = 2 * 1024 * 1024 * 1024 +PINNING_POLL_INTERVAL = 10 +PINNING_TIMEOUT = 45 * 60 + + +def namespace_metadata_request( + api_key: str, + region: str, + namespace: str, + method: str, + payload: dict[str, Any] | None = None, + api_base_url: str | None = None, +) -> dict: + base_url = api_base_url or f"https://{region}.turbopuffer.com" + url = f"{base_url.rstrip('/')}/v1/namespaces/{quote(namespace, safe='')}/metadata" + req = Request( # noqa: S310 + url, + data=dumps(payload).encode() if payload is not None else None, + method=method, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + ) + try: + with urlopen(req, timeout=60) as resp: # noqa: S310 + return loads(resp.read().decode() or "{}") + except HTTPError as e: + detail = e.read().decode(errors="replace") + msg = f"Failed to update TurboPuffer namespace metadata: {e.code} {detail}" + raise RuntimeError(msg) from e + + +def wait_for_namespace_pinning( + api_key: str, + region: str, + namespace: str, + replicas: int | None, + api_base_url: str | None = None, + timeout: int = PINNING_TIMEOUT, +) -> dict: + deadline = time.monotonic() + timeout + while True: + meta = namespace_metadata_request(api_key, region, namespace, "GET", api_base_url=api_base_url) + pinning = meta.get("pinning") + if replicas is None: + if pinning is None: + return meta + else: + status = pinning.get("status", {}) if isinstance(pinning, dict) else {} + if pinning and pinning.get("replicas") == replicas and status.get("ready_replicas") == replicas: + return meta + if time.monotonic() >= deadline: + msg = f"Timed out waiting for TurboPuffer pinning state on namespace {namespace}" + raise TimeoutError(msg) + log.info("Waiting for TurboPuffer pinning state on %s: %s", namespace, pinning) + time.sleep(PINNING_POLL_INTERVAL) class TurboPuffer(VectorDB): @@ -34,23 +100,40 @@ def __init__( self.region = db_config.get("region", "") self.api_base_url = db_config.get("api_base_url") self.namespace = db_config.get("namespace", "") + self.multitenant_namespace_prefix = db_config.get("multitenant_namespace_prefix", "vdbbench_mt_") + self.multitenant_tenant_labels: list[str] = kwargs.get("multitenant_tenant_labels", []) + self._multitenant_touched_tenants: set[str] = set() + self._ns_cache = {} + self.pin_namespace = db_config.get("pin_namespace", False) + self.pin_replicas = db_config.get("pin_replicas", 1) + self.pin_timeout = db_config.get("pin_timeout", PINNING_TIMEOUT) + self._pinning_applied = False self.db_case_config = db_case_config self.metric = db_case_config.parse_metric() self._vector_field = "vector" self._scalar_id_field = "id" self._scalar_label_field = "label" + self._scalar_payload_label_field = db_config.get("scalar_payload_label_field", self._scalar_label_field) self.with_scalar_labels = with_scalar_labels + self.expr = None if drop_old: - log.info(f"Drop old. delete the namespace: {self.namespace}") tmp_client = self._create_client() - ns = tmp_client.namespace(self.namespace) - try: - ns.delete_all() - except Exception as e: - log.warning(f"Failed to delete all. Error: {e}") + if self.multitenant_tenant_labels: + for tenant in self.multitenant_tenant_labels: + try: + tmp_client.namespace(self._namespace_name_for_tenant(tenant)).delete_all() + except Exception as e: + log.warning(f"Failed to delete multitenant namespace {tenant}. Error: {e}") + else: + log.info(f"Drop old. delete the namespace: {self.namespace}") + ns = tmp_client.namespace(self.namespace) + try: + ns.delete_all() + except Exception as e: + log.warning(f"Failed to delete all. Error: {e}") tmp_client = None def _create_client(self) -> tpuf.Turbopuffer: @@ -59,61 +142,227 @@ def _create_client(self) -> tpuf.Turbopuffer: client_kwargs["base_url"] = self.api_base_url return tpuf.Turbopuffer(**client_kwargs) + def _apply_namespace_pinning(self): + if not self.pin_namespace or self._pinning_applied: + return + for namespace in self._target_namespaces_for_pinning(): + namespace_metadata_request( + self.api_key, + self.region, + namespace, + "PATCH", + {"pinning": {"replicas": self.pin_replicas}}, + self.api_base_url, + ) + meta = wait_for_namespace_pinning( + self.api_key, + self.region, + namespace, + self.pin_replicas, + self.api_base_url, + self.pin_timeout, + ) + pinning = meta.get("pinning", {}) + status = pinning.get("status", {}) if isinstance(pinning, dict) else {} + log.info( + "TurboPuffer pinning requested for %s: replicas=%s ready_replicas=%s", + namespace, + pinning.get("replicas", self.pin_replicas) if isinstance(pinning, dict) else self.pin_replicas, + status.get("ready_replicas"), + ) + self._pinning_applied = True + + def _target_namespaces_for_pinning(self) -> list[str]: + if self.multitenant_tenant_labels: + return [self._namespace_name_for_tenant(tenant) for tenant in self.multitenant_tenant_labels] + return [self.namespace] + @contextmanager def init(self): self.client = self._create_client() + self._ns_cache = {} self.ns = self.client.namespace(self.namespace) + self._apply_namespace_pinning() yield + def supports_multitenant(self) -> bool: + return True + + def set_multitenant_context(self, tenant_labels: list[str]) -> None: + self.multitenant_tenant_labels = tenant_labels + + def _namespace_name_for_tenant(self, tenant: str | None) -> str: + if tenant is None: + return self.namespace + return f"{self.multitenant_namespace_prefix}{tenant}" + + def _namespace_for_tenant(self, tenant: str | None): + name = self._namespace_name_for_tenant(tenant) + ns = self._ns_cache.get(name) + if ns is None: + ns = self.client.namespace(name) + self._ns_cache[name] = ns + return ns + def optimize(self, data_size: int | None = None): # turbopuffer responds to the request # once the cache warming operation has been started. # It does not wait for the operation to complete, # which can take multiple minutes for large namespaces. - self.ns.hint_cache_warm() + warmed_namespaces = self._warmup_target_namespaces() + for namespace in warmed_namespaces: + self._namespace_for_tenant(namespace).hint_cache_warm() + if not warmed_namespaces: + log.info("TurboPuffer cache warmup skipped") + return log.info(f"warming up but no api waiting for complete. just sleep {self.db_case_config.time_wait_warmup}s") time.sleep(self.db_case_config.time_wait_warmup) + def _warmup_target_namespaces(self) -> list[str | None]: + if not self.multitenant_tenant_labels: + return [None] + policy = getattr( + self.db_case_config, + "multitenant_warmup_policy", + TurboPufferMultitenantWarmupPolicy.NONE, + ) + if policy == TurboPufferMultitenantWarmupPolicy.ALL: + return self.multitenant_tenant_labels + return [] + def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: + vectors = [embedding.tolist() if hasattr(embedding, "tolist") else embedding for embedding in embeddings] + if tenant_labels_data is not None: + inserted = 0 + successful_tenants: dict[str, int] = {} + for tenant in sorted(set(tenant_labels_data)): + self._multitenant_touched_tenants.add(tenant) + idxs = [i for i, label in enumerate(tenant_labels_data) if label == tenant] + try: + upsert_columns = { + self._scalar_id_field: [metadata[i] for i in idxs], + self._vector_field: [vectors[i] for i in idxs], + } + if self.with_scalar_labels: + upsert_columns[self._scalar_payload_label_field] = [labels_data[i] for i in idxs] + self._namespace_for_tenant(tenant).write( + upsert_columns=upsert_columns, + distance_metric=self.metric, + disable_backpressure=self.db_case_config.disable_backpressure, + ) + except Exception as e: + msg = ( + "TurboPuffer multitenant insert failed for " + f"tenant={tenant} after writing {inserted} rows; " + f"successful_tenants={successful_tenants}; " + f"failed_tenant_count={len(idxs)}" + ) + err = PartialInsertError( + msg, + inserted_count=inserted, + successful_tenants=successful_tenants, + failed_tenant=tenant, + failed_tenant_count=len(idxs), + cause=e, + ) + log.warning(f"Failed to insert. Error: {err}") + return inserted, err + inserted += len(idxs) + successful_tenants[tenant] = len(idxs) + return inserted, None try: if self.with_scalar_labels: self.ns.write( upsert_columns={ self._scalar_id_field: metadata, - self._vector_field: embeddings, - self._scalar_label_field: labels_data, + self._vector_field: vectors, + self._scalar_payload_label_field: labels_data, }, distance_metric=self.metric, + disable_backpressure=self.db_case_config.disable_backpressure, ) else: self.ns.write( upsert_columns={ self._scalar_id_field: metadata, - self._vector_field: embeddings, + self._vector_field: vectors, }, distance_metric=self.metric, + disable_backpressure=self.db_case_config.disable_backpressure, ) except Exception as e: log.warning(f"Failed to insert. Error: {e}") + return 0, e return len(embeddings), None + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile in { + PayloadProfile.IDS_ONLY, + PayloadProfile.SCALAR_LABEL, + PayloadProfile.VECTOR, + } + + def poll_insert_readiness(self, expected_count: int) -> dict: + if getattr(self, "multitenant_tenant_labels", []): + unindexed_by_tenant = {} + tenant_labels = ( + sorted(getattr(self, "_multitenant_touched_tenants", set())) or self.multitenant_tenant_labels + ) + for tenant in tenant_labels: + metadata = self._namespace_for_tenant(tenant).metadata() + if not isinstance(metadata, dict): + metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else vars(metadata) + index = metadata.get("index", {}) + if not isinstance(index, dict): + index = index.model_dump() if hasattr(index, "model_dump") else vars(index) + unindexed_by_tenant[tenant] = metadata.get("unindexed_bytes", index.get("unindexed_bytes", 0)) + max_unindexed_bytes = max(unindexed_by_tenant.values(), default=0) + return { + "fully_searchable": max_unindexed_bytes <= TURBOPUFFER_SEARCHABLE_UNINDEXED_BYTES, + "fully_indexed": max_unindexed_bytes == 0, + "additional_parameters": { + "disable_backpressure": self.db_case_config.disable_backpressure, + "max_unindexed_bytes": max_unindexed_bytes, + }, + } + metadata = self.ns.metadata() + if not isinstance(metadata, dict): + metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else vars(metadata) + index = metadata.get("index", {}) + if not isinstance(index, dict): + index = index.model_dump() if hasattr(index, "model_dump") else vars(index) + unindexed_bytes = metadata.get("unindexed_bytes", index.get("unindexed_bytes", 0)) + return { + "fully_searchable": unindexed_bytes <= TURBOPUFFER_SEARCHABLE_UNINDEXED_BYTES, + "fully_indexed": unindexed_bytes == 0, + "additional_parameters": {"disable_backpressure": self.db_case_config.disable_backpressure}, + } + def search_embedding( self, query: list[float], k: int = 100, timeout: int | None = None, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant: str | None = None, ) -> list[int]: - res = self.ns.query( - rank_by=("vector", "ANN", query), - top_k=k, - filters=self.expr, - ) + query_kwargs = { + "rank_by": ("vector", "ANN", query), + "top_k": k, + "filters": self.expr, + } + if payload_profile == PayloadProfile.VECTOR: + query_kwargs["include_attributes"] = [self._vector_field] + elif payload_profile == PayloadProfile.SCALAR_LABEL: + query_kwargs["include_attributes"] = [self._scalar_payload_label_field] + res = self._namespace_for_tenant(tenant).query(**query_kwargs) return [int(row.id) for row in res.rows] if res.rows is not None else [] def prepare_filter(self, filters: Filter): @@ -122,7 +371,7 @@ def prepare_filter(self, filters: Filter): elif filters.type == FilterOp.NumGE: self.expr = (self._scalar_id_field, "Gte", filters.int_value) elif filters.type == FilterOp.StrEqual: - self.expr = (self._scalar_label_field, "Eq", filters.label_value) + self.expr = (self._scalar_payload_label_field, "Eq", filters.label_value) else: msg = f"Not support Filter for TurboPuffer - {filters}" raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/zilliz_cloud/cli.py b/vectordb_bench/backend/clients/zilliz_cloud/cli.py index a8d177ee5..030a00661 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/cli.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/cli.py @@ -13,6 +13,13 @@ ) +def _use_partition_key(parameters: dict) -> bool: + explicit = parameters.get("use_partition_key") + if explicit is not None: + return explicit + return parameters.get("case_type") == "CloudMultiTenantSearchCase" + + class ZillizTypedDict(CommonTypedDict): uri: Annotated[ str, @@ -20,7 +27,7 @@ class ZillizTypedDict(CommonTypedDict): ] user_name: Annotated[ str, - click.option("--user-name", type=str, help="Db username", required=True), + click.option("--user-name", type=str, help="Db username", default=""), ] password: Annotated[ str, @@ -32,6 +39,16 @@ class ZillizTypedDict(CommonTypedDict): show_default="$ZILLIZ_PASSWORD", ), ] + token: Annotated[ + str, + click.option( + "--token", + type=str, + help="Zilliz API token", + default=lambda: os.environ.get("ZILLIZ_TOKEN", ""), + show_default="$ZILLIZ_TOKEN", + ), + ] level: Annotated[ str, click.option("--level", type=str, help="Zilliz index level", required=False), @@ -58,6 +75,17 @@ class ZillizTypedDict(CommonTypedDict): show_default=True, ), ] + use_partition_key: Annotated[ + bool | None, + click.option( + "--use-partition-key/--no-use-partition-key", + default=None, + help=( + "Use the Zilliz Cloud partition key on the label field. " + "Defaults to enabled for CloudMultiTenantSearchCase and disabled otherwise." + ), + ), + ] @cli.command() @@ -72,12 +100,14 @@ def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + token=SecretStr(parameters["token"]), num_shards=parameters["num_shards"], collection_name=parameters["collection_name"], ), db_case_config=AutoIndexConfig( level=int(parameters["level"]) if parameters["level"] else 1, num_shards=parameters["num_shards"], + use_partition_key=_use_partition_key(parameters), ), **parameters, ) diff --git a/vectordb_bench/backend/clients/zilliz_cloud/config.py b/vectordb_bench/backend/clients/zilliz_cloud/config.py index f0b3fd000..8ab45caa2 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/config.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/config.py @@ -6,16 +6,22 @@ class ZillizCloudConfig(DBConfig): uri: SecretStr - user: str - password: SecretStr + user: str = "" + password: SecretStr = SecretStr("") + token: SecretStr = SecretStr("") num_shards: int = 1 collection_name: str = "ZillizCloudVDBBench" + @staticmethod + def common_long_configs() -> list[str]: + return [*DBConfig.common_long_configs(), "user", "password", "token"] + def to_dict(self) -> dict: return { "uri": self.uri.get_secret_value(), "user": self.user, "password": self.password.get_secret_value(), + "token": self.token.get_secret_value(), "num_shards": self.num_shards, "collection_name": self.collection_name, } diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 94216532f..c249f91c9 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -138,6 +138,8 @@ class LAION(BaseDataset): metric_type: MetricType = MetricType.L2 use_shuffled: bool = False with_gt: bool = True + with_scalar_labels: bool = True + scalar_label_percentages: list[float] = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] _size_label: ClassVar[dict] = { 100_000_000: SizeLabel(100_000_000, "LARGE", 100), } @@ -338,11 +340,16 @@ def data_dir(self) -> pathlib.Path: def __iter__(self): return DataSetIterator(self) + def iter_batches(self, batch_size: int): + return DataSetIterator(self, batch_size=batch_size) + # TODO passing use_shuffle from outside def prepare( self, source: DatasetSource = DatasetSource.S3, filters: Filter = non_filter, + with_train_files: bool = True, + with_scalar_labels: bool = False, ) -> bool: """Download the dataset from DatasetSource url = f"{source}/{self.data.dir_name}" @@ -356,7 +363,7 @@ def prepare( bool: whether the dataset is successfully prepared """ - self.train_files = self.data.train_files + self.train_files = self.data.train_files if with_train_files else [] gt_file, test_file = None, None if self.data.with_gt: gt_file, test_file = filters.groundtruth_file, self.data.test_file @@ -373,12 +380,10 @@ def prepare( local_ds_root=self.data_dir, ) + needs_scalar_labels = filters.type == FilterOp.StrEqual or with_scalar_labels + # read scalar_labels_file if separated - if ( - filters.type == FilterOp.StrEqual - and self.data.with_scalar_labels - and self.data.scalar_labels_file_separated - ): + if needs_scalar_labels and self.data.with_scalar_labels and self.data.scalar_labels_file_separated: self.scalar_labels = self._read_file(self.data.scalar_labels_file) if gt_file is not None and test_file is not None: @@ -401,8 +406,9 @@ def _read_file(self, file_name: str) -> pl.DataFrame: class DataSetIterator: - def __init__(self, dataset: DatasetManager): + def __init__(self, dataset: DatasetManager, batch_size: int = config.NUM_PER_BATCH): self._ds = dataset + self._batch_size = batch_size self._idx = 0 # file number self._cur = None self._sub_idx = [0 for i in range(len(self._ds.train_files))] # iter num for each file @@ -428,7 +434,7 @@ def _get_iter(self, file_name: str): msg = f"No such file: {p}" log.warning(msg) raise IndexError(msg) - return ParquetFile(p, memory_map=True, pre_buffer=True).iter_batches(config.NUM_PER_BATCH) + return ParquetFile(p, memory_map=True, pre_buffer=True).iter_batches(self._batch_size) def __next__(self) -> pd.DataFrame: """return the data in the next file of the training list""" @@ -478,6 +484,7 @@ class DatasetWithSizeType(Enum): CohereSmall = "Small Cohere (768dim, 100K)" CohereMedium = "Medium Cohere (768dim, 1M)" CohereLarge = "Large Cohere (768dim, 10M)" + LAIONLarge = "Large LAION (768dim, 100M)" BioasqMedium = "Medium Bioasq (1024dim, 1M)" BioasqLarge = "Large Bioasq (1024dim, 10M)" OpenAISmall = "Small OpenAI (1536dim, 50K)" @@ -491,6 +498,8 @@ def get_manager(self) -> DatasetManager: return DatasetWithSizeMap.get(self) def get_load_timeout(self) -> float: + if self is DatasetWithSizeType.LAIONLarge: + return config.LOAD_TIMEOUT_768D_100M if "small" in self.value.lower(): return config.LOAD_TIMEOUT_768D_100K if "medium" in self.value.lower(): @@ -501,6 +510,8 @@ def get_load_timeout(self) -> float: raise KeyError(msg) def get_optimize_timeout(self) -> float: + if self is DatasetWithSizeType.LAIONLarge: + return config.OPTIMIZE_TIMEOUT_768D_100M if "small" in self.value.lower(): return config.OPTIMIZE_TIMEOUT_768D_100K if "medium" in self.value.lower(): @@ -514,6 +525,7 @@ def get_optimize_timeout(self) -> float: DatasetWithSizeType.CohereSmall: Dataset.COHERE.manager(100_000), DatasetWithSizeType.CohereMedium: Dataset.COHERE.manager(1_000_000), DatasetWithSizeType.CohereLarge: Dataset.COHERE.manager(10_000_000), + DatasetWithSizeType.LAIONLarge: Dataset.LAION.manager(100_000_000), DatasetWithSizeType.BioasqMedium: Dataset.BIOASQ.manager(1_000_000), DatasetWithSizeType.BioasqLarge: Dataset.BIOASQ.manager(10_000_000), DatasetWithSizeType.OpenAISmall: Dataset.OPENAI.manager(50_000), diff --git a/vectordb_bench/backend/payload.py b/vectordb_bench/backend/payload.py new file mode 100644 index 000000000..49050c85f --- /dev/null +++ b/vectordb_bench/backend/payload.py @@ -0,0 +1,21 @@ +from enum import StrEnum + + +class PayloadProfile(StrEnum): + IDS_ONLY = "ids_only" + VECTOR = "vector" + SCALAR_LABEL = "scalar_label" + + def estimated_bytes_per_query(self, *, k: int, dim: int) -> int: + # Approximate payload size used for cloud leaderboard cost expansion. + # ID + distance is about 20 bytes per hit; vector is float32. + id_distance_bytes = 20 + scalar_label_bytes = 16 + if self == PayloadProfile.IDS_ONLY: + return k * id_distance_bytes + if self == PayloadProfile.VECTOR: + return k * (id_distance_bytes + dim * 4) + if self == PayloadProfile.SCALAR_LABEL: + return k * (id_distance_bytes + scalar_label_bytes) + msg = f"Unsupported payload profile: {self}" + raise ValueError(msg) diff --git a/vectordb_bench/backend/runner/__init__.py b/vectordb_bench/backend/runner/__init__.py index d56fe0ff8..ddee99554 100644 --- a/vectordb_bench/backend/runner/__init__.py +++ b/vectordb_bench/backend/runner/__init__.py @@ -1,9 +1,11 @@ +from .cold_warm_runner import ColdWarmSearchRunner from .concurrent_runner import ConcurrentInsertRunner from .mp_runner import MultiProcessingSearchRunner from .read_write_runner import ReadWriteRunner from .serial_runner import SerialInsertRunner, SerialSearchRunner __all__ = [ + "ColdWarmSearchRunner", "ConcurrentInsertRunner", "MultiProcessingSearchRunner", "ReadWriteRunner", diff --git a/vectordb_bench/backend/runner/cold_warm_runner.py b/vectordb_bench/backend/runner/cold_warm_runner.py new file mode 100644 index 000000000..51bde6ff8 --- /dev/null +++ b/vectordb_bench/backend/runner/cold_warm_runner.py @@ -0,0 +1,120 @@ +import logging +import time + +import numpy as np + +from vectordb_bench.backend.filter import Filter, non_filter +from vectordb_bench.backend.payload import PayloadProfile + +from ... import config +from ..clients import api + +log = logging.getLogger(__name__) + + +class ColdWarmSearchRunner: + def __init__( + self, + db: api.VectorDB, + test_data: list[list[float]], + k: int = 100, + filters: Filter = non_filter, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + query_count: int = 1000, + ): + if query_count <= 0: + msg = "query_count must be positive" + raise ValueError(msg) + if len(test_data) < query_count: + msg = f"query_count={query_count} exceeds test_data size={len(test_data)}" + raise ValueError(msg) + + self.db = db + self.k = k + self.filters = filters + self.payload_profile = payload_profile + self.query_count = query_count + if not self.db.supports_payload_profile(self.payload_profile): + msg = f"{self.db.name} does not support payload_profile={self.payload_profile.value}" + raise NotImplementedError(msg) + + self.test_data = [ + query.tolist() if isinstance(query, np.ndarray) else query for query in test_data[:query_count] + ] + + def _search_embedding(self, emb: list[float]) -> list[int]: + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile) + + def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: + try: + results = self._search_embedding(emb) + except Exception as e: + log.warning(f"Cold/warm search failed, retry_idx={retry_idx}, Exception: {e}") + if retry_idx < config.MAX_SEARCH_RETRY: + return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + + msg = f"Cold/warm search failed and retried more than {config.MAX_SEARCH_RETRY} times" + raise RuntimeError(msg) from e + + return results + + @staticmethod + def _latency_stats(latencies: list[float]) -> dict[str, float]: + return { + "first_query_latency": round(float(latencies[0]), 4), + "p99_latency": round(float(np.percentile(latencies, 99)), 4), + "p95_latency": round(float(np.percentile(latencies, 95)), 4), + "avg_latency": round(float(np.mean(latencies)), 4), + } + + @staticmethod + def _safe_ratio(numerator: float, denominator: float) -> float: + if denominator == 0: + return 0.0 + return round(float(numerator / denominator), 4) + + def _ratio_stats(self, cold_stats: dict[str, float], warm_stats: dict[str, float]) -> dict[str, float]: + return { + "first_query_latency_ratio": self._safe_ratio( + cold_stats["first_query_latency"], + warm_stats["first_query_latency"], + ), + "p99_latency_ratio": self._safe_ratio(cold_stats["p99_latency"], warm_stats["p99_latency"]), + "p95_latency_ratio": self._safe_ratio(cold_stats["p95_latency"], warm_stats["p95_latency"]), + "avg_latency_ratio": self._safe_ratio(cold_stats["avg_latency"], warm_stats["avg_latency"]), + } + + def _run_pass(self, pass_name: str) -> dict[str, float]: + latencies = [] + for emb in self.test_data: + start = time.perf_counter() + self._get_db_search_res(emb) + latencies.append(time.perf_counter() - start) + + if len(latencies) % 100 == 0: + log.debug(f"{pass_name} search_count={len(latencies):3}, latest_latency={latencies[-1]}") + + stats = self._latency_stats(latencies) + log.info( + f"{pass_name} search pass: " + f"queries={len(latencies)}, " + f"first_query_latency={stats['first_query_latency']}, " + f"avg_latency={stats['avg_latency']}, " + f"p99={stats['p99_latency']}, " + f"p95={stats['p95_latency']}" + ) + return stats + + def run(self) -> dict[str, dict[str, float]]: + with self.db.init(): + self.db.prepare_filter(self.filters) + cold_stats = self._run_pass("cold") + warm_stats = self._run_pass("warm") + + return { + "cold_stats": cold_stats, + "warm_stats": warm_stats, + "cold_warm_ratio": self._ratio_stats(cold_stats, warm_stats), + } diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py index 7c8aeb24f..650795535 100644 --- a/vectordb_bench/backend/runner/concurrent_runner.py +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -64,6 +64,10 @@ def __init__( timeout: float | None = None, max_workers: int | None = None, backend: ExecutorBackend = ExecutorBackend.THREADING, + batch_size: int = config.NUM_PER_BATCH, + duration: float | None = None, + with_scalar_labels: bool = False, + tenant_case=None, # noqa: ANN001 ): self.timeout = timeout if isinstance(timeout, int | float) else None self.dataset: DatasetManager = dataset @@ -71,6 +75,10 @@ def __init__( self.normalize = normalize self.filters = filters self.backend = backend + self.batch_size = batch_size + self.duration = duration if isinstance(duration, int | float) else None + self.with_scalar_labels = with_scalar_labels + self.tenant_case = tenant_case effective_workers = max_workers or min(mp.cpu_count(), 4) if not db.thread_safe: @@ -87,6 +95,7 @@ def __getstate__(self): state = self.__dict__.copy() state.pop("_iter_lock", None) state.pop("_dataset_iter", None) + state.pop("_stop_event", None) return state def _create_executor(self) -> TaskExecutor: @@ -109,20 +118,34 @@ def _insert_batch_with_retry( embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, retry_idx: int = 0, ) -> int: """Insert a single batch with retry logic. Returns inserted count.""" - insert_count, error = db.insert_embeddings( - embeddings=embeddings, - metadata=metadata, - labels_data=labels_data, - ) + insert_kwargs = { + "embeddings": embeddings, + "metadata": metadata, + "labels_data": labels_data, + } + if tenant_labels_data is not None: + insert_kwargs["tenant_labels_data"] = tenant_labels_data + insert_count, error = db.insert_embeddings(**insert_kwargs) if error is not None: log.warning(f"Insert failed, try_idx={retry_idx}, Exception: {error}") + if getattr(error, "non_retryable", False): + msg = f"Non-retryable insert failure after {insert_count} inserted rows: {error}" + raise RuntimeError(msg) from error retry_idx += 1 if retry_idx <= config.MAX_INSERT_RETRY: time.sleep(retry_idx) - return self._insert_batch_with_retry(db, embeddings, metadata, labels_data, retry_idx) + return self._insert_batch_with_retry( + db, + embeddings, + metadata, + labels_data, + tenant_labels_data, + retry_idx, + ) msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" raise RuntimeError(msg) return insert_count @@ -132,18 +155,27 @@ def _worker_insert( embeddings: list[list[float]], metadata: list[int], labels_data: list[str] | None = None, + tenant_labels_data: list[str] | None = None, ) -> int: """Worker function: insert a batch with retry.""" db = self._get_thread_db() - return self._insert_batch_with_retry(db, embeddings, metadata, labels_data) + return self._insert_batch_with_retry(db, embeddings, metadata, labels_data, tenant_labels_data) - def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None] | None: + def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None, list[str] | None] | None: """Pull the next batch from the shared dataset iterator. Thread-safe: only one thread reads from the iterator at a time. Returns None when the iterator is exhausted. """ + stop_event = getattr(self, "_stop_event", None) + if stop_event is not None and stop_event.is_set(): + return None + if self._deadline is not None and time.perf_counter() >= self._deadline: + return None with self._iter_lock: + stop_event = getattr(self, "_stop_event", None) + if stop_event is not None and stop_event.is_set(): + return None try: data_df = next(self._dataset_iter) except StopIteration: @@ -158,35 +190,48 @@ def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None] | del emb_np labels_data = None - if self.filters.type == FilterOp.StrEqual: + if self.filters.type == FilterOp.StrEqual or self.with_scalar_labels: + label_field = self.filters.label_field if self.filters.type == FilterOp.StrEqual else "labels" if self.dataset.data.scalar_labels_file_separated: - labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() + labels_data = self.dataset.scalar_labels[label_field][all_metadata].to_list() else: - labels_data = data_df[self.filters.label_field].tolist() + labels_data = data_df[label_field].tolist() + + tenant_labels_data = None + if self.tenant_case is not None and getattr(self.tenant_case, "is_multitenant", False): + tenant_labels_data = self.tenant_case.tenant_labels_for_ids(all_metadata) - return all_embeddings, all_metadata, labels_data + return all_embeddings, all_metadata, labels_data, tenant_labels_data def _worker_loop(self) -> int: """Worker loop: pull batches from the shared iterator and insert them.""" total = 0 - while True: - batch = self._next_batch() - if batch is None: - break - embeddings, metadata, labels_data = batch - total += self._worker_insert(embeddings, metadata, labels_data) + try: + while True: + batch = self._next_batch() + if batch is None: + break + embeddings, metadata, labels_data, tenant_labels_data = batch + total += self._worker_insert(embeddings, metadata, labels_data, tenant_labels_data) + except Exception: + stop_event = getattr(self, "_stop_event", None) + if stop_event is not None: + stop_event.set() + raise return total def task(self) -> int: """Insert entire dataset using concurrent executor. Runs in subprocess.""" count = 0 self._iter_lock = threading.Lock() - self._dataset_iter = iter(self.dataset) + self._stop_event = threading.Event() + self._deadline = None if self.duration is None else time.perf_counter() + self.duration + self._dataset_iter = self.dataset.iter_batches(self.batch_size) with self.db.init(): log.info( f"({mp.current_process().name:16}) Start concurrent insert, " - f"batch_size={config.NUM_PER_BATCH}, max_workers={self.max_workers}" + f"batch_size={self.batch_size}, max_workers={self.max_workers}" ) start = time.perf_counter() diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index b7823af37..bb867b3f1 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -12,6 +12,7 @@ from hdrh.histogram import HdrHistogram from vectordb_bench.backend.filter import Filter, non_filter +from vectordb_bench.backend.payload import PayloadProfile from ... import config from ...models import ConcurrencySlotTimeoutError @@ -45,10 +46,17 @@ def __init__( concurrencies: Iterable[int] = config.NUM_CONCURRENCY, duration: int = config.CONCURRENCY_DURATION, concurrency_timeout: int = config.CONCURRENCY_TIMEOUT, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant_labels: list[str] | None = None, ): self.db = db self.k = k self.filters = filters + self.payload_profile = payload_profile + self.tenant_labels = tenant_labels or [] + if not self.db.supports_payload_profile(self.payload_profile): + msg = f"{self.db.name} does not support payload_profile={self.payload_profile.value}" + raise NotImplementedError(msg) self.concurrencies = concurrencies self.duration = duration self.concurrency_timeout = concurrency_timeout @@ -56,6 +64,15 @@ def __init__( self.test_data = test_data log.debug(f"test dataset columns: {len(test_data)}") + def _search_embedding(self, emb: list[float], tenant: str | None = None) -> list[int]: + if tenant is None: + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile) + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k, tenant=tenant) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile, tenant=tenant) + def search( self, test_data: list[list[float]], @@ -75,6 +92,7 @@ def search( with self.db.init(): self.db.prepare_filter(self.filters) num, idx = len(test_data), random.randint(0, len(test_data) - 1) + tenant_rng = random.Random(mp.current_process().pid or 0) start_time = time.perf_counter() count = 0 @@ -82,7 +100,12 @@ def search( while time.perf_counter() < start_time + self.duration: s = time.perf_counter() try: - self.db.search_embedding(test_data[idx], self.k) + tenant = ( + self.tenant_labels[tenant_rng.randrange(len(self.tenant_labels))] + if self.tenant_labels + else None + ) + self._search_embedding(test_data[idx], tenant=tenant) count += 1 latencies.append(time.perf_counter() - s) except Exception as e: @@ -341,7 +364,7 @@ def search_by_dur( while time.perf_counter() < start_time + dur: s = time.perf_counter() try: - self.db.search_embedding(test_data[idx], self.k) + self._search_embedding(test_data[idx]) success_count += 1 latency_us = int((time.perf_counter() - s) * US_TO_SECONDS) histogram.record_value(min(latency_us, HDR_HISTOGRAM_MAX_US)) diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index be0c6322d..3fc37e0ce 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -2,6 +2,7 @@ import logging import math import multiprocessing as mp +import random import time import traceback @@ -9,6 +10,7 @@ from vectordb_bench.backend.dataset import DatasetManager from vectordb_bench.backend.filter import Filter, non_filter +from vectordb_bench.backend.payload import PayloadProfile from ... import config from ...metric import calc_ndcg, calc_recall, get_ideal_dcg @@ -130,10 +132,19 @@ def __init__( ground_truth: list[list[int]], k: int = 100, filters: Filter = non_filter, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + tenant_labels: list[str] | None = None, + measure_recall: bool = True, ): self.db = db self.k = k self.filters = filters + self.payload_profile = payload_profile + self.tenant_labels = tenant_labels or [] + self.measure_recall = measure_recall + if not self.db.supports_payload_profile(self.payload_profile): + msg = f"{self.db.name} does not support payload_profile={self.payload_profile.value}" + raise NotImplementedError(msg) if isinstance(test_data[0], np.ndarray): self.test_data = [query.tolist() for query in test_data] @@ -141,13 +152,22 @@ def __init__( self.test_data = test_data self.ground_truth = ground_truth - def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: + def _search_embedding(self, emb: list[float], tenant: str | None = None) -> list[int]: + if tenant is None: + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile) + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self.db.search_embedding(emb, self.k, tenant=tenant) + return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile, tenant=tenant) + + def _get_db_search_res(self, emb: list[float], tenant: str | None = None, retry_idx: int = 0) -> list[int]: try: - results = self.db.search_embedding(emb, self.k) + results = self._search_embedding(emb, tenant=tenant) except Exception as e: log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") if retry_idx < config.MAX_SEARCH_RETRY: - return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + return self._get_db_search_res(emb=emb, tenant=tenant, retry_idx=retry_idx + 1) msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" raise RuntimeError(msg) from e @@ -162,20 +182,24 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa ideal_dcg = get_ideal_dcg(self.k) log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {len(ground_truth)}") + log.debug(f"ground truth size: {len(ground_truth) if ground_truth is not None else 0}") latencies, recalls, ndcgs = [], [], [] + tenant_rng = random.Random(0) for idx, emb in enumerate(test_data): + tenant = ( + self.tenant_labels[tenant_rng.randrange(len(self.tenant_labels))] if self.tenant_labels else None + ) s = time.perf_counter() try: - results = self._get_db_search_res(emb) + results = self._get_db_search_res(emb, tenant=tenant) except Exception as e: log.warning(f"VectorDB search_embedding error: {e}") raise e from None latencies.append(time.perf_counter() - s) - if ground_truth is not None: + if self.measure_recall and ground_truth is not None: gt = ground_truth[idx] recalls.append(calc_recall(self.k, gt[: self.k], results)) ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 6b51d1277..3a65c4d04 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -2,6 +2,7 @@ import hashlib import logging import re +import time import traceback from enum import Enum, auto @@ -15,6 +16,7 @@ from .clients import DB, MetricType, api from .data_source import DatasetSource from .runner import ( + ColdWarmSearchRunner, ConcurrentInsertRunner, MultiProcessingSearchRunner, ReadWriteRunner, @@ -56,28 +58,86 @@ class CaseRunner(BaseModel): search_runner: MultiProcessingSearchRunner | None = None final_search_runner: MultiProcessingSearchRunner | None = None read_write_runner: ReadWriteRunner | None = None + cold_warm_search_runner: ColdWarmSearchRunner | None = None def __eq__(self, obj: any): if isinstance(obj, CaseRunner): - return ( - self.ca.label == CaseLabel.Performance - and self.config.db == obj.config.db - and self.config.db_case_config == obj.config.db_case_config - and self.ca.dataset == obj.ca.dataset - ) + key = self.load_reuse_key() + return key is not None and key == obj.load_reuse_key() return False def __hash__(self) -> int: """Hash method to maintain consistency with __eq__ method.""" - return hash( - ( - self.ca.label, - self.config.db, - self.config.db_case_config, - self.ca.dataset, - ) + return hash(self.load_reuse_key()) + + def load_reuse_key(self) -> tuple | None: + if self.ca.label != CaseLabel.Performance: + return None + return ( + self.config.db.value, + self._db_config_hash_key(), + self._db_case_config_hash_key(), + self._collection_name_hash_key(), + self._dataset_hash_key(), + self.ca.with_scalar_labels, + self.ca.is_multitenant, + self._multitenant_routing_hash_key(), ) + @classmethod + def _hashable_value(cls, value: object) -> object: + if isinstance(value, dict): + hashable = tuple(sorted((str(k), cls._hashable_value(v)) for k, v in value.items())) + elif isinstance(value, (list, tuple)): + hashable = tuple(cls._hashable_value(v) for v in value) + elif isinstance(value, (set, frozenset)): + hashable = tuple(sorted((cls._hashable_value(v) for v in value), key=repr)) + elif isinstance(value, Enum): + hashable = value.value + elif hasattr(value, "model_dump"): + hashable = cls._hashable_value(value.model_dump(mode="json")) + elif hasattr(value, "get_secret_value"): + hashable = value.get_secret_value() + else: + hashable = value + return hashable + + def _db_config_hash_key(self) -> object: + db_config = self.config.db_config + if hasattr(db_config, "to_dict"): + return self._hashable_value(db_config.to_dict()) + return self._hashable_value(db_config) + + def _db_case_config_hash_key(self) -> object: + return self._hashable_value(self.config.db_case_config) + + def _collection_name_hash_key(self) -> str | None: + return self._doris_collection_name() + + def _dataset_hash_key(self) -> object: + return self._hashable_value(self.ca.dataset.data) + + def _multitenant_routing_hash_key(self) -> tuple | None: + if not self.ca.is_multitenant: + return None + return ( + getattr(self.ca, "tenant_count", None), + getattr(self.ca, "tenant_prefix", None), + getattr(self.ca, "tenant_id_width", None), + getattr(self.ca, "tenant_distribution", None), + ) + + def _doris_collection_name(self) -> str | None: + if self.config.db != DB.Doris: + return None + case_type_name = self.config.case_config.case_id.name + base = f"{case_type_name.lower()}" + base = re.sub(r"[^a-z0-9_]+", "_", base).strip("_") + if len(base) > 63: + h = hashlib.md5(base.encode(), usedforsecurity=False).hexdigest()[:6] + base = f"{base[:(63-7)]}_{h}" + return base + def display(self) -> dict: c_dict = self.ca.dict( include={ @@ -108,17 +168,7 @@ def init_db(self, drop_old: bool = True) -> None: # Compose a compact, case-unique collection/table name for Doris to avoid cross-case interference collection_name = None try: - if self.config.db == DB.Doris: - # Primary identifier = case-type enum name from CLI (e.g., Performance768D10M) - case_type_name = self.config.case_config.case_id.name - base = f"{case_type_name.lower()}" - # Sanitize to [a-z0-9_] - base = re.sub(r"[^a-z0-9_]+", "_", base).strip("_") - # Cap to 63 chars; add short hash if truncated - if len(base) > 63: - h = hashlib.md5(base.encode(), usedforsecurity=False).hexdigest()[:6] - base = f"{base[:(63-7)]}_{h}" - collection_name = base + collection_name = self._doris_collection_name() except Exception: # If anything goes wrong, fall back silently; Doris will use its default name logic collection_name = None @@ -128,23 +178,66 @@ def init_db(self, drop_old: bool = True) -> None: if "collection_name" in db_config_dict and not collection_name: collection_name = db_config_dict.pop("collection_name") + extra_db_kwargs = {} + if collection_name: + extra_db_kwargs["collection_name"] = collection_name + if self.ca.is_multitenant: + extra_db_kwargs["multitenant_tenant_labels"] = self.ca.tenant_labels() + self.db = db_cls( dim=self.ca.dataset.data.dim, db_config=db_config_dict, db_case_config=self.config.db_case_config, drop_old=drop_old, with_scalar_labels=self.ca.with_scalar_labels, - **({"collection_name": collection_name} if collection_name else {}), + **extra_db_kwargs, ) def _pre_run(self, drop_old: bool = True): try: + self._validate_cloud_cold_latency_config(drop_old) + creates_multitenant_collection = ( + TaskStage.DROP_OLD in self.config.stages or TaskStage.LOAD in self.config.stages + ) + if ( + self.ca.is_multitenant + and self.config.db in {DB.Milvus, DB.ZillizCloud} + and creates_multitenant_collection + and not getattr(self.config.db_case_config, "use_partition_key", False) + ): + msg = "CloudMultiTenantSearchCase requires use_partition_key=True for Milvus/ZillizCloud" + raise ValueError(msg) self.init_db(drop_old) - self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filters) + if self.ca.is_multitenant and self.db is not None: + if not self.db.supports_multitenant(): + msg = f"{self.config.db_name} does not support CloudMultiTenantSearchCase" + raise NotImplementedError(msg) + self.db.set_multitenant_context(self.ca.tenant_labels()) + if self.config.db in {DB.Milvus, DB.ZillizCloud} and not creates_multitenant_collection: + self.db.validate_multitenant_schema() + self.ca.dataset.prepare( + self.dataset_source, + filters=self.ca.filters, + with_train_files=TaskStage.LOAD in self.config.stages, + with_scalar_labels=self.ca.with_scalar_labels, + ) except ModuleNotFoundError as e: log.warning(f"pre run case error: please install client for db: {self.config.db}, error={e}") raise e from None + def _validate_cloud_cold_latency_config(self, drop_old: bool) -> None: + if getattr(self.ca, "label", None) != CaseLabel.CloudColdLatency: + return + if drop_old: + msg = ( + "CloudColdLatencyCase requires an existing cold collection. " + "Run with --skip-drop-old and --skip-load." + ) + raise ValueError(msg) + if TaskStage.LOAD in self.config.stages: + msg = "CloudColdLatencyCase is search-only. Run with --skip-load." + raise ValueError(msg) + def run(self, drop_old: bool = True) -> Metric: log.info("Starting run") @@ -156,6 +249,10 @@ def run(self, drop_old: bool = True) -> Metric: return self._run_perf_case(drop_old) if self.ca.label == CaseLabel.Streaming: return self._run_streaming_case() + if self.ca.label == CaseLabel.CloudInsert: + return self._run_cloud_insert_case() + if self.ca.label == CaseLabel.CloudColdLatency: + return self._run_cloud_cold_latency_case(drop_old) msg = f"unknown case type: {self.ca.label}" log.warning(msg) raise ValueError(msg) @@ -223,6 +320,8 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: if TaskStage.SEARCH_SERIAL in self.config.stages: search_results = self._serial_search() m.recall, m.ndcg, m.serial_latency_p99, m.serial_latency_p95 = search_results + m.payload_profile = self.ca.payload_profile.value + m.payload_estimated_bytes_per_query = self.ca.estimated_payload_bytes_per_query(self.config.case_config.k) except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") @@ -245,10 +344,111 @@ def _run_streaming_case(self) -> Metric: log.info(f"Streaming case got result: {m}") return m + def _run_cloud_insert_case(self) -> Metric: + assert self.db is not None + started = time.perf_counter() + runner_kwargs = {} + if self.ca.is_multitenant: + runner_kwargs["tenant_case"] = self.ca + runner = ConcurrentInsertRunner( + self.db, + self.ca.dataset, + self.normalize, + self.ca.filters, + max_workers=self.config.load_concurrency or None, + batch_size=self.ca.batch_size, + duration=self.ca.duration, + **runner_kwargs, + ) + count = runner.task() + insert_done = time.perf_counter() + readiness_timeout = self.ca.readiness_timeout + readiness_poll_interval = self.ca.readiness_poll_interval + readiness_deadline = None if readiness_timeout is None else time.perf_counter() + readiness_timeout + with self.db.init(): + status = self.db.poll_insert_readiness(count) + searchable_started = time.perf_counter() + while not status["fully_searchable"]: + if readiness_deadline is not None and time.perf_counter() >= readiness_deadline: + msg = ( + "Cloud insert readiness timed out waiting for fully_searchable " + f"after {readiness_timeout}s; last_status={status}" + ) + raise TimeoutError(msg) + time.sleep(readiness_poll_interval) + status = self.db.poll_insert_readiness(count) + indexed_started = time.perf_counter() + while not status["fully_indexed"]: + if readiness_deadline is not None and time.perf_counter() >= readiness_deadline: + msg = ( + "Cloud insert readiness timed out waiting for fully_indexed " + f"after {readiness_timeout}s; last_status={status}" + ) + raise TimeoutError(msg) + time.sleep(readiness_poll_interval) + status = self.db.poll_insert_readiness(count) + return Metric( + inserted_count=count, + insert_rows_per_second=round(count / max(insert_done - started, 0.001), 4), + insert_completion_seconds=round(insert_done - started, 4), + searchable_after_insert_seconds=round(indexed_started - searchable_started, 4), + indexed_after_searchable_seconds=round(time.perf_counter() - indexed_started, 4), + additional_parameters=status.get("additional_parameters", {}), + ) + + def _init_cold_warm_search_runner(self) -> None: + if self.normalize: + test_emb = np.stack(self.ca.dataset.test_data) + test_emb = test_emb / np.linalg.norm(test_emb, axis=1)[:, np.newaxis] + self.test_emb = test_emb.tolist() + else: + self.test_emb = self.ca.dataset.test_data + + self.cold_warm_search_runner = ColdWarmSearchRunner( + db=self.db, + test_data=self.test_emb, + filters=self.ca.filters, + k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + query_count=self.ca.query_count, + ) + + def _run_cloud_cold_latency_case(self, drop_old: bool = True) -> Metric: + log.info("Start cloud cold latency case") + try: + self._validate_cloud_cold_latency_config(drop_old) + m = Metric() + if drop_old: + if TaskStage.LOAD in self.config.stages: + _, load_dur = self._load_train_data() + build_dur = self._optimize() + m.insert_duration = round(load_dur, 4) + m.optimize_duration = round(build_dur, 4) + m.load_duration = round(load_dur + build_dur, 4) + else: + log.info("Data loading skipped") + + self._init_cold_warm_search_runner() + m.additional_parameters = { + "cold_latency": self.cold_warm_search_runner.run(), + } + m.payload_profile = self.ca.payload_profile.value + m.payload_estimated_bytes_per_query = self.ca.estimated_payload_bytes_per_query(self.config.case_config.k) + except Exception as e: + log.warning(f"Failed to run cloud cold latency case, reason = {e}") + traceback.print_exc() + raise e from None + else: + log.info(f"Cloud cold latency case got result: {m}") + return m + @utils.time_it def _load_train_data(self): """Insert train data concurrently and get the insert_duration""" try: + runner_kwargs = {} + if self.ca.is_multitenant: + runner_kwargs["tenant_case"] = self.ca runner = ConcurrentInsertRunner( self.db, self.ca.dataset, @@ -256,6 +456,8 @@ def _load_train_data(self): self.ca.filters, self.ca.load_timeout, max_workers=self.config.load_concurrency or None, + with_scalar_labels=self.ca.with_scalar_labels, + **runner_kwargs, ) runner.run() except Exception as e: @@ -320,7 +522,9 @@ def _init_search_runner(self): else: self.test_emb = self.ca.dataset.test_data - gt_df = self.ca.dataset.gt_data + tenant_labels = self.ca.tenant_labels() if self.ca.is_multitenant else None + measure_recall = getattr(self.ca, "measure_recall", True) + gt_df = self.ca.dataset.gt_data if measure_recall else None if TaskStage.SEARCH_SERIAL in self.config.stages: self.serial_search_runner = SerialSearchRunner( @@ -329,6 +533,9 @@ def _init_search_runner(self): ground_truth=gt_df, filters=self.ca.filters, k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + tenant_labels=tenant_labels, + measure_recall=measure_recall, ) if TaskStage.SEARCH_CONCURRENT in self.config.stages: self.search_runner = MultiProcessingSearchRunner( @@ -339,6 +546,8 @@ def _init_search_runner(self): duration=self.config.case_config.concurrency_search_config.concurrency_duration, concurrency_timeout=self.config.case_config.concurrency_search_config.concurrency_timeout, k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + tenant_labels=tenant_labels, ) def _init_read_write_runner(self): diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 94b13762a..cf5c8fecd 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -19,6 +19,7 @@ from .. import config from ..backend.clients import DB from ..backend.clients.api import MetricType +from ..backend.dataset import DatasetWithSizeType from ..interface import benchmark_runner from ..models import ( CaseConfig, @@ -35,6 +36,20 @@ except ImportError: from yaml import Loader +DEFAULT_DATASET_WITH_SIZE_TYPE = DatasetWithSizeType.CohereMedium.value +SUPPORTED_DATASET_WITH_SIZE_TYPES = "|".join(dataset.value for dataset in DatasetWithSizeType) + + +def copy_if_not_none( + custom_case_config: dict[str, Any], + parameters: dict[str, Any], + key: str, + target_key: str | None = None, +) -> None: + value = parameters[key] + if value is not None: + custom_case_config[target_key or key] = value + def click_get_defaults_from_file(ctx, param, value): # noqa: ANN001, ARG001 if value: @@ -165,6 +180,7 @@ def check_custom_case_parameters(ctx: any, param: any, value: any): # noqa: ARG def get_custom_case_config(parameters: dict) -> dict: custom_case_config = {} + dataset_with_size_type = parameters["dataset_with_size_type"] or DEFAULT_DATASET_WITH_SIZE_TYPE if parameters["case_type"] == "PerformanceCustomDataset": custom_case_config = { "name": parameters["custom_case_name"], @@ -184,14 +200,56 @@ def get_custom_case_config(parameters: dict) -> dict: } elif parameters["case_type"] == "NewIntFilterPerformanceCase": custom_case_config = { - "dataset_with_size_type": parameters["dataset_with_size_type"], + "dataset_with_size_type": dataset_with_size_type, "filter_rate": parameters["filter_rate"], } elif parameters["case_type"] == "LabelFilterPerformanceCase": custom_case_config = { - "dataset_with_size_type": parameters["dataset_with_size_type"], + "dataset_with_size_type": dataset_with_size_type, "label_percentage": parameters["label_percentage"], } + elif parameters["case_type"] == "CloudPayloadSearchCase": + custom_case_config = { + "payload_profile": parameters["payload_profile"], + } + copy_if_not_none(custom_case_config, parameters, "dataset_with_size_type") + if parameters["cloud_filter_rate"] is not None: + custom_case_config["filter_rate"] = parameters["cloud_filter_rate"] + if parameters["cloud_label_percentage"] is not None: + custom_case_config["label_percentage"] = parameters["cloud_label_percentage"] + elif parameters["case_type"] == "CloudColdLatencyCase": + custom_case_config = { + "payload_profile": parameters["payload_profile"], + "query_count": parameters["cloud_cold_query_count"], + } + copy_if_not_none(custom_case_config, parameters, "dataset_with_size_type") + copy_if_not_none(custom_case_config, parameters, "cloud_filter_rate", "filter_rate") + copy_if_not_none(custom_case_config, parameters, "cloud_label_percentage", "label_percentage") + elif parameters["case_type"] == "CloudInsertCase": + custom_case_config = { + "batch_size": parameters["cloud_insert_batch_size"], + "duration": parameters["cloud_insert_duration"], + "dataset_with_size_type": dataset_with_size_type, + } + copy_if_not_none(custom_case_config, parameters, "cloud_insert_readiness_timeout", "readiness_timeout") + copy_if_not_none( + custom_case_config, + parameters, + "cloud_insert_readiness_poll_interval", + "readiness_poll_interval", + ) + elif parameters["case_type"] == "CloudMultiTenantSearchCase": + custom_case_config = { + "tenant_count": parameters["tenant_count"], + "tenant_prefix": parameters["tenant_prefix"], + "tenant_id_width": parameters["tenant_id_width"], + "payload_profile": parameters["payload_profile"], + } + copy_if_not_none(custom_case_config, parameters, "dataset_with_size_type") + if parameters["cloud_filter_rate"] is not None: + custom_case_config["filter_rate"] = parameters["cloud_filter_rate"] + if parameters["cloud_label_percentage"] is not None: + custom_case_config["label_percentage"] = parameters["cloud_label_percentage"] return custom_case_config @@ -436,14 +494,13 @@ class CommonTypedDict(TypedDict): ] task_label: Annotated[str, click.option("--task-label", help="Task label")] dataset_with_size_type: Annotated[ - str, + str | None, click.option( "--dataset-with-size-type", - help="Dataset with size type for NewIntFilterPerformanceCase/LabelFilterPerformanceCase, you can use " - "Medium Cohere (768dim, 1M)|Large Cohere (768dim, 10M)|Medium Bioasq (1024dim, 1M)|" - "Large Bioasq (1024dim, 10M)|Large OpenAI (1536dim, 5M)|Medium OpenAI (1536dim, 500K)", - default="Medium Cohere (768dim, 1M)", - show_default=True, + help="Dataset with size type. When omitted, filter/insert cases use Medium Cohere (768dim, 1M), " + "CloudPayloadSearchCase and CloudColdLatencyCase use LAION 100M, and CloudMultiTenantSearchCase " + f"uses Large Cohere (768dim, 10M). Supported values include {SUPPORTED_DATASET_WITH_SIZE_TYPES}", + default=None, ), ] filter_rate: Annotated[ @@ -464,6 +521,111 @@ class CommonTypedDict(TypedDict): show_default=True, ), ] + payload_profile: Annotated[ + str, + click.option( + "--payload-profile", + type=click.Choice(["ids_only", "vector", "scalar_label"]), + help="Response payload profile for CloudPayloadSearchCase and CloudColdLatencyCase", + default="ids_only", + show_default=True, + ), + ] + cloud_filter_rate: Annotated[ + float | None, + click.option( + "--cloud-filter-rate", + type=float, + default=None, + help="Optional int filter rate for CloudPayloadSearchCase and CloudColdLatencyCase", + ), + ] + cloud_label_percentage: Annotated[ + float | None, + click.option( + "--cloud-label-percentage", + type=float, + default=None, + help="Optional label percentage for CloudPayloadSearchCase and CloudColdLatencyCase", + ), + ] + cloud_cold_query_count: Annotated[ + int, + click.option( + "--cloud-cold-query-count", + type=int, + default=1000, + show_default=True, + help="Number of serial queries per cold/warm pass for CloudColdLatencyCase", + ), + ] + cloud_insert_batch_size: Annotated[ + int, + click.option( + "--cloud-insert-batch-size", + type=int, + default=5000, + show_default=True, + help="Insert batch size for CloudInsertCase", + ), + ] + cloud_insert_duration: Annotated[ + float | None, + click.option( + "--cloud-insert-duration", + type=float, + default=None, + help="Optional insert duration in seconds for CloudInsertCase", + ), + ] + cloud_insert_readiness_timeout: Annotated[ + float | None, + click.option( + "--cloud-insert-readiness-timeout", + type=float, + default=None, + help="Optional readiness polling timeout in seconds for CloudInsertCase", + ), + ] + cloud_insert_readiness_poll_interval: Annotated[ + float | None, + click.option( + "--cloud-insert-readiness-poll-interval", + type=float, + default=None, + help="Optional readiness polling interval in seconds for CloudInsertCase", + ), + ] + tenant_count: Annotated[ + int, + click.option( + "--tenant-count", + type=int, + default=1000, + show_default=True, + help="Tenant count for CloudMultiTenantSearchCase", + ), + ] + tenant_prefix: Annotated[ + str, + click.option( + "--tenant-prefix", + type=str, + default="tenant_", + show_default=True, + help="Tenant label prefix for CloudMultiTenantSearchCase", + ), + ] + tenant_id_width: Annotated[ + int, + click.option( + "--tenant-id-width", + type=int, + default=4, + show_default=True, + help="Zero-padding width for CloudMultiTenantSearchCase tenant IDs", + ), + ] class HNSWBaseTypedDict(TypedDict): diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index eca3dbc52..13c9687c7 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -39,7 +39,7 @@ from ..backend.clients.tencent_elasticsearch.cli import TencentElasticsearch from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB -from ..backend.clients.turbopuffer.cli import TurboPuffer +from ..backend.clients.turbopuffer.cli import TurboPuffer, TurboPufferUnpin from ..backend.clients.vectorchord.cli import VectorChordGraph, VectorChordRQ from ..backend.clients.vespa.cli import Vespa from ..backend.clients.weaviate_cloud.cli import Weaviate @@ -83,6 +83,7 @@ cli.add_command(AliSQLHNSW) cli.add_command(Doris) cli.add_command(TurboPuffer) +cli.add_command(TurboPufferUnpin) cli.add_command(Chroma) cli.add_command(Zvec) cli.add_command(Endee) diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 0d4119e93..8b603be24 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -163,7 +163,7 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non return c_results = [] - latest_runner, cached_load_duration = None, None + latest_loaded_reuse_key, cached_load_duration = None, None for idx, runner in enumerate(running_task.case_runners): case_res = CaseResult( metrics=Metric(), @@ -171,7 +171,10 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non ) drop_old = TaskStage.DROP_OLD in runner.config.stages - if (latest_runner and runner == latest_runner) or not self.drop_old: + reuse_key = runner.load_reuse_key() + if reuse_key is not None and reuse_key == latest_loaded_reuse_key: + drop_old = False + if not self.drop_old: drop_old = False num_cases = running_task.num_cases() try: @@ -182,14 +185,12 @@ def _async_task_v2(self, running_task: TaskRunner, send_conn: Connection) -> Non f"result={case_res.metrics}, label={case_res.label}" ) - # cache the latest succeeded runner - latest_runner = runner - - # cache the latest drop_old=True load_duration of the latest succeeded runner - cached_load_duration = case_res.metrics.load_duration if drop_old else cached_load_duration + if drop_old and TaskStage.LOAD in runner.config.stages and reuse_key is not None: + latest_loaded_reuse_key = reuse_key + cached_load_duration = case_res.metrics.load_duration # use the cached load duration if this case didn't drop the existing collection - if not drop_old: + if not drop_old and reuse_key is not None and reuse_key == latest_loaded_reuse_key: case_res.metrics.load_duration = cached_load_duration if cached_load_duration else 0.0 except (LoadTimeoutError, PerformanceTimeoutError) as e: log.warning(f"[{idx+1}/{num_cases}] case {runner.display()} failed to run, reason={e}") diff --git a/vectordb_bench/metric.py b/vectordb_bench/metric.py index 3634b2114..5a7c14e82 100644 --- a/vectordb_bench/metric.py +++ b/vectordb_bench/metric.py @@ -29,6 +29,15 @@ class Metric: conc_latency_p99_list: list[float] = field(default_factory=list) conc_latency_p95_list: list[float] = field(default_factory=list) conc_latency_avg_list: list[float] = field(default_factory=list) + payload_profile: str = "ids_only" + payload_estimated_bytes_per_query: int = 0 + + inserted_count: int = 0 + insert_rows_per_second: float = 0.0 + insert_completion_seconds: float = 0.0 + searchable_after_insert_seconds: float = 0.0 + indexed_after_searchable_seconds: float = 0.0 + additional_parameters: dict = field(default_factory=dict) # for streaming cases st_ideal_insert_duration: int = 0 diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index a7e7c09f1..dc1709cc0 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -1,8 +1,9 @@ import logging import pathlib +from dataclasses import asdict from datetime import date, datetime from enum import Enum, StrEnum -from typing import Self +from typing import Any, ClassVar, Self import ujson @@ -149,6 +150,7 @@ class CaseConfigParamType(Enum): dataset_with_size_type = "dataset_with_size_type" filter_rate = "filter_rate" + payload_profile = "payload_profile" insert_rate = "insert_rate" search_stages = "search_stages" concurrencies = "concurrencies" @@ -276,6 +278,68 @@ class TestResult(BaseModel): file_fmt: str = "result_{}_{}_{}.json" # result_20230718_statndard_milvus.json timestamp: float = 0.0 + sensitive_output_fields: ClassVar[set[str]] = {"api_key", "password", "token"} + + @classmethod + def _redact_sensitive_fields(cls, value: Any) -> Any: + if isinstance(value, dict): + return { + key: ( + "**********" + if key.lower() in cls.sensitive_output_fields and item + else cls._redact_sensitive_fields(item) + ) + for key, item in value.items() + } + if isinstance(value, list): + return [cls._redact_sensitive_fields(item) for item in value] + return value + + @staticmethod + def _output_metrics_for_case(case_result: CaseResult) -> dict: + metrics = asdict(case_result.metrics) + case_id = case_result.task_config.case_config.case_id + + if case_id == CaseType.CloudInsertCase: + return { + "inserted_count": metrics["inserted_count"], + "insert_rows_per_second": metrics["insert_rows_per_second"], + "insert_completion_seconds": metrics["insert_completion_seconds"], + "searchable_after_insert_seconds": metrics["searchable_after_insert_seconds"], + "indexed_after_searchable_seconds": metrics["indexed_after_searchable_seconds"], + "additional_parameters": metrics["additional_parameters"], + } + + if case_id == CaseType.CloudColdLatencyCase: + return { + "insert_duration": metrics["insert_duration"], + "optimize_duration": metrics["optimize_duration"], + "load_duration": metrics["load_duration"], + "payload_profile": metrics["payload_profile"], + "payload_estimated_bytes_per_query": metrics["payload_estimated_bytes_per_query"], + "cold_latency": metrics["additional_parameters"].get("cold_latency", {}), + } + + return metrics + + @staticmethod + def _output_case_config_for_case(case_result: CaseResult) -> dict: + case_config = case_result.task_config.case_config + + if case_config.case_id in {CaseType.CloudInsertCase, CaseType.CloudColdLatencyCase}: + return { + "case_id": case_config.case_id.value, + "custom_case": case_config.custom_case, + } + + return case_config.model_dump(mode="json") + + def model_dump_for_output(self) -> dict: + output = self.model_dump(mode="json", serialize_as_any=True) + for idx, case_result in enumerate(self.results): + output["results"][idx]["metrics"] = self._output_metrics_for_case(case_result) + output["results"][idx]["task_config"]["case_config"] = self._output_case_config_for_case(case_result) + return self._redact_sensitive_fields(output) def flush(self): db2case = self.get_db_results() @@ -314,8 +378,8 @@ def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): log.info(f"write results to disk {result_file}") with pathlib.Path(result_file).open("w") as f: - b = partial.model_dump_json(exclude={"db_config": {"password", "api_key"}}) - f.write(b) + f.write(ujson.dumps(partial.model_dump_for_output(), indent=2)) + f.write("\n") def get_case_config(case_config: CaseConfig) -> dict[CaseConfig]: if case_config["case_id"] in {6, 7, 8, 9, 12, 13, 14, 15}: @@ -360,27 +424,31 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: task_config["case_config"] = cls.get_case_config(case_config=case_config) case_result["task_config"] = task_config - - if trans_unit: - cur_max_count = case_result["metrics"]["max_load_count"] - case_result["metrics"]["max_load_count"] = ( - cur_max_count / 1000 if int(cur_max_count) > 0 else cur_max_count - ) - - cur_latency = case_result["metrics"]["serial_latency_p99"] - case_result["metrics"]["serial_latency_p99"] = ( - cur_latency * 1000 if cur_latency > 0 else cur_latency - ) - - # Handle P95 latency for backward compatibility with existing result files - if "serial_latency_p95" in case_result["metrics"]: - cur_latency_p95 = case_result["metrics"]["serial_latency_p95"] - case_result["metrics"]["serial_latency_p95"] = ( + metrics = case_result.get("metrics") + if ( + metrics + and CaseType(case_config.get("case_id")) == CaseType.CloudColdLatencyCase + and "cold_latency" in metrics + ): + metrics.setdefault("additional_parameters", {})["cold_latency"] = metrics.pop("cold_latency") + + if trans_unit and metrics: + if "max_load_count" in metrics: + cur_max_count = metrics["max_load_count"] + metrics["max_load_count"] = cur_max_count / 1000 if int(cur_max_count) > 0 else cur_max_count + + if "serial_latency_p99" in metrics: + cur_latency = metrics["serial_latency_p99"] + metrics["serial_latency_p99"] = cur_latency * 1000 if cur_latency > 0 else cur_latency + + # Handle P95 latency for backward compatibility with existing result files. + if "serial_latency_p95" in metrics: + cur_latency_p95 = metrics["serial_latency_p95"] + metrics["serial_latency_p95"] = ( cur_latency_p95 * 1000 if cur_latency_p95 > 0 else cur_latency_p95 ) - else: - # Default to 0 for older result files that don't have P95 data - case_result["metrics"]["serial_latency_p95"] = 0.0 + elif "serial_latency_p99" in metrics: + metrics["serial_latency_p95"] = 0.0 return TestResult.model_validate(test_result) def display(self, dbs: list[DB] | None = None): From 9871e319983c554b6ccc536468af9fd6744bab79 Mon Sep 17 00:00:00 2001 From: Zijun Yang <37757768+zpatronus@users.noreply.github.com> Date: Wed, 17 Jun 2026 20:09:07 +0800 Subject: [PATCH 33/49] perf(hologres): use binary float4[] dumper and cache search query templates (#800) --- .../backend/clients/hologres/hologres.py | 96 +++++++++++++------ 1 file changed, 66 insertions(+), 30 deletions(-) diff --git a/vectordb_bench/backend/clients/hologres/hologres.py b/vectordb_bench/backend/clients/hologres/hologres.py index 9fecc5c01..f5cbf698a 100644 --- a/vectordb_bench/backend/clients/hologres/hologres.py +++ b/vectordb_bench/backend/clients/hologres/hologres.py @@ -2,6 +2,7 @@ import json import logging +import struct from collections.abc import Generator from contextlib import contextmanager from io import StringIO @@ -9,6 +10,8 @@ import psycopg from psycopg import Connection, Cursor, sql +from psycopg.adapt import Dumper +from psycopg.pq import Format from ..api import VectorDB from .config import HologresConfig, HologresIndexConfig @@ -16,6 +19,35 @@ log = logging.getLogger(__name__) +class HoloFloat4Array: + """Lightweight wrapper for float arrays - 1 object per query instead of 768 Float4 wrappers.""" + + __slots__ = ("data",) + + def __init__(self, data: list[float]): + self.data = data + + +class HoloFloat4ArrayDumper(Dumper): + """Custom Dumper that serializes float arrays to PostgreSQL binary float4[] format.""" + + format = Format.BINARY + oid = 1021 # PostgreSQL OID for float4[] (_float4) + + def dump(self, obj: HoloFloat4Array) -> bytes: + # PostgreSQL binary array format: + # Header: ndim(4) + has_null(4) + OID elemtype(4) + dim_size(4) + lower_bound(4) = 20 bytes + # For each element: itemlen(4) + float32(4) = 8 bytes per element + header = struct.pack(">iiIii", 1, 0, 700, len(obj.data), 1) + return header + b"".join(struct.pack(">if", 4, x) for x in obj.data) + + +# Register HoloFloat4ArrayDumper globally so it's available for all connections and cursors. +# This must happen at module load time, before any connections/cursors are created, +# because psycopg cursors snapshot the adapter map at creation time. +psycopg.adapters.register_dumper(HoloFloat4Array, HoloFloat4ArrayDumper) + + class Hologres(VectorDB): """Use psycopg instructions""" @@ -99,6 +131,31 @@ def init(self) -> Generator[None, None, None]: self._set_search_guc() + self._search_query_no_filter = sql.SQL(""" + SELECT id + FROM {table_name} + ORDER BY {distance_function}(embedding, %b) + {order_direction} + LIMIT %s; + """).format( + table_name=sql.Identifier(self.table_name), + distance_function=sql.SQL(self.case_config.distance_function()), + order_direction=sql.SQL(self.case_config.order_direction()), + ) + + self._search_query_with_filter = sql.SQL(""" + SELECT id + FROM {table_name} + WHERE id >= %s + ORDER BY {distance_function}(embedding, %b) + {order_direction} + LIMIT %s; + """).format( + table_name=sql.Identifier(self.table_name), + distance_function=sql.SQL(self.case_config.distance_function()), + order_direction=sql.SQL(self.case_config.order_direction()), + ) + try: yield finally: @@ -329,34 +386,6 @@ def insert_embeddings( log.warning(f"Failed to insert data into table ({self.table_name}), error: {e}") return 0, e - def _compose_query_and_params(self, vec: list[float], topk: int, ge_id: int | None = None): - params = [] - - where_clause = sql.SQL("") - if ge_id is not None: - where_clause = sql.SQL(" WHERE id >= %s ") - params.append(ge_id) - - vec_float4 = [psycopg._wrappers.Float4(i) for i in vec] - params.append(vec_float4) - params.append(topk) - - query = sql.SQL(""" - SELECT id - FROM {table_name} - {where_clause} - ORDER BY {distance_function}(embedding, %b) - {order_direction} - LIMIT %s; - """).format( - table_name=sql.Identifier(self.table_name), - distance_function=sql.SQL(self.case_config.distance_function()), - where_clause=where_clause, - order_direction=sql.SQL(self.case_config.order_direction()), - ) - - return query, params - def search_embedding( self, query: list[float], @@ -368,6 +397,13 @@ def search_embedding( assert self.cursor is not None, "Cursor is not initialized" ge = filters.get("id") if filters else None - q, params = self._compose_query_and_params(query, k, ge) - result = self.cursor.execute(q, params, prepare=True, binary=True) + q = HoloFloat4Array(query) + + if ge is not None: + params = (ge, q, k) + result = self.cursor.execute(self._search_query_with_filter, params, prepare=True, binary=True) + else: + params = (q, k) + result = self.cursor.execute(self._search_query_no_filter, params, prepare=True, binary=True) + return [int(i[0]) for i in result.fetchall()] From fad979c2eca6b72ba0f02916136f702860e5bd88 Mon Sep 17 00:00:00 2001 From: Yuanzhan Gao Date: Fri, 26 Jun 2026 18:27:35 +0800 Subject: [PATCH 34/49] Add full-text search benchmark support (#794) * Add FTS support (#713) ## Context VDBBench did not have a dedicated native full-text search benchmark path. This PR adds FTS as a first-class benchmark workload so BM25-based text search can be evaluated through the same task, runner, dataset, frontend, and result pipeline used by the rest of VDBBench. ## Summary - Add full-text search benchmark support centered on BM25 text retrieval. - Introduce FTS performance cases that load text documents, run text queries, and report comparable performance results. - Wire FTS through backend execution, dataset preparation, runner orchestration, Streamlit task generation, and result formatting. - Use manifest-driven FTS ground truth so recall is measured against generated mathematical BM25 neighbors rather than semantic relevance labels. ## Backends Covered - Milvus: native BM25 full-text indexing/search configuration and execution path. - Zilliz Cloud: FTS routing through the Milvus-compatible API with Cloud sparse auto-index handling, sharing the Milvus optimize/compaction path. - ElasticCloud / Elasticsearch: BM25 text indexing/search path with configurable BM25 k1/b support. - Vespa: BM25 schema/query path plus Vespa feed-client loading for large FTS document ingestion. - Turbopuffer: namespace-based full-text benchmark path. ## Testing Infra Touched - Dataset layer: add MS MARCO and HotpotQA FTS dataset definitions, document/query loading, and S3-hosted mathematical BM25 ground-truth loading. - Case layer: add FTS performance case definitions, payload profiles, and task assembly support. - Runner layer: support FTS document loading plus serial recall and concurrent text-query search execution while preserving the existing backend insert contract. - Backend layer: route Vespa FTS loading through its backend insert path, where the Vespa feed client is managed for high-throughput ingestion. - Frontend layer: expose FTS cases and generate backend-specific FTS task configs from Streamlit. - Result layer: format FTS benchmark outputs alongside existing VDBBench results. ## Datasets Supported - MS MARCO: small 100K, medium 1M, large 8.8M documents. - HotpotQA: small 100K, medium 1M, large 5.2M documents. ## Metrics - Search metric type: BM25. - Accuracy metric: recall@k against generated mathematical BM25 ground truth. - Performance metrics: serial latency p95/p99, concurrent QPS, load duration, optimize duration, inserted count, payload profile, batch size, and load concurrency. Signed-off-by: jamesgao-jpg Co-authored-by: Denise2004 <3087753261@qq.com> --- README.md | 12 +- docs/release/2026-06-full-text-search.md | 117 ++ install/requirements_py3.11.txt | 4 +- pyproject.toml | 4 +- tests/test_cloud_payload_case.py | 1 + tests/test_milvus.py | 27 +- vectordb_bench/__init__.py | 2 +- vectordb_bench/backend/assembler.py | 18 +- vectordb_bench/backend/cases.py | 58 +- vectordb_bench/backend/clients/__init__.py | 16 + vectordb_bench/backend/clients/api.py | 94 + .../backend/clients/elastic_cloud/config.py | 109 +- .../clients/elastic_cloud/elastic_cloud.py | 101 +- vectordb_bench/backend/clients/milvus/cli.py | 45 + .../backend/clients/milvus/config.py | 156 ++ .../backend/clients/milvus/milvus.py | 238 ++- .../backend/clients/pgdiskann/cli.py | 2 +- .../backend/clients/pgvector/cli.py | 2 +- .../backend/clients/turbopuffer/config.py | 12 + .../clients/turbopuffer/turbopuffer.py | 92 +- .../backend/clients/vespa/config.py | 50 + vectordb_bench/backend/clients/vespa/vespa.py | 345 +++- .../backend/clients/zilliz_cloud/config.py | 26 +- vectordb_bench/backend/data_source.py | 51 + vectordb_bench/backend/dataset.py | 513 ++++- vectordb_bench/backend/payload.py | 4 + vectordb_bench/backend/result_collector.py | 116 +- .../backend/runner/concurrent_runner.py | 101 +- vectordb_bench/backend/runner/mp_runner.py | 53 +- .../backend/runner/serial_runner.py | 85 +- vectordb_bench/backend/task_runner.py | 154 +- vectordb_bench/backend/utils.py | 26 + vectordb_bench/backend/workload.py | 6 + vectordb_bench/cli/cli.py | 37 +- vectordb_bench/cli/vectordbbench.py | 3 +- .../fig/homepage/full_text_search.png | Bin 0 -> 134647 bytes .../components/check_results/filters.py | 11 +- .../frontend/components/check_results/nav.py | 1 + .../components/run_test/caseSelector.py | 3 +- .../components/run_test/generateTasks.py | 23 +- .../components/welcome/welcomePrams.py | 14 +- .../frontend/config/dbCaseConfigs.py | 202 +- .../frontend/pages/full_text_search.py | 365 ++++ vectordb_bench/frontend/pages/qps_recall.py | 5 +- vectordb_bench/interface.py | 8 +- vectordb_bench/metric.py | 8 + vectordb_bench/models.py | 15 + vectordb_bench/restful/format_res.py | 4 +- ...lt_20260626_fts_standard_elasticcloud.json | 1622 ++++++++++++++++ ...ult_20260626_fts_standard_turbopuffer.json | 1646 +++++++++++++++++ .../result_20260626_fts_standard_vespa.json | 1514 +++++++++++++++ ...ult_20260626_fts_standard_zillizcloud.json | 1622 ++++++++++++++++ 52 files changed, 9541 insertions(+), 202 deletions(-) create mode 100644 docs/release/2026-06-full-text-search.md create mode 100644 vectordb_bench/backend/workload.py create mode 100644 vectordb_bench/fig/homepage/full_text_search.png create mode 100644 vectordb_bench/frontend/pages/full_text_search.py create mode 100644 vectordb_bench/results/FullTextSearch/ElasticCloud/result_20260626_fts_standard_elasticcloud.json create mode 100644 vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json create mode 100644 vectordb_bench/results/FullTextSearch/Vespa/result_20260626_fts_standard_vespa.json create mode 100644 vectordb_bench/results/FullTextSearch/ZillizCloud/result_20260626_fts_standard_zillizcloud.json diff --git a/README.md b/README.md index 3cdceddc0..72e7f8ca7 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,11 @@ Prepare to delve into the world of VDBBench, and let it guide you in uncovering VDBBench is sponsored by Zilliz，the leading opensource vectorDB company behind Milvus. Choose smarter with VDBBench - start your free test on [zilliz cloud](https://zilliz.com/) today! **Leaderboard:** https://zilliz.com/benchmark + +## 🎈 Announcement 🎈 + +**June 2026 update:** Full Text Search has landed in VectorDBBench. We now benchmark BM25-style retrieval across supported backends, starting with MS MARCO and HotpotQA datasets, payload profiles, recall, QPS, and load metrics ready to compare. See the [VectorDBBench Full Text Search Release Note](docs/release/2026-06-full-text-search.md) for the full rollout details and caveats. + ## Quick Start ### Prerequirement ``` shell @@ -798,7 +803,7 @@ Now we can only run one task at the same time. ### Client Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, VectorChord, Redis, Chroma, CockroachDB, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. ### Benchmark Cases -We've developed lots of comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into four main types: +We've developed lots of comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into several main types: #### Capacity Case - **Large Dim:** Tests the database's loading capacity by inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until fully loaded. The final number of inserted vectors is reported. - **Small Dim:** Similar to the Large Dim case but uses small-dimension vectors (SIFT 500K vectors, 128 dimensions). @@ -810,6 +815,11 @@ We've developed lots of comprehensive benchmark cases to test vector databases' #### Filtering Search Performance Case - **Int-Filter Cases:** Evaluates search performance with int-based filter expression (e.g. "id >= 2,000"). - **Label-Filter Cases:** Evaluates search performance with label-based filter expressions (e.g., "color == 'red'"). The test includes randomly generated labels to simulate real-world filtering scenarios. +#### Full Text Search Performance Case +- **FullTextSearchPerformance:** Measures BM25-style text retrieval over raw text documents. The case inserts documents, runs the backend optimization or index-readiness step, then measures recall, latency, and QPS for text queries. +- **Datasets:** The initial FTS benchmark uses MS MARCO and HotpotQA in small, medium, and large corpus sizes. +- **Ground truth:** Recall is computed against generated mathematical BM25 ground truth, not semantic relevance labels. +- **Payload profiles:** FTS supports IDs-only responses and text payload responses so users can compare pure retrieval throughput against response-size overhead. #### Streaming Cases - **Insertion-Under-Load Case:** Evaluates search performance while maintaining a constant insertion workload. VDBBench applies a steady stream of insert requests at a fixed rate to simulate real-world scenarios where search operations must perform reliably under continuous data ingestion. diff --git a/docs/release/2026-06-full-text-search.md b/docs/release/2026-06-full-text-search.md new file mode 100644 index 000000000..33e335eb1 --- /dev/null +++ b/docs/release/2026-06-full-text-search.md @@ -0,0 +1,117 @@ +# VectorDBBench Full Text Search Release Note + +June 2026 + +Full text search support adds BM25-style text retrieval workloads to VectorDBBench. The goal is to compare database full text search paths with the same benchmark harness used for vector workloads: repeatable datasets, explicit load/search stages, structured result JSONs, and frontend result visualization. + +## Context + +VectorDBBench has historically focused on dense vector search and vector-oriented cloud cases. That leaves a gap for systems that also expose native text retrieval, sparse search, or BM25 ranking. Users evaluating retrieval systems often need to compare text-only retrieval before deciding whether to use dense vector, sparse vector, hybrid, or reranking layers. + +The Full Text Search benchmark covers that text-only layer. It measures end-to-end behavior for indexing raw text, optimizing the backend, searching with BM25-style ranking, and validating recall against mathematical ground truth. The benchmark intentionally separates this from semantic relevance labels: recall is computed against generated BM25 top-k ground truth, not human relevance judgments. + +The benchmark also records payload behavior. Some applications return only document IDs, while others return text fields in the search response. VectorDBBench models both paths through explicit payload profiles so throughput and latency can be interpreted together with the response shape. + +## Who we tested this round + +This round focuses on full text search support across the following backends: + +- Milvus, using its full text search BM25 path. +- Zilliz Cloud, using the cloud full text search path. +- Elasticsearch, using its BM25 text search path. +- Vespa, using BM25 ranking over indexed text fields. +- turbopuffer, using its full text search namespace path. + +The benchmark is designed to keep the workload shape consistent while still recording backend-specific behavior. BM25 parameters and analyzer settings are read from dataset manifests when available, applied when the backend exposes matching controls, and recorded as unapplied parameters when a backend does not expose the same control. + +## The new tests we added + +### FullTextSearchPerformance + +**Purpose.** FullTextSearchPerformance measures BM25-style full text search as a first-class benchmark case. It answers the baseline question: after a backend indexes the same text corpus, what QPS, latency, and mathematical recall does it deliver for text queries? + +**How it works.** The case loads raw text documents, builds the backend text index, runs the backend optimize path, executes optional serial recall checks, and then runs concurrent search. The result metric records load duration, insert duration, optimize duration, QPS, serial latency, concurrent latency, recall, payload profile, inserted count, and additional parameters such as manifest BM25 settings. + +Example: run MS MARCO small on Milvus with IDs-only responses. + +```bash +vectordbbench milvusfts \ + --case-type FTSBm25Performance \ + --dataset-with-size-type "MS MARCO Small (100K documents)" \ + --uri "$MILVUS_URI" \ + --payload-profile ids_only \ + --load-concurrency 0 \ + --num-concurrency 40,80 \ + --task-label fts-milvus-msmarco-small-ids +``` + +### Full text search datasets and math ground truth + +**Purpose.** Full text search recall should measure whether an implementation returns the mathematically expected BM25 neighbors for the indexed corpus. Human relevance labels are useful for IR evaluation, but they are not a direct correctness target for a database BM25 implementation. + +**How it works.** FTS datasets provide raw text for document insertion and query execution. The ground-truth artifacts provide top-k neighbor IDs generated under a declared BM25/analyzer contract. Each dataset artifact can include a build manifest with BM25 parameters such as `k1`, `b`, and `avgdl`, plus analyzer settings. VectorDBBench loads those values before backend initialization so index construction can use the dataset contract where the backend supports it. + +Example: use a larger dataset while keeping the same FTS case type. + +```bash +vectordbbench elasticcloudhnsw \ + --case-type FTSBm25Performance \ + --dataset-with-size-type "HotpotQA Medium (1M documents)" \ + --host "$ELASTIC_HOST" \ + --port "$ELASTIC_PORT" \ + --password "$ELASTIC_PASSWORD" \ + --payload-profile ids_only \ + --load-concurrency 0 \ + --num-concurrency 40,80 \ + --task-label fts-elastic-hotpotqa-medium-ids +``` + +### Payload-aware full text search + +**Purpose.** Payload-aware FTS measures the cost of returning text fields, not only document IDs. This matters for applications where search results immediately include snippets, source text, or other fields needed by downstream ranking and display layers. + +**How it works.** The FTS case supports `payload_profile` values including `ids_only` and `text`. IDs-only runs are the recall baseline because they can run serial search against the ground truth without paying text-return overhead. Text payload runs measure concurrent search throughput and latency for larger response bodies; they can skip serial recall when the same indexed namespace or collection has already been validated by the IDs-only run. + +Example: run Vespa with text payload responses and concurrent search only. + +```bash +vectordbbench vespa \ + --case-type FTSBm25Performance \ + --dataset-with-size-type "MS MARCO Medium (1M documents)" \ + --uri "$VESPA_URI" \ + --port "$VESPA_PORT" \ + --payload-profile text \ + --load-concurrency 0 \ + --skip-search-serial \ + --num-concurrency 40,80 \ + --task-label fts-vespa-msmarco-medium-text +``` + +## Result artifacts + +Committed FTS result JSONs live under `vectordb_bench/results/FullTextSearch//`. Published artifacts are consolidated so each backend directory contains one backend-level JSON with multiple case results in its `results` list, instead of a pile of one-case files. That keeps the release payload readable while preserving the per-dataset and per-payload records needed by the dashboard. + +Use the result collector to consolidate split FTS run outputs before committing published examples: + +```bash +python -m vectordb_bench.backend.result_collector \ + vectordb_bench/results/FullTextSearch \ + --merge-by-db \ + --task-label fts_standard \ + --replace +``` + +The default collector behavior still groups normal benchmark files by `run_id`. The `--merge-by-db` mode is intended for curated FTS result publication, where each backend should present the latest benchmark matrix as a single artifact. The Full Text Search frontend reads the latest backend-level result file from each backend directory and displays the committed matrix by backend, dataset, payload profile, load duration, QPS, recall, and latency. + +## Caveats + +This release note introduces the FTS benchmark path; it is not a complete benchmark report. Detailed raw outputs, machine setup, backend deployment scripts, and rerun notes should live in separate experiment reports when needed. + +Important caveats: + +- Mathematical BM25 ground truth is not the same as semantic relevance evaluation. A high recall score means the backend matched the declared BM25 ranking contract, not that it matched human relevance labels. +- Analyzer behavior can materially affect recall and ranking. Tokenization, lowercase filters, stop words, stemming, token length limits, and field normalization should be recorded with each dataset manifest and backend result. +- BM25 parameter support differs across products. Some backends expose `k1` and `b`, some expose average field length controls, and some compute or hide those values internally. VectorDBBench records applied and unapplied parameters so results can be interpreted correctly. +- IDs-only and text payload runs answer different questions. IDs-only is the cleanest recall and throughput baseline; text payload runs expose response-size overhead. +- Load duration includes backend-specific insert and optimize behavior. Products may differ in whether optimize means force merge, compaction, warmup, or index deployment readiness. +- Result JSONs under `vectordb_bench/results/FullTextSearch` are curated examples for the frontend. They should not be treated as the full historical experiment archive. diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 7208439bb..3d42528c3 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -14,6 +14,7 @@ pytz streamlit-autorefresh streamlit>=1.23.0 streamlit_extras +tornado>=6.0 tqdm s3fs psutil @@ -22,10 +23,11 @@ plotly environs pydantic>=2.0,<3 scikit-learn -pymilvus<3.0.0 +pymilvus>=2.6.15,<3.0.0 clickhouse_connect pyvespa mysql-connector-python PyMySQL packaging hdrhistogram>=0.10.1 +ir_datasets diff --git a/pyproject.toml b/pyproject.toml index 3bbba8ac0..bf9552cc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "pyyaml", "pytz", "streamlit>=1.47,<2", # 1.47 fixes streamlit#11660 + "tornado>=6.0", "tqdm", "s3fs", "oss2", @@ -38,9 +39,10 @@ dependencies = [ "environs", "pydantic>=2.0,<3", "scikit-learn", - "pymilvus<3.0.0", # with pandas, numpy + "pymilvus>=2.6.15,<3.0.0", # with pandas, numpy "hdrhistogram>=0.10.1", "ujson", + "ir_datasets", ] dynamic = ["version"] diff --git a/tests/test_cloud_payload_case.py b/tests/test_cloud_payload_case.py index d227dd176..d3d97fb60 100644 --- a/tests/test_cloud_payload_case.py +++ b/tests/test_cloud_payload_case.py @@ -32,6 +32,7 @@ def search_embedding(self, query: list[float], k: int = 100, **kwargs) -> list[i def test_payload_profile_estimates_response_bytes(): assert PayloadProfile.IDS_ONLY.estimated_bytes_per_query(k=10, dim=768) == 200 assert PayloadProfile.VECTOR.estimated_bytes_per_query(k=10, dim=768) == 30_920 + assert PayloadProfile.TEXT.estimated_bytes_per_query(k=10, dim=0) == 5_320 def test_cloud_payload_case_defaults_to_laion_100m(): diff --git a/tests/test_milvus.py b/tests/test_milvus.py index dfc88cad8..8dcef4f1c 100644 --- a/tests/test_milvus.py +++ b/tests/test_milvus.py @@ -23,11 +23,18 @@ class TestMilvusOptimize: - def _milvus(self, *, compact_side_effect: Exception | None = None): + def _milvus( + self, + *, + compact_side_effect: Exception | None = None, + is_fts: bool = False, + is_gpu_index: bool = False, + ): milvus = Milvus.__new__(Milvus) milvus.name = "Milvus" milvus.collection_name = "test_collection" - milvus.case_config = SimpleNamespace(is_gpu_index=False) + milvus._is_fts = is_fts + milvus.case_config = SimpleNamespace(is_gpu_index=is_gpu_index) milvus.client = MagicMock() milvus.client.compact.side_effect = compact_side_effect milvus.client.compact.return_value = 0 @@ -44,6 +51,22 @@ def test_optimize_compact_uses_safe_force_merge_target_size(self): milvus.client.compact.assert_called_once_with("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) milvus.client.refresh_load.assert_called_once_with("test_collection") + def test_optimize_compacts_fts_collections(self): + milvus = self._milvus(is_fts=True) + + milvus._optimize() + + milvus.client.compact.assert_called_once_with("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) + milvus.client.refresh_load.assert_called_once_with("test_collection") + + def test_optimize_skips_gpu_index_compaction(self): + milvus = self._milvus(is_gpu_index=True) + + milvus._optimize() + + milvus.client.compact.assert_not_called() + milvus.client.refresh_load.assert_called_once_with("test_collection") + def test_optimize_skips_property_style_permission_denied(self): error = RuntimeError("permission denied") error.code = SimpleNamespace(name="PERMISSION_DENIED") diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 9b3cbdff8..3e5c1e69e 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -17,7 +17,7 @@ class config: LOG_FILE = env.str("LOG_FILE", "logs/vectordb_bench.log") DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", AWS_S3_URL) - DATASET_SOURCE = env.str("DATASET_SOURCE", "S3") # Options "S3" or "AliyunOSS" + DATASET_SOURCE = env.str("DATASET_SOURCE", "S3") # Options "S3", "AliyunOSS", or "IR_DATASETS" DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") NUM_PER_BATCH = env.int("NUM_PER_BATCH", 100) LOAD_CONCURRENCY = env.int("LOAD_CONCURRENCY", 0) # 0 = cpu_count diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index b1177f7f4..fe7f0ddc6 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -6,6 +6,7 @@ from vectordb_bench.models import TaskConfig from .cases import CaseLabel +from .dataset import FtsDatasetManager from .task_runner import CaseRunner, RunningStatus, TaskRunner log = logging.getLogger(__name__) @@ -24,7 +25,18 @@ def assemble(cls, run_id: str, task: TaskConfig, source: DatasetSource) -> CaseR c_cls = task.case_config.case_id.case_cls c = c_cls(task.case_config.custom_case) - if type(task.db_case_config) is not EmptyDBCaseConfig: + if c.label == CaseLabel.FullTextSearchPerformance and not task.db.init_cls.supports_full_text_search(): + msg = f"{task.db.value} does not support full-text search" + raise ValueError(msg) + + # Auto-select data source based on dataset type + actual_source = DatasetSource.IR_DATASETS if isinstance(c.dataset, FtsDatasetManager) else source + + if ( + type(task.db_case_config) is not EmptyDBCaseConfig + and not isinstance(c.dataset, FtsDatasetManager) + and hasattr(c.dataset.data, "metric_type") + ): task.db_case_config.metric_type = c.dataset.data.metric_type return CaseRunner( @@ -32,7 +44,7 @@ def assemble(cls, run_id: str, task: TaskConfig, source: DatasetSource) -> CaseR config=task, ca=c, status=RunningStatus.PENDING, - dataset_source=source, + dataset_source=actual_source, ) @classmethod @@ -50,6 +62,7 @@ def assemble_all( streaming_runners = [r for r in runners if r.ca.label == CaseLabel.Streaming] cloud_insert_runners = [r for r in runners if r.ca.label == CaseLabel.CloudInsert] cloud_cold_latency_runners = [r for r in runners if r.ca.label == CaseLabel.CloudColdLatency] + fts_runners = [r for r in runners if r.ca.label == CaseLabel.FullTextSearchPerformance] search_filter_runners = [*perf_runners, *cloud_cold_latency_runners] @@ -78,6 +91,7 @@ def assemble_all( all_runners.extend(cloud_insert_runners) for v in db2runner.values(): all_runners.extend(v) + all_runners.extend(fts_runners) return TaskRunner( run_id=run_id, diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index b93bd04c3..0253168e0 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -9,7 +9,14 @@ from vectordb_bench.base import BaseModel from vectordb_bench.frontend.components.custom.getCustomConfig import CustomDatasetConfig -from .dataset import CustomDataset, Dataset, DatasetManager, DatasetWithSizeType +from .dataset import ( + CustomDataset, + Dataset, + DatasetManager, + DatasetWithSizeType, + FtsDatasetManager, + FtsDatasetWithSizeType, +) log = logging.getLogger(__name__) @@ -58,6 +65,7 @@ class CaseType(Enum): NewIntFilterPerformanceCase = 400 CloudPayloadSearchCase = 500 + FTSBm25Performance = 503 CloudInsertCase = 600 CloudColdLatencyCase = 700 CloudMultiTenantSearchCase = 800 @@ -86,6 +94,7 @@ class CaseLabel(Enum): Streaming = auto() CloudInsert = auto() CloudColdLatency = auto() + FullTextSearchPerformance = auto() class Case(BaseModel): @@ -901,6 +910,52 @@ def filters(self) -> Filter: return LabelFilter(label_percentage=self.label_percentage) +class FtsPerformanceCase(Case): + """Base class for full-text search BM25 performance cases.""" + + label: CaseLabel = CaseLabel.FullTextSearchPerformance + dataset: FtsDatasetManager + + filter_rate: float | None = None + + @property + def filters(self) -> Filter: + return non_filter + + def estimated_payload_bytes_per_query(self, k: int | None) -> int: + if k is None: + k = config.K_DEFAULT + return self.payload_profile.estimated_bytes_per_query(k=k, dim=0) + + +class FTSBm25Performance(FtsPerformanceCase): + case_id: CaseType = CaseType.FTSBm25Performance + dataset_with_size_type: FtsDatasetWithSizeType = FtsDatasetWithSizeType.MSMarcoSmall + + def __init__( + self, + dataset_with_size_type: FtsDatasetWithSizeType | str = FtsDatasetWithSizeType.MSMarcoSmall, + **kwargs, + ): + if not isinstance(dataset_with_size_type, FtsDatasetWithSizeType): + dataset_with_size_type = FtsDatasetWithSizeType(dataset_with_size_type) + dataset = dataset_with_size_type.get_manager() + name = f"FTS BM25 Performance - {dataset_with_size_type.value}" + description = ( + f"This case tests native BM25 full-text search performance on {dataset_with_size_type.value}. " + "It measures index building time, recall, serial latency, and search QPS." + ) + super().__init__( + name=name, + description=description, + dataset=dataset, + dataset_with_size_type=dataset_with_size_type, + load_timeout=dataset_with_size_type.get_load_timeout(), + optimize_timeout=dataset_with_size_type.get_optimize_timeout(), + **kwargs, + ) + + type2case = { CaseType.CapacityDim960: CapacityDim960, CaseType.CapacityDim128: CapacityDim128, @@ -926,6 +981,7 @@ def filters(self) -> Filter: CaseType.NewIntFilterPerformanceCase: NewIntFilterPerformanceCase, CaseType.LabelFilterPerformanceCase: LabelFilterPerformanceCase, CaseType.CloudPayloadSearchCase: CloudPayloadSearchCase, + CaseType.FTSBm25Performance: FTSBm25Performance, CaseType.CloudInsertCase: CloudInsertCase, CaseType.CloudColdLatencyCase: CloudColdLatencyCase, CaseType.CloudMultiTenantSearchCase: CloudMultiTenantSearchCase, diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 4be8d0424..4fd50871c 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -490,11 +490,19 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _milvus_case_config.get(index_type) if self == DB.ZillizCloud: + if index_type == IndexType.FTS: + from .zilliz_cloud.config import ZillizCloudFtsConfig + + return ZillizCloudFtsConfig from .zilliz_cloud.config import AutoIndexConfig return AutoIndexConfig if self == DB.ElasticCloud: + if index_type == IndexType.FTS: + from .elastic_cloud.config import ElasticCloudFtsConfig + + return ElasticCloudFtsConfig from .elastic_cloud.config import ElasticCloudIndexConfig return ElasticCloudIndexConfig @@ -590,6 +598,10 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _cockroachdb_case_config.get(index_type) if self == DB.Vespa: + if index_type == IndexType.FTS: + from .vespa.config import VespaFtsConfig + + return VespaFtsConfig from .vespa.config import VespaHNSWConfig return VespaHNSWConfig @@ -634,6 +646,10 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return DorisCaseConfig if self == DB.TurboPuffer: + if index_type == IndexType.FTS: + from .turbopuffer.config import TurboPufferFtsConfig + + return TurboPufferFtsConfig from .turbopuffer.config import TurboPufferIndexConfig return TurboPufferIndexConfig diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index 37a5c71dc..c5474fba7 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -14,6 +14,7 @@ class MetricType(StrEnum): COSINE = "COSINE" IP = "IP" DP = "DP" + BM25 = "BM25" HAMMING = "HAMMING" JACCARD = "JACCARD" @@ -33,6 +34,7 @@ class IndexType(StrEnum): IVF_RABITQ = "IVF_RABITQ" Flat = "FLAT" AUTOINDEX = "AUTOINDEX" + FTS = "FTS" ES_HNSW = "hnsw" ES_HNSW_INT8 = "int8_hnsw" ES_HNSW_INT4 = "int4_hnsw" @@ -147,6 +149,35 @@ def index_param(self) -> dict: def search_param(self) -> dict: raise NotImplementedError + def apply_fts_manifest( + self, + bm25_params: dict[str, float], + analyzer_params: dict, + ) -> tuple["DBCaseConfig", dict]: + """Apply FTS dataset manifest parameters to this case config. + + Full-text search datasets may provide BM25 and analyzer settings used to + build the mathematical ground truth. Backends that can reproduce those + settings should return an updated config with supported parameters + applied. Unsupported parameters must be reported in the returned metadata + instead of being silently ignored. + + Args: + bm25_params(dict[str, float]): BM25 parameters from the dataset + manifest, such as k1, b, and avgdl. + analyzer_params(dict): analyzer settings from the dataset manifest. + + Returns: + tuple[DBCaseConfig, dict]: updated config and a report describing + applied and unapplied BM25/analyzer parameters. + """ + return self, { + "applied_bm25_params": {}, + "unapplied_bm25_params": dict(bm25_params), + "applied_analyzer_params": {}, + "unapplied_analyzer_params": dict(analyzer_params), + } + class EmptyDBCaseConfig(BaseModel, DBCaseConfig): """EmptyDBCaseConfig will be used if the vector database has no case specific configs""" @@ -243,6 +274,16 @@ def need_normalize_cosine(self) -> bool: def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: return payload_profile == PayloadProfile.IDS_ONLY + def has_text_field(self) -> bool: + return False + + def supports_document_payload_profile(self, payload_profile: PayloadProfile) -> bool: + if payload_profile == PayloadProfile.IDS_ONLY: + return True + if payload_profile == PayloadProfile.TEXT: + return self.has_text_field() + return False + def poll_insert_readiness(self, expected_count: int) -> dict: return {"fully_searchable": True, "fully_indexed": True, "additional_parameters": {}} @@ -255,6 +296,59 @@ def supports_multitenant(self) -> bool: def validate_multitenant_schema(self) -> None: return None + @classmethod + def supports_full_text_search(cls) -> bool: + """Return whether this client implements the full-text search API. + + Backends that return True must implement insert_documents and + search_documents for raw text documents. + """ + return False + + def insert_documents( + self, + texts: list[str], + doc_ids: list[str], + **kwargs, + ) -> tuple[int, Exception | None]: + """Insert raw text documents for full-text search cases. + + Args: + texts(list[str]): raw text documents to index. + doc_ids(list[str]): stable document IDs aligned with texts. + **kwargs(Any): backend or runner specific insert parameters. + + Returns: + tuple[int, Exception | None]: inserted document count and an optional + error. Implementations should return the count of successfully + inserted documents even when reporting a partial failure. + """ + msg = f"{self.name or self.__class__.__name__} does not support full-text document insert" + raise NotImplementedError(msg) + + def search_documents( + self, + query: str, + k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + **kwargs, + ) -> list[str]: + """Search full-text documents and return ranked document IDs. + + Args: + query(str): raw query text. + k(int): number of nearest documents to return. Defaults to 100. + payload_profile(PayloadProfile): response payload shape requested by + the benchmark. The API still returns document IDs only; use + supports_document_payload_profile to reject unsupported profiles. + **kwargs(Any): backend or runner specific search parameters. + + Returns: + list[str]: ranked document IDs for the query. + """ + msg = f"{self.name or self.__class__.__name__} does not support full-text document search" + raise NotImplementedError(msg) + @abstractmethod def insert_embeddings( self, diff --git a/vectordb_bench/backend/clients/elastic_cloud/config.py b/vectordb_bench/backend/clients/elastic_cloud/config.py index be2c2dce8..5b04e3366 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/config.py +++ b/vectordb_bench/backend/clients/elastic_cloud/config.py @@ -7,36 +7,62 @@ class ElasticCloudConfig(DBConfig, BaseModel): - _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"cloud_id", "host"}) + _extra_empty_skip: ClassVar[frozenset[str]] = frozenset( + {"cloud_id", "scheme", "host", "user", "user_name", "password"} + ) - # Elastic Cloud connection. Takes precedence when set. cloud_id: SecretStr | None = None - # Self-hosted / host-based connection (used when cloud_id is not provided). - scheme: str = "https" - host: str = "" + scheme: str | None = None + host: SecretStr | None = None port: int = 9200 - user: str = "elastic" - password: SecretStr + user: str | None = None + user_name: str | None = None + password: SecretStr | None = None + use_ssl: bool = False + verify_certs: bool = True @model_validator(mode="after") def _check_connection_target(self) -> "ElasticCloudConfig": has_cloud_id = bool(self.cloud_id and self.cloud_id.get_secret_value()) if not has_cloud_id and not self.host: - msg = "ElasticCloudConfig requires either cloud_id or host to be set." + msg = "Either cloud_id or host must be set" raise ValueError(msg) return self + def _auth_user(self) -> str: + return self.user_name or self.user or "elastic" + def to_dict(self) -> dict: - auth = (self.user, self.password.get_secret_value()) if self.cloud_id and self.cloud_id.get_secret_value(): + if not self.password: + msg = "password is required when cloud_id is set" + raise ValueError(msg) return { "cloud_id": self.cloud_id.get_secret_value(), - "basic_auth": auth, + "basic_auth": (self._auth_user(), self.password.get_secret_value()), } - return { - "hosts": [{"scheme": self.scheme, "host": self.host, "port": self.port}], - "basic_auth": auth, + + if not self.host: + msg = "Either cloud_id or host must be set" + raise ValueError(msg) + + host = self.host.get_secret_value() + if host.startswith(("http://", "https://")): + url = host + else: + scheme = self.scheme or ("https" if self.use_ssl else "http") + url = f"{scheme}://{host}:{self.port}" + + config = { + "hosts": [url], + "verify_certs": self.verify_certs, } + if self.password: + config["basic_auth"] = (self._auth_user(), self.password.get_secret_value()) + elif self.user_name or (self.user and self.user != "elastic"): + msg = "password is required when user_name is set" + raise ValueError(msg) + return config class ESElementType(StrEnum): @@ -108,3 +134,60 @@ def search_param(self) -> dict: return { "num_candidates": self.num_candidates, } + + +class ElasticCloudFtsConfig(BaseModel, DBCaseConfig): + number_of_shards: int = 1 + number_of_replicas: int = 0 + refresh_interval: str = "30s" + use_force_merge: bool = True + metric_type: MetricType = MetricType.BM25 + bm25_k1: float | None = None + bm25_b: float | None = None + + def apply_fts_manifest( + self, + bm25_params: dict[str, float], + analyzer_params: dict, + ) -> tuple[DBCaseConfig, dict]: + updates = {} + applied_bm25_params = {} + + if "k1" in bm25_params: + updates["bm25_k1"] = bm25_params["k1"] + applied_bm25_params["k1"] = bm25_params["k1"] + if "b" in bm25_params: + updates["bm25_b"] = bm25_params["b"] + applied_bm25_params["b"] = bm25_params["b"] + + return self.model_copy(update=updates), { + "applied_bm25_params": applied_bm25_params, + "unapplied_bm25_params": {k: v for k, v in bm25_params.items() if k not in applied_bm25_params}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": dict(analyzer_params), + } + + def index_param(self) -> dict: + text_mapping = {"type": "text"} + if self.bm25_k1 is not None or self.bm25_b is not None: + text_mapping["similarity"] = "vdbbench_bm25" + return { + "properties": { + "doc_id": {"type": "keyword"}, + "text": text_mapping, + }, + } + + def search_param(self) -> dict: + return {} + + def similarity_settings(self) -> dict: + if self.bm25_k1 is None and self.bm25_b is None: + return {} + + bm25_settings = {"type": "BM25"} + if self.bm25_k1 is not None: + bm25_settings["k1"] = self.bm25_k1 + if self.bm25_b is not None: + bm25_settings["b"] = self.bm25_b + return {"similarity": {"vdbbench_bm25": bm25_settings}} diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index ae3350ddf..552965cfb 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -6,9 +6,10 @@ from elasticsearch.helpers import bulk from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile from ..api import VectorDB -from .config import ElasticCloudIndexConfig +from .config import ElasticCloudFtsConfig, ElasticCloudIndexConfig for logger in ("elasticsearch", "elastic_transport"): logging.getLogger(logger).setLevel(logging.WARNING) @@ -30,7 +31,7 @@ def __init__( self, dim: int, db_config: dict, - db_case_config: ElasticCloudIndexConfig, + db_case_config: ElasticCloudIndexConfig | ElasticCloudFtsConfig, indice: str = "vdb_bench_indice", # must be lowercase id_col_name: str = "id", label_col_name: str = "label", @@ -47,6 +48,10 @@ def __init__( self.label_col_name = label_col_name self.vector_col_name = vector_col_name self.with_scalar_labels = with_scalar_labels + self._is_fts = isinstance(db_case_config, ElasticCloudFtsConfig) + self.text_col_name = "text" + if self._is_fts: + self.id_col_name = "doc_id" from elasticsearch import Elasticsearch @@ -59,6 +64,13 @@ def __init__( client.indices.delete(index=self.indice) self._create_indice(client) + @classmethod + def supports_full_text_search(cls) -> bool: + return True + + def has_text_field(self) -> bool: + return bool(getattr(self, "_is_fts", False) and getattr(self, "text_col_name", None)) + @contextmanager def init(self) -> None: """connect to elasticsearch""" @@ -71,6 +83,18 @@ def init(self) -> None: del self.client def _create_indice(self, client: any) -> None: + if self._is_fts: + mappings = self.case_config.index_param() + index_settings = { + "number_of_shards": self.case_config.number_of_shards, + "number_of_replicas": self.case_config.number_of_replicas, + "refresh_interval": self.case_config.refresh_interval, + } + index_settings.update(self.case_config.similarity_settings()) + settings = {"index": index_settings} + client.indices.create(index=self.indice, mappings=mappings, settings=settings) + return + mappings = { "_source": {"excludes": [self.vector_col_name]}, "properties": { @@ -149,6 +173,38 @@ def insert_embeddings( log.warning(f"Failed to insert data: {self.indice} error: {e!s}") return (0, e) + def insert_documents( + self, + texts: Iterable[str], + doc_ids: list[str], + **kwargs, + ) -> tuple[int, Exception | None]: + if not getattr(self, "_is_fts", False): + msg = "ElasticCloud full-text insert requires ElasticCloudFtsConfig" + raise RuntimeError(msg) + assert self.client is not None, "should self.init() first" + docs = list(texts) + if len(docs) != len(doc_ids): + msg = f"Mismatch between texts ({len(docs)}) and doc_ids ({len(doc_ids)}) lengths" + raise ValueError(msg) + actions = [ + { + "_index": self.indice, + "_id": str(doc_ids[i]), + "_source": { + self.id_col_name: str(doc_ids[i]), + self.text_col_name: docs[i], + }, + } + for i in range(len(docs)) + ] + try: + result = bulk(self.client, actions) + return result[0], None + except Exception as e: + log.warning(f"Failed to insert FTS docs: {self.indice} error: {e!s}") + return 0, e + def prepare_filter(self, filters: Filter): self.routing_key = None if filters.type == FilterOp.NonFilter: @@ -203,6 +259,47 @@ def search_embedding( ) return [h["fields"][self.id_col_name][0] for h in res["hits"]["hits"]] + def search_documents( + self, + query: str, + k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + **kwargs, + ) -> list[str]: + if not getattr(self, "_is_fts", False): + msg = "ElasticCloud full-text search requires ElasticCloudFtsConfig" + raise RuntimeError(msg) + if not self.supports_document_payload_profile(payload_profile): + msg = f"ElasticCloud does not support document payload_profile={payload_profile.value}" + raise NotImplementedError(msg) + assert self.client is not None, "should self.init() first" + source = [self.text_col_name] if payload_profile == PayloadProfile.TEXT else False + filter_path = ["hits.hits._id", f"hits.hits.fields.{self.id_col_name}"] + if payload_profile == PayloadProfile.TEXT: + filter_path.append(f"hits.hits._source.{self.text_col_name}") + search_kwargs = { + "index": self.indice, + "query": {"match": {self.text_col_name: query}}, + "size": k, + "_source": source, + "docvalue_fields": [self.id_col_name], + "filter_path": filter_path, + } + if payload_profile != PayloadProfile.TEXT: + search_kwargs["stored_fields"] = "_none_" + res = self.client.search(**search_kwargs) + doc_ids = [] + for hit in res.get("hits", {}).get("hits", []): + if hit.get("_id") is not None: + doc_ids.append(str(hit["_id"])) + continue + + fields = hit.get("fields", {}) + values = fields.get(self.id_col_name, []) + if values: + doc_ids.append(str(values[0])) + return doc_ids + def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases.""" assert self.client is not None, "should self.init() first" diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 146d23aed..48ad45608 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -795,3 +795,48 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): ), **parameters, ) + + +class MilvusFTSTypedDict(CommonTypedDict, MilvusTypedDict): + """TypedDict for Milvus FTS command parameters.""" + + drop_ratio_search: Annotated[ + float | None, + click.option( + "--drop-ratio-search", + type=float, + help="Drop ratio for search (optional, for performance tuning)", + required=False, + default=None, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusFTSTypedDict) +def MilvusFTS(**parameters: Unpack[MilvusFTSTypedDict]): + """Run FTS (Full-Text Search) benchmark on Milvus using BM25. + + This command uses the MS MARCO dev/small dataset for FTS testing. + """ + from .config import MilvusConfig, MilvusFtsConfig + + # Set default case_type to large dataset if not specified + if parameters.get("case_type") == "Performance1536D50K": # Default from CommonTypedDict + parameters["case_type"] = "FTSBm25Performance" + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=MilvusFtsConfig( + drop_ratio_search=parameters.get("drop_ratio_search"), + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index 054a3fddb..b069685fa 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -492,8 +492,164 @@ def index_param(self) -> dict: } +class MilvusFtsConfig(BaseModel, DBCaseConfig): + """ + 1. inverted_index_algo: Index algorithm selection + - "DAAT_MAXSCORE" (default): Suitable for high k values or queries with many terms, balanced performance + - "DAAT_WAND": Suitable for small k values or short queries, faster + - "TAAT_NAIVE": Dynamically adapts to collection changes (e.g., avgdl), but slower + 2. bm25_k1: BM25 term frequency saturation control [1.2, 2.0], default None + - None: Do not override Milvus product default + - Higher values: Increase importance of term frequency in document ranking + - Recommended range: 1.2-1.8, adjust based on query characteristics + 3. bm25_b: BM25 document length normalization control [0.0, 1.0], default None + - None: Do not override Milvus product default + - 1.0: No length normalization, longer documents have advantage + - 0.0: Full normalization, shorter documents have advantage + - 0.75: Balanced length normalization, commonly used default + 4. analyzer_tokenizer: Tokenizer type, default "standard" + - "standard": Standard tokenizer, suitable for English text + - "whitespace": Split by whitespace characters + - "keyword": No tokenization, preserve original text + 5. analyzer_enable_lowercase: Enable lowercase conversion, default True + - True: Convert all text to lowercase, improve matching rate + - False: Preserve original case + 6. analyzer_max_token_length: Maximum length of individual tokens, default 40 + - Limit length of overly long words + - Set to None to disable this limit + 7. analyzer_stop_words: Stop words list, default None + - Comma-separated stop words, e.g., "of,to,the,and,or" + - These words will be filtered out and not participate in indexing/search + 8. drop_ratio_search: Ratio of minimum values to ignore during search [0.0, 1.0], default None + - 0.0: Keep all values, highest recall + - 0.1-0.3: Improve search speed by 10-20%, slight impact on recall + """ + + index_type: str = "SPARSE_INVERTED_INDEX" + metric_type: MetricType = MetricType.BM25 + inverted_index_algo: str = "DAAT_MAXSCORE" # DAAT_MAXSCORE | DAAT_WAND | TAAT_NAIVE + bm25_k1: float | None = None + bm25_b: float | None = None + analyzer_tokenizer: str = "standard" + analyzer_enable_lowercase: bool = True + analyzer_max_token_length: int | None = None + analyzer_stop_words: str | None = None + drop_ratio_search: float | None = None + + @staticmethod + def _manifest_filter_list(analyzer_params: dict) -> list: + filters = analyzer_params.get("filter") or [] + if isinstance(filters, list): + return filters + return [filters] + + def _analyzer_manifest_updates(self, analyzer_params: dict) -> dict: + if not analyzer_params: + return {} + + updates = {} + tokenizer = analyzer_params.get("tokenizer") + if tokenizer: + updates["analyzer_tokenizer"] = tokenizer + + filters = self._manifest_filter_list(analyzer_params) + updates["analyzer_enable_lowercase"] = "lowercase" in filters + + length_max = None + stop_words = None + for item in filters: + if not isinstance(item, dict): + continue + if item.get("type") == "length": + length_max = item.get("max") + elif item.get("type") == "stop": + configured_stop_words = item.get("stop_words") + if isinstance(configured_stop_words, list): + stop_words = ",".join(str(word) for word in configured_stop_words) + elif configured_stop_words: + stop_words = str(configured_stop_words) + + updates["analyzer_max_token_length"] = length_max + updates["analyzer_stop_words"] = stop_words + return updates + + def apply_fts_manifest( + self, + bm25_params: dict[str, float], + analyzer_params: dict, + ) -> tuple[DBCaseConfig, dict]: + updates = {} + applied_bm25_params = {} + + if "k1" in bm25_params: + updates["bm25_k1"] = bm25_params["k1"] + applied_bm25_params["k1"] = bm25_params["k1"] + if "b" in bm25_params: + updates["bm25_b"] = bm25_params["b"] + applied_bm25_params["b"] = bm25_params["b"] + updates.update(self._analyzer_manifest_updates(analyzer_params)) + + return self.model_copy(update=updates), { + "applied_bm25_params": applied_bm25_params, + "unapplied_bm25_params": {k: v for k, v in bm25_params.items() if k not in applied_bm25_params}, + "applied_analyzer_params": dict(analyzer_params), + "unapplied_analyzer_params": {}, + } + + def analyzer_param(self) -> dict: + analyzer_params = {} + if self.analyzer_tokenizer: + analyzer_params["tokenizer"] = self.analyzer_tokenizer + + filters = [] + if self.analyzer_enable_lowercase: + filters.append("lowercase") + + if self.analyzer_max_token_length: + filters.append({"type": "length", "max": self.analyzer_max_token_length}) + + if self.analyzer_stop_words: + stop_words = [word.strip() for word in self.analyzer_stop_words.split(",") if word.strip()] + if stop_words: + filters.append({"type": "stop", "stop_words": stop_words}) + + if filters: + analyzer_params["filter"] = filters + + return analyzer_params or {"tokenizer": "standard"} + + def sparse_index_param(self) -> dict: + params = { + "inverted_index_algo": self.inverted_index_algo, + } + if self.bm25_k1 is not None: + params["bm25_k1"] = self.bm25_k1 + if self.bm25_b is not None: + params["bm25_b"] = self.bm25_b + + return { + "index_type": self.index_type, + "metric_type": self.metric_type.value, + "params": params, + } + + def index_param(self) -> dict: + return {**self.sparse_index_param(), "analyzer_params": self.analyzer_param()} + + def search_param(self) -> dict: + + params: dict = {} + if self.drop_ratio_search is not None: + params["drop_ratio_search"] = self.drop_ratio_search + return { + "metric_type": self.metric_type.value, + "params": params, + } + + _milvus_case_config = { IndexType.AUTOINDEX: AutoIndexConfig, + IndexType.FTS: MilvusFtsConfig, IndexType.HNSW: HNSWConfig, IndexType.HNSW_SQ: HNSWSQConfig, IndexType.HNSW_PQ: HNSWPQConfig, diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 063faf3bd..6ac1f5817 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -6,17 +6,18 @@ from contextlib import contextmanager from typing import Any -from pymilvus import DataType, MilvusClient, MilvusException +from pymilvus import DataType, Function, FunctionType, MilvusClient, MilvusException from vectordb_bench.backend.filter import Filter, FilterOp from vectordb_bench.backend.payload import PayloadProfile from ..api import VectorDB -from .config import MilvusIndexConfig +from .config import MilvusFtsConfig, MilvusIndexConfig log = logging.getLogger(__name__) MILVUS_LOAD_REQS_SIZE = 1.5 * 1024 * 1024 +MILVUS_FTS_BATCH_SIZE = 1000 MILVUS_FORCE_MERGE_TARGET_SIZE_MB = ((1 << 63) - 1) // (1024**2) @@ -27,15 +28,23 @@ class Milvus(VectorDB): FilterOp.StrEqual, ] - def __init__( + @classmethod + def supports_full_text_search(cls) -> bool: + return True + + def has_text_field(self) -> bool: + return bool(getattr(self, "_is_fts", False) and getattr(self, "_text_field", None)) + + def __init__( # noqa: PLR0915 self, dim: int, db_config: dict, - db_case_config: MilvusIndexConfig, + db_case_config: MilvusIndexConfig | MilvusFtsConfig, collection_name: str = "VDBBench", drop_old: bool = False, name: str = "Milvus", with_scalar_labels: bool = False, + fts_batch_size: int | None = None, **kwargs, ): """Initialize wrapper around the milvus vector database.""" @@ -43,23 +52,40 @@ def __init__( self.db_config = db_config self.case_config = db_case_config self.collection_name = collection_name - self.batch_size = int(MILVUS_LOAD_REQS_SIZE / (dim * 4)) self.with_scalar_labels = with_scalar_labels - self._primary_field = "pk" - self._scalar_id_field = "id" self._scalar_label_field = "label" self._scalar_payload_label_field = self._scalar_label_field self._multitenant_partition_key_field = self._scalar_label_field + self._scalar_labels_index_name = "labels_idx" + self._is_fts = isinstance(self.case_config, MilvusFtsConfig) + + if self._is_fts: + self.batch_size = fts_batch_size or MILVUS_FTS_BATCH_SIZE + self._primary_field = "doc_id" + self._text_field = "text" + self._sparse_field = "sparse_vector" + self._sparse_index_name = "sparse_vector_idx" + self._doc_id_sort_index_name = "doc_id_sort_idx" + self._main_index_name = self._sparse_index_name + self._sort_index_name = self._doc_id_sort_index_name + self._sort_index_field = self._primary_field + else: + self.batch_size = int(MILVUS_LOAD_REQS_SIZE / (dim * 4)) + self._primary_field = "pk" + self._scalar_id_field = "id" + self._vector_field = "vector" + self._vector_index_name = "vector_idx" + self._scalar_id_index_name = "id_sort_idx" + self._main_index_name = self._vector_index_name + self._sort_index_name = self._scalar_id_index_name + self._sort_index_field = self._scalar_id_field + self.multitenant_tenant_labels: list[str] = kwargs.get("multitenant_tenant_labels", []) if self.multitenant_tenant_labels: self._multitenant_partition_key_field = "labels" if self.with_scalar_labels: self._scalar_payload_label_field = "scalar_label" - self._vector_field = "vector" - self._vector_index_name = "vector_idx" - self._scalar_id_index_name = "id_sort_idx" - self._scalar_labels_index_name = "labels_idx" client = MilvusClient( uri=self.db_config.get("uri"), @@ -75,31 +101,59 @@ def __init__( if not client.has_collection(self.collection_name): schema = MilvusClient.create_schema() - schema.add_field(self._primary_field, DataType.INT64, is_primary=True) - schema.add_field(self._scalar_id_field, DataType.INT64) - schema.add_field(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) - - if self.multitenant_tenant_labels: + if self._is_fts: + analyzer_params = ( + self.case_config.analyzer_param() + if hasattr(self.case_config, "analyzer_param") + else self.case_config.index_param().get("analyzer_params", {"type": "english"}) + ) + schema.add_field(self._primary_field, DataType.VARCHAR, max_length=512, is_primary=True) schema.add_field( - self._multitenant_partition_key_field, + self._text_field, DataType.VARCHAR, - max_length=256, - is_partition_key=True, + max_length=65535, + enable_analyzer=True, + enable_match=True, + analyzer_params=analyzer_params, + ) + schema.add_field(self._sparse_field, DataType.SPARSE_FLOAT_VECTOR) + if self.with_scalar_labels: + schema.add_field(self._scalar_label_field, DataType.VARCHAR, max_length=256) + schema.add_function( + Function( + name="text_bm25_emb", + function_type=FunctionType.BM25, + input_field_names=[self._text_field], + output_field_names=[self._sparse_field], + params={}, + ) ) + else: + schema.add_field(self._primary_field, DataType.INT64, is_primary=True) + schema.add_field(self._scalar_id_field, DataType.INT64) + schema.add_field(self._vector_field, DataType.FLOAT_VECTOR, dim=dim) - if self.with_scalar_labels: - is_partition_key = db_case_config.use_partition_key - log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") - if not self.multitenant_tenant_labels or ( - self._scalar_payload_label_field != self._multitenant_partition_key_field - ): + if self.multitenant_tenant_labels: schema.add_field( - self._scalar_payload_label_field, + self._multitenant_partition_key_field, DataType.VARCHAR, max_length=256, - is_partition_key=is_partition_key and not self.multitenant_tenant_labels, + is_partition_key=True, ) + if self.with_scalar_labels: + is_partition_key = db_case_config.use_partition_key + log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") + if not self.multitenant_tenant_labels or ( + self._scalar_payload_label_field != self._multitenant_partition_key_field + ): + schema.add_field( + self._scalar_payload_label_field, + DataType.VARCHAR, + max_length=256, + is_partition_key=is_partition_key and not self.multitenant_tenant_labels, + ) + log.info(f"{self.name} create collection: {self.collection_name}") index_params = self._build_index_params() @@ -119,17 +173,31 @@ def __init__( def _build_index_params(self): index_params = MilvusClient.prepare_index_params() - vec_idx = self.case_config.index_param() - index_params.add_index( - field_name=self._vector_field, - index_name=self._vector_index_name, - index_type=vec_idx.get("index_type", ""), - metric_type=vec_idx.get("metric_type", ""), - params=vec_idx.get("params", {}), - ) + if self._is_fts: + sparse_idx = ( + self.case_config.sparse_index_param() + if hasattr(self.case_config, "sparse_index_param") + else self.case_config.index_param() + ) + index_params.add_index( + field_name=self._sparse_field, + index_name=self._main_index_name, + index_type=sparse_idx.get("index_type", ""), + metric_type=sparse_idx.get("metric_type", ""), + params=sparse_idx.get("params", {}), + ) + else: + vec_idx = self.case_config.index_param() + index_params.add_index( + field_name=self._vector_field, + index_name=self._vector_index_name, + index_type=vec_idx.get("index_type", ""), + metric_type=vec_idx.get("metric_type", ""), + params=vec_idx.get("params", {}), + ) index_params.add_index( - field_name=self._scalar_id_field, - index_name=self._scalar_id_index_name, + field_name=self._sort_index_field, + index_name=self._sort_index_name, index_type="STL_SORT", ) if self.with_scalar_labels: @@ -219,7 +287,7 @@ def _wait_for_segments_sorted(self): def _wait_for_index(self): while True: - info = self.client.describe_index(self.collection_name, self._vector_index_name) + info = self.client.describe_index(self.collection_name, self._main_index_name) if info.get("pending_index_rows", -1) == 0: break time.sleep(5) @@ -236,7 +304,7 @@ def _optimize(self): try: self.client.flush(self.collection_name) - if self.case_config.is_gpu_index: + if getattr(self.case_config, "is_gpu_index", False): log.debug("skip force merge compaction for gpu index type.") else: try: @@ -269,6 +337,8 @@ def optimize(self, data_size: int | None = None): def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" + if self._is_fts: + return False if self.case_config.is_gpu_index: log.info("current gpu_index only supports IP / L2, cosine dataset need normalize.") return True @@ -309,7 +379,56 @@ def insert_embeddings( return insert_count, e return insert_count, None + def insert_documents( + self, + texts: Iterable[str], + doc_ids: list[str], + **kwargs, + ) -> tuple[int, Exception | None]: + """Insert documents into a Milvus BM25 full-text collection.""" + if not self._is_fts: + msg = "insert_documents is only valid in FTS mode" + raise RuntimeError(msg) + assert self.client is not None + + docs = list(texts) + if len(docs) != len(doc_ids): + msg = f"Mismatch between texts ({len(docs)}) and doc_ids ({len(doc_ids)}) lengths" + raise ValueError(msg) + + batch_size = kwargs.get("batch_size", self.batch_size) + labels_data = kwargs.get("labels_data") + + insert_count = 0 + try: + for batch_start_offset in range(0, len(docs), batch_size): + batch_end_offset = min(batch_start_offset + batch_size, len(docs)) + rows = [] + for i in range(batch_start_offset, batch_end_offset): + row = { + self._primary_field: str(doc_ids[i]), + self._text_field: docs[i], + } + if self.with_scalar_labels: + row[self._scalar_label_field] = labels_data[i] if labels_data is not None else "" + rows.append(row) + + res = self.client.insert(self.collection_name, rows) + insert_count += res["insert_count"] + if batch_start_offset // batch_size % 10 == 0: + log.debug( + f"{self.name} batch insert progress: {batch_end_offset}/{len(docs)} " + f"({batch_end_offset / len(docs) * 100:.1f}%)" + ) + except MilvusException as e: + log.info(f"{self.name} insert error: {e}") + return insert_count, e + return insert_count, None + def prepare_filter(self, filters: Filter): + if self._is_fts: + self.expr = "" + return if filters.type == FilterOp.NonFilter: self.expr = "" elif filters.type == FilterOp.NumGE: @@ -332,7 +451,7 @@ def poll_insert_readiness(self, expected_count: int) -> dict: self.client.flush(self.collection_name) stats = self.client.get_collection_stats(self.collection_name) count = int(stats.get("row_count", stats.get("num_entities", 0))) - progress = self.client.describe_index(self.collection_name, self._vector_index_name) + progress = self.client.describe_index(self.collection_name, self._main_index_name) return { "fully_searchable": count >= expected_count, "fully_indexed": progress.get("pending_index_rows", -1) == 0, @@ -374,3 +493,42 @@ def search_embedding( res = self.client.search(**search_kwargs) return [result[self._primary_field] for result in res[0]] + + def search_documents( + self, + query: str, + k: int = 100, + timeout: int | None = None, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + ) -> list[str]: + """Search a Milvus BM25 full-text collection and return document IDs.""" + if not self._is_fts: + msg = "search_documents only valid in FTS mode" + raise RuntimeError(msg) + if not self.supports_document_payload_profile(payload_profile): + msg = f"{getattr(self, 'name', 'Milvus')} does not support document payload_profile={payload_profile.value}" + raise NotImplementedError(msg) + assert self.client is not None + + output_fields = [self._primary_field] + if payload_profile == PayloadProfile.TEXT: + output_fields.append(self._text_field) + + res = self.client.search( + collection_name=self.collection_name, + data=[str(query)], + anns_field=self._sparse_field, + search_params=self.case_config.search_param(), + limit=k, + output_fields=output_fields, + ) + + hits = res[0] if res else [] + doc_ids = [] + for hit in hits: + entity = hit.get("entity", hit) if isinstance(hit, dict) else hit + if isinstance(entity, dict): + doc_ids.append(str(entity.get(self._primary_field))) + else: + doc_ids.append(str(getattr(entity, self._primary_field))) + return doc_ids diff --git a/vectordb_bench/backend/clients/pgdiskann/cli.py b/vectordb_bench/backend/clients/pgdiskann/cli.py index f1e44e2f2..4f372c779 100644 --- a/vectordb_bench/backend/clients/pgdiskann/cli.py +++ b/vectordb_bench/backend/clients/pgdiskann/cli.py @@ -80,7 +80,7 @@ class PgDiskAnnTypedDict(CommonTypedDict): click.option( "--reranking-metric", type=click.Choice( - [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD", "DP"]], + [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD", "DP", "BM25"]], ), help="Distance metric for reranking", default="COSINE", diff --git a/vectordb_bench/backend/clients/pgvector/cli.py b/vectordb_bench/backend/clients/pgvector/cli.py index 2d56c4815..7da41d5c6 100644 --- a/vectordb_bench/backend/clients/pgvector/cli.py +++ b/vectordb_bench/backend/clients/pgvector/cli.py @@ -109,7 +109,7 @@ class PgVectorTypedDict(CommonTypedDict): click.option( "--reranking-metric", type=click.Choice( - [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"]], + [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD", "BM25"]], ), help="Distance metric for reranking", default="COSINE", diff --git a/vectordb_bench/backend/clients/turbopuffer/config.py b/vectordb_bench/backend/clients/turbopuffer/config.py index ed28675f8..cdb7e8328 100644 --- a/vectordb_bench/backend/clients/turbopuffer/config.py +++ b/vectordb_bench/backend/clients/turbopuffer/config.py @@ -60,3 +60,15 @@ def index_param(self) -> dict: def search_param(self) -> dict: return {} + + +class TurboPufferFtsConfig(BaseModel, DBCaseConfig): + metric_type: MetricType = MetricType.BM25 + time_wait_warmup: int = 60 + disable_backpressure: bool = False + + def index_param(self) -> dict: + return {} + + def search_param(self) -> dict: + return {} diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py index e28e38ae3..38103f1a1 100644 --- a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -1,6 +1,7 @@ """Wrapper around the TurboPuffer vector database over VectorDB""" import logging +import os import time from contextlib import contextmanager from json import dumps, loads @@ -12,6 +13,7 @@ import turbopuffer as tpuf from vectordb_bench.backend.clients.turbopuffer.config import ( + TurboPufferFtsConfig, TurboPufferIndexConfig, TurboPufferMultitenantWarmupPolicy, ) @@ -91,7 +93,7 @@ def __init__( self, dim: int, db_config: dict, - db_case_config: TurboPufferIndexConfig, + db_case_config: TurboPufferIndexConfig | TurboPufferFtsConfig, drop_old: bool = False, with_scalar_labels: bool = False, **kwargs, @@ -109,12 +111,14 @@ def __init__( self.pin_timeout = db_config.get("pin_timeout", PINNING_TIMEOUT) self._pinning_applied = False self.db_case_config = db_case_config - self.metric = db_case_config.parse_metric() + self._is_fts = isinstance(db_case_config, TurboPufferFtsConfig) + self.metric = None if self._is_fts else db_case_config.parse_metric() self._vector_field = "vector" self._scalar_id_field = "id" self._scalar_label_field = "label" self._scalar_payload_label_field = db_config.get("scalar_payload_label_field", self._scalar_label_field) + self._text_field = "text" self.with_scalar_labels = with_scalar_labels self.expr = None @@ -138,6 +142,9 @@ def __init__( def _create_client(self) -> tpuf.Turbopuffer: client_kwargs = {"api_key": self.api_key, "region": self.region} + max_retries = os.getenv("TURBOPUFFER_MAX_RETRIES") + if max_retries is not None: + client_kwargs["max_retries"] = int(max_retries) if self.api_base_url: client_kwargs["base_url"] = self.api_base_url return tpuf.Turbopuffer(**client_kwargs) @@ -177,6 +184,26 @@ def _target_namespaces_for_pinning(self) -> list[str]: return [self._namespace_name_for_tenant(tenant) for tenant in self.multitenant_tenant_labels] return [self.namespace] + def __getstate__(self): + state = self.__dict__.copy() + state.pop("client", None) + state.pop("ns", None) + state.pop("_ns_cache", None) + return state + + def __setstate__(self, state: dict): + self.__dict__.update(state) + self.client = self._create_client() + self._ns_cache = {} + self.ns = None + + @classmethod + def supports_full_text_search(cls) -> bool: + return True + + def has_text_field(self) -> bool: + return bool(getattr(self, "_is_fts", False) and getattr(self, "_text_field", None)) + @contextmanager def init(self): self.client = self._create_client() @@ -345,6 +372,42 @@ def poll_insert_readiness(self, expected_count: int) -> dict: "additional_parameters": {"disable_backpressure": self.db_case_config.disable_backpressure}, } + def insert_documents( + self, + texts: list[str], + doc_ids: list[str], + **kwargs, + ) -> tuple[int, Exception | None]: + if not getattr(self, "_is_fts", False): + msg = "TurboPuffer full-text insert requires TurboPufferFtsConfig" + raise RuntimeError(msg) + assert self.ns is not None, "should self.init() first" + + docs = list(texts) + if len(docs) != len(doc_ids): + msg = f"Mismatch between texts ({len(docs)}) and doc_ids ({len(doc_ids)}) lengths" + raise ValueError(msg) + + text_field = self._text_field + try: + self.ns.write( + upsert_columns={ + self._scalar_id_field: [str(doc_id) for doc_id in doc_ids], + text_field: docs, + }, + schema={ + text_field: { + "type": "string", + "full_text_search": True, + } + }, + disable_backpressure=self.db_case_config.disable_backpressure, + ) + except Exception as e: + log.warning(f"Failed to insert FTS docs. Error: {e}") + return 0, e + return len(docs), None + def search_embedding( self, query: list[float], @@ -365,6 +428,31 @@ def search_embedding( res = self._namespace_for_tenant(tenant).query(**query_kwargs) return [int(row.id) for row in res.rows] if res.rows is not None else [] + def search_documents( + self, + query: str, + k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + **kwargs, + ) -> list[str]: + if not getattr(self, "_is_fts", False): + msg = "TurboPuffer full-text search requires TurboPufferFtsConfig" + raise RuntimeError(msg) + if not self.supports_document_payload_profile(payload_profile): + msg = f"TurboPuffer does not support document payload_profile={payload_profile.value}" + raise NotImplementedError(msg) + assert self.ns is not None, "should self.init() first" + + query_kwargs = { + "rank_by": (self._text_field, "BM25", query), + "top_k": k, + } + if payload_profile == PayloadProfile.TEXT: + query_kwargs["include_attributes"] = [self._text_field] + res = self.ns.query(**query_kwargs) + rows = getattr(res, "rows", None) or [] + return [str(row.id) for row in rows if getattr(row, "id", None) is not None] + def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: self.expr = None diff --git a/vectordb_bench/backend/clients/vespa/config.py b/vectordb_bench/backend/clients/vespa/config.py index 3d4a1deaf..006160a0b 100644 --- a/vectordb_bench/backend/clients/vespa/config.py +++ b/vectordb_bench/backend/clients/vespa/config.py @@ -49,3 +49,53 @@ def parse_metric(self, metric_type: MetricType) -> VespaMetric: return "hamming" case _: raise NotImplementedError + + +class VespaFtsConfig(BaseModel, DBCaseConfig): + metric_type: MetricType = MetricType.BM25 + bm25_k1: float | None = None + bm25_b: float | None = None + bm25_avgdl: float | None = None + feed_client_command: str = "vespa" + feed_client_connections: int | None = None + + def apply_fts_manifest( + self, + bm25_params: dict[str, float], + analyzer_params: dict, + ) -> tuple[DBCaseConfig, dict]: + updates = {} + applied_bm25_params = {} + + if "k1" in bm25_params: + updates["bm25_k1"] = bm25_params["k1"] + applied_bm25_params["k1"] = bm25_params["k1"] + if "b" in bm25_params: + updates["bm25_b"] = bm25_params["b"] + applied_bm25_params["b"] = bm25_params["b"] + if "avgdl" in bm25_params: + updates["bm25_avgdl"] = bm25_params["avgdl"] + applied_bm25_params["avgdl"] = bm25_params["avgdl"] + + return self.model_copy(update=updates), { + "applied_bm25_params": applied_bm25_params, + "unapplied_bm25_params": {k: v for k, v in bm25_params.items() if k not in applied_bm25_params}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": dict(analyzer_params), + } + + def index_param(self) -> dict: + return {} + + def search_param(self) -> dict: + return {} + + def rank_properties(self) -> list[tuple[str, str]]: + properties = [] + if self.bm25_k1 is not None: + properties.append(("bm25(text).k1", str(self.bm25_k1))) + if self.bm25_b is not None: + properties.append(("bm25(text).b", str(self.bm25_b))) + if self.bm25_avgdl is not None: + properties.append(("bm25(text).averageFieldLength", str(self.bm25_avgdl))) + return properties diff --git a/vectordb_bench/backend/clients/vespa/vespa.py b/vectordb_bench/backend/clients/vespa/vespa.py index 1f2e1b883..d10d906a8 100644 --- a/vectordb_bench/backend/clients/vespa/vespa.py +++ b/vectordb_bench/backend/clients/vespa/vespa.py @@ -1,24 +1,56 @@ import datetime +import json import logging import math +import shutil +import subprocess +import tempfile +import threading +import time from collections.abc import Generator from contextlib import contextmanager +from typing import Any from vespa import application +from vectordb_bench.backend.payload import PayloadProfile + from ..api import VectorDB from . import util -from .config import VespaHNSWConfig +from .config import VespaFtsConfig, VespaHNSWConfig log = logging.getLogger(__name__) +VESPA_DOC_COUNT_POLL_INTERVAL_SEC = 2 +VESPA_DOC_COUNT_TIMEOUT_SEC = 1800 +VESPA_FEED_OUTPUT_TAIL_CHARS = 4000 + + +def _document_id_from_hit(hit: dict) -> str | None: + fields = hit.get("fields") + if isinstance(fields, dict) and "id" in fields: + return str(fields["id"]) + + document_id = hit.get("id") + if document_id is None: + return None + + document_id = str(document_id) + if document_id.startswith("id:") and "::" in document_id: + return document_id.rsplit("::", 1)[-1] + return document_id + + +def _tail_text(value: str, limit: int = VESPA_FEED_OUTPUT_TAIL_CHARS) -> str: + return value[-limit:] if len(value) > limit else value + class Vespa(VectorDB): def __init__( self, dim: int, db_config: dict[str, str], - db_case_config: VespaHNSWConfig | None = None, + db_case_config: VespaHNSWConfig | VespaFtsConfig | None = None, collection_name: str = "VectorDBBenchCollection", drop_old: bool = False, **kwargs, @@ -26,7 +58,9 @@ def __init__( self.dim = dim self.db_config = db_config self.case_config = db_case_config or VespaHNSWConfig() + self._is_fts = isinstance(self.case_config, VespaFtsConfig) self.schema_name = collection_name + self._text_field = "text" client = self.deploy_http() client.wait_for_application_up() @@ -35,8 +69,13 @@ def __init__( try: client.delete_all_docs("vectordbbench_content", self.schema_name) except Exception: + if self._is_fts: + raise drop_old = False log.exception(f"Vespa client drop_old schema: {self.schema_name}") + else: + if self._is_fts: + self._wait_for_document_count(client, 0, "drop_old") @contextmanager def init(self) -> Generator[None, None, None]: @@ -54,8 +93,39 @@ def init(self) -> Generator[None, None, None]: >>> self.insert_embeddings() """ self.client = application.Vespa(self.db_config["url"], port=self.db_config["port"]) - yield + self._reset_fts_feed_client() + try: + yield + self._finish_fts_feed_client() + finally: + self._cleanup_fts_feed_client() + self.client = None + + def __getstate__(self): + state = self.__dict__.copy() + # Multiprocessing search uses spawn, so DB instances are pickled before + # workers run. Vespa client and feed-client handles are process-local. + state.pop("client", None) + state.pop("_feed_proc", None) + state.pop("_feed_stdout_file", None) + state.pop("_feed_stderr_file", None) + state.pop("_feed_lock", None) + return state + + def __setstate__(self, state: dict): + self.__dict__.update(state) self.client = None + self._feed_proc = None + self._feed_stdout_file = None + self._feed_stderr_file = None + self._feed_lock = threading.Lock() + + @classmethod + def supports_full_text_search(cls) -> bool: + return True + + def has_text_field(self) -> bool: + return bool(getattr(self, "_is_fts", False) and getattr(self, "_text_field", None)) def need_normalize_cosine(self) -> bool: """Wheather this database need to normalize dataset to support COSINE""" @@ -84,6 +154,173 @@ def insert_embeddings( self.client.feed_iterable(data, self.schema_name) return len(embeddings), None + def insert_documents( + self, + texts: list[str], + doc_ids: list[str], + **kwargs, + ) -> tuple[int, Exception | None]: + if not self._is_fts: + msg = "Vespa full-text insert requires VespaFtsConfig" + raise RuntimeError(msg) + assert self.client is not None + + if len(texts) != len(doc_ids): + msg = f"Mismatch between texts ({len(texts)}) and doc_ids ({len(doc_ids)}) lengths" + raise ValueError(msg) + + try: + self._write_fts_feed_batch(texts, doc_ids) + except Exception as exc: + log.warning("Vespa feed failed for schema %s", self.schema_name, exc_info=True) + return 0, exc + + return len(texts), None + + def _reset_fts_feed_client(self) -> None: + self._feed_proc = None + self._feed_stdout_file = None + self._feed_stderr_file = None + self._feed_written_count = 0 + self._feed_lock = threading.Lock() + + def _ensure_fts_feed_client(self) -> subprocess.Popen: + if self._feed_proc is not None: + return self._feed_proc + command = self.case_config.feed_client_command + if shutil.which(command) is None: + msg = ( + f"Vespa feed client command {command!r} was not found. " + "Install the Vespa CLI or set VespaFtsConfig.feed_client_command." + ) + raise RuntimeError(msg) + + connections = self.case_config.feed_client_connections or 8 + cmd = [ + command, + "feed", + "-", + "--target", + self._feed_target(), + "--connections", + str(connections), + "--inflight", + "0", + "--progress", + "0", + ] + log.info("Start Vespa feed client: %s", " ".join(cmd)) + + # These files must stay open until the feed client exits; Popen writes to them after this method returns. + self._feed_stdout_file = tempfile.TemporaryFile(mode="w+", encoding="utf-8") # noqa: SIM115 + self._feed_stderr_file = tempfile.TemporaryFile(mode="w+", encoding="utf-8") # noqa: SIM115 + self._feed_proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=self._feed_stdout_file, + stderr=self._feed_stderr_file, + text=True, + ) + return self._feed_proc + + def _write_fts_feed_batch(self, texts: list[str], doc_ids: list[str]) -> None: + lines = [] + for doc_id, text in zip(doc_ids, texts, strict=True): + operation = { + "put": self._vespa_document_id(str(doc_id)), + "fields": {"id": str(doc_id), "text": text}, + } + lines.append(json.dumps(operation, ensure_ascii=False, separators=(",", ":"))) + + with self._feed_lock: + proc = self._ensure_fts_feed_client() + returncode = proc.poll() + if returncode is not None: + msg = f"Vespa feed client exited before all documents were written: returncode={returncode}" + raise RuntimeError(msg) + assert proc.stdin is not None + proc.stdin.write("\n".join(lines)) + proc.stdin.write("\n") + proc.stdin.flush() + self._feed_written_count += len(lines) + + def _finish_fts_feed_client(self) -> None: + if self._feed_proc is None: + return + with self._feed_lock: + proc = self._feed_proc + if proc.stdin is not None and not proc.stdin.closed: + proc.stdin.close() + returncode = proc.wait() + assert self._feed_stdout_file is not None + assert self._feed_stderr_file is not None + self._feed_stdout_file.seek(0) + self._feed_stderr_file.seek(0) + stdout = self._feed_stdout_file.read() + stderr = self._feed_stderr_file.read() + + count = self._feed_written_count + metrics = self._parse_feed_metrics(stdout) + ok_count = int(metrics.get("feeder.ok.count", count)) if metrics else count + error_count = int(metrics.get("feeder.error.count", 0)) if metrics else 0 + response_error_count = int(metrics.get("http.response.error.count", 0)) if metrics else 0 + + if returncode != 0 or error_count or ok_count != count: + msg = ( + "Vespa feed client failed " + f"returncode={returncode}, written={count}, ok={ok_count}, " + f"feed_errors={error_count}, response_errors={response_error_count}, " + f"stdout_tail={_tail_text(stdout)!r}, stderr_tail={_tail_text(stderr)!r}" + ) + raise RuntimeError(msg) + if response_error_count: + log.warning( + "Vespa feed client reported %d HTTP response errors, but all %d documents were accepted", + response_error_count, + ok_count, + ) + + log.info("Vespa feed client inserted %d docs; metrics=%s", ok_count, metrics) + self._feed_proc = None + + def _cleanup_fts_feed_client(self) -> None: + proc = getattr(self, "_feed_proc", None) + if proc is not None and proc.poll() is None: + proc.kill() + proc.wait() + for file_attr in ("_feed_stdout_file", "_feed_stderr_file"): + file_obj = getattr(self, file_attr, None) + if file_obj is not None: + file_obj.close() + setattr(self, file_attr, None) + self._feed_proc = None + + def _vespa_document_id(self, doc_id: str) -> str: + return f"id:{self.schema_name}:{self.schema_name}::{doc_id}" + + def _feed_target(self) -> str: + target = str(self.db_config["url"]).rstrip("/") + port = self.db_config.get("port") + if port is not None and f":{port}" not in target.rsplit("/", 1)[-1]: + target = f"{target}:{port}" + return target + + def _parse_feed_metrics(self, output: str) -> dict[str, Any]: + text = output.strip() + if not text: + return {} + try: + return json.loads(text) + except json.JSONDecodeError: + start = text.rfind("\n{") + if start >= 0: + try: + return json.loads(text[start + 1 :]) + except json.JSONDecodeError: + pass + log.warning("Failed to parse Vespa feed client metrics: %s", _tail_text(output)) + return {} + def search_embedding( self, query: list[float], @@ -123,6 +360,45 @@ def search_embedding( result = self.client.query({"yql": yql, "input.query(query_embedding)": query_embedding, "ranking": ranking}) return [child["fields"]["id"] for child in result.get_json()["root"]["children"]] + def search_documents( + self, + query: str, + k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + **kwargs, + ) -> list[str]: + if not self._is_fts: + msg = "Vespa full-text search requires VespaFtsConfig" + raise RuntimeError(msg) + if not self.supports_document_payload_profile(payload_profile): + msg = f"Vespa does not support document payload_profile={payload_profile.value}" + raise NotImplementedError(msg) + assert self.client is not None + + selected_fields = "id" + if payload_profile == PayloadProfile.TEXT: + selected_fields = f"id, {self._text_field}" + yql = f"select {selected_fields} from {self.schema_name} where userQuery()" + result = self.client.query( + { + "yql": yql, + "query": query, + "type": "any", + "ranking": "bm25", + "hits": k, + "default-index": self._text_field, + } + ) + children = result.get_json().get("root", {}).get("children", []) or [] + ids = [] + for child in children: + if not isinstance(child, dict): + continue + doc_id = _document_id_from_hit(child) + if doc_id is not None: + ids.append(doc_id) + return ids + def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases. @@ -132,6 +408,35 @@ def optimize(self, data_size: int | None = None): Time(insert the dataset) + Time(optimize) will be recorded as "load_duration" metric Optimize's execution time is limited, the limited time is based on cases. """ + if self._is_fts and data_size is not None: + assert self.client is not None + self._wait_for_document_count(self.client, data_size, "post_insert") + + def _document_count(self, client: application.Vespa) -> int: + result = client.query({"yql": f"select id from {self.schema_name} where true", "hits": 0}) + root = result.get_json().get("root", {}) + fields = root.get("fields", {}) + if "totalCount" not in fields: + msg = f"Vespa count query did not return totalCount for schema {self.schema_name}: {root}" + raise RuntimeError(msg) + return int(fields["totalCount"]) + + def _wait_for_document_count(self, client: application.Vespa, expected_count: int, stage: str) -> None: + deadline = time.monotonic() + VESPA_DOC_COUNT_TIMEOUT_SEC + last_count = None + while True: + last_count = self._document_count(client) + if last_count == expected_count: + log.info("Vespa %s document count reached %d", stage, expected_count) + return + if time.monotonic() >= deadline: + msg = ( + f"Timed out waiting for Vespa {stage} document count to reach " + f"{expected_count}; last_count={last_count}" + ) + raise TimeoutError(msg) + log.info("Waiting for Vespa %s document count: current=%d expected=%d", stage, last_count, expected_count) + time.sleep(VESPA_DOC_COUNT_POLL_INTERVAL_SEC) @property def application_package(self): @@ -151,6 +456,38 @@ def _create_application_package(self): ValidationID, ) + tomorrow = datetime.date.today() + datetime.timedelta(days=1) + + if self._is_fts: + fields = [ + Field("id", "string", indexing=["summary", "attribute"]), + Field( + "text", + "string", + indexing=["index", "summary"], + index="enable-bm25", + stemming="none", + ), + ] + return ApplicationPackage( + "vectordbbench", + [ + Schema( + self.schema_name, + Document(fields), + rank_profiles=[ + RankProfile( + name="bm25", + first_phase="bm25(text)", + inherits="default", + rank_properties=self.case_config.rank_properties(), + ), + ], + ) + ], + validations=[Validation(ValidationID.fieldTypeChange, until=tomorrow)], + ) + fields = [ Field( "id", @@ -185,8 +522,6 @@ def _create_application_package(self): ) ) - tomorrow = datetime.date.today() + datetime.timedelta(days=1) - return ApplicationPackage( "vectordbbench", [ diff --git a/vectordb_bench/backend/clients/zilliz_cloud/config.py b/vectordb_bench/backend/clients/zilliz_cloud/config.py index 8ab45caa2..ee5ecf645 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/config.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/config.py @@ -1,7 +1,7 @@ from pydantic import SecretStr from ..api import DBCaseConfig, DBConfig -from ..milvus.config import IndexType, MilvusIndexConfig +from ..milvus.config import IndexType, MilvusFtsConfig, MilvusIndexConfig class ZillizCloudConfig(DBConfig): @@ -46,3 +46,27 @@ def search_param(self) -> dict: "level": self.level, }, } + + +class ZillizCloudFtsConfig(MilvusFtsConfig): + index_type: str = IndexType.AUTOINDEX.value + level: int = 1 + + def sparse_index_param(self) -> dict: + params = {} + if self.bm25_k1 is not None: + params["bm25_k1"] = self.bm25_k1 + if self.bm25_b is not None: + params["bm25_b"] = self.bm25_b + + return { + "index_type": self.index_type, + "metric_type": self.metric_type.value, + "params": params, + } + + def search_param(self) -> dict: + return { + "metric_type": self.metric_type.value, + "params": {"level": self.level}, + } diff --git a/vectordb_bench/backend/data_source.py b/vectordb_bench/backend/data_source.py index 139d2e308..942a80409 100644 --- a/vectordb_bench/backend/data_source.py +++ b/vectordb_bench/backend/data_source.py @@ -1,13 +1,23 @@ import logging +import os import pathlib import typing from abc import ABC, abstractmethod from enum import Enum +import ir_datasets from tqdm import tqdm from vectordb_bench import config +# Set ir_datasets to use tmp directory for both home and temp +# This ensures all downloaded files and temporary data are stored in /tmp +ir_datasets_home = pathlib.Path(config.DATASET_LOCAL_DIR) / "ir_datasets" +ir_datasets_tmp = pathlib.Path(config.DATASET_LOCAL_DIR) / "ir_datasets_tmp" +os.environ.setdefault("IR_DATASETS_HOME", str(ir_datasets_home)) +os.environ.setdefault("IR_DATASETS_TMP", str(ir_datasets_tmp)) + + logging.getLogger("s3fs").setLevel(logging.CRITICAL) log = logging.getLogger(__name__) @@ -18,6 +28,7 @@ class DatasetSource(Enum): S3 = "S3" AliyunOSS = "AliyunOSS" + IR_DATASETS = "IR_DATASETS" def reader(self) -> DatasetReader: if self == DatasetSource.S3: @@ -26,6 +37,9 @@ def reader(self) -> DatasetReader: if self == DatasetSource.AliyunOSS: return AliyunOSSReader() + if self == DatasetSource.IR_DATASETS: + return IRDatasetsReader() + return None @@ -155,3 +169,40 @@ def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: return False return True + + +class IRDatasetsReader(DatasetReader): + """Reader for ir_datasets based datasets""" + + source: DatasetSource = DatasetSource.IR_DATASETS + remote_root: str = "" # Not used for ir_datasets + + def __init__(self): + self.ir_datasets = ir_datasets + + def read(self, dataset: str, files: list[str], local_ds_root: pathlib.Path): + """ + Download FTS dataset using ir_datasets API + + Args: + dataset: ir_datasets dataset name + files: Expected output files (ignored, not used) + local_ds_root: Local directory (not used, ir_datasets handles its own cache) + """ + log.info(f"Downloading FTS dataset '{dataset}' using ir_datasets") + + try: + # Load dataset using ir_datasets - this will download if needed + # ir_datasets handles caching automatically + # Actual data download happens lazily when iterating + self.ir_datasets.load(dataset) + log.info(f"Successfully loaded dataset: {dataset}") + + except Exception: + log.exception(f"Failed to download FTS dataset '{dataset}'") + raise + + def validate_file(self, remote: pathlib.Path, local: pathlib.Path) -> bool: + """For ir_datasets, we don't validate against remote files""" + # ir_datasets handles its own caching and validation + return local.exists() and local.stat().st_size > 0 diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index c249f91c9..6a6f262cc 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -4,15 +4,23 @@ >>> Dataset.Cohere.get(100_000) """ +import json import logging import pathlib +import types +import typing +from abc import ABC, abstractmethod +from collections.abc import Iterator +from dataclasses import dataclass from enum import Enum from typing import Any, ClassVar, NamedTuple +import ir_datasets import pandas as pd import polars as pl from pyarrow.parquet import ParquetFile -from pydantic import field_validator +from pydantic import Field as PydanticField +from pydantic import PrivateAttr, field_validator from vectordb_bench import config from vectordb_bench.base import BaseModel @@ -140,7 +148,7 @@ class LAION(BaseDataset): with_gt: bool = True with_scalar_labels: bool = True scalar_label_percentages: list[float] = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] - _size_label: ClassVar[dict] = { + _size_label: ClassVar[dict[int, SizeLabel]] = { 100_000_000: SizeLabel(100_000_000, "LARGE", 100), } @@ -150,7 +158,7 @@ class GIST(BaseDataset): dim: int = 960 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: ClassVar[dict] = { + _size_label: ClassVar[dict[int, SizeLabel]] = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), } @@ -162,7 +170,7 @@ class Cohere(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: ClassVar[dict] = { + _size_label: ClassVar[dict[int, SizeLabel]] = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), @@ -200,7 +208,7 @@ class Bioasq(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: ClassVar[dict] = { + _size_label: ClassVar[dict[int, SizeLabel]] = { 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), } @@ -236,7 +244,7 @@ class Glove(BaseDataset): dim: int = 200 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = False - _size_label: ClassVar[dict] = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} + _size_label: ClassVar[dict[int, SizeLabel]] = {1_000_000: SizeLabel(1_000_000, "MEDIUM", 1)} class SIFT(BaseDataset): @@ -244,7 +252,7 @@ class SIFT(BaseDataset): dim: int = 128 metric_type: MetricType = MetricType.L2 use_shuffled: bool = False - _size_label: ClassVar[dict] = { + _size_label: ClassVar[dict[int, SizeLabel]] = { 500_000: SizeLabel( 500_000, "SMALL", @@ -261,7 +269,7 @@ class OpenAI(BaseDataset): metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA with_gt: bool = True - _size_label: ClassVar[dict] = { + _size_label: ClassVar[dict[int, SizeLabel]] = { 50_000: SizeLabel(50_000, "SMALL", 1), 500_000: SizeLabel(500_000, "MEDIUM", 1), 5_000_000: SizeLabel(5_000_000, "LARGE", 10), @@ -532,3 +540,492 @@ def get_optimize_timeout(self) -> float: DatasetWithSizeType.OpenAIMedium: Dataset.OPENAI.manager(500_000), DatasetWithSizeType.OpenAILarge: Dataset.OPENAI.manager(5_000_000), } + + +# FTS Dataset Translator Pattern +@dataclass +class FtsQuery: + """Internal representation of an FTS query.""" + + query_id: str + text: str + + +@dataclass +class FtsDocument: + """Internal representation of an FTS document.""" + + doc_id: str + text: str + + +FTS_GT_FILE = "neighbors.parquet" +FTS_BUILD_MANIFEST_FILE = "build_manifest.json" +FTS_MATH_GT_FILES = (FTS_GT_FILE, FTS_BUILD_MANIFEST_FILE, "manifest.json") + + +class FtsDatasetTranslator(ABC): + """Abstract base class for converting ir_datasets schema to internal format. + + This translator pattern allows easy extension to support new datasets + (BEIR, TREC, etc.) without modifying core code. + """ + + @property + @abstractmethod + def ir_datasets_name(self) -> str: + """Return the ir_datasets dataset name. + + Example: 'msmarco-passage/dev/small' + """ + + @abstractmethod + def translate_query(self, ir_query: typing.Any) -> FtsQuery: + """Convert ir_datasets query to internal FtsQuery format.""" + + @abstractmethod + def translate_document(self, ir_doc: typing.Any) -> FtsDocument: + """Convert ir_datasets document to internal FtsDocument format.""" + + def load(self) -> typing.Any: + """Load ir_datasets dataset.""" + return ir_datasets.load(self.ir_datasets_name) + + def iter_queries(self, dataset: typing.Any) -> Iterator[FtsQuery]: + """Iterate over queries in the dataset.""" + for q in dataset.queries_iter(): + yield self.translate_query(q) + + def iter_documents(self, dataset: typing.Any) -> Iterator[FtsDocument]: + """Iterate over documents in the dataset.""" + for doc in dataset.docs_iter(): + yield self.translate_document(doc) + + +class MSMarcoTranslator(FtsDatasetTranslator): + """Translator for MS MARCO passage retrieval dataset.""" + + @property + def ir_datasets_name(self) -> str: + return "msmarco-passage/dev/small" + + def translate_query(self, ir_query: typing.Any) -> FtsQuery: + return FtsQuery(query_id=str(ir_query.query_id), text=ir_query.text) + + def translate_document(self, ir_doc: typing.Any) -> FtsDocument: + clean_text = ir_doc.text.replace("\t", " ").replace("\n", " ") + return FtsDocument(doc_id=str(ir_doc.doc_id), text=clean_text) + + +class HotpotQATranslator(FtsDatasetTranslator): + """Translator for BEIR HotpotQA.""" + + @property + def ir_datasets_name(self) -> str: + return "beir/hotpotqa/test" + + def translate_query(self, ir_query: typing.Any) -> FtsQuery: + return FtsQuery(query_id=str(ir_query.query_id), text=ir_query.text) + + def translate_document(self, ir_doc: typing.Any) -> FtsDocument: + title = getattr(ir_doc, "title", "") or "" + text = getattr(ir_doc, "text", "") or "" + clean_text = f"{title} {text}".replace("\t", " ").replace("\n", " ").strip() + return FtsDocument(doc_id=str(ir_doc.doc_id), text=clean_text) + + +class FtsBaseDataset(BaseModel): + """Base class for FTS datasets - completely independent from BaseDataset. + + FTS datasets are text-based and use TSV files instead of parquet files. + They don't have vector dimensions; native full-text search uses BM25. + + """ + + name: str + size: int + metric_type: MetricType = MetricType.BM25 + with_gt: bool = True + with_remote_resource: bool = False + gt_neighbors_field: str = "neighbors_id" + + _size_label: ClassVar[dict[int, SizeLabel]] + + @field_validator("size") + @classmethod + def verify_size(cls, v: int): + if v not in cls._size_label: + msg = f"Size {v} not supported for the FTS dataset, expected: {cls._size_label.keys()}" + raise ValueError(msg) + return v + + @property + def label(self) -> str: + """Get size label (SMALL, MEDIUM, LARGE, etc.)""" + return self._size_label.get(self.size).label + + @property + def full_name(self) -> str: + return f"{self.name} FTS ({self.label})" + + @property + def dir_name(self) -> str: + return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() + + +class MSMarcoFts(FtsBaseDataset): + name: str = "MS MARCO" + with_gt: bool = True + with_remote_resource: bool = False + + _size_label: ClassVar[dict[int, SizeLabel]] = { + 100_000: SizeLabel(100_000, "SMALL", 1), + 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), + 8_841_823: SizeLabel(8_841_823, "LARGE", 1), + } + + @property + def dir_name(self) -> str: + return f"msmarco_{self.label}_{utils.numerize(self.size)}".lower() + + +class HotpotQAFts(FtsBaseDataset): + name: str = "HotpotQA" + with_gt: bool = True + with_remote_resource: bool = False + + _size_label: ClassVar[dict[int, SizeLabel]] = { + 100_000: SizeLabel(100_000, "SMALL", 1), + 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), + 5_233_329: SizeLabel(5_233_329, "LARGE", 1), + } + + +class FtsDatasetManager(BaseModel): + """Manager for FTS datasets - independent from DatasetManager. + + Handles FTS dataset preparation using Translator pattern for extensibility. + + Similar to DatasetManager, but for text-based FTS datasets: + - queries_data: loaded queries (similar to test_data in vectors) + - gt_data: loaded ground truth (similar to gt_data in vectors) + - translator: dataset-specific translator for schema conversion + - _ir_dataset: ir_datasets dataset object for direct access + """ + + data: FtsBaseDataset + _translator: typing.Any = PrivateAttr() + + queries_data: list[FtsQuery] | None = None + gt_data: list[list[str]] | None = None + bm25_params: dict[str, float] = PydanticField(default_factory=dict) + analyzer_params: dict[str, typing.Any] = PydanticField(default_factory=dict) + _ir_dataset: typing.Any = PrivateAttr(default=None) + + def __init__(self, **data): + super().__init__(**data) + # Initialize translator based on dataset name + if isinstance(self.data, MSMarcoFts): + self._translator = MSMarcoTranslator() + elif isinstance(self.data, HotpotQAFts): + self._translator = HotpotQATranslator() + else: + msg = f"No translator available for dataset: {self.data.name}" + raise TypeError(msg) + + def __eq__(self, obj: any): + if isinstance(obj, FtsDatasetManager): + return self.data.name == obj.data.name and self.data.size == obj.data.size + return False + + def __hash__(self) -> int: + return hash((self.data.name, self.data.size)) + + @property + def data_dir(self) -> pathlib.Path: + """Get local data directory for this FTS dataset, following vector dataset structure""" + return pathlib.Path( + config.DATASET_LOCAL_DIR, + self.data.name.lower(), + self.data.dir_name, + ) + + def _download_math_gt_files(self) -> None: + DatasetSource.S3.reader().read( + dataset=self.data.dir_name.lower(), + files=list(FTS_MATH_GT_FILES), + local_ds_root=self.data_dir, + ) + + def _load_math_gt_data(self) -> list[list[str]]: + p = pathlib.Path(self.data_dir, FTS_GT_FILE) + if not p.exists(): + msg = f"No such file: {p}" + raise FileNotFoundError(msg) + gt_rows = pl.read_parquet(p)[self.data.gt_neighbors_field].to_list() + # FTS math GT stores dense document row IDs, not original ir_datasets doc IDs. + # FtsDocumentIterator assigns these same row IDs during insertion. + return [[str(doc_id) for doc_id in row if str(doc_id) != "-1"] for row in gt_rows] + + def _load_build_manifest(self) -> dict[str, typing.Any]: + p = pathlib.Path(self.data_dir, FTS_BUILD_MANIFEST_FILE) + if not p.exists(): + msg = f"No such file: {p}" + raise FileNotFoundError(msg) + manifest = json.loads(p.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + msg = f"Invalid FTS build manifest: {p}" + raise TypeError(msg) + return manifest + + def _validate_build_manifest(self, manifest: dict[str, typing.Any]) -> None: + source_ir_dataset = manifest.get("source_ir_dataset") + if source_ir_dataset is not None and source_ir_dataset != self._translator.ir_datasets_name: + msg = ( + f"{self.data.full_name} manifest source_ir_dataset={source_ir_dataset!r} " + f"does not match {self._translator.ir_datasets_name!r}" + ) + raise ValueError(msg) + + for field_name in ("doc_limit", "indexed_doc_count"): + value = manifest.get(field_name) + if value is None: + continue + if int(value) != self.data.size: + msg = f"{self.data.full_name} manifest {field_name}={value} does not match size={self.data.size}" + raise ValueError(msg) + + query_count = manifest.get("query_count") + if query_count is not None and self.queries_data is not None and int(query_count) != len(self.queries_data): + msg = ( + f"{self.data.full_name} manifest query_count={query_count} " + f"does not match loaded query count={len(self.queries_data)}" + ) + raise ValueError(msg) + + def _load_manifest_params(self) -> None: + manifest = self._load_build_manifest() + self._validate_build_manifest(manifest) + bm25 = manifest.get("bm25") or {} + analyzer = manifest.get("analyzer") or {} + self.bm25_params = { + key: float(bm25[key]) for key in ("k1", "b", "avgdl") if key in bm25 and bm25[key] is not None + } + self.analyzer_params = analyzer if isinstance(analyzer, dict) else {} + + def prepare( + self, + source: DatasetSource | None = None, + filters: Filter | None = None, + ) -> bool: + """Prepare FTS dataset for testing using Translator pattern. + + Directly uses ir_datasets API without generating TSV files: + 1. Downloads dataset using ir_datasets (if needed) + 2. Loads dataset object using translator + 3. Loads queries from ir_datasets and mathematical ground truth from S3 + + Args: + source: Data source to download from (should be IR_DATASETS for FTS) + filters: Optional filters (not used for FTS) + + Returns: + bool: True if preparation successful, False otherwise + """ + log.info(f"Preparing FTS dataset: {self.data.full_name}") + + try: + # Download dataset if needed (ir_datasets handles caching) + if source is not None: + reader = source.reader() + if reader is not None: + dataset_name = self._translator.ir_datasets_name + # reader.read() will download the dataset if needed + reader.read(dataset_name, [], self.data_dir) + + # Load dataset using translator + self._ir_dataset = self._translator.load() + log.info(f"Successfully loaded ir_datasets dataset: {self._translator.ir_datasets_name}") + + # Force ir_datasets lazy document cache work before timed insert. + for idx, _ in enumerate(self._translator.iter_documents(self._ir_dataset), start=1): + if idx >= self.data.size: + break + + # Load queries from ir_datasets and mathematical ground truth artifacts by row order. + if self.data.with_gt: + # Load queries using translator + self.queries_data = list(self._translator.iter_queries(self._ir_dataset)) + log.info(f"Loaded {len(self.queries_data)} queries into memory") + + self._download_math_gt_files() + self._load_manifest_params() + self.gt_data = self._load_math_gt_data() + if len(self.queries_data) != len(self.gt_data): + msg = ( + f"{self.data.full_name} query count {len(self.queries_data)} " + f"does not match ground truth row count {len(self.gt_data)}" + ) + raise ValueError(msg) # noqa: TRY301 + log.info(f"Loaded mathematical ground truth for {len(self.gt_data)} queries into memory") + + except (TypeError, ValueError): + log.exception("Invalid FTS dataset configuration") + raise + except Exception: + log.exception("Failed to prepare FTS dataset") + return False + else: + log.debug(f"{self.data.name}: FTS dataset prepared") + log.info(f"FTS dataset preparation completed: {self.data.full_name}") + return True + + def iter_batches(self, batch_size: int = config.NUM_PER_BATCH): + """Return an iterator for streaming FTS document batches.""" + return FtsDocumentIterator(self, batch_size=batch_size) + + def __iter__(self): + """Return iterator for streaming document batches. + + Similar to DatasetManager.__iter__() which returns DataSetIterator. + This enables batch-by-batch processing of documents without loading + all documents into memory at once. + + Example: + >>> manager = FtsDataset.MSMARCO.manager(100_000) + >>> for batch in manager: + >>> print(f"Processing {len(batch)} documents") + """ + return self.iter_batches() + + +class FtsDocumentIterator: + """Iterator for streaming FTS document batches using Translator pattern. + + Similar to DataSetIterator for vector datasets, but reads directly from ir_datasets + using translator. Yields batches of FtsDocument objects for memory-efficient + processing of large datasets. + """ + + def __init__(self, dataset: FtsDatasetManager, batch_size: int = config.NUM_PER_BATCH): + self._ds = dataset + self._batch_size = batch_size + self._finished = False + self._doc_count = 0 # Track total documents processed + self._docs_iter = None + + def __iter__(self): + return self + + def __next__(self) -> list[FtsDocument]: + """Return the next batch of documents. + + Returns: + list[FtsDocument]: List of FtsDocument objects + + Raises: + StopIteration: When all documents have been read + """ + if self._finished: + raise StopIteration + + # Initialize iterator on first call + if self._docs_iter is None: + if self._ds._ir_dataset is None: + error_msg = "ir_datasets dataset not loaded. Call prepare() first." + log.error(error_msg) + raise RuntimeError(error_msg) + + log.info("Starting to iterate documents using translator") + self._docs_iter = self._ds._translator.iter_documents(self._ds._ir_dataset) + + # Read batch with proper error handling + try: + batch = [] + for _ in range(self._batch_size): + if self._doc_count >= self._ds.data.size: + self._finished = True + if batch: + return batch + raise StopIteration # noqa: TRY301 + try: + doc = next(self._docs_iter) + doc.doc_id = str(self._doc_count) + batch.append(doc) + self._doc_count += 1 + except StopIteration: + self._finished = True + if batch: + return batch + raise + except Exception as e: + log.debug(f"Skipping malformed document: {e}") + continue + + except StopIteration: + self._finished = True + raise + except Exception: + log.exception("Error reading documents from translator") + raise + else: + return batch + + def __enter__(self): + """Enter context manager.""" + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> None: + """Exit context manager.""" + + def __del__(self): + """Cleanup when iterator is destroyed.""" + + +class FtsDataset(Enum): + MSMARCO = MSMarcoFts + HOTPOTQA = HotpotQAFts + + def get(self, size: int) -> FtsBaseDataset: + return self.value(size=size) + + def manager(self, size: int) -> FtsDatasetManager: + return FtsDatasetManager(data=self.get(size)) + + +class FtsDatasetWithSizeType(Enum): + MSMarcoSmall = "MS MARCO Small (100K documents)" + MSMarcoMedium = "MS MARCO Medium (1M documents)" + MSMarcoLarge = "MS MARCO Large (8.8M documents)" + HotpotQASmall = "HotpotQA Small (100K documents)" + HotpotQAMedium = "HotpotQA Medium (1M documents)" + HotpotQALarge = "HotpotQA Large (5.2M documents)" + + def get_manager(self) -> FtsDatasetManager: + return { + FtsDatasetWithSizeType.MSMarcoSmall: FtsDataset.MSMARCO.manager(100_000), + FtsDatasetWithSizeType.MSMarcoMedium: FtsDataset.MSMARCO.manager(1_000_000), + FtsDatasetWithSizeType.MSMarcoLarge: FtsDataset.MSMARCO.manager(8_841_823), + FtsDatasetWithSizeType.HotpotQASmall: FtsDataset.HOTPOTQA.manager(100_000), + FtsDatasetWithSizeType.HotpotQAMedium: FtsDataset.HOTPOTQA.manager(1_000_000), + FtsDatasetWithSizeType.HotpotQALarge: FtsDataset.HOTPOTQA.manager(5_233_329), + }[self] + + def get_load_timeout(self) -> float: + if self in {FtsDatasetWithSizeType.MSMarcoSmall, FtsDatasetWithSizeType.HotpotQASmall}: + return config.LOAD_TIMEOUT_768D_100K + return config.LOAD_TIMEOUT_DEFAULT + + def get_optimize_timeout(self) -> float: + if self in {FtsDatasetWithSizeType.MSMarcoSmall, FtsDatasetWithSizeType.HotpotQASmall}: + return config.OPTIMIZE_TIMEOUT_768D_100K + return config.OPTIMIZE_TIMEOUT_DEFAULT + + @property + def is_advanced(self) -> bool: + return self in {FtsDatasetWithSizeType.MSMarcoLarge, FtsDatasetWithSizeType.HotpotQALarge} diff --git a/vectordb_bench/backend/payload.py b/vectordb_bench/backend/payload.py index 49050c85f..04cba3b85 100644 --- a/vectordb_bench/backend/payload.py +++ b/vectordb_bench/backend/payload.py @@ -5,6 +5,7 @@ class PayloadProfile(StrEnum): IDS_ONLY = "ids_only" VECTOR = "vector" SCALAR_LABEL = "scalar_label" + TEXT = "text" def estimated_bytes_per_query(self, *, k: int, dim: int) -> int: # Approximate payload size used for cloud leaderboard cost expansion. @@ -17,5 +18,8 @@ def estimated_bytes_per_query(self, *, k: int, dim: int) -> int: return k * (id_distance_bytes + dim * 4) if self == PayloadProfile.SCALAR_LABEL: return k * (id_distance_bytes + scalar_label_bytes) + if self == PayloadProfile.TEXT: + # Approximate returned document text. FTS metrics still use IDs only. + return k * (id_distance_bytes + 512) msg = f"Unsupported payload profile: {self}" raise ValueError(msg) diff --git a/vectordb_bench/backend/result_collector.py b/vectordb_bench/backend/result_collector.py index a4119a5fd..a550df27a 100644 --- a/vectordb_bench/backend/result_collector.py +++ b/vectordb_bench/backend/result_collector.py @@ -1,26 +1,128 @@ +import argparse import logging import pathlib +import sys +from datetime import date +from typing import Literal from vectordb_bench.models import TestResult log = logging.getLogger(__name__) +ResultGroupBy = Literal["run_id", "db"] + class ResultCollector: + @staticmethod + def _group_key(file_result: TestResult, group_by: ResultGroupBy) -> str: + if group_by == "run_id": + return file_result.run_id + + if not file_result.results: + return file_result.run_id + return file_result.results[0].task_config.db.value + @classmethod - def collect(cls, result_dir: pathlib.Path) -> list[TestResult]: + def collect( + cls, + result_dir: pathlib.Path, + group_by: ResultGroupBy = "run_id", + trans_unit: bool = True, + ) -> list[TestResult]: reg = "result_*.json" results_d = {} + if group_by not in {"run_id", "db"}: + msg = f"Unsupported result grouping: {group_by}" + raise ValueError(msg) + if not result_dir.exists() or len(list(result_dir.rglob(reg))) == 0: return [] - for json_file in result_dir.rglob(reg): - file_result = TestResult.read_file(json_file, trans_unit=True) + for json_file in sorted(result_dir.rglob(reg)): + file_result = TestResult.read_file(json_file, trans_unit=trans_unit) + key = cls._group_key(file_result, group_by) - # Group result files of the same run_id into one TestResult - if file_result.run_id in results_d: - results_d[file_result.run_id].results.extend(file_result.results) + # Default behavior groups files from the same run. FTS publish artifacts can + # opt into DB grouping so one backend owns one consolidated result file. + if key in results_d: + results_d[key].results.extend(file_result.results) else: - results_d[file_result.run_id] = file_result + results_d[key] = file_result return list(results_d.values()) + + @classmethod + def merge_by_db( + cls, + result_dir: pathlib.Path, + task_label: str = "fts_standard", + replace: bool = False, + dry_run: bool = False, + ) -> list[pathlib.Path]: + source_files = sorted(result_dir.rglob("result_*.json")) + merged_results = cls.collect(result_dir, group_by="db", trans_unit=False) + output_files = [] + + for result in merged_results: + if not result.results: + continue + + db = result.results[0].task_config.db.value + db_lower = db.lower() + merged = TestResult( + run_id=f"{task_label}_{db_lower}", + task_label=task_label, + results=result.results, + timestamp=result.timestamp, + ) + file_name = merged.file_fmt.format(date.today().strftime("%Y%m%d"), task_label, db_lower) + output_file = result_dir.joinpath(db, file_name) + output_files.append(output_file) + + if dry_run: + log.info("Would write %s with %s case results", output_file, len(result.results)) + else: + merged.write_db_file(result_dir.joinpath(db), merged, db_lower) + + if replace and not dry_run: + output_file_set = set(output_files) + for source_file in source_files: + if source_file not in output_file_set and source_file.exists(): + source_file.unlink() + + return output_files + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Collect or consolidate VDBBench result JSON files.") + parser.add_argument("result_dir", type=pathlib.Path) + parser.add_argument("--merge-by-db", action="store_true", help="write one consolidated result file per backend") + parser.add_argument( + "--replace", action="store_true", help="remove source result files after merged files are written" + ) + parser.add_argument("--task-label", default="fts_standard", help="task label used for merged result files") + parser.add_argument("--dry-run", action="store_true", help="show planned output files without writing") + return parser.parse_args() + + +def main(): + args = _parse_args() + logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s") + + if args.merge_by_db: + output_files = ResultCollector.merge_by_db( + args.result_dir, + task_label=args.task_label, + replace=args.replace, + dry_run=args.dry_run, + ) + for output_file in output_files: + sys.stdout.write(f"{output_file}\n") + return + + for result in ResultCollector.collect(args.result_dir): + sys.stdout.write(f"{result.run_id}\t{len(result.results)}\n") + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py index 650795535..194fbf53c 100644 --- a/vectordb_bench/backend/runner/concurrent_runner.py +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -19,15 +19,16 @@ import numpy as np from vectordb_bench.backend.filter import Filter, FilterOp, non_filter -from vectordb_bench.backend.utils import kill_proc_tree, time_it +from vectordb_bench.backend.utils import kill_proc_tree, time_it, timeout +from vectordb_bench.backend.workload import WorkloadKind from ... import config -from ...models import PerformanceTimeoutError +from ...models import LoadTimeoutError, PerformanceTimeoutError from .executor import AsyncExecutor, ThreadExecutor if TYPE_CHECKING: from vectordb_bench.backend.clients import api - from vectordb_bench.backend.dataset import DatasetManager + from vectordb_bench.backend.dataset import DatasetManager, FtsDatasetManager from .executor import TaskExecutor @@ -68,9 +69,10 @@ def __init__( duration: float | None = None, with_scalar_labels: bool = False, tenant_case=None, # noqa: ANN001 + workload_kind: WorkloadKind = WorkloadKind.VECTOR, ): self.timeout = timeout if isinstance(timeout, int | float) else None - self.dataset: DatasetManager = dataset + self.dataset: DatasetManager | FtsDatasetManager = dataset self.db = db self.normalize = normalize self.filters = filters @@ -79,6 +81,8 @@ def __init__( self.duration = duration if isinstance(duration, int | float) else None self.with_scalar_labels = with_scalar_labels self.tenant_case = tenant_case + self.workload_kind = workload_kind + self._prefetched_fts_batch = None effective_workers = max_workers or min(mp.cpu_count(), 4) if not db.thread_safe: @@ -115,21 +119,14 @@ def _get_thread_db(self) -> api.VectorDB: def _insert_batch_with_retry( self, db: api.VectorDB, - embeddings: list[list[float]], - metadata: list[int], - labels_data: list[str] | None = None, - tenant_labels_data: list[str] | None = None, retry_idx: int = 0, + **insert_kwargs, ) -> int: """Insert a single batch with retry logic. Returns inserted count.""" - insert_kwargs = { - "embeddings": embeddings, - "metadata": metadata, - "labels_data": labels_data, - } - if tenant_labels_data is not None: - insert_kwargs["tenant_labels_data"] = tenant_labels_data - insert_count, error = db.insert_embeddings(**insert_kwargs) + if getattr(self, "workload_kind", WorkloadKind.VECTOR) == WorkloadKind.FULL_TEXT: + insert_count, error = db.insert_documents(**insert_kwargs) + else: + insert_count, error = db.insert_embeddings(**insert_kwargs) if error is not None: log.warning(f"Insert failed, try_idx={retry_idx}, Exception: {error}") if getattr(error, "non_retryable", False): @@ -140,28 +137,24 @@ def _insert_batch_with_retry( time.sleep(retry_idx) return self._insert_batch_with_retry( db, - embeddings, - metadata, - labels_data, - tenant_labels_data, retry_idx, + **insert_kwargs, ) msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" raise RuntimeError(msg) return insert_count - def _worker_insert( - self, - embeddings: list[list[float]], - metadata: list[int], - labels_data: list[str] | None = None, - tenant_labels_data: list[str] | None = None, - ) -> int: + def _worker_insert(self, **insert_kwargs) -> int: """Worker function: insert a batch with retry.""" db = self._get_thread_db() - return self._insert_batch_with_retry(db, embeddings, metadata, labels_data, tenant_labels_data) + return self._insert_batch_with_retry(db, **insert_kwargs) + + def _next_batch(self) -> dict | None: + if getattr(self, "workload_kind", WorkloadKind.VECTOR) == WorkloadKind.FULL_TEXT: + return self._next_fts_batch() + return self._next_vector_batch() - def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None, list[str] | None] | None: + def _next_vector_batch(self) -> dict | None: """Pull the next batch from the shared dataset iterator. Thread-safe: only one thread reads from the iterator at a time. @@ -201,7 +194,40 @@ def _next_batch(self) -> tuple[list[list[float]], list[int], list[str] | None, l if self.tenant_case is not None and getattr(self.tenant_case, "is_multitenant", False): tenant_labels_data = self.tenant_case.tenant_labels_for_ids(all_metadata) - return all_embeddings, all_metadata, labels_data, tenant_labels_data + insert_kwargs = { + "embeddings": all_embeddings, + "metadata": all_metadata, + "labels_data": labels_data, + } + if tenant_labels_data is not None: + insert_kwargs["tenant_labels_data"] = tenant_labels_data + return insert_kwargs + + def _next_fts_batch(self) -> dict | None: + stop_event = getattr(self, "_stop_event", None) + if stop_event is not None and stop_event.is_set(): + return None + if self._deadline is not None and time.perf_counter() >= self._deadline: + return None + with self._iter_lock: + stop_event = getattr(self, "_stop_event", None) + if stop_event is not None and stop_event.is_set(): + return None + batch = self._prefetched_fts_batch + if batch is not None: + self._prefetched_fts_batch = None + else: + try: + batch = next(self._dataset_iter) + except StopIteration: + return None + + doc_ids = [] + texts = [] + for doc in batch: + doc_ids.append(doc.doc_id if hasattr(doc, "doc_id") else str(doc["doc_id"])) + texts.append(doc.text if hasattr(doc, "text") else doc["text"]) + return {"texts": texts, "doc_ids": doc_ids} def _worker_loop(self) -> int: """Worker loop: pull batches from the shared iterator and insert them.""" @@ -211,8 +237,7 @@ def _worker_loop(self) -> int: batch = self._next_batch() if batch is None: break - embeddings, metadata, labels_data, tenant_labels_data = batch - total += self._worker_insert(embeddings, metadata, labels_data, tenant_labels_data) + total += self._worker_insert(**batch) except Exception: stop_event = getattr(self, "_stop_event", None) if stop_event is not None: @@ -227,11 +252,14 @@ def task(self) -> int: self._stop_event = threading.Event() self._deadline = None if self.duration is None else time.perf_counter() + self.duration self._dataset_iter = self.dataset.iter_batches(self.batch_size) + if getattr(self, "workload_kind", WorkloadKind.VECTOR) == WorkloadKind.FULL_TEXT: + # Prime lazy ir_datasets document preparation before timed backend inserts. + self._prefetched_fts_batch = next(self._dataset_iter, None) with self.db.init(): log.info( f"({mp.current_process().name:16}) Start concurrent insert, " - f"batch_size={self.batch_size}, max_workers={self.max_workers}" + f"batch_size={self.batch_size}, max_workers={self.max_workers}, workload={self.workload_kind}" ) start = time.perf_counter() @@ -259,6 +287,13 @@ def task(self) -> int: @time_it def _insert_all_batches(self) -> int: """Performance case only: run task() in subprocess with timeout.""" + if getattr(self, "workload_kind", WorkloadKind.VECTOR) == WorkloadKind.FULL_TEXT: + # FTS datasets come from ir_datasets, whose loaded objects may contain + # lambdas or handles that cannot be pickled by ProcessPoolExecutor. + # Keep FTS loading in this process and use threads to overlap the + # I/O-bound insert RPCs instead of Python CPU-bound work. + with timeout(self.timeout, lambda: LoadTimeoutError(self.timeout)): + return self.task() with concurrent.futures.ProcessPoolExecutor( mp_context=mp.get_context("spawn"), max_workers=1, diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index bb867b3f1..bf81d2a7e 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -13,6 +13,7 @@ from vectordb_bench.backend.filter import Filter, non_filter from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.workload import WorkloadKind from ... import config from ...models import ConcurrencySlotTimeoutError @@ -48,15 +49,29 @@ def __init__( concurrency_timeout: int = config.CONCURRENCY_TIMEOUT, payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, tenant_labels: list[str] | None = None, + workload_kind: WorkloadKind = WorkloadKind.VECTOR, ): self.db = db self.k = k self.filters = filters + self.workload_kind = workload_kind self.payload_profile = payload_profile self.tenant_labels = tenant_labels or [] - if not self.db.supports_payload_profile(self.payload_profile): + if self.workload_kind == WorkloadKind.FULL_TEXT: + self._search_func = self.db.search_documents + elif self.workload_kind == WorkloadKind.VECTOR: + self._search_func = self._search_embedding + else: + msg = f"Unsupported search workload: {workload_kind}" + raise NotImplementedError(msg) + if self.workload_kind == WorkloadKind.VECTOR and not self.db.supports_payload_profile(self.payload_profile): msg = f"{self.db.name} does not support payload_profile={self.payload_profile.value}" raise NotImplementedError(msg) + if self.workload_kind == WorkloadKind.FULL_TEXT and not self.db.supports_document_payload_profile( + self.payload_profile + ): + msg = f"{self.db.name} does not support document payload_profile={self.payload_profile.value}" + raise NotImplementedError(msg) self.concurrencies = concurrencies self.duration = duration self.concurrency_timeout = concurrency_timeout @@ -64,6 +79,21 @@ def __init__( self.test_data = test_data log.debug(f"test dataset columns: {len(test_data)}") + def __getstate__(self): + state = self.__dict__.copy() + state.pop("_search_func", None) + return state + + def __setstate__(self, state: dict): + self.__dict__.update(state) + if self.workload_kind == WorkloadKind.FULL_TEXT: + self._search_func = self.db.search_documents + elif self.workload_kind == WorkloadKind.VECTOR: + self._search_func = self._search_embedding + else: + msg = f"Unsupported search workload: {self.workload_kind}" + raise NotImplementedError(msg) + def _search_embedding(self, emb: list[float], tenant: str | None = None) -> list[int]: if tenant is None: if self.payload_profile == PayloadProfile.IDS_ONLY: @@ -73,6 +103,18 @@ def _search_embedding(self, emb: list[float], tenant: str | None = None) -> list return self.db.search_embedding(emb, self.k, tenant=tenant) return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile, tenant=tenant) + def _search_once(self, query: list[float] | str, tenant_rng: random.Random | None = None): + if self.workload_kind == WorkloadKind.FULL_TEXT: + if self.payload_profile == PayloadProfile.IDS_ONLY: + return self._search_func(query, self.k) + return self._search_func(query, self.k, payload_profile=self.payload_profile) + tenant = ( + self.tenant_labels[tenant_rng.randrange(len(self.tenant_labels))] + if tenant_rng is not None and self.tenant_labels + else None + ) + return self._search_func(query, tenant=tenant) + def search( self, test_data: list[list[float]], @@ -100,12 +142,7 @@ def search( while time.perf_counter() < start_time + self.duration: s = time.perf_counter() try: - tenant = ( - self.tenant_labels[tenant_rng.randrange(len(self.tenant_labels))] - if self.tenant_labels - else None - ) - self._search_embedding(test_data[idx], tenant=tenant) + self._search_once(test_data[idx], tenant_rng=tenant_rng) count += 1 latencies.append(time.perf_counter() - s) except Exception as e: @@ -364,7 +401,7 @@ def search_by_dur( while time.perf_counter() < start_time + dur: s = time.perf_counter() try: - self._search_embedding(test_data[idx]) + self._search_once(test_data[idx]) success_count += 1 latency_us = int((time.perf_counter() - s) * US_TO_SECONDS) histogram.record_value(min(latency_us, HDR_HISTOGRAM_MAX_US)) diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 3fc37e0ce..fc46aed23 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -1,4 +1,4 @@ -import concurrent.futures +import concurrent import logging import math import multiprocessing as mp @@ -11,9 +11,10 @@ from vectordb_bench.backend.dataset import DatasetManager from vectordb_bench.backend.filter import Filter, non_filter from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.workload import WorkloadKind from ... import config -from ...metric import calc_ndcg, calc_recall, get_ideal_dcg +from ...metric import calc_ndcg, calc_recall, calc_recall_fts, get_ideal_dcg from ...models import LoadTimeoutError from .. import utils from ..clients import api @@ -25,6 +26,9 @@ class SerialInsertRunner: + # FTS insert is intentionally not implemented here. FTS performance loading + # goes through ConcurrentInsertRunner; serial FTS insert can be added later + # if a capacity or serial-load FTS case needs it. def __init__( self, db: api.VectorDB, @@ -128,23 +132,39 @@ class SerialSearchRunner: def __init__( self, db: api.VectorDB, - test_data: list[list[float]], + test_data: list, ground_truth: list[list[int]], k: int = 100, filters: Filter = non_filter, payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, tenant_labels: list[str] | None = None, measure_recall: bool = True, + workload_kind: WorkloadKind = WorkloadKind.VECTOR, ): self.db = db self.k = k self.filters = filters + self.workload_kind = workload_kind self.payload_profile = payload_profile self.tenant_labels = tenant_labels or [] self.measure_recall = measure_recall - if not self.db.supports_payload_profile(self.payload_profile): + if workload_kind == WorkloadKind.FULL_TEXT: + self._search_func = self.db.search_documents + self._use_fts_metrics = True + elif workload_kind == WorkloadKind.VECTOR: + self._search_func = self._search_embedding + self._use_fts_metrics = False + else: + msg = f"Unsupported search workload: {workload_kind}" + raise NotImplementedError(msg) + if self.workload_kind == WorkloadKind.VECTOR and not self.db.supports_payload_profile(self.payload_profile): msg = f"{self.db.name} does not support payload_profile={self.payload_profile.value}" raise NotImplementedError(msg) + if self.workload_kind == WorkloadKind.FULL_TEXT and not self.db.supports_document_payload_profile( + self.payload_profile + ): + msg = f"{self.db.name} does not support document payload_profile={self.payload_profile.value}" + raise NotImplementedError(msg) if isinstance(test_data[0], np.ndarray): self.test_data = [query.tolist() for query in test_data] @@ -161,25 +181,36 @@ def _search_embedding(self, emb: list[float], tenant: str | None = None) -> list return self.db.search_embedding(emb, self.k, tenant=tenant) return self.db.search_embedding(emb, self.k, payload_profile=self.payload_profile, tenant=tenant) - def _get_db_search_res(self, emb: list[float], tenant: str | None = None, retry_idx: int = 0) -> list[int]: + def _get_db_search_res( + self, + query: list[float] | str, + tenant: str | None = None, + retry_idx: int = 0, + ) -> list[int]: try: - results = self._search_embedding(emb, tenant=tenant) + if self.workload_kind == WorkloadKind.FULL_TEXT: + if self.payload_profile == PayloadProfile.IDS_ONLY: + results = self._search_func(query, self.k) + else: + results = self._search_func(query, self.k, payload_profile=self.payload_profile) + else: + results = self._search_func(query, tenant=tenant) except Exception as e: log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") if retry_idx < config.MAX_SEARCH_RETRY: - return self._get_db_search_res(emb=emb, tenant=tenant, retry_idx=retry_idx + 1) + return self._get_db_search_res(query=query, tenant=tenant, retry_idx=retry_idx + 1) msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" raise RuntimeError(msg) from e return results - def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: + def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, ...]: log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") with self.db.init(): self.db.prepare_filter(self.filters) test_data, ground_truth = args - ideal_dcg = get_ideal_dcg(self.k) + ideal_dcg = None if self._use_fts_metrics else get_ideal_dcg(self.k) log.debug(f"test dataset size: {len(test_data)}") log.debug(f"ground truth size: {len(ground_truth) if ground_truth is not None else 0}") @@ -188,7 +219,9 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa tenant_rng = random.Random(0) for idx, emb in enumerate(test_data): tenant = ( - self.tenant_labels[tenant_rng.randrange(len(self.tenant_labels))] if self.tenant_labels else None + self.tenant_labels[tenant_rng.randrange(len(self.tenant_labels))] + if self.workload_kind == WorkloadKind.VECTOR and self.tenant_labels + else None ) s = time.perf_counter() try: @@ -201,11 +234,15 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa if self.measure_recall and ground_truth is not None: gt = ground_truth[idx] - recalls.append(calc_recall(self.k, gt[: self.k], results)) - ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) + if self._use_fts_metrics: + recalls.append(calc_recall_fts(self.k, gt, results)) + else: + recalls.append(calc_recall(self.k, gt[: self.k], results)) + ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) else: recalls.append(0) - ndcgs.append(0) + if not self._use_fts_metrics: + ndcgs.append(0) if len(latencies) % 100 == 0: log.debug( @@ -215,10 +252,22 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) - avg_ndcg = round(np.mean(ndcgs), 4) cost = round(np.sum(latencies), 4) p99 = round(np.percentile(latencies, 99), 4) p95 = round(np.percentile(latencies, 95), 4) + if self._use_fts_metrics: + log.info( + f"{mp.current_process().name:14} search entire test_data: " + f"cost={cost}s, " + f"queries={len(latencies)}, " + f"avg_recall={avg_recall}, " + f"avg_latency={avg_latency}, " + f"p99={p99}, " + f"p95={p95}" + ) + return (avg_recall, p99, p95) + + avg_ndcg = round(np.mean(ndcgs), 4) log.info( f"{mp.current_process().name:14} search entire test_data: " f"cost={cost}s, " @@ -231,13 +280,13 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, floa ) return (avg_recall, avg_ndcg, p99, p95) - def _run_in_subprocess(self) -> tuple[float, float, float, float]: + def _run_in_subprocess(self) -> tuple[float, ...]: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self.search, (self.test_data, self.ground_truth)) return future.result() @utils.time_it - def run(self) -> tuple[float, float, float, float]: + def run(self) -> tuple[float, ...]: log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: msg = "empty test_data" @@ -246,11 +295,11 @@ def run(self) -> tuple[float, float, float, float]: return self._run_in_subprocess() @utils.time_it - def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: + def run_with_cost(self) -> tuple[tuple[float, ...], float]: """ Search all test data in serial. Returns: - tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost + tuple[tuple[float, ...], float]: search metrics and cost """ log.info(f"{mp.current_process().name:14} start serial search") if self.test_data is None: diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 3a65c4d04..6f9025d4a 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -7,7 +7,9 @@ from enum import Enum, auto import numpy as np +from pydantic import PrivateAttr +from .. import config from ..base import BaseModel from ..metric import Metric from ..models import PerformanceTimeoutError, TaskConfig, TaskStage @@ -24,6 +26,7 @@ SerialSearchRunner, ) from .utils import kill_proc_tree +from .workload import WorkloadKind log = logging.getLogger(__name__) @@ -54,12 +57,15 @@ class CaseRunner(BaseModel): db: api.VectorDB | None = None test_emb: list[list[float]] | None = None + test_texts: list[str] | None = None serial_search_runner: SerialSearchRunner | None = None search_runner: MultiProcessingSearchRunner | None = None final_search_runner: MultiProcessingSearchRunner | None = None read_write_runner: ReadWriteRunner | None = None cold_warm_search_runner: ColdWarmSearchRunner | None = None + _fts_manifest_report: dict = PrivateAttr(default_factory=dict) + def __eq__(self, obj: any): if isinstance(obj, CaseRunner): key = self.load_reuse_key() @@ -139,19 +145,21 @@ def _doris_collection_name(self) -> str | None: return base def display(self) -> dict: + dataset_include = { + "name": True, + "size": True, + "label": True, + "metric_type": True, + } + if self.ca.label != CaseLabel.FullTextSearchPerformance: + dataset_include["dim"] = True c_dict = self.ca.dict( include={ "label": True, "name": True, "filters": True, "dataset": { - "data": { - "name": True, - "size": True, - "dim": True, - "metric_type": True, - "label": True, - }, + "data": dataset_include, }, }, ) @@ -161,8 +169,20 @@ def display(self) -> dict: @property def normalize(self) -> bool: assert self.db + if self.is_fts: + return False return self.db.need_normalize_cosine() and self.ca.dataset.data.metric_type == MetricType.COSINE + @property + def workload_kind(self) -> WorkloadKind: + if getattr(self.ca, "label", None) == CaseLabel.FullTextSearchPerformance: + return WorkloadKind.FULL_TEXT + return WorkloadKind.VECTOR + + @property + def is_fts(self) -> bool: + return self.workload_kind == WorkloadKind.FULL_TEXT + def init_db(self, drop_old: bool = True) -> None: db_cls = self.config.db.init_cls # Compose a compact, case-unique collection/table name for Doris to avoid cross-case interference @@ -185,7 +205,7 @@ def init_db(self, drop_old: bool = True) -> None: extra_db_kwargs["multitenant_tenant_labels"] = self.ca.tenant_labels() self.db = db_cls( - dim=self.ca.dataset.data.dim, + dim=getattr(self.ca.dataset.data, "dim", 0), db_config=db_config_dict, db_case_config=self.config.db_case_config, drop_old=drop_old, @@ -193,6 +213,24 @@ def init_db(self, drop_old: bool = True) -> None: **extra_db_kwargs, ) + def _apply_fts_manifest_params(self) -> None: + bm25_params = dict(getattr(self.ca.dataset, "bm25_params", {}) or {}) + analyzer_params = dict(getattr(self.ca.dataset, "analyzer_params", {}) or {}) + self.config.db_case_config, manifest_report = self.config.db_case_config.apply_fts_manifest( + bm25_params=bm25_params, + analyzer_params=analyzer_params, + ) + self._fts_manifest_report = { + "fts_manifest": { + "bm25": bm25_params, + "analyzer": analyzer_params, + }, + **manifest_report, + } + + def _fts_manifest_additional_parameters(self) -> dict: + return dict(self._fts_manifest_report) + def _pre_run(self, drop_old: bool = True): try: self._validate_cloud_cold_latency_config(drop_old) @@ -207,6 +245,13 @@ def _pre_run(self, drop_old: bool = True): ): msg = "CloudMultiTenantSearchCase requires use_partition_key=True for Milvus/ZillizCloud" raise ValueError(msg) + + if self.is_fts: + self.ca.dataset.prepare(self.dataset_source) + self._apply_fts_manifest_params() + self.init_db(drop_old) + return + self.init_db(drop_old) if self.ca.is_multitenant and self.db is not None: if not self.db.supports_multitenant(): @@ -245,7 +290,7 @@ def run(self, drop_old: bool = True) -> Metric: if self.ca.label == CaseLabel.Load: return self._run_capacity_case() - if self.ca.label == CaseLabel.Performance: + if self.ca.label in {CaseLabel.Performance, CaseLabel.FullTextSearchPerformance}: return self._run_perf_case(drop_old) if self.ca.label == CaseLabel.Streaming: return self._run_streaming_case() @@ -293,11 +338,18 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: m = Metric() if drop_old: if TaskStage.LOAD in self.config.stages: - _, load_dur = self._load_train_data() + count, load_dur = self._load_data() build_dur = self._optimize() + m.inserted_count = count m.insert_duration = round(load_dur, 4) m.optimize_duration = round(build_dur, 4) m.load_duration = round(load_dur + build_dur, 4) + m.additional_parameters.update( + { + "num_per_batch": config.NUM_PER_BATCH, + "load_concurrency": self.config.load_concurrency, + } + ) log.info( f"Finish loading the entire dataset into VectorDB," f" insert_duration={load_dur}, optimize_duration={build_dur}" @@ -306,7 +358,7 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: else: log.info("Data loading skipped") if TaskStage.SEARCH_SERIAL in self.config.stages or TaskStage.SEARCH_CONCURRENT in self.config.stages: - self._init_search_runner() + self._init_search_runners() if TaskStage.SEARCH_CONCURRENT in self.config.stages: search_results = self._conc_search() ( @@ -319,9 +371,17 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: ) = search_results if TaskStage.SEARCH_SERIAL in self.config.stages: search_results = self._serial_search() - m.recall, m.ndcg, m.serial_latency_p99, m.serial_latency_p95 = search_results - m.payload_profile = self.ca.payload_profile.value - m.payload_estimated_bytes_per_query = self.ca.estimated_payload_bytes_per_query(self.config.case_config.k) + if self.is_fts: + m.recall, m.serial_latency_p99, m.serial_latency_p95 = search_results + else: + m.recall, m.ndcg, m.serial_latency_p99, m.serial_latency_p95 = search_results + if hasattr(self.ca, "payload_profile"): + m.payload_profile = self.ca.payload_profile.value + m.payload_estimated_bytes_per_query = self.ca.estimated_payload_bytes_per_query( + self.config.case_config.k + ) + if self.is_fts: + m.additional_parameters.update(self._fts_manifest_additional_parameters()) except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") @@ -443,8 +503,11 @@ def _run_cloud_cold_latency_case(self, drop_old: bool = True) -> Metric: return m @utils.time_it + def _load_data(self): + return self._load_train_data() + def _load_train_data(self): - """Insert train data concurrently and get the insert_duration""" + """Insert vector or FTS train data concurrently and get insert duration.""" try: runner_kwargs = {} if self.ca.is_multitenant: @@ -457,20 +520,22 @@ def _load_train_data(self): self.ca.load_timeout, max_workers=self.config.load_concurrency or None, with_scalar_labels=self.ca.with_scalar_labels, + workload_kind=self.workload_kind, **runner_kwargs, ) - runner.run() + return runner.run() except Exception as e: raise e from None finally: runner = None - def _serial_search(self) -> tuple[float, float, float, float]: + def _serial_search(self) -> tuple[float, ...]: """Performance serial tests, search the entire test data once, calculate the recall, serial_latency_p99, serial_latency_p95 Returns: - tuple[float, float, float, float]: recall, ndcg, serial_latency_p99, serial_latency_p95 + tuple[float, ...]: vector cases return recall, ndcg, p99, p95; + FTS cases return recall, p99, p95. """ try: results, _ = self.serial_search_runner.run() @@ -502,6 +567,14 @@ def _optimize_task(self) -> None: self.db.optimize(data_size=self.ca.dataset.data.size) def _optimize(self) -> float: + if self.is_fts: + try: + with utils.timeout(self.ca.optimize_timeout, PerformanceTimeoutError): + _, duration = self._optimize_task() + return duration + except PerformanceTimeoutError: + log.warning(f"VectorDB optimize timeout in {self.ca.optimize_timeout}") + raise with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self._optimize_task) try: @@ -514,6 +587,11 @@ def _optimize(self) -> float: log.warning(f"VectorDB optimize error: {e}") raise e from None + def _init_search_runners(self): + if self.is_fts: + return self._init_fts_search_runner() + return self._init_search_runner() + def _init_search_runner(self): if self.normalize: test_emb = np.stack(self.ca.dataset.test_data) @@ -536,6 +614,7 @@ def _init_search_runner(self): payload_profile=self.ca.payload_profile, tenant_labels=tenant_labels, measure_recall=measure_recall, + workload_kind=WorkloadKind.VECTOR, ) if TaskStage.SEARCH_CONCURRENT in self.config.stages: self.search_runner = MultiProcessingSearchRunner( @@ -548,6 +627,45 @@ def _init_search_runner(self): k=self.config.case_config.k, payload_profile=self.ca.payload_profile, tenant_labels=tenant_labels, + workload_kind=WorkloadKind.VECTOR, + ) + + def _init_fts_search_runner(self): + fts_dataset = self.ca.dataset + + if fts_dataset.queries_data is None or fts_dataset.gt_data is None: + msg = "FTS dataset is missing queries or ground truth. Call prepare() before initializing search." + raise ValueError(msg) + test_texts = [q.text for q in fts_dataset.queries_data] + ground_truth = fts_dataset.gt_data + if len(test_texts) != len(ground_truth): + msg = f"FTS query count {len(test_texts)} does not match ground truth row count {len(ground_truth)}" + raise ValueError(msg) + + log.info(f"FTS test will use {len(test_texts)} queries for testing") + self.test_texts = test_texts + + if TaskStage.SEARCH_SERIAL in self.config.stages: + self.serial_search_runner = SerialSearchRunner( + db=self.db, + test_data=test_texts, + ground_truth=ground_truth, + filters=self.ca.filters, + k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + workload_kind=WorkloadKind.FULL_TEXT, + ) + if TaskStage.SEARCH_CONCURRENT in self.config.stages: + self.search_runner = MultiProcessingSearchRunner( + db=self.db, + test_data=test_texts, + filters=self.ca.filters, + concurrencies=self.config.case_config.concurrency_search_config.num_concurrency, + duration=self.config.case_config.concurrency_search_config.concurrency_duration, + concurrency_timeout=self.config.case_config.concurrency_search_config.concurrency_timeout, + k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + workload_kind=WorkloadKind.FULL_TEXT, ) def _init_read_write_runner(self): diff --git a/vectordb_bench/backend/utils.py b/vectordb_bench/backend/utils.py index 432f0d1d1..8c9c99a1f 100644 --- a/vectordb_bench/backend/utils.py +++ b/vectordb_bench/backend/utils.py @@ -1,7 +1,10 @@ import contextlib import logging import signal +import threading import time +from collections.abc import Callable +from contextlib import contextmanager from functools import wraps import psutil @@ -88,6 +91,29 @@ def inner(*args, **kwargs): return inner +@contextmanager +def timeout(timeout_seconds: float | None, exc_factory: Callable[[], Exception]): + if ( + timeout_seconds is None + or not hasattr(signal, "SIGALRM") + or threading.current_thread() is not threading.main_thread() + ): + yield + return + + def _handle_timeout(signum, frame): # noqa: ANN001, ARG001 + raise exc_factory() + + previous_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, _handle_timeout) + previous_timer = signal.setitimer(signal.ITIMER_REAL, float(timeout_seconds)) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, *previous_timer) + signal.signal(signal.SIGALRM, previous_handler) + + def compose_train_files(train_count: int, use_shuffled: bool) -> list[str]: prefix = "shuffle_train" if use_shuffled else "train" middle = f"of-{train_count}" diff --git a/vectordb_bench/backend/workload.py b/vectordb_bench/backend/workload.py new file mode 100644 index 000000000..e3c958b20 --- /dev/null +++ b/vectordb_bench/backend/workload.py @@ -0,0 +1,6 @@ +from enum import StrEnum + + +class WorkloadKind(StrEnum): + VECTOR = "vector" + FULL_TEXT = "full_text" diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index cf5c8fecd..d7854be6c 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -18,8 +18,9 @@ from .. import config from ..backend.clients import DB -from ..backend.clients.api import MetricType -from ..backend.dataset import DatasetWithSizeType +from ..backend.clients.api import IndexType, MetricType +from ..backend.dataset import DatasetWithSizeType, FtsDatasetWithSizeType +from ..backend.payload import PayloadProfile from ..interface import benchmark_runner from ..models import ( CaseConfig, @@ -250,9 +251,30 @@ def get_custom_case_config(parameters: dict) -> dict: custom_case_config["filter_rate"] = parameters["cloud_filter_rate"] if parameters["cloud_label_percentage"] is not None: custom_case_config["label_percentage"] = parameters["cloud_label_percentage"] + elif parameters["case_type"] == "FTSBm25Performance": + dataset_with_size_type = parameters["dataset_with_size_type"] + if dataset_with_size_type not in {dataset.value for dataset in FtsDatasetWithSizeType}: + dataset_with_size_type = FtsDatasetWithSizeType.MSMarcoSmall.value + custom_case_config = { + "dataset_with_size_type": dataset_with_size_type, + "payload_profile": parameters.get("payload_profile", PayloadProfile.IDS_ONLY.value), + } return custom_case_config +def select_cli_db_case_config(db: DB, db_case_config: DBCaseConfig, case_type: str) -> DBCaseConfig: + if case_type != CaseType.FTSBm25Performance.name: + return db_case_config + + fts_case_config_cls = db.case_config_cls(IndexType.FTS) + if isinstance(db_case_config, fts_case_config_cls): + return db_case_config + fts_db_case_config = fts_case_config_cls() + if hasattr(db_case_config, "disable_backpressure") and hasattr(fts_db_case_config, "disable_backpressure"): + fts_db_case_config.disable_backpressure = db_case_config.disable_backpressure + return fts_db_case_config + + log = logging.getLogger(__name__) @@ -499,7 +521,10 @@ class CommonTypedDict(TypedDict): "--dataset-with-size-type", help="Dataset with size type. When omitted, filter/insert cases use Medium Cohere (768dim, 1M), " "CloudPayloadSearchCase and CloudColdLatencyCase use LAION 100M, and CloudMultiTenantSearchCase " - f"uses Large Cohere (768dim, 10M). Supported values include {SUPPORTED_DATASET_WITH_SIZE_TYPES}", + f"uses Large Cohere (768dim, 10M). Supported vector values include " + f"{SUPPORTED_DATASET_WITH_SIZE_TYPES}. For FTSBm25Performance, supported default UI datasets include " + f"{FtsDatasetWithSizeType.MSMarcoSmall.value}|{FtsDatasetWithSizeType.MSMarcoMedium.value}|" + f"{FtsDatasetWithSizeType.HotpotQASmall.value}|{FtsDatasetWithSizeType.HotpotQAMedium.value}.", default=None, ), ] @@ -525,8 +550,8 @@ class CommonTypedDict(TypedDict): str, click.option( "--payload-profile", - type=click.Choice(["ids_only", "vector", "scalar_label"]), - help="Response payload profile for CloudPayloadSearchCase and CloudColdLatencyCase", + type=click.Choice([profile.value for profile in PayloadProfile]), + help="Response payload profile for payload and FTS cases", default="ids_only", show_default=True, ), @@ -797,7 +822,7 @@ def run( task = TaskConfig( db=db, db_config=db_config, - db_case_config=db_case_config, + db_case_config=select_cli_db_case_config(db, db_case_config, parameters["case_type"]), case_config=CaseConfig( case_id=CaseType[parameters["case_type"]], k=parameters["k"], diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 13c9687c7..5a89c018a 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -17,7 +17,7 @@ from ..backend.clients.lindorm.cli import LindormHNSW, LindormIVFBQ, LindormIVFPQ from ..backend.clients.mariadb.cli import MariaDBHNSW from ..backend.clients.memorydb.cli import MemoryDB -from ..backend.clients.milvus.cli import MilvusAutoIndex +from ..backend.clients.milvus.cli import MilvusAutoIndex, MilvusFTS from ..backend.clients.oceanbase.cli import OceanBaseHNSW, OceanBaseIVF from ..backend.clients.oss_opensearch.cli import OSSOpenSearch from ..backend.clients.pgdiskann.cli import PgDiskAnn @@ -57,6 +57,7 @@ cli.add_command(Test) cli.add_command(ZillizAutoIndex) cli.add_command(MilvusAutoIndex) +cli.add_command(MilvusFTS) cli.add_command(AWSOpenSearch) cli.add_command(OSSOpenSearch) cli.add_command(PgVectorScaleDiskAnn) diff --git a/vectordb_bench/fig/homepage/full_text_search.png b/vectordb_bench/fig/homepage/full_text_search.png new file mode 100644 index 0000000000000000000000000000000000000000..fa9b19af8215c2a7fd1c777dbfc8c4d54da3a1d3 GIT binary patch literal 134647 zcmY(r1C%9A&@I}wZF}0bZFAc0p0;hJ$R_Hy%>u_ebI6O2Jb)&wMVDwZh>Rsj-7QVRotE(#1@B8B?x zj9gckKQ9I*3T841Xld%O&mPI==smy!~8zj`9*Z>Oazk*Q^nQ@&}5Zt=f^nX|U zSDyz2X2r-w@TLHoJCdjSe;WUHuM!!AUqdHm+(e8C3=UYN7m4wI7pJfwd(r{P+h@G( z%B>kkGzF%<|98hnR1mYI3(8$Oo&VE?5saT11l`6l(~2!oW9o-AZ3yN6ec1n7ltTva z&oc*Fj`D}rmO)`G|IfSr*KLCkr1J4e$M$|-$E&NWq2!r+Q(!^Mve#DWB8<@*2rq^eLTFq$j4-&AD{m5Nlw*%WvP z;G3C{Pn1Zp7Od-lV|7)M@#l%iL?)FsV#r8NPR_n2zcb(V)dGumzz}}P z2-pcryBpg-kpoG$C%IF{{d<9G3Q~xrv~Z3wG0;8=zQXO^fnwZP521*JFl@4A3%88d za~5DC1bxZc9v(M+kc1(ywjF1s7qvaUy!62ebT*mPwvt6NON$9(P%Z-k8I!IqF7Vj% zi}Ukyo13Hf>np2NAvfpJlqkXz+&n7q@Je*Vd&Dv46cw@*gb`3dj|;6nk7#(Exc{G2 z_F9Dt(ovKxo`+!SdGEbpmdRzc^g-ZWwjIPm;(H5Hq5ZM3v9_=X7Yzml%yVBVk%>KZ z-lw7Vz_z;8GKnS`N*|Uunm{^9VX(rO3U|tmX__wMnXdDknEv}bvf{5vOGZYH@?XE@ z7&uK;R6YSO@qO8H`+1x3GaAU^bl9)yyyiBWw2(Xb_rl+GH;fAjj{{a;!ESYI6 zR+Yv`s78Z8RKzD!I*e9}+MK(tnm16rvwxX++l;wWwtQZ;?zkQ%S>kx_8RgoOhu_qj zdcL0mDr-8YI#9rh&{29pP*;we-q}e;Fpz)d43Qg#+lWV;F$EE0 z3=MLvj3WVLKJ)8Os4&duVhSqRG$R#WtN_YmwF5UXBD?F`QJUXvA7l_D;KZ^KQg#-NOqXj(ohya^V^vff+33wyj&PJMYV_6mprpqsJ-=d_g~;h`j#fc7XbU z+2(`T4b)eF5Vik&e*gqaf6}YxeW`X=&u!a-^6{d6gn%?bL(c=8-sSzQOd+$4ZlJ*= zvKg*uJ@4o9WQu*`@-RtZ+q!lu-?@&qRXOl?uIJ6a7c7>L@*L0eF<$PEOG3W4!z8rV zqcr_(!!Udz4-?=Fp1ojHEJvU73cs&iNC68=%M5-W?nOECGeDn;yzj>{zsDj%K7f#T zT{ZKB(6Wu56?f#0rpNvNlms-0S_WPegr<1!B>Nx7S!TSSo1d>9pXartIq_yyU8gBF zzt;gm1Kqkpzpo2@BxZL4k@BAxzn_=)i(2E}fnJD`Al)su9Uojxz&&t|v zY4=OL)iL{Quj}?n4P&ieRmkE{&>PN4Q!Com)k9I3JB|}{w5CJ+5~7oKlg3fvLxJJV z(PGdH-LLzhxVEp;JU4>(+}w$>4m=lCjnA7-3Ys`^&#T57ZA51SheWYmqpx=d9>~k{ z`vZTt*hPrRY6SxUAGrb$NY>afr}1s9-jdr@^r}=ed*lVaXlGzeSpsxyzYlV~E`63WFsC#Aq7#?U=deCc_aiD)6xGnI>rR~}gH(`YyUoO5Vw&gb zJ?o-I;6&nt@9tIQvm`O&py%nzkMH9Kx8~=|L8W_plhyD+9?guxVbdwvPX-JY97qDtcOx$$cD_mS zf8O)P2f`W1f+D>#-a`?TaSc)oM;;+QD3Gi~l^+ukrQ@C%$!SuaXSNJEkBQU_8853` z;XTo5Gxgs&0&Y56`orTIu|V*0IL4TuWT##)E92sM`=gBx)VY!wFho}inZ{h6m7i|I z6L)D(jvh-X9ys|QsCM6q1>boKRrB7zzAT-VQe_V4zN_yswN+RiGk(9p8stcq@j-|X zdK@CB5*h3m9k0o8Y=J=rk{Rk|3Hb3p%rj2l`#jD?%Nu16Y=KoClZ-VYU6S{Q^`=ZteK(UU9_L%8aq&7w4us#=CZE22A{%W(1yk!TgUM| zuUeK}Q7Gh2T0zYB8A%{()&6BU8IHw=-6Iv^T@wdu#B-?MuI{|<7<56sY>2KAKCbEc z_M8kxi*T0bJ}Z7&(Z}6)TT)9?$Y2H+!JmWnyQpqu9`3MEf)(dHkGJ*dpX8lyvi2tx zgf$Rqy4D@t;z}h2eh)Wz=jh$jywYcwfspKrr}V4(3o{-J>oTytYVX+0(1Tvb4iXSQ zk~iY2L#u-@BTJ$nt zBz^Hq+8>T4F!IvcR9ENB(kdIPChzHT;vpMqD>9|y(Z(RpdpuwyFc($AMcgMX<|qEa z410r@As9?xgV(%Cua`(hCN&wOD=O#Zdp~8kf4QHi)pg$sfrETd9th)jKzkNH$@fUe z(FMf|fVvN@GxmrQMrp(IVo-D+4WvR_M8ariOhJ_Jkip)4t+K796jupGGC8>2-`7laO#%IhOdoW;!!>Q&bE&EM7#n4Rm={LUTp@K27*GIr0=@ud5O zysOA)zAd5eD{;~W))()7xQL+uz*03lKYV`>_7fhnx}2*@;i!DhGD^k$Hzk{;` zSqhD|tQtjy5qO421GKyyzyt)?b-y7D!fHe4lhH@87(^1oD@(@Kv2F=!uAwhnu5zTC z!fWXJJc(iIzV<;n)wb=!46B795+WFr%0+uK3NKd@c;EJA*>==V*92Yl5gB?_`Su>S zK1}m8j^XXCjR!j#jTr~fH6=#<-fh@4ba90@0J{ekw>)Ku_Y|V$S}KyJ&|Kus99W(~E@4H*?n zcp;{PZGb^K6{`G#kbZb1v=Ta%!48A>JXLEya9#YpN#3;;qPu+cw^yStt7=V4)zI;} z#K0I})G?bbTQ+R>ENI=alR!s1W!fBZ2k5DLQj^Y3Nb%OmnDnrei= z>+*7~anBykf%z83B`CXI8C`o=8>Y0A9OX^~kuSMTb9z?|&flnV;A9Y0A$R~8E(#wE z8Pb*KIrxC`L{O2QE!_s-EL;M;Z0vkL*8p{46D7tgvq|)62<+MPs%wU@;OuyqqvY#AVvUnCf<90a`jihAkLv6BDn0&S zEpc$D3o?d5zT!vS(BD_*0$?U^QJZQW5I!*1{MF+!Vnn42cak^P8&RwKL2EAUvlC$Y ziW3uj&4heNTt%gZUdVk88K||v_k$@2_P73skFWS zMiJ=4DT`rpC4PshzeRu&42L)%$Uh%3_`P-Re9=dlU>}CWARkxif1Nvw;2#8{KQr~Y zO47gf;&Zjj9OU}GSJ-o=lgCjZs@jbwQ%@!xI5YUL5g0P79^ffm;><$jhxKOzjqpG) zo?+5D;t5mUYY@J?gXPaKqAK{UfRp(fh_c)2;mteQp2!PhZ=hJ%(aSn`5w&*R`zV%NubY5wK!N3WX|z_v@$%O zGzo)EF{DyP>0}GNDYjPbAQ=G3KxJmF_S#!HX#Q5KIIy}_sif-n7R;pS`Gn~79%Iv+ zQ|QkarYlr>_|cX{Kv?8;kDvacM9#6IxQayoIfFZ~?n}i=3x?<`$CE(ep|)1l3g`t! z9DlyuySF6#j_2jenxJ$&-%ap)PRqR$YuK|_wz4Gu-(vU7u+}T^q*hErCGoz29qr zgG1>2CmmiRD7Q|Im;^}5#+9}ls7)S~zu=xmkeM#h@&3-w*_t{QM|H&~ha+hS<Kdg5VLnc;{>r1R~x!)l@!$T4x?+81Dd5jlex(%Cb9eyiaZ5O1g1cP^td0Ut` zkOIchDWH^^BaDuR+QD}V(c)SodTo-@3n#7It6*5I5bAmJAZQ0`KcGkf-mNM^YfYGw z?rW|Ls;er~u=a&extC-s)E~Mgv*N_hDX_Ga5h`{!B zdRcaeKH3&aL-|#xBv?Jp?*SD1UKk>3uX$h@bb{li(CSd*1iE9zAL1}iwB|#%E*{D# z@{$4OR;`**-d83^n&y!R?pQ}Q()=-wWJSMWk*(g^J~tZUjaNe?{0 z&me`F0V9|MbZ$S^qcFBp2;v5yK|#k`(zkaN9S0zR(oG-ZjGqJrcys2#N{z;S6p}0Z z(#v5bqp^Oq8wrzv7g}c$73r!G!pg_`J}M{dNiCF4To`lvz^Uiq85-=8X^uHEzT|BZ z|KS$QlsN`z8iqSxlrux;zor65$ZWV32z0W92?5)jS97>%#Xj|}fKc#mKYgZ3n8(qh3#p{T>S z1n`j0&^+ocFq?SFYx5pV3*Mgbfp^Ja9KrPT6!DKje%ky#B;OGP!W!-sYd>iq6-?Ez zi9fzweH)hWd~@!sm73$5zdHY+;q%A9L-SBb@FXX6(wYbkPaP?v(z@k=FUmK@DWEgu zVa=@sTCI&hECnq#H9s?H!9(ze`*-Xeh$!#gn6I#~IgTu>VBzGKjZ#q>S`{y_9O3Xk zO(28+HCf-HZb3U#l^pLQaD3dLw|~b1$1`DR91|q9D|6r_~YV#wmBe_d(6i zsoMIfd=GW*-B}s#)vFc@!cv$xmm-Y1;LzEb8wuwNs(|x+uVv!*PUcLmzl|7&KpvJ_ zNuSpd`oIT1Rw?08qHlNRPno`-%Ns(%gt-d~%G=KfMXe)1Imp&zJMSpZKD8}WRGMq& zycLqmCtV~Y{A>=p^%-vvRI%pXd{SDQX}J9)6I#W>qzla397A>~61XFtnn!^4Ii&2T zA>Tw>)iC)kLU)a4&^on#znR#K?m(S1sDu{UO-Ca{9L8cnDv~2p{|q;u5%@l3*$mE^ ze{=SkfYwx%O_mcP&3(RGLYdV7{pG20oj;!A{5OQ*iLyqoB-h6wP0fEkj|^(j^oUpv zT>s{i494(1QG;+GzLsJFzhQK1rZSEiiS|q^Nso5uI97O-$VmupNRMKQ{c4&qw;%|! zA`u;yAa!Z2`$UngI8tFJFcDPV(yIce+78hoLYw@PJ`ZO3*B4DC?q{(os{(lH+o{R6 zHHpu!-3CRMF=X-xD$&~sec_%QAdwhkrL4RBaMsKza6_B>H26}dJ=%q1&RX&u1x(ty zRl}$&2rK38sIhykszv=KBYFW?Mx{xmISbHrOiTnEI(43mEg==?hET->NvOclBw}(?;&C+4BdxG^8qJyVN!P(or&-9{X00Y9tE57}Z=MAhj}@5KbvE%2rWI87QqOuhMcaj7yXe(w9duA$QM&|y=Y zO1W2RwV$^j*wjRh?*!!5P0V|lZ0bd3kl?cQjF((=^qi&1fF32`yO6_!C06Oms0Vzu zL_fh?Vc;AO{-4}N5QdXR68=FgDLEKcgGDCv6zR(3sM>$)JFb7#B_mHo=kk_diZVe; z;zY(!5j61!tWHTc|0Y8DU+0r$X72P^1e{&EpypY%KW^!xlKIg4?5rQvW;)J4of2aQ zFBo&`+UY~ZroTRXfO>~$EjlUj_F$Gx3Xc41`7ND9$-PRVQltEnvUe@xb2|}~+#U`R zcI>i>9qAeV_?3HBlor4EIP}$VNV@hih*{rPCV?B&Oa8AUicegdS$7`Iw6trZkqByP zAgPvS_@IA#*3d_xYbCxHT$+YQFN~t%DUXAshCxMCwoXsw0$Y&ON@tf^oXyqp3LB1a z2nkYbA4CdVMPis5H!dEZPMc|kU5}~4lkXzYSQ^SDVGKBNrZ<1Bp_&xjw9o<4^%1X@ z#5)#I0|b1)HXc2cAh}T+srb*pP3-P3;t({S5{{zrTO!XzG-dj)VR5A3XY24`(bCsi zK^|pf6>P~e4IjPsWb$rn-lSh1m>IS3rQG1fi#wa4f^1;xUCYz4eH=e~Ps`Hx+3%?9eOEH-x#|NBQ*;SF8mKQeX z()t1*2?*qmUbgz0Y>D^!b=p`U_HUNzu*+^g4KRw&NnO&`3Cr~7K+NQ@x%<}>UQhMB zk+}WjrmJxv;kZ~I>46YgdDj+ZUu%hC0b;P`{9o=WEj-CqQNgQtLc_a`eTs~D@FoLq zXlD2JBzT0(N02Th`lR+}FFgt^pSLCMRu6ZCPeVdBl;G=V=s;nGxb$0F$D2CHP{-Ek@t<}(axc#^e}e}B$T{*=TUknN^Y{hg$~65tuWxnHRs3E|M3srv2rWaiYJY* z?kU``t`so*6vF6XBpeblXJ?C$5ar5}Di_z&!Dt#eivn0G09# zrCP8L9xKoasjdR&x}p|iXL zcEf3@G!cic7D3PBel45!8d=`FNiz~fE&f)JJ}5#qOYCb4sKGE%T&689ZcSFx2@Dq7 zR1+UU5W=ISw5X_X%fI%VRv;Fj{Fu4?1LA@f@?S<;0q=rl=o>eVA>}6$0&Daa)J9iz z>vWc35|ZyS*WCVUJcWF0V1oG@Nn87!$}x9?;&X$ef9zFhSV)fGWsW3@nFkvBT#OIcQah_C`QfLObKg|j&wPo6E5EzJBE>{sA_Bjet`8zJ=D3&B(z$+o>8#Qbbxbg@WEGhd{f3PZ+U!3wHx`^+#ED-O!^@^2p0=G+IxKJW8LM z|4Q{SdTW7}&V%wAQX#hmVY)K=*QKt;#}b;TiMRGu^Dedm~O2`mqv*6`*Tzl4<#vs0DlxX zvju=2{wR)Dp8YCcJon6*?^T6}4d(?QKek9`N}0`IM1baM-rhl7{ue-_iMdEF`FCS+Af5y#|NbOHq6I@=Cf6GEBLKt^fhN z&uo0@Y0G#ZFjD(w84oOlGoa(?oX#N(ooIZXRMd0vS^=|r{=5qdg>L|mA#i`&9})G@ipW`-}gD=hm{Pc=I4r#z~_)~PBPi$cLPRVdmj~(^lXmLXKI`q2Lpn(x8p2ApGTz*NZ)seh>z>exZHu8%ak}2!JsqU1tpFSDbS>%cPEl4 zZYnaGn_RDJ-G<_jX8m%sWCPmNS(JXdrcM&@kU2<|qO@%(9P6>$?8zwoMLZuj3ui;# ze&rFNvCzIT(by%201FlFNqu^}%h=_4yUdF16vr4`4%89)92dKy$P zQ06p!*QI408xi^uf`^#1+J6=M%Y(>N2vcQ*Jof^oQS){VN^F=zUO8iZAGxoTgcYKdF+T+os5D^ zRAQ$x_L<$kRAwm60<$zs>qJeL`B$FW1r?KET?9X3!{xirZ5_AmlE&w+eFQb#)F#OZ zacn!F0_U%1EOIb_$oIV6NjNWjcXK^{8-4|wz%RhEXe3V{-LRDNGNJ!z)@kF0!{+#J zi@$jVQJ%+Xt{+2AU5BY}G(1zpn(uITz-|3b#h7FJ@_FNW@Z|G>zA!kF7(u-iGYW3K z(7K%Q(RzJY3)ko4%8mDaRP~cI0jp8RfmV0@9T1mH3JzE9OeyD7#s z4yXO1C>y73p6_7bvLMHygxwTq`84aQs}#HEEkU(26wRq#U_@j3xBeD9mCQ6~bvvu; zao3wY@d9CZzIv}Dsz^qH>_KGCk?Z)S*^SLq2Un#!!GwE!A{ckok0cGM*jSH!n3wyx z_tF48_*WN(|GtCgNjNfTlab0X08b+X1m#bfcNFwBdRe7e71ut3V1RX>500V#q%~AAWpJ za$YQ?>ASiZK$NlHK<@E`1gq7hI0r#;^+4HdKVK@4@=J}q3|vCi#nw6 zsEDGYljS7OkDq0P6Fbf~6<#V`W?^n_YHg6raX*662zLX(jCd6s6dqm$RsFMAt`uCr z`Pfe@BPNkA%{df{fDYc*&!gVg(mba0D_Cg7z9S-6R3br9xty#C(YE_6rB&Cy58Q$Y zLS0g#C;afq>9UdHhxs7lL-DAHe%q`j9woU@8-OTbJ zC4lZfAJ5ib$VfKSR-c<$G_;$KZ?CfVH*Wl&-Wv>Jyi6*)*&2*yW*LrpyNW8o1QvJE zDV#0cMuRYE%BF`9%*@5>z~&6!3tmwM=JzYgvj5k6#*d40T@`VtO9N+J8EW$( z*E5#Q-?jsX4!s0~`8*c$zIE%`5K0#;5dyk>76A7GV507~y$C-6EP&Uw8DH1$H-SQ3 z7W$Lfl;pAf7!EU%ivREtG+zLK|KETVNyLfgv#!29V8-Hc1%lF%YiQ*3sw)ueQ;sGf zbXw z!xuV+@46Zg$oFyC$aB?%`h@48B3AM68Ku`s8FiLGRSU?1NypL4AS!`64odbBcbr3bUdQZo=tnGx2+=m z#v~zv@BSDm$S5zu%H^*@AupAXQI*gdi;br9|F|-%C-OX3XuJRezGHAyIADLJ9!pl* z=TsK<`gP;ycM#KB=p)bn=B5AIkBPv!8_(c-2iOO*2O+%50N5$pUN;fTtA85n0Va?G ze$%C7@|VO61z<(bN1#&rJXPu+u^|6bu#zC~AoIjC4H!BCfTCoOo((KBydVAm+RJTk z5H%c36dDx)4S|iAg8#Y^Ttb0D2exwCX<^iCA|$>uk}b$B2FIrTzzP3LI{@7d>Z9$jMmV_HQy6SqEPR7U?ZKW6(>3N&7>;BkE0+0$wAc$4dFKUjH7;Zd|QZx$s zC4Xl8y0d`wxe-zJkTLCGba$1Sqtv1s76i~Rz%!2)c3@BPor%Wu3l8#)XS)mB6K$IZ zXe=l9_S79$97g%1IYD9_4j3dQDhTlZO^ zus@ww&zSn&O!^OxYp;y*FBJ+`-y1vIf;X_CP&vsJO0%5Yy!`0S`Psf8QZ~&?>KeK( z=)s|kT{j%vckp`f5dr`fiZ=UvvAR96=^!v92}X0EroDQH$Rzv9?}tX?*f;K>wv$}1 z+vm+puYMag=>%KRV?60q}gB9g~AJuK_6UxubLATCkhi zG0Jt+-&-h{TsV)hy3cx%{7-OG^acxqo+o8nH%qbJvpRJ7B6P1EpO=3Cs@0nmd6_{I z+9m)@UIDPLIsiPjyX((lF+9J|){(fK^YWrt-dJ=f}oXM=1Z0Oq_q90U@*)YK8`*vCiARFR9?DLPkRP-#Da6GI` z_Y^a@zGeK2eg0USB){%)qKhc-3AIiw2S8$X_&+Y4YTEWA<=EHYto12CpQ2I0W0-v2 zalM;J86pGLZe9=K%G`+XtLuw*F4VBU^oq2AL<|gwh;&pU>5UNh#s_u1&pUfanDRS~ zAt#!^E`mzJ*kx5lMuk33EWt?0-^2fNmW^NK`H+IB20?6X?r0%G7{lYP*8AuefS5Q6XiHcMiJY*x2j7MK z4}=EJclS#>U)hPL(?mS-ME<-o7HqrRyS+K>?yWz^iFnKwXF+a9uUDe~IDP z4)4ykdmJ7H1H!q>A0{>XR}5alh6A;t|-eLPp;Vcx^^S` z0I1p7kUx!6?23jdAQvcgl}wSr@)FBPrc^5cfLp>G!kUo9Or)daWyg=O8|rV`Z%61k zDU1qYi@$jNg_Oh2LL{%G6#yb!j&xubln9%D1(8ET=W(VrfLJ7pg;E^1cq*CEF}^Yc z&|9(IX9o;DpN^hF=j`B|FPlzhGpASiq>Pe(`JRB2pj{>2DHA&mVtLL}YNe52VlOqB z(oqW$j8SFeK595G!12**@={ z*4$D!(K_Z}BoTi^bE;+WTXIz=vOrVIgx<|WP7+#HG$zU$zhtELQMk%mxe&`CU$YmC zsX+WS`u(NnXR31ntH2gOIpoNb1#mNHux#7*EcDKoH1xmTS9%b{!P_}m z_wL(M#O{Zj8#qG#^4kyz4+E@nEN&JQrGC+h+k#%) z!YlE(P||(^M#z5M;W)y$FaJH^ceCq9JgR0UhdG;HA-Drga#sOT9LM~%o^Al z!7_P7Y2{agd=mY~#g2;_c$M~>?hj~@wbMx|(DNk*6uTH0Qv0KcF$u!CoQD(zyIk1g zU7Jf!e^S)(yK087U}u!2&iC;Ouk7{ciXcQEfPlO%iaUrD)l2rdoze6Ld?Q*aWOLq~ zSJsNR$Kkdz{bLjC)5bz0H6#KFL*!3~|GjD^2*HAdGJ@rNQOz-*a2h4>{klK|I#Wv8 zM-gdqW?7D#Wz&KUfpQL1cZ){oNqd=NUCoB4La6%+Fj-C0+(kgCUFE`(gdAHCwph@) zXt?wt>8hLYl8Gb?RUrQ@dFrKDp6fOM(_4isw*1(IC=g6ASrwvz6g*~HXI4m5tpi&xEE>zbP~4{d$i3~%@8<)>5(VqL7Zb{Ieua5^93;*2=t3J=L?(L80rQTD zEUawO(ExMPn?7@bF^@HIB6Q7N{ZG9bLni1971kF$TO@>JNLk{+&N@sp%x+w=rMhI& zbRtExP%@RE=s$2s;rqP+<1}4YK`QTf!ST1;HM<_4)2jM5gD7%deU#Qk6|c0V{-MTa+;6-b5i^0P_Zw;*Xezf$cT=D*_rW z<$SQ;0|txPWN!<`^IlM=}YINMQptTSmT zTH&E%7zfotDx&E+^(RjvY-td%#qT}@NOVm2Uv8E|q6}0@1uiHhWI{`oL_pBu8j49m z(X}|<7Hk#*HtOCE#N*JY6hN1N)d}0;&5w!+-Zp@+F=zGq9l$atPpr8*TJg51lZkE^ zpWK3}imIh(E(t?SV#-_ya_bD(^^3)-;wEEFX4#P~GN40m#&ZHAL1Nm#&%O?ekC$aT zqS(HYhX{ov-ki=|C@BzyXGCXbU#x3`jwdk?HUx#^*Ii~k9!3c<;k(~vQlgN-y!3$z$eB} zvmW=Qo>IS6Ntn(eGb;x{1gr^S8UTS&943-i$Fp4G{SDcJ@qV~tQ>YyvL_Pa$dvr zJTR79@}y_q2Jmru`CdNJ$j$Z8YJt>~!)$T0@LD)v5Shumwmr-rbP-;1cs+KMik1WS z3TD}O;sdZjJ3|MhVa!A6XR%R%Xdq;YTE$IKAtqf$(RC2mS>}mhZ;K}a^3E_pR)t%L z_ne;}rKF>%an)d()TqYM%b~R)OyKLBX~;tG&U_^!^Ux9*yO2wXd>fFVT~hR5f=HgB z`|Xhv5JH{Hg%1no4=MNFB8g0CqA#9JfHITs%2c5;1C?H0v3kQ{f3MbrtEG9bJ*3@n z{H1|dE}W9(_<_SDn1iE_33HMOF1HsS>?gx!kSKSVYZ=#*`7p~C3#bF^LrYB-? zw@2N#*XLxu3K*ZvI>$tN%FsBL(`|tB@an2<^nn4f{~etS;0819@H+ovRqzM#(B@c6 z$ZugsDO0{P6h}*EJr<-N_lL-@_q)*90fYnSsRx3DP^6eNC4Pl^_|u);iZ3D!a#$@& z>IKZeCGO7wlxSlYp6TXhL_LFJa)#!_M17F*lUyt-v!ywdq^=bdL+08e^6;V}C*{B) zJ>%n4hDHX(9#(E$dq4T!myD0cktx6J4@LK{*WTVouiXGh5N31~ahW${5?3Q2GN5zt zA3%IQJptwv87>ZV?QKvj@YHHEnt+J;kGprdg$yyKy&#DNO+H1sI^yQwKDA3kD%2}j z+tad+6mSYuUQr;jcoC^Bo+=z?p6^CHwbY&heDPcXJW*P)Y-YJBPUi3sNo+z~J@Pp* zx5UEU7BwRDvnL(o@7e-7KsMPd(x}87rs3qOR)!n;kcCo&w#(s>Cyk5^r{owr98g$Il2O%IOG>()oJhvOM+ zej}>-=S}Mufo%s|!V9}@uA~4iei4o}6I!jA^tb0vdxl-U> z4q<_M?31c^o1BM+MILUuPpQt@Mv4&n=T!c*|17>OU9R65iK6H>R#eE#ab@weA6tJH zxtB3LW+4Rj{ zdiP(1&kb<1P$~-y9&TxU4w{@ZXY@on8BoXM({t4iRu<6=HkRP##mnn0P5+A6GS0Qv z8(u5Q@?LwQ7=F9`Yu5LC>k$cKb8~f>aet`Rb=^u}!W`BC?1Ox&aJVzYLoE!caLiD) zVF{vVe!e;Q(k56WZm+$*nVnbjr)ym2vAL`U**E0L(&&cmTo=>gR?Y9tmv!yPu=jqB zisFh}6wUpSseyR--T`DLMhB)<34Mex5%EU`lK>)v$XnibL=a=+ApLpDKF}uId`i$& zz_tN$Tl_}MV^s?H4N%=5fq)zk$mkl4_(g~!1h$CkYsA}&Q04+ntpdbJ4}Fh@Bf(@) z`DoI`qZSzDNZ>pR9L=C|LDc;;&OxN})qAK(m>Ga)AF(!^0QQ#bNP!1-30EUcOeQ-M z)jELsSnjBO(d0@z+boVC(OuzeHaU#j{W%^*S8dboZMH5@%8n>RpR+YKt;jjMzDz&K z6fdIO)LfS5;kpcY6NArTHqVZl>wceS*Ta&Og5~!87CUp@sX#?5m$(+)WL-G8Umz+H ztgbMs@$a z;tsDjYyd8ySQiBi!dAb;$O5%1!=Fm;!rO3-fl!H*W41lN!G2(8?+B({ z!O8;WBu>%D-X@|=GI{jCd$NI7`odnpKqGAwSjAO`N-LU#|C_LXn|Vn|Jwr z7TOn4%Lld0eNi>SNf`}kB;o7~nN+B_Y&;wt0C~F=H!he3P*x1!pdt>k*=_JNBDn0R zTj|<;!B~^L-rIX-c)%NzfW#MLI7{!bJIm2eEmm-a4B#nqs$)DXbT6yj0rOv0;m9KR zEW$qd)>uGBx0o*C9*P);vE70WNC~#M|AK`3!hn*oQRe_Kj7Y)#BY$< z3l_d84x_e!LS`o@zhsJbB(UaDSQ@$F!)SSKy5*sCQW`9(Q)@Q-L=wjOSFHnKko%KI z3{{>~(>A&pzSOBfmbGE$QsNxz(LWMAPNqob_;Be_D@--gDTq3U$#uFdo+BG z;~o7xu^Wanh#iZ>+U!yZml~p<1P%enoNL14Bqyx7H^!HqNQ(+bEgXahXFGOWaH-^k z2#()3O%399e8J!gq+}>Ud}f&o%y5Eye{A<%p-1TtpkGZODZoP_TJ9 z%=?=TZz|O2Oj3kqG5(k#TQUq+WiM2fF%=pnFX%6Ih6;s7|B31nPrTB4MOp{(&6X}BDNBIU6qs4FRJ44M`m;X+{7!ztI}wLTiQj~d+W)!>E%#1?LPR-RJW zpM6T|LfY5md%g)g^Z^n7@F-o0C!Cz7gP4XBa2Sx7s(L7q9aK>dS2mqJ`0U79R`rL_>fs%v9$uZSw-*kLXk5B=m<)FDrq%VdwyqZe$(0_0cr@p zawQOzP^THbJo>|?1$jE3wMgU%USzmVrvz1<)z0Jk4@}Cytv*5aEAq{I<3A4RHu>7| ziqRlgE;V1#wEepEupz(flQLNJKp?SF_a;dtY{(&Uz(j#os}NrIj|tJcsTj zO>+D3iHja}yakYyS&CPPvEbDFjNJ1_l-*`{#PxZlA_-te0I@)zs6y*Sj&tnpzlbB% zrQBVbr5{sgEZ5yUj4uPoa`ogkythh;|MOn}unpPoNO1OKK=#b0{8%w!D9Xefd|#!1 zrJkPiRlAqQwdC|(JKu>=KEpAG{8^Ny1asM?trj^4$B(1P9nq4wh*aeJU*e0<0?OoL z|FovkBox8%FOy1OPdGWEb#iv~P>s+;&aBCief4J3vg3S zCwTU|nkHcX(gsPVEDDe$nHWyIW4OKxX~T>#@>WAQt@0$(g4adL2X{b*IGt5W&fqB# zBQK=ebAq!qS|99lgOwoxWX3)t3S>6|(r*>PY_y^<==Lrou?12Hufa*p&59dze27Cb zY3+AKDpkot5iP8v3{nYDC3mC9#2C>)2%*VBG@C1kSLZzgz2^>#ONRg%tk$*!qW24> za)3y#K2v9C8{xPMA;+UHy25ZN zqGG*+Qch4VwL4HQ^N`idNizDcBGWl3*rj=+lrmoB(`;>+9|qnLcR?A4WctR!JwloW z#w@$`1%Y3n;UC5O&RmtPVo19~3ozI4zr_}r~$v7bg4LiJ$V;5+*M_q*S8<}i~&I;1*tyU=-Y^6i1bT~ObXFp$}(eKbdoB>vj zgvAH0qmn||+d`gp;jT< z;ZWm<{gcslX+){Qhm^h%4sw)+fyzWBn@Pz%O?oSKH{vgq7Bxp#lsFg&cQa&CyVwT# z^);~wU5p=H%Rv#|e2Ej8KtkI#D;~{Qcxaw3uL)oRzHR;bG|qGhxJ$I144! ze|(K42t1>Bq7E?vKq8f(dQGd({X-+a9!d0I%MEDS4N(?x;Sh4c&>({cyP!XA1)p3E z`{_rV%bj^X=fJ1h0|{rWk2_S@`bwc(o`*k0EFME!a}iXiK9@WGQYha^cwaGe5Azk7 z&^13hfuh(&gVXpYD>4*Y6cYEzGFvJj#X!@R#6|~mk?g?_E;XKN!Z zcWDU1gRBvT@BfZ33Wv6n@#kkwpoxk0B?R4SLalCd@}X5Y+fYPhhi-wSpW5?ok9>`r zWo<$9^TkTsOLibNf=Vl&(NMO9q&Rj9`MdaTKdE3L#P?+*N>e1 zI-f_iEfemAE!)(TS-FpR@jLzkE-(1ZhtjYc00ngr2PG8JDH7swzte*j* zji;$Vfk1qSloUXBFN{!V7`?N)B$%D={v^@yV)_KvTof{WkSoqWq)Qp!Hf#=0Zh+}f6 z&k2N$GwdKo`d`wK0tz zVwVvX76s@)!IUP{7`+3F`@{o{?-4x1JFq2EYM`6;F_Fk_N#{KjxArW%_ zaa4^E#6X{r5foMXfSqfc^Z}BG!jffMgE$gbhp^R?UC91-fePszhHD=akCb$1sCtyS zXT4&Mj4$^rEixzp(US!y)vHF>{v`j5yDV-l(!z!kQz3hd+nH!@B>TnkWQu* ztY|nq+4ZoHA2!mKZ8h7S7^tV3C~)#%FIWXeL~>jGHK^fxi$T(CamSVZyR>f=XzyGD z8T_vGy?!sJTQuvG@ti_5=Rta@{eXB7WY_Jv7x3nt#~hJ!y1z1J%KT~^`pZY=K_-Re zu2e5_n0nv&HJ^ki*!#9fSa&ySql*;cj`1i#t3ZGO;9iM?%?sxYzhhJCn&c=`Uglw{tU1@1FGVB;r z%X#@J-?vSbLP!ToKeGu4AKgN-lcl=#+NGFzn&cCaKROwOMj&UfG>^>QmYNg~Jv}zk zWiVnrOZ66~-NrIL9a3(W*=!c&ia1Pre7(xYX$E99FmUgk*|D*SNU00#@>MkS5F$<4 zAGUQZGDzEC+tbFCb!A!$qRDhuNXvrp1@T}bMM9s++hh;Wb{s}4czfi`McOx&zgfsp zC3UM7NuiO=45k%2UY{gv?NSw8aC5^(mZ=3@NEhwdR@c_{>dhP)W!fjW=Yqft6D^DC z%dQZW#*& zg0UbiEc$0DCS&_jxBHShVBep^R3obh7%O{Ct9>yDEE&j1*gae?kgzS**RSCJLnfZy zFH~C)*=V3hPmkIoOiU^|p62MvzLqbi(7pg{HyZU;A7PXsm#{>7oT00EoF3dRdD^gQ zx*{V9+|y>-olm{XUl8%p_nu;)+C4F<7?>Wv1%N%kf8ccQUPLc7`$A}zn+01l7E*~% zm|gtc>*i~Aa(NIbP`^n3lBvXSmvSG?S;c}WV8~;glj$?_@&|b3VIoZX8`-R-F@G+s) zaMwQh9dwG|Nhvv-td3m3;jGR|RkRPYsV>X*d#*N4Lu#&SryBZj_gqfLe(Y%}();I< zlr#zjS55>iOai?ct?k2E-sB)MO%cLJrMRx49AVMK($@ zfh<&k$5s@!V7Hh5$oNN;itd}Iv%RZ^V@6RBI-_(H#FbDcL4A=F9;@^aBd{WbX<@7YKBL)?s0PUzhV1fArM;%)RX3k|(K(HwIN+U4O z=vrDey(LHxJ4mNWr3z9X7Hu1$<3|UK4G=7{bUyy(=V+nAhb6Mpp+wWwLPB;qC<}R9 zLz^2)5QA0y^gP$-GaAizf_5!YKeFNF4_p#6G;5+`G z=c?(IP3-$jNxmKBmV1zj9o}wCYu@fnQ$!of#-+}uvmnlQY?{vlkLZlW}1!)H{Ab`wtR`JA&GO=KEuJ6 z?aT~(NH{1ueWgtG=H$f!(0~>J)aHAd`7mahIbU$Pb zTyUr;ACNMV1R~64wC=iGmfn~4hun7lO%<|)1S^n^LZywTy*a_6pj@g^4mQhtvsX@S5H}1289aSrUPx zVp6K@nF9Q2DoUi_{c-I+^QjDjaEZzqiG5>M*HT9Wl_HydkE2i;!JFFm3kuBXtN#xD zf+%>x^}Py>xa_iKIT(d%Qr^>WPrhw2z&c6958&pWzBU<8KPYwQMIM$$Agq*V|x{@*!U8Ye& zgV~aul5uiH5+dOsORDL#mBnl7t!iq2e^KACWyaOSO^97Vru{;54(c(vz`dS+20zhBlWNp?#2{bNicC zs6q>_(v5?ZxKPRC%xSZ+UB7@^ILP7XagP}fcld&~3l~$tY*zZgft@$E z2lU{My;d#nfjnqwcP+5_O1Y*CqeBJzKaV8#C6Jy9K?+LcV{%MeoJ zF-G+j{}P7y(|t-YD@xqJf#H&Jw=t(_0J&BB0v&(Ag`wes7dE)Vb7)f!xdi(c)h)P7 zQbZ49X_qEB^9BJ_IU>r*?>-OFGXr@rYHL~{!iIYO%Lm!nhhUX&8_k>#`Ns?qo4sp#n9ZdiTgWISFsv(tmhwl*oSG`u1he2UZ+}upe(mcynyUL%0QGK=u(=z^duds20a$pPhqkdvki(kCd@>J?v9VLG7g) zU-INIB$f1@q4`$|d>E-(tr3AR0`<>4H(~1te<3l|)S=;*z7IR{j8<3l>@!A$iuzA+ z@u`1UU0Hzj=tYPO$mVv)3AseNUD%jLljyRE>$*ZCR9b~s4?=8PiVjV-+zh;UPw@~l+pts?AK;U3(}u%>rSr!0RVJ@t%9 zne@Ke!-BvWP*HFP=hriLvqC9z?9>)*S*1(a5KV#$Du}0j@uLj&$bTHr>$Ih`&Tuv1 zUSB5+IJp$fv7*B|J3FI6;)?m+-pV2n#s2r)d0HA z2GB(tzNrcaBk}IW0B!ZnoMHyv^va6+S@Xd}I_EhcOZJeGf)fDB==eUj0(dd^qq!0U z1O#w_tD#DzIYRzkwzLR-?=9nd^#&j)4P$-- zq%hTYiquIt?tn0P4lr0<*VwKA$ng$aubrQJ# zJkP+p1r(M(J9wLXK%yku<81T1Q!vC)4RUokwd3Vh(EDkXa{F8%JzP+Qr}baZT^F)i zBJ};!kBSj#)lWLsj_hpS&xHVx6i{+8v_-<9-X2-93IMUot=^D2PrqKBw|6{Co3iv z!cg>+fMKjw6XV>>?H|VM8-g^rixTuU0^y><(!L@s5%Kn1;E;7jaSmN=t>=%U`XS!k zqje%Mw3vLdi7B^Bf{ZN}+a9l1ceYf0!qDFh+-#w4ZWtmv8;BpB^8D9DHUVN87`hC}I(#*`KyctAIZAP0m?vmq9SPL0+QN@%?+X)CCP% zma);lYcksw9Fi`6c6#2C=KQlf_az*~!FUFDe3*T*wl@qdo=L~c{>pL0n*JaR{CDfd z7nR9B$A_vqUgqJKcn*3?liuFr3Dk;Q<==@(3=5zLnrv8_C*Ks019(;EKW3fp^|n@h zUMKP0e;w3}y#U$WO|v`%huUG=rdHngP$V^R!7Z77oz4F{Pt_1mu&aLrOe-%JaiV%Y zo!%|zcziVj8J#^}`^+Vd4eWfEVraJSe7V(hS+57w>wW=iOc$$2WqUim0+;0iD8eoMD@T$Wz?Q862yhcw{2o9e&)z>- z?L8D-R+;{KJ;}c~;x1off)g05$Y7g}L+7NbiA>(R5JA3UOWmiH6lO!zNcr19JZ1w| z5)>Pob5{VZ2}rCdfqE39_}Ci5K`oybgt07-+htQ=qx{+5-$#?z;{bV?pb=A75Ek}pF0l>tVt(K5EG7Jax_IHc?vzxx+PW7oJl49K)|ON zth29gXs9b-$}5%*!(+Fg_pF{bX0hTO<5^&vLqMd&VW)8IE2b@@I{A;yohM?7Nd-yM zVn*JhBS;Qjm=S@;!}Scbw6xp=LjJ<~Q0Oik;Nws83oRIbpU9$vj4LeB38|)-F8`OT zvx6`#RGo>#@)xGL<@sTK-Frs3Q!&>6S|KQ44+>dW8Y-92QS@gNPSwDG=HPG#yJ#1< zKC8}WCsr7k)6@j0fNSDRURkhVBTO^F}%+Sk8rrVEn*v8N{9cAyvz_ zJlC#jZxkvT9>DYNR!Kki_Up3f-+iI-OAi=mf+WxZZAqk*5m*#(7YO)plv{(X;| z3+YOm-2L`@T(?O>&)zQs-leCmx{!MTBf_=jSEwT$EpPwutiU?!#?(8-Z9a9**x&ZV zjPVHh4&BZ(yOZPJM!{k8EBolwveD^&{O7>+@3I)!O`5L7aKi~x6cM4Di7Ov$>~jQ-avA5m8DE|3T>NAa6s{JHyf#uKnI-lr_&NhfXrWWM;Og zElpy#EraE6RYTEi z0@U!y86^;r0S(<-7k*uKI+$Zfu!E^ZYwPzV9_@NtGC@-{c%je9@tXQS=IQ02dyoU( zqaI)Lr;P_)t78_|f~hcIYY}0MT~o@w`ZM<6CayhX?f(sD|CvYhMRG^aNz1sT0xUg) zweaRA7aSIyH(-8E$5QN`FDV0siLLC%#}UxafeJ5d0nhvQP@cChcL#K8)lBGdnF7KF zVrUd%;$Bd92B192?Vg`twG|)}ZZBR?ZFGhxbq9l-ct#U3Cgq$%PzrwXWN=$kU7wpD zU)^?`L7^grSne);QYYU6cijF|gnL|L_1|ks=~EXZ#1whn+d zC+H)j7!R_UUr7V;aXAA7gEk<^zcUcoy8xNp`7RkrcY?TT1ljdws4&JGa(}?&N`=QlO~Vx53t(A+m3O38#LhV3pj?hE8c)0~ zFhu;<7~F@xMJiYhcqM1G2EIvpsrq>Upy&;=Jm$`)^>ZSN!yizQ>D<==2Y+s|Pa9V+ z0rh6iySFk+79%wek2Ik<%gW}*Vn&}>EHf+V3uwiRh%);HFxIWP1;|)`%kGCjsqR7} zH?Y@(t5naJycSp`IcXQCw6Q*P%D%{oZTsdgJlS#wDQuL+wop7xdo|@gMA{7$CTkpB zug?D-y}ra3c0@TDV&+JPH)IaLeH4_5%jasVs)~w=?hrB|`1>bmJcSJ4D9!mpz)>9o zS{wiu8?Xb^8AbPRiv+_1 zs!fx`Dsjb)eVV1K7@}|h<&3nzLhEjxWgxv5axiJs;&|sXTEop`R8bZWr2{X3HAI$Y zP5^)t1=Lbxiy-R%!?Y)S?yesF#}upe@}@3nAR=wKM8^!G$?&0!rT>)iWj*gC6 zu3P;PP<0<~fN~_Bt07YAyKu&iQ1-K}kRmVVO}V1kT1O1~qm+!#dk{Tzq0k}1bW%a~ zL0mlo$U%+N*WpmrN!=LC$u+`Bf^_!grpV|lgY(Dy$&tM%`awOA545#i1D~s6D`WzP zA<{@Lr9*49aGZ>N=gF-&|6QO!9|jt*FWCyXP%_&j@;+@&z@CvDqv-lWx*m{@*6f-5 zrPBBA^c=K0H4F!2Fm-X%n?I=>Wr|Log@t*XIl-eX!ngKsA*-kxfcAz*k@=Kuqs- z5WR{Z`c2_eIgPvb+NbK@|GIu%$i;#0Xu%wPYi4>APl^`Up)*if;O(c)6108P2j6Yx z>y7Xn0DO(x6-&e3@WGvgA(xN)(5d|=6dc@HLlX}0zJ*D3!_0&9fi>mh*M$NHFDESr zm>#POHyWw7SRp*%du^e}@jEP5;;1muq3y59*)~i=y%d84R7<-?5J9}AxkHafBfVh% z_C)}${KC+`(wo9d?IpTRqTKCvK-NZX7)C0m@3y?8t48!Q}k{=>tT$=m%;m1mhK_i1Y}2IGyB`t zsKHC6$8kI}Hy>UwSr?8Y;BiD`v&>FrHc2vpAZV1iufxkF*go9R~x)ax4FZ}P3 zVT4+iysjmtQEu9d8LklnBkOsXKKsAiV?sY~L;HkxdARuqq##EGLp(uZ2d;!(eVc$E zLOamD0T*W%m4O6i7=K}`a^CVXq`E=fjKbG(|Iz%d1|!m56& z#x&a*9YTXvGl%$^QrG}r1hz}<3(6Fw~fT2&kC(ln~ zwNY3oxZu2qM`x$K8SbsGCkvGMatmLF_ItN%(Ir>b=`lq;bL`f>Ti-K%92=^Btiyn_ zTjr1(k8wp6mY}!(Pc)&}&zHFw_va94xeXJWi<&A&%;au+0#;W>lAXt@zR4&#s{IJ~ zE?x3CMf}o|e+oP=Pm4l{G!s?)>)I|ivBOk{I9V`~-0CVXjPLun3N@yFJ8*h8{rtH) zGY{%`evZE!^)6yP>B{79IpLr-Ps{+V4JpWWX@TtE&F%}#@oc0>LmW}dvWsdxDsUM~ zbMsy(JWaN3ii=^U@{IYFg#}`~RB++cAADaetn?h>tv*M%6$A`bf80kjD0Y}W0TfaH zON4x+2agz@7Y;FfPcbx1*NhIyy*~^g!lKNOJi2?eaS=-j<6A&aAlEgIE%;Tv<%in^ z(Pe_Vt?c6QX6SZ>DbfF-985fn&6C-usSXG8Z4+ykb$=6h$>zdk7k z6Kr7D;(ISEpvqoL2dapyl_+)W!7} z2x>+p$xzL#t$hdV>_XaOcXNd7IE$j1YTZRB2fB}b+UAoA{U^ef!wIhPh)K|nq37f? z0pilv@zbIjGwF{7?KdP;^8(z3=Di!v0}9-b6IQp$Q_9H6+UvDf({q`hr==sUul!kF z1H7hWXyDL_G0gBZdO{ny%A>_jgsD=Kd5|I~bILxj$)fbSqt z!}yX49>f$#3`1trcFliH@Mp7rp_!7>kL>QV{dqIz`V2yA+XIpjgox2oC~<1AAL^OL)ZMZ$X4w00yrP#LAdpRleJK zG}GFvAtQr>zZd!l{IG%b?A~4x%^jz-ad#v)l9;+4Uu$0uJX7_U1Zh zH6`$AQLR)%QJQ?c<+MIcT}0arfR5YO^n!D5JwUg9@M+9=Sv57~RXEnw@)=k*b;UUk zO%&t197EfMLAibCUhvar++?;noH+}?GwgerMsC!1urBe(N6qWT^>~ zTo>fLFY7AZ9P-GPn@V0|fq3X0?}o(lPmAU=-pc4t+C~UBNEzNE=2Nvz1pL=?X$hA6 zFT0BAoYvc4)FP)h7_^$E>M|L<4o4f;?T>-BGI{KcjEc0k=Vw!+yGh#r&YA5yA|hvd zkLF>WC>Yf|vRhkCC#%f~a0!l@jCU~0dUkVVdjz_Tw=xHv0>3k>JfBvJI{sw*zzR%c z!Yl6TtZpRqey!|>!z{Rrt1fTU{@nm%GdJ)XU#G4!K7}VWh3!4amS|BTOP~i!Q(Vrm z%Dr(=9{CJV%D~dX=)c7@ccu9befz)lLj>*X%hcS?2U%}1{0*90!9_7JHXP;^Z+>S= zrK1eLxhJ{u3D+Bb6iLAKsq5E*jON*j`(m%)1M}?ApBv@u=O;O@TpvBFr`yv}M0MGH zss_6M4rkQpoCmM1UUe5BmHcAx*TG7K@o_K!>^IdMJey)Qko?A8e4xbQ;?H$p-hESP^+!rKcn#-N=eP#o}j^JFgBO*_qWq=T1?wU|B;w!Hea z&Au%9TvD&nYkNQJNg)Yxm1@XtXxL6YJ9dGP7cRt0d~Ba`Kf#6Xxl`RLSp7L0Gn$w~ z!(G&~{}G$L5<>=9E>`sSvBTq2c$D*);44_~QP$7*Hu_2;_Do3`wPl{|e?`o%OK#6= zZ^D&pD2N5&k}q}5`-A23emZ2bzOMKAz*eb5=bi--*?X7hWTw@uWWWIJEWAUr$Kqx| zGZMB}s7*3YO@eLcoqWR#toFf_F-%_kCnH%KE<0P|tWJKkC#Q|-1J*GG4BTRD2`lX(K6Y@iiNu5C zvn$UfLwf4tw|*90v>%X3iVQnXXSPYW&xeF=Y3Y5{Y{%om7j-NJiP*3jl4CX;#?)`d z{@p+CYp6HaFltm%vnPsN-6eVd@C;v(OSvzmD?@D&Z7=p^wpP|N49$u=YV=X+V<5Mt z47J~JZxo&(Lrt>A$UiTG2u}1BTNqh&NKGUKCV_v_um`KRcTUL%JWCq6*K}!E)Ke+m zDJi4ZyDo&>R0w9=dqMp>o8ik$3gJ|3?Xp$9#<>0vvBaC@_p<{J9F8QdN$(Ugg82Se zKr$MxV2y4jP{fD&%ke>|-vU!!!Yavv&pKr>FgaozP2fWZ+UySJd2rlv$)@a~vGb{; z76)LL^}eBqKU8zDyM_ywsYH6oHJ4l$*5U+7h@a;SdiJ5YSXnyeXDqNbCD5|>pfWe> z)?*g8SvO1|9WuQf$6ZHfDMb~hQe3R5RK&@mH*Y=kqP9`SzgG|Abb3)h6r$6ub zkAU*7Spews0yOsVI^f+*9IX0g{z!wdwm}FWabja*A>c5XcCM9Gwd;NGC<@>J^D`#& zF=b8XOnJO)Kr&fUaD1#RrHxB@vBWq-Gpl&1_Zcun@X+JZrvKDT+n)$rX}$FnGA#?&QWL)_X-AljsS<_ z@bIi`>yvW$S=)7Gw9Z4#>%*lsr|0#+D5DSMp5o9*A}!$V7{R_}|03(xXg4sazr0yl zsy8>nen*SkrPrN1*<$=nIP%Nc?8x>xHN>~UQX*!W*d@X9&=~OS@B+G$iL5i3 z1vo7?xC{|Xv7~asBOnya>NIOnZgc7!35{sh7(nFm&gG}fz8rV{IW8&6__IU!%C)yz zMyTnqiy}6ex#~6(R`$lG$Prh52*|0cLn8@|U2AbV-T=xC<&>Xxd4Qvg>BB$nMtTB5 znLO$zXJq|}V9gSYoqox=CLSLc46D}Ioj~W(vn4y!u3f$pU5fZpiQRm~;4*fgp(Pme z+yY0j1s;5~q3{mHhw@zv@WOX~7VNNK$%KnN-`}6ooIzgzkO4w)S0=KPz;9nS+EU^B z2+~+h8TN4iFc3fQ^#kI@4ZCHQUn#w&^H5LFWXu7;fC)ygt+iUttzq_8r%)Txr-Vv1 zJOEO{A`k29>*Z3YZ{A!N5tBq&DaH5*_ zw4Fu#2o0ja`1^k8s-xy9_6*8j3(6J@c#A7Fa)e25X7&dhSa2_ML3RnUG$K3g?=qD# z>h2crf=ea#j$-$HK{_@AGf;*V;BuBEH~~bX_G9gH|6=A$cnuBd?fuDYUE>C948yh_ zS+t*hTs%aL9;cSNtMHOD3{>L$Z#F{%C50mu!~?+wu^f@MrDv3{L}e72)Us(#^w#mKMO}>;VG*x#p86FLzoCwzabWos>8{1|kHb_3 z1OIS+j}A00W41Dh1Q{*C%Kn9%Q;hHqRaqz}ENiTylyic#g9MURKJImz$QxMwjKkV5 zn>4&_Lt!vqOle0#<+E7w3WeMu;X(}~C>E93GbCzgD8XREN`qIX zaXp*Ko}$$S?mf?ig}4A!dM{Iu!E606El(Q-IOGls<;&0KuYbF*aR-}gw?7mBFq5k$44|0_`T9(YQ+nr3g-!dp9OG(}-LWdrVNk<} zztLNU;+#_X_>yuFP;61brfks>l9q zCW4h`JB2*MS}Mc_oSYmI8~?E2|?O z2hq{ppSy(+^bd}P#>BvRJsb296wTZwUBF~;tu?9pvOYpi=&~Dtr`xR-!bGrHUA^Y{A#OL0ljRfGfa4}W{T73DTz=iA~8biyV zv5QKOsbhBn1-Wfo^6)rF{9!`9PO%x zLLO@>1J`Z~FfrAH$gVKi@@zRy2S1pb0jWsBsI-WN|3G9^(p%F*(oA=tod0=L_shM$xg<7td`Am-gl^2Ua{8Z?D^3j z)Lu%UBR_Z9SNf;^1XqyE2e^poxg*hF`0PN0=F`K=a?E7kDwRFqWI{xQy`>WCPh6d!C_~_PIePqGRWRs6C`ld5Q$`Y+s6oQr^xM& z4hCA7nhfQwKoB1$A3Kd9n7}Bky)=%Y)BY2m6{VJ?o^5eiSy{{>O(LxtJW_Oovoksi zptg13)x$Jp%^%f}2QBnU3M5Cx*$beF+Tk0bJ$_)%nQzB=!55rW)iYQn=n|&~4ChY_ zk&ottPAudc<|5Z%vje-u@7u8jv`Ww;Wago82VpO#)Xz`GtTzS5U^&QjpSQ8>)+rq* z@(xlKC6(!`6|G;tW3Xzn8|qRFz~sGaQItz0vf-Hhsn!ww=^NhMni2+*A`4T+_XaeT zMayW>#27ji7+Q_WdHe`{;q2&-{1Z#m7{}lacxeDIcYIEIBC_Z@sUFHi0S>sXBZpQm z%4A4l7b=!a$9|OnO&lKL6X1*(E6y9NyYQxjMT5-r+{R}Q9ZLvUM8-av=hsc8-r8B^ zEY{{qCghj9msn$!YmnpO;2j?Q`WEl{)Ifae;)I@KwfY3bvfL$WQGoiRste>ZA|@c; ztK#_`dx~W!&l2QPX@ljNPXKImZ$RhTiSZ)US#M$}w? zW_fr!1J!xN25|oH%&E|30!&O7Vm`mcxcMz3OMY0A*}L;eR&xdyjRZ<_&KiA;D0+d; z5h^eBl}6?t!`WGI4u50P%gq~%F^eCh)jqH^J91E~%-CMwbCTosAgr&`%w^Uuue;f{Z5#VX25$4mW%*1! zNu1$FX;YF7B&drWDw*tdXfw>BL`TNO0bqB8&cX(P5x|?N+70D09-R{om|HH_8Uq0l z^jcQYu)Msyw+Pb210B#3O__&)XG1e9E!UCH&4NS;?&}9sI!6TTqB-m(4HRA_a3ngl zi`e-~Cz$oX(%z-)$ZY->pTpl0=F!D~s11!o?WWIqMZOyzm9{{$bbr=ltb)SHXpH}B zqt#<~94<82*WiyF6_CT#G@_O|9ER%V5$>!HvL#3XO!lr-%ryB^J!>o=q88`b#W3W{_F$=O zF_7tqzB$?z+tA~&;Gn=4j3dV#0e7t~n{V%gUsTL9Q2$UJCL#`gAI*~0_(=AzW-y`YGd|2jPl94P4`hrkFzEXg_( zb>h8k%5h-(pbW0(F~)@e)hrUJnR~K*Pm&{Ud>i^19aR8zN5YG7uF81vp7??G?feMv zb;rlZ?r~67?UHzYH_Ba>z~m3FbJ7UA-c(q6iQ{)5PRt)ubz8ivfl=pgQNPSuBb-WC zw_s+K2qS!;j-o>R8)->xj(%~mSN_w_IXPx-9>@9-ZbxEw&C~d|J9r)Vdo+!{k;7ikJF1Em_9cKIn$Wn^JU5jzko4?f^Yw+jkHl^LSjS}m$CRi%w*`nMUi1Jh zvH=bC^`xNBM25AQR=q}%K1g9_(PYQy(?3^3>d#%slyypI56r6p_LkMBzZ;m3SB`z^ zy_ne5@ElmY$xC(;*aw9T-<^{j1SZup4UCRPL+ZR98S1>QFa+MU$|VfTy*W@ITj-M# z!=);{=kd*F1Xz`Um_*5;wHfHq9OHb0nr{I#>Gc}-jXoY29w}E_avQH9iIVi01B%My zdI4KZwBmKRb`#70L>&CP=16}=0FCaoiqvPczQJIFymeMqs^`^PiKF}(IFlKI>ie8H zumY9LEvVsR{q6Uk?qdVSAKQeeB zf**}t6y2&?(R_}sdlXIx`zgg{h%=iVi&{_Q+_#Q22;oR!%B(hYZLOcoVWJde;MTv> zmelewB0XRNW<|Ebu;{tpj@AYurmxb7v3S^NMAFieK@-VTWL!M10)prjrP(CV)zc8H zGK*bZU*zPXv+By2cWBn4UJS5UGzmVLhY&HLp-jP!2dhOmePt`M+H&;e`N(*gx@J#UiYKF(f)d{-RKL#lc%U~gO$zF^|bbm9>M z?;#5@Mn-cB8xen6|MWa9YomV*z4#&Y8SSO9!$>yh!Q@O}XUvG~bBTriie z#>$zMe3y|0H7jv%%MzFFNiMN4VV8Hx8;=PTMZ0pqa z<(E+t#rm`>VR8@*?mScZ)_9IniJc4Q#IqegnP|LZ;KG;@OID$Eq6Ln?0crO?+r+=? z2l2-ko$+1tY=!Tmm;50z)6Cx;hw@**99Z+qEtIZL%*qwcZd#^TsdgH|=tG#GD;J;_ z0x2PsWSr8S*G+<%G7aW&EJy~uXAm`tIREhtp&;K*-iQ?d6tvmLhLj`2g*s3^#U<*9 z%$sjRrLJx>!`>b@cQgV{j!)$fFIm40)%;;cR)Y2|tbL07WwwMvT}gzZ$h}T7f>l7D z7)CxTVWi((VRZjW6xmKvQo|J76iuDNe4ZBdA{jBBHB94x%c;6Hoc}skppRGdb^mS^E6kV zNiW1G%vdLeOk`2T`HpuovTRvtoOA4A{gxHWC$wymKKf}0Re{a2iK3rIN6p!)5ap1# z8Y0Az_*W%myh;q@q&$9jZvvQa z{@-=#v=g=Zt4<90rWS>W?uZK}SX?ER;wEnaxGiYuTHVyGio_#1KoL@D#|@3M2Dbnq zXoyy8{0p}*M)GH=Tl1wD$6DstiE5qQuQ9ssc2NpyB~7fmTgW(RY*8;-{^Y#uE>_YL zqXhp8Q+Jh%eT1~D`-3Sb_aRagLH#}7*xiRg{U~-UngNP5l3T;Q6j@Vs7E-SCF}ptE zq!WpPFPUY{!((wDKHH8G&^wX-(u*By5+offn@?!SoZAgGQA`3h(?6M=EvO1z{rDy*%L}ngvLGxAr~PaBG&fEtKss& zHw?0?NgA z?N^C);dhnl8WnDwf`hJaFDG_wd;KqpS^Z> z{e(rY;Rv*(naEr<)Y2LpkOZ3WHZGRCT`d7MIWNCwlj+pKQp9(VSHh&vKDt)sNB;k_ zUIDyMB?f}JPd~D+5)jgHdus3lR|S`pDJOjHB)q-7eHqSvK9uuupKSOVFc7_^x!GcV zyl^_S6v`)73AFxrc|N#{jKYkqM?k_lZ$0;;t!hKnUj@3cc27$-t5k@VsaAPk_G1a& zl*o17Z=0m#)Y`$z10wmP#68YyYT3|^8m1eYODN4S4}FoRM;J=@I5bxPlUQYrwMkt7?tz5{P90nsA$wa7+v(b!9RCzOorvK zUI7^?ApN3p15) zYd7T^)&rrleRaEuRkOAw>FWQsA>Ah+%u3f@GrL}{#bB!~e4q6B^XLCZ)inp?`LF$I zwX|idmTkLMtCnrswrwmdTgy+jv1Qw~jpg3Eo%1{Aynk=q&-d2%zOIihTj~A7L1buG z!@Sh7tqP=ajfT?@zE+E!!Gv$YL&h~%dt)L{j{v25C1vA|D$sa(Y7Oe}es)Jtf))5_ zkW>l%3rGmzv6w4=-t&3Eyma_5LQ@q4EBjlm_spp|jcyL}A5ygd0sO)>nX44Bh?;Q^ z<0lg?DdfWrBl0_@CP^;eME=%9dcNb+mNQ|-hwV5E7;cy68yz6$bt@Tq6Ht`Dr?6US zzMz?i{rmPyhy9~8crAk>(;43M@ z)hw%GWOaCFjg)BuxCl^)l0AT;RXOEk@hJ2Q&OxR~v=A||jaTdY^za$^5 zu*Mtzo>OJxvVS=+$v{D~hb`=LqR!i8>61bmkgmnewbm*nZ}~|%c;1mAZxPTpLU+T= z%B4<;HhI&$!!=xBq(mjo#4cZs4o9irqgZ}S7}NJ>BApFEiAs#iDkKLgi#k%#zKNwU zF2p2(lQ@DTn&Ow1*1dTdg2J>&g7yrKIk{~fT@5Zcgf6%83<8!&p8nXTmF)8m@vfX3 zk*wufV*?SKLaU{!V=mvcEWItkNo1jcH7eIpEA%*ofmoHrAk_oC&!El)9O(m;^X7N(=%%B^tn8F;^?;B>~Ow%Fk}nHoI13vB?K%Uah9sFdU)AviQDH-g(ghiPW^3 zO?rh)1$M4R*F1H%_hZ4RLy3>As1PaTvI~t>DtpsU(uRL_Q-&JkWaku3>I6%4j=IyV z1*EadMRracY9#gok(o`w`cPQ;k0O)WT=F*1bUpJg!Gi3j6mG``PVRR+w!KZA}lm^@p!3=(R>E_9>y(2Lk=^VD-66EHD zGYJ0s#xWq7Qj?JrC8Ycr$KnKS7^O;@F>3f13UDr^gq8!~D93@8doT2vwDTw9)S=OWpZjlO$F{>}Lp>}5}- z?8P3XjY4f6ss@=@Ga5RR_H>MkXOb!LvjKVthdSlX6_`gZj%Y^MbWjzJ3SzGX(HmO& zuLp1;x)XqB#q)!0i1pLONhJ+#4b_$!5N~dr62j{&okc3)(2ta$5|mN~43fdA<~Lvhk8XygzL%*Ay~jg*lHq)q6X29=XVa*!EgUhgPdjAUAQvdsx21xhaunC7 z+%N#mC#pxEc#@7LbV)`M*rRUQ2A@!7CJrJASvP)qlsi@>KG4L6qUs3grQOx1t&^Qg zB!Q9SgV^$l28AJXmb(|tW}p@mDP-zuf0eI@<@Cb@<>#h}62^)EuaPkegd@SQL3;=p z3!k4ye~k`8{#3DimB7R}hw-FgqN6EJdt`8Lf_UA=#wKRFEp37c9T`*Cf@(-7qlm(% zU`MOrxo&%r9To%KI7A-46(Xilp|J2dB?~Yit!j(lwoClK|>Le;kf(3{I2?WEL_qH(vC`E{{1Vl)1k`Gi`EP+TjjNiQc9Up zS|&59PB7Be>f^B$`F>M%Ab}P+wQd_O-cX6JNW_e?vR67QP(7u@aZUpbx&_LoL9GA8i4iL_i70~5{zf$Q~wws}`A;@Y8&kD>Ny(wlsm*L4hXCLk< zeQD+@CgygMh}F!S ztTrp0P?F_4540s34c53=SVmj|yR$S&UG`1x0HLSjx)EexU?3Uqy=pM&h1F0g3^0FN%ve%i$eOfwOs@vMh?1WMRhAnmu5|n~7wwvXfIZhgg>PQogA$ zQsJ)U_xg?IBajag=|g#+$)l=95pad zfyF~%;d2s`35Z?dgdu76EM=5=0X})s@J)ST72$izh3zGBWn@!!hM+v%>;g9{!*y^+ zDv_w|Fv6+{Z+|>mCeWbj6T!*;U4g!x0EAB25ft6z`yR5*|a3T-?smD zCDMy}^fFP&wZhZfLh-9#`vML#GuyHB@Ov~zzc5-J-xX-d0w$s$+Yhz%+f_M%3yC50 zOmLX6-L{+;MBYJfannar*pte86T@^=8t)Dh2LV#F861R$_hs5b%891!#zNwTa3}hg zbk^`!JdF&zX}c7hi`|Lk3UUonRh;_$q+-c|a*!5zkU{7TDv#2(L&Q?6R;&YoB!ChF z$yVQpAF1U*iWQOcl3A!aok4MyKt27v-`2sl(d^lkAxIc-!0Nig&NLPKsNeP~Avs*# zDUkR~kLe9aFs5vh{{6&*L4Md{tPMF#MaZdJo$B7+cVX~`>$<Hhzo+Lbmvx⁢l4wCC^Xv?JHUp-cj zulD4V#9=RLDh8eWEZPYVSFKg}DK3gpGE4)o3>nx?*SPG(eOMZ9@27M4!mm5!H)WYQ z*)t0?>OYVBHzWF#wS*>0|vVPmwJ7kg-JPSpbCr-BDgf60JTbjO-$ z_J9+rH3Gk(9(6MF+gC-yMPD4PFqk+(KieQq?1T2FIY#fCN6<8Z^^Q7gz%}>npd(`H zKry@Y6i1!FM>2(0fa8bs!}9fwc9utNOtR-tc%;mQD)5=mkF#6WwGQwxA z-<-@~ubyIf%Ch}^3-o1~jMVmnVEl%-ecu&Bl@Vl;LS9A%nq07!Kn#*7LUEw6b66_T zheg%M6Ru4@d3nsj5U-|4`_Xp7>@3gc>$LV>%PIC01enu;+LB0bGkm(7-pyu7Bg#UW zEyZYEx}S-qN9F!Os}^jx8(j8p^2AzJi14)t=JX_G$HC-f33{u@hzJ5>!EJvz4Z0)` zV8{zy>-oq2IX26ATVNJb;*e2r4ShRM-DR{Nm1kvojbgcbYJ{Yllg&|#JDls*SuTP{ zX0tn_r)v4GR|%S_2nLsYQZCL$4`h_fkzBOw$KL<;DN&Lv)N8k1-m+hcn~jysJT;L( zK!09#QE8fEz;U%YX;tX69wfLM|Dv4Sis*FK!71)%VRVrg)MgB(CdI)kE$G)NP%Kum z2XJ`|7t}4jlS8RotVnY0AQzBTGVwNi={1^BhTvA5+@G@xlHSo)7h52PIVOi3?iv7l zSB=47ZEUQE*xU1HUg%1%XwvgZ{v#6_s)90foC2I}v&qHj9lx##1hs{c7rJ;%65ZU7 z0m_}gm;tl0l2e*n1%rH5pms>cfUGt3>GPGnA$6CXn{JQ>roN&SKzu#KOslu#pWSqwfr55|N}0DiN_yScdH3$!+S4w3H22H?>Z&%EuqF}GuAFYJ zm6Vl#^*6==+o0V7I7eePeKb$2#Gj4Qam8pg=wH9d(Hmf(l2^fVvi1%~!ciZ6(p`E- zot2n41M%S@d$AwyBuUq*@XaCU(&TRRvCLBEPiC5?BIq5~%=v;V z4$I+NXbQ@7g6GI|C6WPXTQ{g*;6L5{x+8%F;&Of6$@|t6Ypi=mY-v{x2o%2?NWw-$ zKezLwf6`?T&zbJcpKcs=gftI1AW2;ja4;GK`vDJNznw3_*)tPy78j{!mjyE5aUS<< z-q*SKY3w$5L=7e758Yzq8Q->%6SE$d_kLG8FPat7s#QY{aY8yJ_r+KpOy;w%d0qkF z4xh;=&Ku2_5Lr>uGPqp@M*a{#69*0eMM}|KJJ_!k%zOU+dUJ4(+xw)A( zXH_ggYUF#p;$zoFICC5GsKgZ>89X5ftyJSDV_!Ops+D2Ovwksm4H12ALq@^M=+#Qs{4*B_Ooa45};dNR<9N0Vl{pUc_Zv2-O z_!A^R7;**?%b91PrG*Q#CoJ;aR9bBQ{-=TPfxPUug?o^izbJ}&OaiM8Rc z>gIoyhW}uY)RCRFW-2+RnQ&+pP?D3`o2yjs^crzL1{UePQ=!3#8Q@pyFduG6DI4A=dENa z0mCsI;e3!12DBnSQu}G3NFIJZum~8B=iSY#_+ph zyw88|(YMpZ35AhNbH4cssj>L~7ZBcw*1t{mxk;sehuS?PpW&Y-26)%nU|_6L+x{r4 zll^zeK-a8iU-OJ0@%TUYL0Hx(WcYfhJAjV$s zu*o<22Y>=l$|dCF>@)L{*8{_WHN&eh=l*=J|NbkW))Rn%8F3Htjo1I@FmFdowudC6 zh(rC&zT+ZHNAchF`FE}&A%)ko2W~h1cP%gwC*i;kJGU{*)BpGU0I6f}n>L(~r0;*9 zbW;_`%D6|VC;6p1wg3O$5y0M{V6bIFv!DMx_OLvpu6_S@s(#a~6QBW`Ig-QCkc%IG z+EkqpFC`KCNugh(Nmuis>xYpf+5fvEFc1pgJ2MUKGxzS>kM?P1%)cbT7cF&mb^;ix zG7ScE$_K?UJDf(`q2jW7O{xERe*g)ad2bxEKpk#iseEz%6eW^>m)4(zBVeZYdYm-Z zK@#w>q;2qSXld~P)Msa&d)zth!Cm~jmP;5M|9heUONZizJ6*N|3ZLQMl1)h*Qrcex z#BvS*e0hEC$aCKL?hObA=EdD^ey5L$i1@k%nxRTLx^W*msHXa#OAg#abQpp-1vcA6 zBNB7lM%-nDqXtQJGsT*P;{vHPJRi@Z4j(YGy~mf?GL6T_$Cl>i>n3HY{5+Tc_dx$$ zPl0ot+r;}6!TCgSN^F%f;UZ<5=7R$Ls#^0IkS5)q{oH6eo%ZyM&?8ZfFYE7+A*pKq zML7QRZ1_ILI{tvSI_ELKbch;cz%t_2ssOlM0Wu~W9qb#X6Sa;RUc@I>5sV!=( zev749IrBDQvuMgR;+DcLd?LZ7T+p*LDV}r|J2Wj;{QVS#{zTQN9IM)U&sw!f5vAE@ zbije<1S1cARKFA3gmnVxM)*=~Ueombh(L9^N%ec#ypchx?BE<|ta=PEHIkxIPoE8`eo~VIPvv)^N&XA+sT% zrx-LYbf}Gj&?n=P!Oi=UE&=ihkm-0aM|Wt|1B^jBd{lm%YufMO0T z0(*QO9@L~Xjtf(dm^QyI2#LVA+0C>m+btpu@v6I|lR9@5XGS9V)M~R(T>5QyA%&@R z#vRelud|{>$(~)|stm=3UJN9@h{@WhcHh6uH%wtJG&q>#YuiojSgX|2kg+jglc zOcb2Sq!_6gv5FeS_>r)KP3ME1b-^-P@nJrW+*s-0hEPMFrU#qpd^=tD>F8@D9DBU}yB2CNbrXo5bQY zoE#0riSkQ_PDo7}+Jqex9(2u$i`ug}VTld@ukXM^LZVDoXU1^H>XG@VV}q26+%-pJ z1N%DougSvF0E3;5-k-*mpkoZz19A1n$79qAsL(<*%XYL zNXcD<;BLG@AUg8sC@4s81@0dapEIp^PW?yJzK^!cMT>M-vR@%;VPCW@*qyM7P@WpY z&qSM)UCgH^LLR2KA|h+T6jV0JUF9JyU|HiFwpLcFi%T4b63i=W<}Xp&jXWoL@DgRc zYJPTuey<Doh}{Lz z={d0Uta&uW3L?3ZBApY_+i!;JYzgI$`OVL?B9#aEsM3CLts`{c=@9oVkQd}&zo4Fy zVJe45b%!mH4`@uVnE4>%ahz~~#}d~C*E{~Vv(|3h2C!Q{EI%*|(`ki2g!J?!^R%>I za|cs)#5vp^kd(j zn*u4xiDh4m$TWSF{3pd?rauO^r#aQlX{zbS+7x*mr`s;X$|2oJA&-xxd!G(ZS=S## zNJ8fcO<0~>8Mzwi`j9+BOpF~gNSw*R-xQ?qJ=ho+WL1y>!*l(=rV(ZHdA6L0IS*}{ zer(SVib7Ckh^EG{NO2sLtOyT`RwcRT$wXo`5G^EkO}VNcmEPbgx)UwSzL~Orr0e?#LIG#p97JL zsjqke3I>lmo{rwI5p>-$_^s0&qM^MwZE1(9I$0mKlME8cWTtxF4;AD*WC|+i9WfF? za5rzlhY#eQeJZy3Ws zh*)7jBswd}*aTO~Ku0WaC#xHhi3sb@qw6`qQzoH{Zi|#AvM_o^TU4S;1tuNEf64rJ z+fC*KJMh*nSVj3_oSbFb8B>U4q#cfbnedN_LiioAxI$PVCepS3cKjn zP7?tV9W8(9?z;ttf+=_rEkQP_11Z{|4SlLn?$Th3oMlbgvhU%0tckuxw0s1{ytZ2| zOs=j=bS6d%eN(1&8!D6;H8q*GWRl z-ag7S`i&B{3agVWqTu5Oa}1k!CRT*6_g#EzGF!4!+hDW4>LL|b3z!jkVbI{?xctn7 zdi*$rc8BVN*c|t8{!LfKpmq6iSI{;Z@Le>OfsE)cRb$t_(ptR+wl>AAWVW44H)^A| zScj;aRFy1#yL-<*K%W3uib~#`$4u(og$a`f7Gy^kx*TLHsN|PjYECQ7PWZ2PeO?`% zJ4pTL-*Zh}18T>1e?ilCsRJQyYfL86^aUO>>M5z?D2`TEA|t7~L;quMg@Jg)@&kSx z(jfFSSVV*i4!4M7?Y3S*1+PZmFfZJB9jM*gZ`Gd&-FNBtd9TNJrjgXy)W||+!JMd4 znYs-td)wPH-&_%kyz@QLbuHXh3kQ5|ulO#Gh7_-p5_Xz(QwL~A3Fd%OYYy6ou<-rw z|5vngd_|rwGp|;qfe2$XGJp!qL`Ha!xk2l`dBEH_a?AIAAR zu{RvPMF*5anUyrq+fb;Rm|$4<5ym$K_Xf!&lZ8k(s!864$Ljn~zvH(M*lBHK-wpZ8 z8-7udZce3C)voI!I}ZR3b|G+EK0jc-4eBEr#ON8v&HG@=mZsC{dI1dh3ixcAk4?1)>7oN^FM0_VvYIJf8Ym(hV&4Rj3E~(9W!uR?M!vIWhNezkP=n}T5&Yj1TmyT%gE&wM#o471 zlME79Ktdvsz!??!c(5QwbuCAnQejtTGPYB-lP}=QBOYbg+sq72$$+I%0YyO;JBk1! ze_V|?%k3eFo2!%%ONtbjQ_@4Sg6b0k#u)KSE`P-T)gAtWdVq&@S_1_|H2-e4U0o`X z{0x;tB?|Z_cBP=WdM$;$YFP9o_Bhtk&1CU}bz7KMhS=O%`QlhurUdhp8}YK^MM-?5 z;vYCsL{eu-(%{AOiP%GOb-{A&ibH9}nujJ(P!i(Vm@f)d2` z#CtE|>sIuZRm_>;wmjhj*}IH1Yc|T&W#e6X`LG@7?2~+3P8yh6P6%tW#zN+S`2cPC z>Mk0)pprTg(UQInjQi{8Qr&XNvjJC*odr4r4K^4Q%?3*d^f?sr@h=j2M9RX=HI81p zP35~+i&Vm^No+n4dG7w5kVxa-EFtbmzW)m!=tLu4FD$K^;OG9-c5?gbzHuAC=Z< zJ3U<#cl!16pRL=vtlUiDKdnQD;54T@&Z?QD=G>vNi_?(!EH`|oFgM5Ef=f3Olz-!te=OF z_m$;c>2wigWlIehLp`r&d=+gkpV>b|L?qN@E%<@w)yg9!j{s`2v=J8-VEP^4`^NTj zZxffju2j}(b`+z_y1z^+@2Th;GwKx%7VZm1)bh4PQ!7915eW|F=58h)-q1X1U31$q z;rn9)xEg8YphEwk{o1k(w54SMy{XwoEVNN)4&TDA*+{uku$~=7ccA>2ovp)r`5*NzEAWF|^A(dAjt|+5BMFKWTK7>4i|Qf=7+l>ME9>XFLV) z@4!{WeRI!Wb5q;^QZYZZ@4i+p1@bSqCjoqq3%N(iuiH))9=TZuy1i-oO5)&z^A_TG(FU50rNFA;KIBd~E z+TOjU`BwW7@pV5sd?&Tq$Klurn*Om9`jsYYfY6CLq2%~6?oV5E6DWY-+><-3Y%5xa zv#_Efm)JsR*67SrMWq6+O&X3mCJFgW>0K}l6o8rdnyQ$bkHmrAyu8Q4x~soyk6aKw zsK7uigM(kiV^T4C1GbrJN>-zQlg*tU1IE+GWc3Ln(E0YH88=f4of3?!;jBD7lI23* zF)Jf}kB>K!MBKaijdRGxkiZPPz5%g!8WbPjls^mMowwlg8tZol_AB9p#!*TJbmnPh z#VL>Kq@h}zrPHNFnus}$mJh{fLsj~W(xvwKvkOyZqM7QOyXG2_np)As;tC!BVF4Jl zMF8dCa7V}_#2ZAA5A#8ZhJ2^M9*ayasHiwAH_>^}Zy*7+XVr4dfriZS{9M0GaF?eL zO(Z4C6UIy;R9=5$PFALrF`dj48e_lE%RB09I^~J~P31D|`L^OG4+`416|04R9n(x! zlP`@_A`)6;@DK1Lv=>b64Yz7Z{f->ija?Fy+ruSUjy9_)am7CRLT{-);jONsV%RCNH5$t@M$q-fC;1 zv%03n;c9mT_a1`KzstM~aKXvb_1S4sB}9F>Ocqy^Xh8`wtmoyJEPn^@$EIg*?ApY= z|ECn<6W?Iyj4AGQr&D3|{F>-iN?$|atBpW{W0~U4$b&1Xi(;C2YKF&qb9HrQ?IqkY zhMDpAGqt0@U@-hVbqnK6Q>ByfzRJF^QVJ0HDh40x{vxzcKhAk`O;{5{?114=0tG%K zsGC7do{{lfEc57NG=qVi)*N_vmnoObf&vwL(nG(HjKX(uZ2RC?V4FA($^TQFnYutO1`fzY?v8z( zUABzFu78Y7FS|J+$@W^o#{OV2W6qXzdQ6qaEdErWdAZprU*e3x71Ircb*F|qSt!WJ z(Jq5`0~3iZK_~4FAY@5`PbkYT@&^P#y>K(J0gaT1xDIq2gst;H2I3zEa&$W&%lkZnr(mmSrxKG7jVKa?|=v%9DSK+mnr8 zY6;^Me#WZ>SF@O8A&Wvw4>lk*po@f*=B>_6t*j)F@Bi+U?U8`qm#xaSK&t2-bV6&C zT*J=Pn`dPm9mL}@O*%|C&M54uW@pp3Y5sq1^E@+zLg#j^ znR_k%3W$&cI?qRwl9^1a?bwIVc;xXo67k3q^3Y3_#xj9bzaN!R-wAJOdxirezX zu_Hvz>q>Rn`whgir#_#jzPAhvU>Oxx_rHN0(Ce6J&3k~f5IB?ObxSxalubuKh?+gx zfF(}{00_{9cd>F#50L6AQAycYtuf6pmSa&P8!*o(fsWpFxbKO146oNO?Gx zo_h9Zca8_T#u1hIXOjq2j%k&`-Ulbs<@K<$hL^hSc{}6JWt=rV!s8qmpW2}=mjz~< zjN*D0ol!J#xVV{%!5kW>z`uE4eAjup;`6aus<&ERQCHpST?JCUfKM=6j@WF4M@~yg z(w@l#Km5-h#L5v^TBaWn;?ET`rsjI4Lfg+ZUsg;0daOQDGV%x}Vp#oCbZ9VWHtARwRFWofg&kW^f^Dw_Z07UOkf~>9 zr=uC8^i%u$6f}{kQ2dnuTBe+BK}>8^kPHja01zBNRAtTfkoZ_Yb`U8f0xVphuPHK_ zad%Yk1or{LU}d}Tkju3W2y%2WrsKf4btqD~@lT;{nrXK&U|w zdyIa)L)^!{hqaS2n%<6ItO3{4u2NkFR7h+kwdGWXTE8cX%%lr1#ZVw`VZk?-&XHm{ zmi}r0dC{BB$S)3KrX`A|+A&E?{+0SPzUai_kua++nSvoE(XajuyQm5dsvxE+yvLp_ zLyf^_6`tPmVbrBvfl4XGa2H^)$g>P3GghxMU4B^&BgNVlYYCmmTLEt-3h~wFAm1__uvIB1aE!y4J*An9*u!K+d7D7# z!+w^qE?4kkgxdnLl)=O1D3x+Kk$Ylb1GfqRTM@O$*cY3(T)CedQ?ht@u0<$ukc}g} z&vtLp6m|eVddnX`2AZyigNmK0?@9_(35TbryWc-UivI{sQQ*xof|Ap$QLp^ME_x!c zRqKx;=$7GZ?Q_-btU2K_SSvL0r83Sw`C$^}*`fwR&O*sff&1gpzsZmydYrK}X~aBU zz3A_nIW_9heLvL)Wtv;uo1cpmsSrOXhMf+%`Qh^b!2k}AyLK(iX6#vGxgKrXu~&0t z8hthV@7dnMb)!AEiFsHB1%)hkl`FJs91kV|o?@@UV@2&)v&Q)2nV^VB*W9mZA*sC^ zrW?-`EsxR`93C*F^&DCHGPlK>hr3*Zc(BmU{nHOATle^U95el&$dY7Z?6iwBjq#oj zK;w$<2^K|5CR>SILi_K5)@Y_1bu<=sqB*{N$$oR@1BTolnHvT+p#I|@WtBa#;bW@pD3`c+uj6k^Zb63P?reEPGNUb)5JsA%;9MIP9 z*i~Vg2&6|>+T!|$J|P+QHD^R8xoNQCHG7DXEawU&z{10laKV^d$gcSjF5K`s7VYy; z*>aD#W{2$BWB2jKf-Whb-iYT&sesx8#TSNc$4Q#Mn@SIUnVN|EshQrkHWOVuo$+`rBV=BZ^VXRab;Hh_a^30*U zy$gQ&(ur)V|2({<^%02&TllH8F+;E+m=y+1{Lk-e4@0+06yizv~D$kYsToXx(6JF-$=O^?X1DQ&H zX!iv$Lbbgt&z%RIp7n8ooB=#j7aNc=&U`@8d%$-mk^WFd(oK+$p}&VG4DML?Zyn?2 z6b@6NqpX91($bDX8O`ZuW~)>A60@Z|anWdy>SmIKQwU6B4&n_UX5NuE%LqINZ;N+5tf%4=z8PCuyiKJTQsG>JR$5|>Z( z(Ql+n76(Uj@+i};S1l<$sX=^p{+vO#_1P*Zad0@>+-ik~rJ?X`-*D^tyb2~XWXzl( zegjNRlyz~*>F`OecKbQc=W?pqURv28LGQJ+JEBopo5tPXakcBc`th4P4oGou!7>qD z33o)sr98>=(Ybf4bM^8scQaN?#Ias^INkaqha{7^j$wT!c$g8S2Q%=l*S42OSs@y0 zZuVWtd@}YNO^JMoY!BDt8%XxmgIL%2-JDndP)DVLF#NSkm67P6OU4BOf2*d-%m!dA z>-vmraQkLD3jS=}sdyTQsvNtOsA28)bkSGQv>QiTD6nq&0SZJ$*e;-%FD0kQJaV$m zT>1>7$cD1>b-y;EUokh*Iu(>A4*FT3Uh7}iOV?)e)HpDsEj8{mNs)$Kk~$R?jG;7x zqn3NCrBt$P?_saltSJS<2hk-3wLdXMOuRHv^XJcAO8`gKh_b-Wi#!hQ(WnXBf@Z!d zz~5~O+%ckk&;6udAiuhA8Ur2d)TkS4m_(~lYc$064UGh{>)sMtT_u?NoU@U)kWp7R zP&d8?MA%Z0bh^MAVu{ZIQ$(&wp32h>S_NBj{Hl&-&(o;v`rH}!pCR!Ga?LlML zvD(()8%b`0Pm8iKcEd#F|g%WLI$NxqwandR9&@0X6e++=6uL{UI`I-!A?I*y8bL~70cYyR#_Qf> zj*3Sk3XPMK7=qh=T<9WVF;}Klxt@*5UZGWIo{$tim=J^bbf2ZaaXnMveRGEzb#ihN z+{3DceztKYS282qT57rD>SHCh{&^;xHJwx~#mn zQ@7_7usE-brZY^OhdJgA4lsGoh_lo_Q0=%Z-}&`x_Qmr9aiP-s=hXPyrA;rue4PA| zlm-uKQ+tRk!2ihtNac(W7`vG#v4lu-Z0%oH@HQ{y=%vIVdPfeD7BrKXj&4~{wPVcx zII{l30oWH5e7_!{j_znvWm!~bQ5FrCCw;{bYJkYv%22D+1^?NYX48H@1x&hXb-U7j zT%1F(D$mV*$9i2il0fTz8eSss$zy6QYwG1Uy8q7E@&LLb0RtKxurRy#Jd?zzhW*nkvVT z!%CK7j);JO@=JU`AUCjo)0mupXF+fNGr#2hDbgoDJg~<|ca&c+@b|yh0i@l9JTO`T zAk}Cr&^H4giHY;0qAck!*XKw3C!J=XMPmuZdEZhpwr&hydScnbyTM@690gj5Y&05e zdBGSBr?fvbQQVDnG7un)G zG4bdUAFAzP4(6q>UOO}~+++NpSZ#jN^gJ3C>MxJQtMYH|?qeUwAH6wd`etwwgvI2h9Q@xzCW#3Fy9NDUxLsjXv6=MjIlQm;|9DZZ$!h2?#!^7;LD4HWY31plbksQ8g60K$NY&n`ID6)mt} z2frW2+jf(jg{M?JOs3dfI6B_H8?GuDmQ(u+$qIh;=pghE1S?!MZKq_}Xxf|A0;VuX zB;jZlEBeR#{8`}}+vB2p4d8nz^m!=UlHq{u4gR)H|FRhl3|GC`P3!QOQNm{>B(Z7S z5WZw)Cm{X0HMsKtVAEk%V`1c({mZ_?kPe5%r9l1Pqu}n}u zZ2X2_vj;*Tx8`jRiwk)H2}5u%;OckzF>f_r>GxcKG+iwTcD1_ZGsX`+67c4ac3|Li zEH$;fHzu@B&IuK(Ga-eT5)K1KHT$|g%Yv{C7q&n0^GFL2^yyNkDba~;>!F6>ZrW|{ zhwd;~6EJ(kXT;^HMfytEVpy|hB8W+HC>*P`dXCaMPV?|}_YB_<4{uo-TS_#+c62ID z$V`ukIX@sL37=HG^Daj(0`hVMa`h7a!u}ZG2A}5?D}Vk0f@F zS;Nygfe_Bmdq}$cvT@d3KhMj7LO_`9wYC7H=K+KFd?ti2LkQ!*L256+rV&{d1dlk$ zPa#)hO(u%*2rD3if?X*4V_N#{T@A=0blUm|`B7>W)L)w8A};?E6)R#KAx+u&yf@bG z>BDbP4vH#hUK6Gv`+6eZEh@f#b;1*q&<2&D;}AWmzT&RSNIQ-|8fVzLEI0*z(GT2; zWeX@;yvM0~rKvBv8jD)6qJ(YpIg&CGd`PUUqa6fp7`)nTs?wt3KtlbYOtaXFrMZ z_6G(0$N4^gFcqG?WP>Qq+hl`r1g|rXhFEBqker;V@>8BTI4Ldo9q~7EHJOc!-LF{; zgC8us;H&O~LP3l!)`miX{{ZFtULBh4_#t?3tLZ#lB<^lI2F`*NBM9~oGaWVaHDw-U z&LVhye&gROHGjEqW-M^#%DzeArEfR-R$>^?X+cy~C(0q{vIyR40= zl=B<|vv~~;2~r>P>Aj%vTL9O*OC>roVUV z1PC7Wo*!-s)9vBhnw#TS*!;gH0wTm){_Kw_>7F`Edl$(ssmT-c#Fjn#mhgwYc)P!f z+?5w34#lMoiL?XGfsp5#`Dod)-eJYj*w-yWaYe@cqZP=P-Y2a1yxl|6*XssZ^EdkM zG29o{%U5+&boV^c1JEFPztlWZAUeAM?}DSVQ527P52{T0%kb+*nnx%tR5{(0WBhXa-TifUDSe8*HV1$S0ct* z704LovyS=uqHULB+_ehwj;GfJZp5!m;nmMs&0c>J6;T9WARz)1g+svLMA>}_Jkg+_ zgyb%|{IlK<^@V6nbm;a#Od?bIPkNvt`Hd4IK~OfVwcH$M*1WE9*ZeU?MQNPdX0sJJB8u3i;w&h$w=oHzwG?mIZzJ{Js^ZD$#4GYJ;v;=s&K zoxM6{ zqY9;dRwO>&H2QsRNH+uP+1{Kys1#s5TSm1>CZgwB=bIC8NocN2bIY(7yu8)=$NP@` zZQU$H1gIGtES>48LWtu`QN!MBY;b=BgrFM&Tv^%jv7j$!BEQKtv9g>i3ETbP+?8Aq zLBID6L4F!$r5O9xPS0Jkd(~seGdoREak#(lw_yfR*hX#3_<<0D7}D>@K;#d9n7Aru z$Yc!Gs_1}(f!noD1N?J___m^-@%wRViZoSfiqcDs>T7MhF)rUXY^k~JEgHaA1j7Wv zL@+_h0<&Nkk^?#!`#_QN=dQkURGON0Kc9!4oa@=y*ruLx?6!Y+5fGZEo9d{fpJRU_ zB8+~I&cenkH4>C7>xO8;3Wj#}^Ai}_6Bp9_jxUAsGll9kZme$h zd&Zg8)==#!hOP>|>h7uF(S=qghaNyx)0El28$H)>v>y(G+~iNr2jV_=8SC)$X=8$u z1Rw<9NU%k&IX*+T^9IrM0dDF>gDQT;fZwNkGf=<3CSnS>059en{I(snu;z7 z!8T~bsAOM&TbnJj^PX?3bqtE~|JZuR@VufeTsvyi*bN(}vDq}XZQG5VG`8(Dwr$(C zZ5wBG@3YVM=ey?rcfE71$!CmlKWGtM=m6ba9*m-Y@`@yOE*(*9XG;;6yenUyGeLQ$ zyNYHta%IhTz%~ONX(tfh=PwDQuw%54Q;1;;>%T7v>w`SBXL!UEjnq)$lg+Y{SzjGs z@GW&eNh>a*NgNqUqAADhb!h-XxGnsZoz;eS2G+GUTzooadG1%@0L(r34TkF38tgYX z51jK1fF$FKrJOmfOxSZ52t7wAJHW!OkRN_$-oe)bpS$i9+kDa_|3xHzSQ^5gxRgk+ z7elYt=p$IPJ1YjE0F`Q3NNB-)F%NO4~6*3hPjAuv>ligLz}?dtof zwh@EpJVq%zKl?Q$g}+`fGM0~|bucn??pI{!|1_)w=%Q;4y03BLTAID#ufpcjwE;qO zhd-Irua_G+SKbu!|$ zK_oyxUgCxJm$U{4{nkMSy!L-h*A%%b0%b#3a2&ETRtDDyTiZ)VXXg_*1UsSOa?R)h zVC?GX@LKZ`3l?QK3ZIJWfkf2yUgtC`T@9-Ah<&7;Ir$0qdY6+*&gefIVc?;p9_)g; z+{n-=Yi?jzA3y*16c?nbh|nGBc1h{;?m@^%c!P7xW`@&#T>CbH;8NsHOU03VeszqT z2OdSc$@&JzK_e5W0-1oK5H-m37}a@|`@+RmT=DMc;d_F$t6}&c-l08K+{Ep)f1~?f z3q%5CDICr-w~4m|_v1kx+tSQE>nn2*SZ)v}?UlwdTBO*^?T3A{0Qt+o#87lFzw%?T z%40zg3ETmBwGG=$;p@qM1fLcoB$B|BX>b=LFEIIVK>DuG3Bmr`dVN6N#KxV>0mV_c zr5agos&2zet}e3R zFwT#aA{&EFbAcz>8{s316D&0uP(*9*oG-=wEg>CVy0KJ z`QOlifG|W%b^~X5ri{x*pQJ+nt>VulR*B}4Z`p<%MYM*;Ths%i5E1%4RE6W`aCMwy z#6bCVYV<%ksOXa;F#1x~kbLok(f{{1w~?-{S}kO}`#J9aTuEp(U9^uixmZTAv_`S}4~4H9bcs{DU%)i3-H z9U4`Y(dT|@f#jUqcQb%K$Mw_pI|@m;NI1)pp*&7ia(KFFW3XU;f&>XyV;_ zSf6LM5aS2-A`#KU{p&LpFJdm-|LwD?H-zC2-whwc)OoH>m`UHFl(XF)MGEo`%a@** zcKNT2=v0ywSt3Rdje*!vi;JZo7SlOprM1Z#pXmWM(8yRIz>>L;KZwO1k zdP%&$2_cw(KhWzy=p*`6B%l8__^>}_f=j!6>?z7ae1(0>X`~bDW&HDFwddyZ;m-vM zkg6MspRfqXyQ#z{ARwY^WXLWy~h35Yj#=0-T&8X8whHTdvhf$ zz*b81FA*w%z44X?#Tca*OHs|#-%#^$Ow#o~U_8+>?JW_eC&nrXP+a*bescUb|98g-CdoJz8|fH=2YtR#}7#tuCLy zqp_+`k$GNtQ+nbS)U84S29GWwPRGA}`#-CUEHaXKe#mNx>7M%c$lc_xj&maTFoxdc zUCd#n_-BJD->OV=E}CELX-8G&JUd#(^h3K4QZ(_@w~wO<>C+`EN=vEYhPSc{%lL_6 zD9~VIf_GoNNKlHEOAE~GC&pE&0~6DyS{16dU*{=Hd(CPo79EiiS}g6$0@a>23gnhY zc$F$@eZ@Ra@az`7*v7mKuiHkzLXzO-6M|6?+lOZXTgN+X>~G%g*E$Jb#gbAZbwdS4 z1eBAW;|r9EAw2t8?m`NMi^Yx9-%!7o`cx=*4gXS64r2TP-V6Xc`0%=fO$u-0`mJ*i~%u#(fh>m!(+01*S4-&4-2HeW_$ zFPQI+Dac%DAfH6xnQOjUQ0kuP(}IUDE8Y{tW(?slc~1*{=h%CBGp4(4Zfg47^VtKx z1u}v-)#aylu_93n))_qZV{O(uoHbO$E+{(!4F;GB?JE+PhYQyPr9Qb>-Z!crVg~!U zBuPZbl+SO8Y^wb4!ph&p?tRgF#9lT`-~@bRglSeWmcuElH zLzucjnE6w}*`DQca*9U`FRL#X(+&*o(E@v|KrnCn(NP@b{q>^RpA$O74-Pc=jX=d| z+{iw3rKasboPqFf&PRWt=+tWJ^7F?Vv}Szab|?(!lz;rt;Y~;NulY-##C6iD83#g8 zG6|66j`x%m?2Ao;o~ud}Ph!bJ4ok(EjulGxhZ{%y;VwjEvzbZwuyvGRV|jnTH#++U}S<18rP?8MrB1kygd+y2|PhdkLIP8l2Ov|m9zy@0Lh`zFrG@1 z7jSh=tg_mse@I}=iotCX+`gOPt>5!KLSnmqNl1(%ayjwq>Uc?bX}z?IMBy`BJdvT8 zivO*EY_DSO^OK~oln2M_EAkQu^mpNt&73r=_kKMCyD4^cXh~^;=`qr{VL)Aa+LWP5 z{pHDdt7DKln83}0AJzjdn{QP6tSElx6j>`?8F(^*VX!dISe(dsFTGQq`RPV2yw*J8 zKM`iN%&3+{jGeN8stnlo9r~=PKHry3cYFHS8|zEKvDi9FnB#r+Sn(N7 zO&GVvK7_4I-_F3cA&lY4mD;?UmQbZfrpWr8UDe3nYyf{)R+|j@las(#iT^QZgjcc0RqV;7i+G7NdYN$dAU_Ygw(R-u#BuSy>{9yv}(D~vs!c4>RX1Ivb24==NUVU;XcxEHg7JLdA2x7~jITcFP2SO7u zR>?U;kDxsR?E*$}gppyGXIZ}!y&q+cLgIFy8U9qGOx5Jbu#_|LMBRzV*pbDdioZfN z5{6nfn13!7OZ1{%LxwrdcNKV7;+{XB>(B2T%V<=R_@Z9!l}L+MDqF@EFDwF$ z+b?LKeC}E_qf8LxS5~GkqdYI)xF9X!eXeo?1>7Ece*6I+;a9o-HK88W%R8BZ!0&V| zY^-S2PMB=|Ta*>mOv=@MvF^wY$?tE*8t*oI)+ISu>h00MsLYG0z~*F-6nUY7u)!=q zUbX}BRCmvoYX>9N>x_(`j&XRrc4~~M*q_gByfGsNuyJs9ZT|vBf!_Y#marOL(Bb6U z#Ph>Az|(--zaZz5fAa z%LG5<+- z)P@LiUq?Zb7Fy)7n?+BYh(ciJn~=BnfGRW+T0~7fur+s;4)DU`(ym0wHxoF(nx~M8 zfirPq(HF=tMt*0kf)zz9?m-usriv9jiwWZVfy4YO4?b0xyegW%s<}k%N7KGOONA{4 zW24ve?C%6|;f~lw4^G5Vl>KrqlB#n`=Gd8fZYFgWA@ej#Xygj5_3V9|4oTvz7JXx3 zs>Hyy26{_L=~{^wxwE}!I)E98b+e!ChtP8S5JWw#m8(&^Q@zLYVfzY*myX?EHY;S> zSkvOJkt~A19%PMO`Vf_kVhaiukfTkcg}nHVT>Nie9>e|Z$8An#g;_}+Ut+~8_xOwi z2}Jm7UT3o93s4JXI1Y`a{)mrEv*c(hGDqg>MN(7JOhsPMPu*2&A(i2Pmv5;)zDK+Q z#9H(}e;*Y~(1dSN+kfz}$&Ru8ETBgbR?k^kq>;;4o)Fm}ssc6AuBxE(SxI8qBsBWV z8SK-5@mnEyo>Pv725>6)+D?#whU}>a95^5#{5Q+|29Q^)7@9R{2g500IZ!~ES}!|Z ze@?z$M|o=wsWE2)(sT__0-xx>b3dnGx@+Bf4lp&>Dm5!!_`cbN_Go~TT0zErF^m!W zCF?m`LOsefi}aIrNK^aEG3h9&ss&oeRS3mPp%*tJf*iU&!l_fP4^GSvWqJJCJ{+U@ zV2TyuTehaLXtkZL3IvUqa&&qX4s1Sh^4T8qw^9eY^q2?AP{EtsXWyT#ch>S% zxM6huZwUx;u^fhh-)Kk?gO->C*q4dz$ljGX6xjBlL`y#FeWQG8CNLfKAHyQ1pqT55 zBa%OmO@cj`z=Dn)QUJnD)%ngT1cPcm!uRz(CoFylR%)>595?L3K@`DhwoOjUI^M@@ zPFq||TFoMRuhY#7_xpJj8SeW&%?f|5c_4<|?rKq>Z2CBi6Uw z8%Pclx41I2tW-T!e69`!B(fV~EQD}FhQKG6DdKd_$^B;Oshs7i@SeljgD3jw8Y7+I zmMhJar>=$ZPL%OQXbhsX#vpz$h%_1!43F|^BvxP^u8EO4yzCO-6B1+hNh!JSc`$eRb@}N8@Vx5l8e(=7Df@|Fq;_AR%{<9^Z-Fx`C{i}6Z^U$5_i7)% zqT^Z>+hplhZcOEG_Fowp>*{2@b1f}yx;8~Y&xD8+g*-d>!S)bP=aJ4gl zjUEL_HR343AL6|Fu&wXqpnb_+e0LV2!?*Yw*{cku*wPNgi(Gg6WU4V>&J$NdJ)b5c@9j8x@ zen6kFs;*w50K|(QNDCX85{J0~;ly8CZf*`QoDkIjUl9Sn0keH|>I2^0ri5{fSF-Fo zaLv4XWYzRMmN#{w#Vlt3$$t;}*Wf_Gh$DxqF<3C1EuYd=^sk}-YBO)sv8jy>I+sE^ zQS}+$J*zCI6*vO~eSjSB&uAe`S&p`43mJ&?Kk9OG!|rOO+wz4p-$+nmJn&`E^raQ4 zNBT<1eqV85zR)iW!g%l{ne zx7t%>p0Ct zYL&iAL}mE4D?sYhtk&>TO_p-YRow=4;Hx#>#;A?iYxP z?Hk#zwNx2eETQr85oyfYuNo4e0dsUPnL8kqbP4Vf4g-Ntmccu$tT(;(u*6=rp8Op_ zDzVv6#Ze!T!gX;sLq(_UISxPB`LtF?)A&1=y{NI{F2UZ*efSdJ{Y}R*md$IYsUq{j zvf^&#G)#@G8{8?L6uEmOA!D-rsb?7QLSt)DjLWe-dH1Zkl`71;6n8=S-+IIhED0HF zQ-)mOq=H-TK7Xvc#QFa4OG6RbE5l)u2-p0Or@--U+XzGZOC0HjLRj*CIQwUoJ5=x8?6i=scKew22?V15HaCZ?lL73yurMRy)H>Staiv>C zc972=Qb~f6Ca9a;>C^{Mqtbk$i-`AkAB*I$|LO#Ce!Nx?JmxbvWX!-FIvXF9)!Xov zAw?wtq}+F&Z)K%5EJwN&o9Epd^SphtLt}EOEx-0~!TZhg^7++kJIv&~<50V^eGi3P ze>|0GurA~I^&CB?M8NYLpNom>`d;td`ndt63}8EZV92+OFSs_Jr8zSON{65xUUezA*PJJL z08d6jMx(*=M(vKa(*_M|OG(m(W#hESip+sP2S#)e@Ii>uOu2yX3uT??VY9|t zxd=V<-T&2*aY$mSj(DP7U|x+8>-%JzCp)cH&l_cU3-}IbcRe zeawAx36v-#K64C$+WPA4;Pn@2OEvPg#>C^u?-kE(%ggdgmg4E?f=tF#)#W1Tx8@FZ zLN)ww*H!2J_@^8L@AeBE6Z@gSQG|vk4C4QEECD~>EOz|VqaM|NTLUWv*c3VUU$wzp z)A<6qkXB=)J%fhn<%XX6aa9rkDxH>!f;@kuCDO20g7)sRR^wjD-k|q#cB)IVlVh=K z-DjJMja?O7niv*|yfhE1QPo2BQyP`Mf13pK1Xa4+da4Fq2jf2zYl;AZKSR@77B&k8 z8LJq2PBoq+>K^LqM}Ilk;uUu%?`_~#gqfvobFtnaE~>K)A9J@+xsU-L_@l*Pn@>`T zWz`Iq1f?^0{|%)P)QQQ~*))Znb+x`y5q{Jnnfi0SGT?6wND=>Sz#arhZAfSIB}hK- zk$;r^wv*As3nUNbC?ST_NX=(k=1KKTSYcTR-He5fuYA4mOSuz6)F+)H>~vK-wwD;% z#>9|501YM2oZWMyZNB{K|D9DF{t0eJRR{*K$V*$$RlG`IEA06waplZ=7W22_6;>%2 zZM5p}xZCcsm6SN#M`V@ERz1DYmri|J^gPV25mri`Uoq{qi}wVV`DKjlS={#oV+k=| zlxs&^|L;}}DA*vt2!RJ3(nAB3(f|-NSfTPV|^85@LL&D28 zQ1ldFy(7{kBUh;7#5)mp`EL*eSYn8Pnaw818#E0Zs35`IY=2bv1W3%AruTvO#vaQU z3!p~OJO|sb9ITOT;@RxsK*qz-T?p8(O z4&2T2>G%J-@j~>k8$$GNp21yJMA`NFvJLZKc?i8tj1!_$V@kx+PgtF)7Y}Rk+)nj} z8>e9=U$g@B`)eDL>3_Jg+pE|g8iL!mpD@SO`fhh4`5;b`w<9*bRig1-*a707vooy-S?MD|4}d3Q}q5#QCjdOn!RDsuHwp+%>Ky9ii-HnhZ?LXUu1CD zdHHwdctJLM$C@@~=Bc=Wj@ROSSG0CxM4q^8f6cR|~V>IS8Mfq3HESl*rD8%+>XpTyka~xD z>_K1%VxkW42H|~4!1|#>uUu-?t0ia4C^ngVLWJ@Ooz)PCqg0hG{pLR%>vN~IIt(%z z<>wc^df(isbT(x#t>cXvdf6^3$?&|cDJ!tOo*fr{AuTO0-+xTfP8Z8Bluqv0s_M08 zFxYOKw6HWcH#1wdYFtP7x%J0C43nVuma4Mz<|IGqO;7VS`TZe<$2rMW-jh;E0GAVi zq7C(MI^xRR=p*CV^HHa*b6VA8gKfsL5;aP%Wx7-UXj7p#J|N~>Inoe)895ksZ>rdu zTL;zK^VRn$a;_H)_8^*E9@3Zz!=>9iAfs>qMQ*2-S1jrF%m1H;&_`&zBR-PLC-ssJwO6X2rnaeyhV1s!ERAvB^xA znIGPr?+61tKA2lU)jVd7u`V-iV(J8#;IP4^O?G#AA|pHYY|RNIiVF=)Dcy% zS7sU2_{-sG3zvE&omK;`O{h@(s(+XI%?VDL)C%1A-Eiz@STx~9y~0Iqu?H1E7| z^Ll2K)&_UBZ9m*f|5()rUV+;0{9g>E>I0hhTtwZ)JMw%XpeBSgWk?C{L|EN((d}|r z$Mq>A>xG>4;r2te?w8wQRuG#VZ@$wE;&9Mssim@it(ow5^n^miDUG1{4u9`JP#0I z=TMWuT*P0`lMHE6Baj1qH01cMYt@_VpQ{~KIxg4!!v(4$B0h;dH@Ov>_=xLa+8!|P z_yZ-+*$0~Lf0jKv34eB-1w~cj`Di1zJ+yAm|K$bIuXc(VnU@`AxI)R6K9w$ws=h|J zv*$eOTsv{M++wga>_lQ#v2AiR)OMt2oPqAWBCZNtIx2sSt$MlLmN^QR>E0o=sqr9; z8DVB4HG?onOY3-!2sq4m4S0WC=DHF`9&Fklz>)s7-*?;Lq@6F{eE9JSU>Jg$xwsg1 ztB4V7F45T5q{eOgo-F7(SLSCPo|$CcXAV22Iy|1kCfgGizIKWH1k(c(`2x0mc0!Md z%E^(RcuH(FLu}sLR6nxDVtD6X;8jlNI;qp%NdSVHZ-~N(xwE`kbD=Z@y0SuwY3SOS`@W0DyY<_qM03SORF+H4 zHa6SZ)uI$zio>3q$fhe$^w0k3vD(1UkXHM8mMt4XlHLJ1m(}-AxG4MT0|YhzJu*cy zmU{A*2)X)IkZT-6;12$|O@%fI2Rrtuf5iAUB@lv~Gnk76-D9&-xpXN`S%|uPIYROJ zTOcOQ%D2D*O5MHOU+p)NI(RF@Fl7AcT*qCD-=URhZJvRoW`_$AnUx%dFCnkH`AHpj z3png{QX(i>pLZs0{QH$`>rF@dhvZf*!-dk^_1KD8-av8d=iOo8z#UK|#$Yf2*Qg@& zIYb<#HthN?Kk0#M&(d+dv z4C2l%EoEh|*xbDR?G&CF>98641MB&=^HLO}otQ>7q;DOD2&zg*D5+v$&)W(S# zEau3U7n6X4|An}BEA-n~OOF~uu3RHDFnhGYFl1ul) zv0A6gVZct#R_fe>^cLTYOxBAC7;2 z+yr*V11R*(hO%I@U8C#pTAp5&MAmedfx&fEv~47rfv^4Y)aW#>3E5b(?!!eDV*&g> z@S|=LAQY4wqzWo?|J*rt66u$;a%BzAfzY}3QX7iW>|_u-26lxY)T!k`j3+*ff~P-j z@X;scJmsIk>+;{bAA)N-udf&=YT+L;M5dp{ZD6*2eiHGXRC}5P21N`L#@8-eQYP^K zCN49mPCi3Zqtz(Di4s9VrBbfbazC#J6rXQ*Nqat#aU)X5=F+@;C*gc33qwkDCJN`u z*#2vUzH%{n6>ziPixSmg{YicdS4fuz@OPh`(4vP58;sDsYH4Xf;WI|Xo#%LeC1jEyl(-bY?t5!ONTWQbUd$TwcRH=_U)S=#D`nYbSAT0 zc0^o1o)2}Jb|blpbmO6^T}#Uw+-}F&=sMr*`lbLij@AAE4w~S7!653_S<3}bZQOWT zaZ90~MDF)5Z9K&!6Ci~Ud|U+;N~Xa1ETFi$n59F4UWpY3$f%CiUE^pXO7TpX#fG!z z7^CB6EAvf7{B)u&EG&eQ{yk7Qfnsec&UV|n`Tp;Bp;7?Tu)Nw@sFr~nZc3)|d&&PF?Rl{;1WW}GFP$6OGW6g>n;&uTIjlpCFscV-S`D=(2gqRGiwkOQpMudABY-1XMIzy;1&JBNFm->#Dvh02QnZVdSLg*Yi-{*i7bjY- z*)T?{f-)IusBBfXfd^vd2OmZC10H zyvdCF!t3o`JKTJ2z_lrpYQA~j6sryUP+Xv#mGs*QJ!6N43>oG))#u(;M z3CP~Brr6Zz5SgQV(aU!&xDxPhT`5jmH4h@{nI5_O9LD&$Es)ci5Yk(=92Yeg zf3d3#&U7q;d%!5+s^-io@Y)HPDSFbaE={Ag<-^dpaeW8wBj{Pec80gVK4n5euN36< zKgM8@4Vp4bOXR)Vt>02=PM>4zmzbvilqoQsn5wsH?z3@?vTB9nc&2e~MmWqpLE){Y z^J^J?^jyc1KB)~?Bpkh0dE&H`jBFybpHqny>d${|J4m(oM;VZGTl)t<ff>H zY%i1jvzR8Qc_}&*5W8sP=a$P%vUUm|D`LiLVW?7ss8(TgG)9qAdtJeK@xci;P$ zn6a1=lA_tn>i8OAhNNgW1%dF~mdhGQ&clO~q40bS@wK3VL0v8RY4NohBVO(#`)lW9 zNdH+%N(K?zMqOLJ<6Bd;3p$$zRgX2K-?wNJ-ydD(NAtz8x&X2!XB;LkB<`B!=2!Bu z`t3Rj+i?-VWIzpT`mPKOtNpjh!zc+K*1^FMQygCl+ry~G-&S;tN5^)*I|vunr|F=j zQ*!e~a}LFM(|vQ?L}0O#?N>W|Py~5Km&!~HYHtH5_*ZQuyU7V%OAfVS zZF>(~m#Hi!v`B;t_gHv*l^x<5*#=ll#&F(>nbhUgyf}l9coi1H#I}nua(u;ks@diX zMWR!lqMyqe`_D8bDEAHBoXI7i6n9_T0)8zm3L;Vb$TqJomH$>2Uv{lMlMU`&?Dwph z`HYYq>4yFXKCvEcG$H)7%+ksRo69=vSmD*ndc_(AKS}H$l(f^ltzCcLmK=A|7=mj& z^>&DdVAJaWF5huu%uxZOO=~Mb4zo|Fi63@OOSzNTdbU*L0(qrD&>JR(} zH?rD0s3O&!EZoez(Hxbby2UgReS^4A9euw(pV_Rz6^9|%IJo}8#>Vngw6Vt!Vl#fo zoP$lB?UcdX003aRC=XA+(laf{2rQ4+#H(2|k@TZP$*`&Xyy`EqxEdqI*L6eLS z2_M|Wv{QD@>*eRirT5!=W+y;Tpy#NdLg{rNCBVJuMm(>5UCS^YcLPACbnS1I z=QLDN+~%O3=y8Sr9x?4iPcC#IYe(M=d#<}urO6804|c0O#XiD;tg_q7H&Ll8h34A| zrkPQ~KL5A@RL+9>1tmt7MG>vOuL;nK#uq0?UsASN+M_+**1Vg-FoNY7yxd?!# z`4W;flH?N2DR)mlz6Ek4zbSzhsaIO?V9qs2Qo`Q^e766AV}7txE9l>cFX7IK+9Sb_ zp!2d|7=L)X^=lr!Kt1)vrPz>2heDV*r`*{xM)7;e2mi+%$=5{^ z(|3PmmG88QHiSaoRTM|=4gBol!J@rekmgd-QA7hz_KF~8=2442{W5SvjIDgG#eQX_ zG3w=fE0cssa3RZGg?`^31@_d0N)LOWSo|ljL(Co@wYj$bTTbK1Wdg4V<=H!KMl9UV z7=Vn#Lu4*&Lw(f-%Wq7SXnk|V3dJxMqbl%0^`s_#S_)SZ$ah|%Ln|{x=zPAk!5<mmAF#w~&Z(>+{hVbz+<3T;X51ufro_C$($OADy=d*QkmGSF=* zuiQfCjd==|CUc{BKF*lJOuyeAOmOZ+3s{a``;F*Sv}SVncyxFG8}nu39}5yR>0O1 z_$7_Z&n!y3&xnerIm}!tlL6kbKRZ7z_@F?zA{!hMzd2GZEEc3BBCG~gs^C=A)fHd- zS3!f;!M2Me-X^KSrJV)%p(lk-p`Zwjh2pNWFgLI#A$#bU7r{&FUZdmNFZ~svc6)fP zsK^P4@zJ5FwlJLk2QhP66XD^9j8!EMz9?Zmrkxjsg4IN$ovA1ZPj~@RN>1yDtcwRB zz&w>-m#5y5QD)O1_gZP;Xr(i1mQm^2#J|eq6X##-*baIa&ZcPaypPo)uH8;eFq!=Q z(2Ebu79<>$5ziObuz&*ZO-!&lA=TgN$6&Z_DsLU${M;S^BAG!h4swS#0cazCyWz%I zrbtzb#Q$h7%a7{B6d0-&cTa^v=SPb{R%^zu`eD7WuVLGE?d6e$V6#yqiIZu_)&vxV zc`8Zmj%scw$Z)@(l!m3Hrv7>!j^cja%PDW%2#k+2cMzg$t4hwayjW{?>p%}9dcuW; z5f94In_62=$>4Nk+>=>KlOzt64nHX{e8M(cj@BV1n4@<3^+mP#oIU3du~3oG4A}!3 z+k5>Qj^ah1A_jK}nHH4bK@q3AS)Szf6n;6+nYv=TgzQvy+z^$BCMEQT{v0d{JZ>qF2l z4g4LrQN5Ve^+gfFEv+}hd8%LdxQgW0j7$rq7^rL({_i zFw1!3E3)Zmq<>=`=Q-G6ykmnyJySv;T)7z15V`agl*^zu4p{tPuS^!{66Qv96m8+c z31iRCVRK(tgV|+U_P8DY2PM}6?qicbZ{ruWJ@LK_Gs)2DSEN(=^9C-hJ70oFTdr;A z*yNDL)GsV3C*xLw{OZ9CB-EpM#?gFBc&q*{PUH=td&X%Yv_WpCH1g5wcgj(0Iuej2 z_Xm!`6f(4>`kH>%oW`n;451fZ6vj-pURs{N(66qSXoykPO5d_YY&Myg=l9a;B;7X! zse+nBWR@D6#!cJT3}Ew;5pg%ER$aFYJu1=AXpI(fzpO{BAE<-?rX42cW&0s$I59|8 zMib5A)yRMl=DXd?UHS3%CJkz3BDr<+g15C{#>FhX5QQt4Q;UATGrLKH56Xa*_cjU8LE3U_ytd)CT6p}(Wa+E@1C3Iquh zsB(>wRTDU;E?y3r!r^xaUrD=X>dWxmQl?H+g0|8cd<~xv7L|oTzT6E52ew{%_%GpL z>z9{vr%x$-oTy+%;jnd7SIb%GHy7nI;eB1NRY5m1Qv&PMSAHw9c_AgYNbb-o7A|5! zHzv-_#QTxLiU)2p3tmJxw1q7viwvWg2j{?Wl@MO0TFuXlPzU+o@aQSrek|xwJI>mA zQf`qAt)k`05-4K-knT9E^9cz!-Bk&!u5xu;1x%uc<$tAZ`eQ90k*3CoLYMH9dT+{J z^U8bE?DFWWwhZk4NKa;*np<9Oa<;@QiM_!~ z(@$^Oy&d#%s`JS&5k#XU0IvMTN;*Vi6&_`cE-sRAP#;fMt=dxs$?!x5nU{T0jOT+3 zErd+dV3xqWT5S-(xL3@-;W3D@-ftS+KjQ)jq*5ocqi?6w(#&4 zeL&ay?UvMm2S9mkJ$5*QW+2&HVRfAY9=dN2K|F7Sym})rvU8w+dM*MupWkjL6$lb( zwO9)X{aTJzEGpfBF{K7#3mkJUT29xDM%4kZrCg?7P1+Y!c=!->s+E6v44{_>X)>e( zl4JsML@&VYz5@#11*8t3(CgIA#Y59AO2J))OduM{jc$DzboK~qKk0uMzGJWd357w| zo~nX;>lE8v1le6GucR;mukCi1w#gBO;6U9Av0Yr9T*Xbq&w>xC0 zA|=lQIAAFaX0_dRJ8^t;)d3t%m`zzzO!X$Yj)Mxi$ApmFH*I%wPs#y)r0$}QpP0CA-PzV zVzT!XYiT_tknE{fOZc~3x%>P;6$Oj`nCX!~cpVc@&VLW9tm7PI)3h9d?I#^oc5>J! zlctX-vwbahmQHIhGBB`bJ*Dqmnokh$bexs)Hed@D?E~skO?c5GpmKklAQR_^#gVDl z3R^A#_dX`P9R#kk-@6fPu`qSYULcJmbGhY{P-}bh?l0~|@zqX9wu6X8SihwdqZCJ&Nnc42SC^s_4 zhF_WYjJ`8q3Mfte1{8v^M2UBbb4RmM?w3%+B-fVGwTbK}ug;7>_YlS#i(Viq6h*0% zhlq=b_QzG=sXZ*LqOPg3{R(#DWW|}~bu<$Do-EscT(^~tTf9~lu;T5811s&Xt;$RCO?@#x1CVCY6H~vZx}txCb~(z+4Mq{j#Mp*y+GU{Jjnu%|U-{}*$W?OfDh?Kf z3gUyfKH-h)W)pvFDPoLEN{}}o>d7-_R10j5O!;;3SA-zrm()S3j_W?UI&Kf9*1=5v z=ya+*7z|`Sklc)9+yH=R2&a&*fl_f?aP%c~v)v0Eo3?HkgGZxLvMF_7s<&Kq+I3!OF`R%W=D5@SvOO#(Z_Wk$I}cEg4&)J;{ImxqS}j(h zm+wp-G_+wdxrKQg9~B6w$`}13?t~lNh>%2qJVA*PJ_-I#ZAr$W()c7bVLTK8mY0{< zdi4-QNY$UoXU311LquLy~0>UeFEIE!imPD3xz@*2aHyGRUC*VNj^nSkx zEC9$V+O9MbL@vDfxysikGj|hIj%-J9bBPS7D!Q-zM?=TZ|U@Nru_^LKWmb>!c_T59Yr@s!Hqq_$;l1Z)@EiO~I zZ*&~o2r{-GR=7tJB>e)USYVAFO;;qLQ2FI_GrT((-hO`Z>J9h?&BhUOH?G;;+%Imv z48-<}s$$XG0K^L~;Z6EH?n4T>o=m_?!(42JJqMZeC@N)Pi_AU#`cv26>Sb{P1HLCaXz+T57a>BKEZJZ z;u=c*D==7slno}W>ON5IYG{8tjhG^4w^^g}WYxcNV3La2jZ?&rAB90Ju!NyiHwtO%Ry8+6Gzo~gT`N-w#N;3!f_x? zC^+=RLWhekLELMPn1IiQ$hIOs_uE=#eGWyc(<)VU+SOA{pepv0TM$3zHVYQrV_oZf z!gZI~uyf)}2;&O7j`B|=x^lWIaVK4oQQK8#jd$t{Y{|%_6codk`EIXV^8(32$9@fA zlM3IEizVX%MhqISa$QC5CqIH&jPJatW|yh1 zN0HIbfp7-603BT)tG>UdV>ll?SAN!u~;G(}U(K-GqM;~9*h6nPX(0*BL8 z8}?Z&Kj$NyU=tYjREjhw=pG0hRrd=Ty96LKbOJSi7z_bEp5eTKuNS5cg)p_0d_+u1 zQ4@pm^A9>>=gv2b5`|30&w^Ao^=)Ue0t6q&HQf57(KO@ zKyO(iWCH3Ly&#xz_Mx9&2#dSjQl4wk*RI8S)r0XQqk>X|NOy;g}_>y8s2jmv| z8ZXQ@6pKo{7Wti1Kkl`0PCD=VyPBu4l^#*KEpzI@Lip7FQtbTfpj%j0A1>w`3;EU`+u_mAgIqy6L1e2I>Sd<)qC`N}C!0oO>5 zo2yODyU*yq;bg}}&S^|x&EWx??_|J8z_XPUnP}Zk%-G?vkP?`};(Ot5lm3`M=jpf|mA-@qh^8i7_L7DC_YbSiI;fS-@&WmjyP&?Q zb1iRHgV6XLpV|)jToWs{8?^R5i2J->lsWr4QiC}iL_o;2TmGl?7UYaPRs*L z`Jn`-wQJvG8(u%Fx@bQkR6HwiKp$=)tdx|8cFx!^o)_@#8_f_H>XxhNX z?cqf$rbM_cBB65x?XAWk^&O5NXU2VToNon!#TtYh&vSrO3{naDo+wPTSOs? z8R-{SqTfG<4qDqStR9W)UMm!a2E*~om~xGX=XKG?u{O3#%HmjIMfKx5^-+yE(BFoe zg;{pff^WoipDW~gV7u9>=moz+*>WYPN^4wax{Th8JryWOzwk;}E8E-Mj5n_JL>pH= z+ZR=@KPUJkEK)(Ru6vXnC$6+`9#MEQ4KC)7COI~=HLv2=I<0ydH*;;a)SM@#S^qjW zAjjqtGQ_o#e}Kxl<|R>j?6Qr9LMZE;=MPxEPab_qzLWck;CZ4W)7r3~@CMk%-SS;c zrZGJRc7DJgEYGeu9*H1M2<~!CEPXr|VvH}lzFHMnX)Uk1KWzu4qzo=Z6V>DjuQhOT z*+s2Kv795&&~RV7NCp<_*Xe>%p)9(dbh8Xb4@NbG`C`LHww95w9LO* zYF6&wQJG%Gw0e&SKf?hL$u_(3-2@1YVhd&>^N>p#Gko;gWkHG3#;5=NndjB*`^BMd ziv5Y}+b%|@5G=g>4IthkmzK<_C&!}0lVgaq2<6+!(wChvl;Zp0a|NGb6W7Q_Jd#=k zn^Ll!i*mx%YSM58>*HgHHuE2TZR^bgjk|-lkX1)QHfiVk>d+xpxX*5yj51pgVP^W) zsW3S;{wbdE1T?R>v;K`rw zbqB@{9|PpSY&~e{Fw&00&#Fb~1}*)DWPc*oiB%Vy|hZ!EJqiN9yN)B|ABUh)|)3qjtSeGljN&ch#hA` z?KvAUxYEigN9U_Ss}fom+cPZH(ukWVmTpr&9g5q+2V-0W!&U(#b6#*SlQnM@?8Y9d{m9v}@gJ4oqz8rR3? zcn{7gGno{&OUQX>+{%`f2;KerH4dWP}`%DPp(5Gd@d_jm7^V`0i<6mfwK7o|mVv7jifv3r1EJUHRjskjDQ=X!qLYlplPDOhvf(kIQgH*@ zWsFQJAUp3&%W4T5;a}1=l*w|uazi|sz##}@3|WYWE&}^)rqq=N1ZF1cJ^@$yc)7+6 z_yeZ?YK~deN<1t?BXs%!>dirn=h(L&W=F&f<(l$(FuoS0F%Gq-t+7CvZzeZQ29;yy+af+?gl z1lkoVjD#ub*P%9Zakr%X%aBgY!b)B#hEc|74h+AUO~>savyO^M3jzoke9Da(H-7q< zsS7@}>y}Xqb!ARmaM&rkddz|a8fTVc?YsBhyUdx5CNB|ji`kw-4?N(ogAQUWr51$- z3l2TB)At(^$&y!%y5@|Xx1YB4c9DdPdX%Bmq?E`L#D9^1Le48vd#a^=F#@VG05vL-D59 zx$6g@ah#CUEM|zQ7&J~TG$}MUH^yU0ijvw}4=FT`Cow00Tw+2@anmtK`2V2CiA5r%awxT46dErU>7f|R+$$oL0}No3}Q}!f&2wSamyf-qDgIKHm)&U&E=q8Z@&2!3s*5xWJOy% zk<8}{W5+b|R-DOUe#gdiZEb9i#PYpLHt)sa(dID?tt~6sJG!>ncH1}KT#hrpwvJX7 zvQFM?3fn#qJL{xvaqFiW&tM}ovfi(&yVQRv0%6mTW-2~gI9TzB=XEh}6)*evYl$00 z(G;NuLw`w!O1~s__fT~4t&0XlGGMJAr{}==)suMV{5ut>YGmVQrk}i|RNB_qJ7XP@8??A1klWE9xnLmvznAe#a zOe@s*1~M%TL*Qh7#WWhmG#6KNzTMuLulQC{tUhd01WP4fa_HLR40}|M^F%!D7cmcT zV+k2UO0%NX@d`%sZY-_5aN~d&<6|GInks(n>qqjt&`v#D4DX@by@;TR&8(ori4 z=)HWyNwD{TIL()>C#9FHGGiq&H)i2y0P+$zbb?e`*=uq1GwA@Q05L3}N+$h`T~y>v zcNC1pR1K^(LFz4mwH5N6nykT7eg>(}h{Tw926mehV{f|grY9bJ(kQw&{^pWRW^aL% zy!Y-0?!NOrTKDuVW=x&3$sKn*h|L_n7k=^cUt(x+&_Vkgbig5;v-Bhtk2E$n-};Y# zy!gV))3@I0Z-2Y-jW^z8_=gjjapPq+hh#uSR9b4Lo06=JCWvRp)ixZ)aE|Fcr?Wec z`A^R4F3CJN%W!grpEoLODm+<4iy9HD+AgF5^f>aVT|o}@UWrtCBCIk~97VJwxE~Vr zQZ->Xon1Lr4<#`bW5%khBa?2j*bmpVm?&oz5JDbO1C7&UP7!?(bkIB3jicMZq(N`L zHmb{tv9=P`$;Dqi|K% zNJLY*maJHbmM)e~HxOmnFJE=-75Q@8XHPo~%V{?jtX$kaVZ!)wSLLbu{>l2*SYULt z7skh$Ud3xtz8XuIFTJ>g3j38a&(tZt`nu~Fm~FoKW~k}=@_?>K6NbJka-#X%YL`}h z;U`O$EWxD%>SxvkW677vG<^JHr~T;1KX~b-muJr0`o8<_U$EdPR%`wBuQz=5v!7YM zd?i^p=%9mOrL4EWJ`Wm4P6^Cv06^(mlqN6l6&fPRy^c6)kAn{% zgJuUcUCik0%$@s{v$0lTT#me^yE^cLd-4G|w%w|t^s2KbkGNnCtK^B|hlw73f|vQenY+-8NhhsZ~cijGo#~(|klMBE36(|v|CA;1zUH#`j zqxxdi|J*s-cdY2fCAFTE`^opefAYyEFI~Kt)elf5R~GNd79&&-op52tvO6BW+*h%> zcjQSE*)=ckvde6gY7amB2&g3Dcr0H!bGsdA5qs{jH?FWTy(*Dq53{-OipJI2&5ydV`gwW zI$f^wfTnTGqUwh^f&X!ruapYEq9pXK(R+-(bl1u^tf$+TZc^%jtyLI7M;i(yT03i1 z@fwaLsO45%WN`_bVAeiVXu}Roo~bHs(NW^%L*#Wd*;#SNZ8tA7Yj-2rh}{a6UTu#w z%tbuJMFWjnYm6X;(4l+683ejFR7-fV#AdEqD9I)dA|I5kFvBisyFv;Oy~_AlCO$-` zgxAhUtZ0;&RBCXU1tFd?l36{E4;)yWJ5@|PvQ}}+pa`_-6kS>?Yc(NzxCKIe;9z(NGHjC_R6Nu2 zbGhzBGB(DFP4N?RHs8W7bY>c%h2@5(D9)NIIFxp+LRpqH$!t=T{Uma>DGO2KN$KR; z^F`O#HJqd zU$Ng5;D`P~2 zaY3pm;zBvsU<8h^F)9qkMPFK}BrvE;$UrC?`0X976UI!4+Ma0p=>;mX@jL@f#!fuY z8;F}iR~J+n_Ff;fzr=>GiIV5WBk2Zr?z|o5?Y2`QaSHj((c4xzGNG%$F&+bTboo^LE=6;lno8WQj^j*02QN zike1M2c0viyh`l$Ewoz2Fy#)4VTKCu2Lw53d;9I>bLPy0K8zbT;huZ$!Ee_V)21Ug zm>N%~8s2z)Ir|mda?7nxJnMjCYq7aSdo_KvI;mBHWN(~CmP1i zhRn>;o6()oo<=u5{{g&xWKJ4BK_nUz6M)HjQWynI97BBkd9q^NHoKO{bOn3V7L-Iq zA(vz6iN(Op=wwYiv#IDl_1^3#))`^sV>b5wvd67F#>9tAWAo;S+dR=XqD2Zuc^7+s zPzZ*ida+JRm#h8~nMLhiG$LK$fV`9eM!Pq625G!o^U z368Wj9>N<&cULBrVC@A`p7Lch!!}>3q-^ga`~{}LOgequ`=c173im6JcLF7#>ptg< z4Mjg~Ib|;&O*xrZG*!s8z5YV7*b&cn#w!K2SOqRYALvVI%G2<3dOQ>${9o-#sg$Tj z#nVnvV0H>_QwD8Q*`GIiC*r~m>D5VzcB3j?S zZm<%N7`OoJS}QjYCD7;%-$c{k!F(^$9DtYtLS(_Lg=heEC_ z&hHTcqJpX7=h9}pB{Le8Xq=Q6xsKNlsw??tkxYqN8i_VFXP~;VI3lCge_^$PtdU+# zNW<#+>Z5VB%)+^bWv`Ywm1wO~{r2g)xJ1-Ym>qJAaVpS_@1(kAuoxHdUKs_PD3$`M#a1o9;Kq-P( z@uc4aRq!nAk(hFYjj)gvJ9Ks-HhQ|cqK!6d(u^ppg?IojLq5t1ZEFlN7#yhTh=rK= z!ABvimrtXcVzdG3NL#FKL0EW?^2fnr)N<^TMh~#P(rk^^u7-W&i;WKG<7L=@tm)}_ z?xKZt&Awg)to^)N6+eeo6|J7EO~W{#Xh*Y7wA)P_{NLx=@l4W5$h;NC{Q@YDDvzn7 z2fj5^S}9}3Ojp5zJn94cq;ZnG@`?=fS*E(FPWN5##vgDKZPIdLNE`MVNpnQP6U!5;?+Q)T`}Pe)@Y*DrGVbM(S9bIHg@zn8c7{l_6m{ z=CgvOyP;Z$`m7JeZoclHtaw=cU)_}qi-k&7lEo8ME0^`GF)_N@gk^TLc4e9xIy&0g zyE>LGdmFcZxl(n)oL zH@hS5Id)^9&Erx|_^`^UvYBNyLotd~ZngaC(+?r3w@e_tdkh~wJkTzfVuet|-{{CC zCvP!!hdDUckK5FIUEEUet3Vn2VT`EFX|H%!Kp=|Zz+&BlF2*#g0hI3#j;19Z_~b+^ zPipOZjT4>jSfr-ns>_B^uHe(J~v>Ne~FlfOr>da(#$9{t7G(^XMet~TrZP}<93Bh_OX}aix?J6pHv!`H8oidHhF_RISos0rE z*;(w)#4_xIgt;*rhf?7%H`Ym~=Fv5Cu9r_k>8R^`qZZ}(i#i2NF1+{A_L;b*6vVLrFDYAg4)e|+DQi>woDpD~jO7S8 zYiXup8YQ@`HJMWPSZPRZCA*xUr7v#xU z%S<@&pu}Q3Okmjyxzy|j*bm2Xp^CZZK!OhT{4Hu9{L6jWlhpUEN!9Yk`V$f^2l8WRml@Hwf*(iQgiXjHS+GG1{578PfjlCPAy zjAHlnZMOKy&o7ihcOq;d<@;sXY%pq;eG|O}9>P$gN-H zwgzv0*YB)w_0Yp*#&+8AX{IrDo2`uq!)#4vM5HOWYE_`FEesZl%yQ87QKX@^r|}V^ zZ$Pw67L^etsp+kHA|_OyHL&*1OS-!9=y5qgB@aviqyKxJrJHBGE010CUFgBl%CH-7v2>&vt{ ziO24|^S(Xy*dvi}*?Ejx(74AQdy=ghUw!!%9*#yEsn|>#5_LGc3O-?=9-`Qw>k)Oc zGC2||hoqy1_R0c01JfetDFv>I=~i&>g~^gw)x#bT+z1MbZZgX#jfGQK-p3j-nN7Q6 z=AKAh#XlcwivHX>`ufAeHz^2A}imL1}ODG!^L)}<~27A+Hi&WXs!sL(^ zny5)uSIJWKt@ohhi%cB>0W*KSk8whHkE3rF&}x`s873Tu^(hjcOd#SojT2>#rwUVq z*l0nm7nyhdjZ(#^Cy+DWvVBXoi)XiTK~8e3rl?^LGAl2xvQ9_(LV2=(#|r*=96O)u>A|H|qS;5jjLUrZzmEMN zGQzMg`SqWsOktk%;TNBKzIpe_dDJA0X`F!(@fXhiG{>)h{acG)U&`KhJROUhA6{?- zmd_|hzWd$pPMSOsh3jY!jKQhnbwS3B)KR9kEx#D4mLgux$+bpt2jh1oT!cOHgEHZ< zejV+qhbA*>j%p}_evI|`n9>ZuDBeCUTi z{Q1I#=QA;S(nn;)S=~`Zv~A zcXV{HNdc9eC73ugfk6^A>*9u8Ef?zK`dOm}>oNL2J&Yp)8ocm>I-6y&n^3(RbFAtC z|);PW)!io3qQ23-)7Q9De)DU;Y~7 z%@2NX-xpqZkv%aPtWdu{@rh4prEyPBf$bL;Em~Caax64qhqSZLJ{z@j-<0=Y<$puJ z7XSn)5sh+`(_XaK5-WlOefyxVvM0u7iQvxO#r0TTmB69`N~ zWQ`1P5k>nGotL zbtN=5>T*p31DolN%W2waA?5GlvX@{8!r+gGw|xKW7hHZ7TlCy{*Y{ujQ`g_O_wtp4 zhp@!|#@D|=bQppK!W`llc<|r~vNvS>ci+7qE2T>=IiLO1&pBr|XoofS2~YU3_4VN# zyQDmv6C;`G18R%0(YK&!<&NBRYTAG<^0J1Lwu-ppCT*wO7@0nqy2*2O!$Osay|N~= zk<96EBM7vUQ8`tYQ)l|ayvXF2`)VCl4VQqTj54Qj@4@u69U5knp``$**3QA)p1wdv z!`*kSfAYpp|N6iE4fuzqyozBBQo8ZRn;2c}$Ux>8 z-XXwSTial#Z1#Xg91eq|b zeoSd*Cd(c_p;dE}G0HlpU?jQ}j!=w!6d8~d5G^LigQ^dgu)4au#GDGE4}I{%|NOY(u#>Lh8*YSh~C zD*BkE#gNX3dJLhgWlY2_o9hvT%y_ec`}1^TdH?tCjK6>9;`-NTpyNt+#aSrwv>Ewn zk(D*JE)Pca?ygH8b?MpXfX%R;p*(5<*~wyRs-+s9I?2Smzypu20_=2!Wex!e_msrp zj1%-6@6)FO)O5$Zv2dV2WJeSB?21NYjmtVfP*!}ZAzZu7f^Ur_NM?Ugl=saROzvLy? z9XPOZ+G!@>_PBZv$G?V^5TR_+7PMKz2%6;|i__f2HWB;ics+LFDBL|$Gt;$wxL1`P zf!GMvHntI0Fj2)Ht`f_L_(|j`IieogQ3?e3_%y_0x&%feZjGV_j`f+4^#;(Q19gCq+&k{EL#+yFHnneH*ihu8W z|4V1CJ7l*n@PbUD)oL*tNw#a3EZ#^+D_F>IeK6#JY>>!lqF zmtA%-+|1*zc_af4i7`eh7hG@wlo<@iwbx!tNNY-iQ$hB&vcmqv4aP3QIififWsd0p z;hn6hoT-*#zF22rdwKF74KY#C8I#&hXgKZ|^d?LPD7nx3(;ne0QOn}-08K+z7y*@v zP?457HtV8p(V_Ow!}!Q<=+FfTT!d+y!nd$QNL43l)vgr)VZ3y4M=;U&lw?5KRWz%7 z4LdASFvn{mMne?1No}y29oT!CbC7R&Y06{WO{q|mUCm*riBw#R`d)cPn=R~O$7@)= zUot==mV5Yvf98<`G7-5<%fS!`PGtk9ZB@yLn-$%r;csRDe}qfSh@I|Ot1oP<-Am-f zT3JhmDfEN0c>0e&=byjq6|cDcu6r-pvzzc3PNVzgU;bZ%@N~8V86?S-S3Z&-6|hSw zV0SQ+BT>kTddvLt~TuqSCw_(MX(K}>*c+n87R zgD|hHu`R|dX|CDEL)>dLd$<~~EI>f1G$WI-@jx*}Up|-@gq{1KLkxE13~Oa(w;Gu!tbxz;D_?f~oz+e%v?mT>z*&V^=y z>S#1dGN)BzoDAt8t2nq3*bOd?YX}1z0*?&vCd1PPiDlOCo%zt|u+@Dm<2C>_Zq!efPaV z5PjkEU%KHHFaPr2e(^o;`M~R6_li!t@#&jCcka38N-9=!KK}8K_O#QMxI=Jq_F#ptPtq9T3{e~-8SzKE zrGVA2?~vdu86*ZX0~l@u$p{^7-3s;K#HG#AcH-)AN|!9OE39hRXoG-JvvO&H0(K>7 zaWZ6_F5zX_M1UV}N!r02LD8D0>5yIT>QQa~!TopK_5I7QybN0KUH{?#V4pZbtK&5d zlflR%u`3P(k*GOvhs4D}Xh#v&9uCPfVG^J4geTZb2sns22tJan&%^6zp?CB;^NwB1 zraa=ysqh;L)j zhS9MKAR5|2e7L$QUOoG462(*|cdTD!IzrRppn)!L@9vOFen~N0MyL{D4*$vhmfbhCG;wBcXXo6 zO(25J30cid{#0i_LU4ljvY~@RLyh2Bst!f257$u~rRbB1ZjZ3RmnBN96WBc2c~Adi zzhDFqFL=REJod5YKk*4qA`aYeDA;0KiKd>Rw7#xwxIXu}&;8Et{650nU-^}{VohWv zZ98`^^3FNuoO$)tS7S#6Q$gZeXgJsj$T*JtAmdh7*9p;O1r}f=9AGWPkj6pA;g&*J z6SfAXGa0d70_VKYmTu+8YNxdfhN4{$bg zATXSwbH()@-g-P+=dii71z7sUHX0zKyQWAQNRoGxCvq;!_PeUg?^uTQ;J{67q48(!n85jmQ zU5Yd&Vji9%QqG}&3cS2UeDyrjh77Z$n!5^~7_gZE=5g$_A)M_ZsPlRR3a&d@J4qUD z!^;!x#=|o&Cj)wY{_;^PE|?M?21A*P2neAS6t-RMP=9ky zg8t?>OEv$=-kpQ}2a&;$qP8QfJie=9Aq?*!0zg=mz^8`PmvM6*JGvbltBL8f4x-OMsPRZULLmwabvhzVRl z>Q2$2t8%~Zx&u~HHCd?Q;K6(6*p(f_j53PXR!;9Ol-m z%NITJitpTU_jwmSGHgZPzUPp?>w@LsXsLc58!Wbhj^odrm>R459O3+{ERBH8No(ZeX5=zXj7UV;c{` zW`k$U9`P2d<3eFcMyv9olgL|A%_pv z9?FZIOFMe2-_1tOeFs;=n*UR;{uec4{`qGg`tU`UUUKbopY`#N{pHi1_KZ%qIRc5) zyp4^)ZMS{v(n~LEwWHp~5H6+Nt~;>S%=xVF0}&A7NKAa7hq|>VeR-^0Hfdkrjg8O$ ze;@hUzV&+!^`8FBYoGn(N3()`_~RePrW;Qv_(e>=aS%pa4{M0;-VHb0z~+6BagcOh z`qGzfx#bq*riO zyklvHWv^McK%=17{-ED#vN;$7zg7!tc+i^AQwo8_)8S~?WRs8W^xzPPape<#5kj9q zk7aO1Wj1hzb+@VAWJJmFG2{hUDS(0peSoZ_Thoa;4mlm?yhTI`sBK;$LLN^6QKGwVUxi; zAa-J=XbZB%MaVeIqH}^~jJ1G0Z0JflVy24R@8xYu?tTC{*)N3w@%zSX6O3pmdC5RZ zw-GhJ`-8iSOZt~xe#PJZ&3AtHH-9r96+iL3=X>m^Bs-Nf$#bypC@H@Ez3<=pt=n*Q zXtz5z-0+j^wepEi-o$3iY?X6hd3nd_J0#yg9|kvK&?}oi`(^{zlN~fCw0?Kpb=Rex z4l|RO;NjfP-~IjHd+g;G;-nT2hTpj5>o0o2Pt5OJdcxx#vtwZq;plh1^BuhTE;|2w z8wZuZw)gGbcgckp;X8~kKdyXeo-7$ho0#PAZQMDb?K^1-H&VbO)y9hvhc;SaL*xLI z89PB4sseLQp-ZqQav@8`p;95CMzLfZm&6+7hyvwEId9+Fgg}eUZb2vo7PknVO7?_j zHyNA~*V!l9Wv>XFPPji%7f8>lm@!ZpqT#e+nxkPu2@?bRiRMuoAc@kGC8cP*hr3X2 z6I4HvGa?COh*J%5P#=wirlH!{g{9vQTCLt-5QIpBMBw!YqgGqnj6%H`N)GcxLicI+ zOFTL`XSBnjgR+R4hN>451lj+|hGy=)=gucRY1f;6>4x?7d|{q-iSVc!Q6u`bU;j5$ zIFGYh=wlRwXtM4*?|jGG6U1aq=NI1mYtTu%&pG?aPrBy7zSW&O=gVPMeD7PgeBl4C z_4a#>T{%+Beh+W9oYZnFCgcredib^Xiz=DagNvhNb~sqVWhZ#UfV(u2zwuXw-oreEAx8!(7E ztvW0ijKG`T{40cY7$xP|XPG5oluKu#+heX`(y`~PGk)?XuP6QsbQmZVEP4L<=Q3Bn`qi)gvw!x_?58YY zn9Z^!+WF_7fA-nCx59Peabm^tQtci*4f{E4sr!*|cS@S<0~>ZJ?a z&fRzI`~Ba2*Xd{Oz*+mMtFB<_`p8Em&Aiwd6Pke$N*|lpI zAw%c7l7nui2ZI0`LlG{VV9&^W(3oYlY@>JZzP8&^tTb%f@&195>w}nL+<=@~-Gkiz z002M$Nkl_*G||RExcF)%x|;YH*`Ns6NW2;(GW8UHo$3ZC36Fiey|;!8uwI0Xu9jN%9vhBE1m3r@ zXiOiGgRnr{cWC4Er4Eu19D1k+QBl~Q6(J)0V-SSa3>~!$>^dAKmux$)jyP^QP9oXu z^ss_}$Ou^@*^yIKUDjZ(7y0!@cb;+PHy0g*Y+B~r2*Eoe+9+gOQEN1i)$x3{$#Kww zyF)C2)Xu{fK4P~5$Ju@E={$nrwjp`omHFQ2XN@hZEl{ZU6ioofr2B(FS`YZ+x%QX8 z{CA)JZ-0B{nY;JwIb*P~L1dQ_w@o5B(<9nO+F4od?cclq^*{SF$_FwzEnVLo4Mv~6 z`Lmz?ub;l~qDMUUxz8nNvlYnm%ChN~`thHBg59JHeS4J`)=1=9@pu{Wrh)Um_*F>84M; z?4>U{dP%4F(`+zcKNHLmp$5Pgt7-3hxBku7*o2qu{z^Y}p&QtLt%n_6>=A(?$T5$S z(m;MN94+*~DR=;|#dH`o6&4UVO5sL+aOp*tzF_H^Qyjv_Xg)sSX1zzuEW#^}zk17e zzW&Ycoqx`r$35Zk*yDcXtKa_Q$N%eGJ9^yXE_=c?KeoKmd*A!s2X=zhe9dcK4H-xN z?CS%o`ObI#doqEgAwE201SEy^UHiKrxgT=s@ZM~T9GD@Ab-f(V5S7OVQU2az!!Kf3(d97D8*u*Gr`sMm6V-lo3 z?e7t%dH3`V^_`|u3|4Rd_KjcsyR|_N<`5C{d;ilP{9x}vX2#ClXCiNT{cB$FAAk=Rv>aa^ z+h09|5YV`ju7YF*5*n{IWqGo?#xB$de=^x&*0en(Pf@L|ul4Z=Z#KL}!^1R!_mem$ zq7h`=Pf-%kcb+Q-8kyIMAQb&5I>K#jGyhWg=2qQM8;Jp7gak*^G;yxBl- zqEs4LcHe#b2BSVs!E_U39B+U|p~jGgn&%S8v9BNZME z?YHBK#eM85IKqN++)hV@06d0w)ngw|RQ7hecJ|rlB1cC2Fh93=-nr*J?P*V9!DA!h zUAs^Xjz+nBCB)&Op~~vT z-t_}_t?loJSue`^Q8tK@zWEKqWY8F`H?p;6w#u)y`Qy%7J6~-k8|`d08m&a}N;F)> zw`yryBzKH42A&}$$q1AJvsYpZHXPpawOi>BP&FU_m;d~Mm%QX22m=3CzKX5RnGSN;JE&)#ULBmd$r{^Bu@dCV(c`AURyMCgDnhm3=PKSuEn`Q!D~ z<&KHDi$ViOC3F`oe9dMD83dti2O#6*;YL{mx;Nf<6Ced9U|w*p1r9XCx4->u zSTQIxMJETgiO6AgiD5I!%P+qec{>{g64soXe1#RuwuPJp-7${cE-$RD=3q%gZr{G` zyV#%rZPN1c%3v^rXhW9*&iug-?t;!EBmsB8t!ryl*W7W(oqP72eT-60`NwsjN>H?% zm|Hn=HJ`8}t)Q`^i;MHnPX~TH^plYvk7`L=8^wf9^HTf6HzIXGP+96RrV|W5aHT~# z%hMcJZ?+SSdq_@Bc>o^+05&X?MGNw|URt{5DNniTs;h{T?bY#&h0ylTJ8$P%XSJ7? zSCK#;IB;lgj%`t#U1yxhq-Ztky^R5E`SsUd_nhZE6ZYnjkGu@uAVT9VFRxd6m}6Ao zp?$14rch(igdme%{7q8m5sCZJn6|#t%ogk7A7LW z+2=lDeyPji90c|2uKRI3#E}#(EuD731?NC66Fdcqk)W-B?}aaX(V4q<&2{I{h7Sj^ zl(QxNaw6%D8@St)+;M~M#C)h>4c#VsP8M>8mU9VQ(Psr?hvo(R^0I^3Y9D`7zuX{B zKn>S^HmV%0rK?1{6q-D@6)PtNz z9EZIoG08I8y>w-z_sws96OsZ^93^MPBzxqGn)zZp_DzQ0Vrp2?WrAqLqWY<0y*`#rU-QifdO0} z^O(o})^EN0Lm&F|8E5Q9FZuMRKb;xz@sHp5^{;>Zna_MC^u)pf)>;ZpKw=Y8(9!PQ zyViGlqpzy;qh>8FY>lQIpv7*B{A=d`k$RUm9cDf@Wzea<2I2_4A zM{cT}-YD@SFKk9=SYX_5yy+$!*Dx9#4b#gnx$xcZe)pgM`CsDefHur%#RYSAnE32x zzsRPs*hcTzvB-pC(})gq=MRy#?c{z4;GKw1F^MxWIcfGa8x19aH$Pkns?fX@>@U}9 z{_YJ;aUuqLAwFa_$$DiXTHEA(y=G9|$LPW#dI@sqkf3X8@%qkiDH$;%YJRw>yf z3f#VUAOvc)0|ySg_q~7c%2&RM1rAlIUNOFr$T+zKgb>PGxn699Y&N}K|Gn>h?<-#M z3JALgX1sl8D+Fq+g>rQu#sy~N2!Rx65R_4r>>gur61SKU+D)B55Ou;=P;abqyek`D z#f&bXv;EMG^N5d!S=|e;n+u2vvF)P072FRvhZO150p;nl)^+812L` z`pQ?na^;m55P^H|-o4#!7d-%AKUJ>9t$(l+Afm8Hy$34$lY2w$KCHfB-85YsM=eWKP8zE8V36L;0DkEo3s z8^d$^g90-mB9zzgzz!031Tp5z&)VcL-Lan#v@eX>n8#pCQ#Kd6xhF4&1P9_k5|(e? z`i)s!#3MT8!`Zy#(tL5yL}sVI==9n5Sv$5WL=r?5`b6k?sylRtz=PMp$qdN@IV zvBD_f2bNTlL`b?;zY#LJ?_E9cgTMdpYhEHo9)~{3kvu zitzCRUziA`mz{|h_BrS5SzbQ0cW)1_guZ|$#zLsoLZe!unKDpy z{^1|Kdc_r2AkTQ$yWaJ@=RNP@i!X-MYc;ot=EEU;_?753QV9kEvp}9r7#kr1i<)rR zQ}AJ~#PevDF!(1&otb1*3ieUufN#DUKi>X(BnW^F8ch_7#i7wia()mBw@hlywZ3x8 zKlXa-zxea7Z3NGM!yA6->CbrLM?d<}7rp33mt1nr%{PDPkN^0O|JA?xW#|T|2V*tk z{NW$|;ZOb4Pr+oauJ-2V+vIn$?R)#3!K$CEy9d!j#;90nF1jd@av)IQP%|3!hhZkx zH?9w%>>D{WJ9#|5?%_Yt^KD6>jJl#D`# zV;qmtda#xaZa;MQnj6(RbA(8Y(}ZY{cSXqN6_&*fx#egI&8ID%udKSAcbd2e>4odU~XZKH85evML(EdoTm#1 z@u=Nx2^|m{N!*?|Hl#=BXE`V5Z8Gkd1MRan-L!xI-X!_E&wTz17_q$l*MIfNPrl|o z?|Jv3^}SHZ&1U$8FMR1EANdFb_lG|8!P8GaodxWZpS&@OLY&B$JAeA8fAXa-eG!E|gBX7{#@?BNjI1rX$ z2-(U8sv033CcR#%xWNQ3iR~^Zr!st}XuFiEvai*4i=6b<*08cAE?+W8u*bm619E3$ z!#Vrxv)E36y>T(&#gHqEf_8_<*u&dyyAAbUzu!X+ef8B>HRSeL3jlXOh`-rhC?yca zagq^J)cQDIL4BE`zQ?;EkMWSEh9FxcS?pkIMxb4gMAO;DX(ea}>^4zq5J~{SxboPf zraaW^;apuO=o|YLArQ_Imu<4$)Oc8nEnz?0X^voZui0A15Q#k{*iNG;PdgpcJ*|xp z$G3g^8_kB6+*%9W_fRv0ND=d0I)aMeGHfNLpa1;lpYqfvp{rS0J*2pbE?$Y%Ca}DJ zd}+32{`kt+{^~~w0S1D`D-m_v!1oCyBCP3@5j)%!qaxXGM;f8xIi@RadSkd9i z$0hGzLC(o`Iwk~=N3%%TA96WlX#5^uqPA zjl;7YksjjA*sbL?)cYip&osv%YV2KJT{`Ci$jLwYzy8B_4zU05`a9nK?_coz=RE3B zSIy6Lnyu)-fz>m1%wKrng?}@*@b#sS7!q6ZHttCo#sm6xC`}Z0J zEwrE^^>BLFMkfyrdp+s2=hilQ$k}>XujTtYF24AsXI=(jC%7T3@|-uk;b)V6QTM!7 z8>q(@U3Bi--iCtj;2CEqbU9kocf8|&8&)DWtb$FQMjXFL)>4oP%;;PGjo)JDA zq>Wbl)o*yiC@Vv3^N11Sd$0Z3f1v^HIXB$!aws;;7Of!-G&EuJM4&5}9A6(#+Fpi_ z2Wltq8}YA(sIqn7+&Hm92owm>GpH}nD~@13%UFmzQb|Nw3}2@{MmyTIHk&>ZV1N?D zAizswVatWk@YZyyYFD)=nKU`!O1uBiu6*K4T2a<~cQNdnNGOPDg+M@Du5vlx0NKVJBf6g~Pf; zp6HvTfswE+C3+lW94Z>tEwS{*9);iwpjU`F$T(It#HmEp(g)k=Xb?cY#zr%_2sjo> zEutB#rddq|Sg!Skc}Xr;qU|=xIFqqOB$A}^v?2{{4&%-|TKT8;Gkn`&?V!Nm*lQ9Y z7mc|NdhDdX&N=WyfYt4KtrkSv0nWi3aD|X@^aDYSFot6m0794+OU4a`DE^3-A)hwr zxswgL*nfLyg!~#z1CoW9NRc(S)`&{kZ|jGc%^9EbTI9+14^ueg89r}f3dasbJ`NEw zLjoWG4HF9=;ar&wv$$~4)beQBkC4Iu5h))@=jn-Eg{6sXuU63tIT9HEfp%U<@ z6{dO_JEYZD*Ve!@z#nE3G7gppvl)PI$vEDJjN=Fy2c|l1_-|vb;|AS{`HdPNXDCK*>HP9x~)2_ZafhYgp~Z6vvfhfc!cv*e;PkXO&_PYQo6>CZ(Oa zCl9vPNF;%sT+3n0b}%K2m7LqzVZ0W!TOE2JF4Jb%M0P>s7iM_q?ml-~h(mn9$sddNdK_1v<-&1lb~fx_#XQg=b-N z*=~~M5dVMf^Dm{Q}>YWc+RJ6lWKKHmxfw&p}Sh5Tla=R}=cVC~1}At~zE z)VkUY7d<5Judm1R3-iO_I$eY&5>AGV=OO=iG#K`n5teCz$!sILLo+ZMLd;o`Vry|H z{2WaMG3rxV=%x5Sv$U}QR{qe8ol@ci1`#9B=5FS(ejP8ww<)>fh1?1H4wMK1 zcR#~aMhzky-)mw-kDdZPgp zSQINtZf$Mf@@>i7?tbYb{NFX-?5!^6QJq<8&6BN|&1{qGAzEiX-_yUBBMszpUIRgQ zJ?+E?(}qKtrQVSCG20&&KzOJs$#{Q&kP?BY)+q@_*@d7HMZ;mg5w-9v0k+6B0iz-p zU_}>?ptg&#M47ELX~h+hp7n!vPMYKg4q~nBTE;jvd*J(}USfQBg*GZ2+n% z(Cva%QoI=m2Fe{M2aZUc5l1a8F7^67P>S|^Z?&O@peO9qfS8z~3J>Wk(xs9Ru)*r$ zNl?}j#Bv~8mNT&jP-=!ZFoYY!%rA+fD#;hydf^sx`1j!D zu-@E|vkht5<6JG+f8ap4)Kvt2#yCV+vtA5)EX!e&fN;TDzvtC)d+_9CQWwo_g>BYI zhD&fexCofj7bNs8ARlUwu$^mQP?)LEtcR11lB1Vknyot7+OJHSG;yxs1XzfQg=!hn zy#SCXmD=pjWzd!(szDxMT244XHm({uA!x-LIngM{HzT7>GCtZ#Ohj`st!)H~MBYh= zc@(vBY{+nZ@fxu|gaL5!c>>t@-ZI`-zPN!I(t3B_sv93@FEtXkUF&5IT+#prwKMP{ zx|qnaHD};re#M@VoMt=M!X${78$&b~O%|=(%RE2!g0%Gdb!UXfc99LEPTp><$Jki; zO~M1ulFGW`&?Z=d7SMBE1359%hR{*Cq=9?23ATR71&#NRQ%3;BEC_Hxi%sM@poKV} zuQVtJNQ*it?h4Q5j)2yuWbYiN|5 zFk`(NiV z(!UV+cN`Rc^%DBpnBN)PrSYzK4NMXM8=ytx4%CZT3(FLjWn-k|#!lx@;q|>VtlvX# zuI8t?{=8gdXGwV$U}~0l8@az0g&A8$V}9n;v$(c`HKvS_9pA-jxC}i4yP_fX?`5nE zX@3?qdpj7PgQ#D4LoXk(c#qm0_BkxdjX^U{$(7AFXN3R2S-`_LiQ*zj7J~WpelZ9a zN6j6CX4oJaVolz6bE=ZlK(Ji)>+Iof)3_fRbwyNXh^iw^s@IHPHg?BhgV{nLM$|bB zoVrG`sB8wiUKE7QUvspyB$3ouHNs46xZ@8Uua9a4oHKM9%Zlb$^OJnFjKhT{G3s-- z>`Iq4Lt@K>5|M)2aB7XDXvEoat(}ZA%yb}pkdy;L#IRvEyNFgHNoia13?o*?CwY2FB%>q>vTCFbT>HwwdsJ*ribqc{hgev*Dk$z?&z)6FcR2W$g8aNQfFs zL&6#71IHcV{HV5+>@e$(Ku1o|F83>pnG^b1qur->LKdh|kmb!J?e!MR+R}QkGdrz4 z%JRlq>TL98j4hzJW1yP>ftcM- zC##UQCqxB?i}zJ+jq>sD^>Seg$IiOp&)=OU_3%lLC`qFuLReZbww#;o?wlc$knq< zFjs)WJW6g4eB=#Fmo;w4nvCzhwCQC6wjoA37Cq?tsj!TKNkR82BE9SDi5f<-qa@z7mCtJ8K2A2bO%~J7vPO(DI}iNt3m%(FM}_-w)2!_W)n; z2amFrs78!gO;+`dY#7vkI!%aER|j@8!aoc`L@Mm!1B6RzlE88mJrH0AuW%GjxZ*EQ%&cOd`4!;W~(&h0r{o@t`pEDoPe{OiE+2QAbBw z22FMi&wYkjltO&bbppWjDGKUTl2!vo1{+7dwIiRi2U~Sb570>azS`%obuRK$Uz-&o zq6#s9uFnIkS;s$}s(!__7OG9d(#>M}Ar4zKt|D|)FjI}S1t2p-!Dof*v&|0R!B>ny zjgyY7;ft^eCR=Njct~R{%t)IJ{vq*kwD&q&(=qj825OcM)J{l*WCCI~Av>2Cet%;})+9zci zrR-Q$W4WCrHKQ6+o>e!%#3w;l9Hb+76+q9-X6&)NG1v&{0e%1_aEqmiaUfv@V9&r@ z#-a^IlYk#AZkS=HX+WM8vDP4f$Sl>t?rKPIg&344gMmE8P=I~A(u}^;l-HENqG>Zr zN-`ISQJ@(ZNGh9u?DXYgdw?Z42o@^@h*eOhoxPi1Pkn~Ncr`hy6%B(WkPiqn3h-IP z=7h|#m&Z(Xw)TpTg<(vbT)QO0)0Qau@*)X|5&+P_zCgLG8|sZXv;@B(@GFB}-NhVK z&w%t;qw{!BCIctwl`+2%i!Y%w`sYEG?gweQn~gBT@Y{_+s~*fZVrRf^1${5y$+pgQ zw%mZgEY<}I08%tuXAOQygFiQ;*&<`sMA6gKkfjaD9Lp&$1f5XbGh+f23`Na=u2x96 zJ_aG|Ix;af!=uOPe$Mz|m*(OhBYtm&#Bz&e8%|`%*W~s9I3+}2jG1v|QM{R$^aiO8 zS4rqy-sE zInN_rIaAqPb_p?2n&9%&=I)%9cb2kXF7I^89QvJoPYr=00i3vKq{ZT!WV9KSqTg17 z5YtMebd&&>L+x0bvGOd7zz=<&k!2=U3W8zC!@)4MXGyivZ`SHTM*MYx9l}6C*VToJ zNCsE4TmWIAT>fkFn90iEGKkGyb1@tYnnYqH3L^EyCe<6|=`#JrZg|RS>V#z~z8oon z(4dCnCpcqg*reSk!q7&-YvP{;EyWszI0P0sNAkotQ1$o;Yg_FN;Uq261>e-t%*%3k zA=J=8GMmR1b`??;Nqq!W%3o1($Qs?aj(E5+4BB9DAfFW-i0Prf?!lPNkT!a5)2*ii zL>@GP2G9fo$_X#30VXFMg~%q*=S+A^&u!fE5={c@yLzesqijT_7qP;yM_a&PnI*y9 zOCE%LrSuwkH0QUoARc+6!Eoe4rfLNRqDVYh*`a75*AtdALacj53I{HvR>V0{RQQ}Y zua|=;XQS_8q31+>uRh?t+2FqeJP9^6N|9D@U7!)mR84D~<4K-p{4hw(4YN5{`Ai!E}XMuV_3>QP5 z`N`ekYA+v#FkS>v=mt^HZ20?47E>RRtXI~CWo?A-OfwBzjrl>Ex58-99~qe(=G`<9 zq(Q-&flygYIha=#KlBvv{e8M!1U5!SY?V5~(i^4`b}<=2K}7+4&@5@#h*`SxR>R5a zc^o63_4F`coip-9e@ENuXbXEs_%iRRB}L@XCH8mt~{MxD+!-hx^ZWMzAme+XNl}`z3-f7Zr%#gAWAzy+HGjyA{X|5twYC0>)eIu_u za@qAUKrn_f1Z0V%9=2;%e9Kma_<-?aj4-$}sAzSvG_G4~Xis*CPzs3 zquU-x%*{59k5Iyo;FjHhj4c#zzWHXbDwY1^CqKz1!7}KFmPXsH$P8*4M-37o2aQht zGqnUO{l`=?(?+6b2Ezt-aX0DCIcrcBvm~sfr7Un+2U#Ye7@+Xw6;l_Lh2}n?w^%V* zYGsK7?WD@^-&X6`$JQtbSrmsuc-^dEfIk}U~q5vT(!$S4+(N9MvFpbCmm zXlfn_ZA$#*gB@Y~ZeyAag5z63Mn(K6OufqmjNs8E)vS64;{XrDs9wj~>j0|Y` z0r(2av{kp5I$ADy!h1Z0jME#0z#X9%B5c|rzYJU5)?hYeCX!TSa$(X#edzSTJ?EiZ;C%axm!?bL zY-mO(7QSTVqK_bF5JtImuNHV=;5Co|P&V5qOV8vE!`a~9&F9*FV=n5T{zErsfKE{Y zE1j?Yw{WLtBzU8FsTuRM398T1dXjPhE(RZG3|u6IAloJc8MU5G1SY`-P%;<@bV|f% z2@+!TsVx{qu-GR^%DWpZD3=4V%+c3$ZVS_>^oR3;`6ZSd%-a2gj zVqNCX)2F~E%pC$Nh8P4hA{@7mA~J{PgH83N%><%`el!lGDC3zH)JWP0st23NNWj#9 zzzg(ByR3DCW+*fHcIw3$sy(1MOB$vHDue_oof8ZUr65Rx6!Bh!RE^e~`6LJ#?1io` zG-RC009ER~nUeTpA&3)`qRMJnmPSIDW48XHNb}<>e&kPv>bho4MgWXIZs6Yt*A@kS zoMC04n#_>=VGTM?7WLS)XM+@?bC_pdquF9hb~0DLh;Wxx78I)IHAJ(jQ_v%6RG=lP z2X2-kJ7cFOAeIl4VwlozydSh?#6PX(VuFUX-qFT$VeQVfeRt$*JwKRp!8cIwMau3z zFwA+;A3^#%2gP4ehB|>0>I2uMivco0_O6N`gYj|mVz%nWddJ&IC(e=*1snwsU?F82 zHAT6H>~hfafH?r+uJoZL*Z|&C17M~Eve4%UvltpkKUoA>2(j@{TgZ(@fg@$%V2^&; zmC9SIkeCW4qhI>)Qo$(oQWUj_)Et6+5>XDbg)%Rx*i7~D>J3rqb>|0%_Wt8=_4Zor z5WWjo?O-QTq#>IKv*9rRFtI>Or%|Eo%|gL+6#M}cky_ofT#Dw>)lsVq7ut;-tvMql zg^fQlelVhvs;Z(h9*|BtN{)eUm8a}hhQZdep7l)X{F%>uX7Ap;d-j~kxh3MXJ+{@j z)0>P_u$&aBkhF&75NU%{)^8BiE5K|;%t!`JQLL;ZxtS$pc?lJ+^rqhe-Xv+b#u;?_$1-IArMooVRp3$m=q)(zYNv0)J0ULwc6d?!3 zLL%j*q(E;J5w@BY8f?L!fS)C#4Knti=oN<&cD4?h1Nfkz37J_~JMgKS9Q7oN3kF`R z%XU*xI$qT7h-ehTl7Njw!wjiZ8K!VYERrTE|HXj8SF29|lwWH%T%>~S#;8`T5r}c! zo6Qhol+mDI=;w75W@*4KV$cIYb2m$4j)fe^TCxx3EmBDO-%q`fswZIp2PHd!=r~%S zS;XqjX49C>x;3z}C+W!ysT(sz!CWA7^*KZvb2V)^1Ak-WEw8zwz8CbM^@+`lyuvMd zjskd2pPJem&`En~8Bq>{jw)+)6XLQBGx)&3UuR%%FdTy)Vgk&?eQw#F5~}0F5#B4q zB0f~lx|kU3opJml^F||uRa_PfGlX5}wlUMK&yA1`Fgz65A5MZbiK#~`sVXK1@XuN= zK$QqThwcdOHx!9eas?AIbmL_wyEF8cJJHo@*bFHxu!3`P%%0K#h)02F z>I{sn0nk8N3*+M!jnQn{D{#E64cK1VF~j<6bQ|ocCemOHb;>YDX&QmK(1Y~K(h!Qw zZ`9y9vDk6lLEjBIw?i&W3-A;9g{mWiZ%q1|_y{ZF(aJTJ=$a#ueKDATDu> zkpxm<)H3acR3wd{ z46(=+505Emtq9|Eq(Lv9L5PK-Wk%q-LxRw9LOpVNAd#*nEt+}U49msDSxo8+8P1~f zEm;1dBg|c&O=r@JG5fBufhVq1p-;R-8P4N&n)XIw`w@yFW%1`BjH0%KqUI8frl z*aE)Q=4|lq75xk~Jd~RjLqn5_V^u3@lj&(4g(4{lPXY>aRVo5!pLu}RF2!ay?)q-K zs7Ef^&JObAfHkt@q>bbyZ!dt*<%OaX*euuik^-mvMW?s$ggsC!$qLQ8p1%$Fchh!3A7P1 z@u4djiMzHI^fvBoH`Dp-dowjOcTX=1v;aLR0=y;_LvvLWhvu{SG#L^0i~jPE-$H!} zytP0`OpEW7#{-ysV>%yplZF@7F{Bs{BH6AS7$8lN1q_UA9B93)!;zQedBiz zDwFlP(`kg=pw?=7eILW|gZFgW8`!eK?;)!&s|1uBh*T&%m07tKHeqvd($r&WQjXpw(j^B_G5^)5GdnO%;>II}>`egou$P?|U^jSbv8g-JX*Uvbg8x;e2Pdl~Tm2TQ=F*CX6Gm4EtDz4rV!P{J&!9xQ5dh3+uLaH%8& z+R7k1atx)C}|nB5l}T>Ds1GDE(H}4uNGb0BeU0geYpCcK4A#kA>Y< zN)Gf5o`!9LoRwx%{3OI6f+%lg!&&I8udS_sn2Lm0(5%_sJTlT*Qc~Q@Eca`1vhOy( zF~G93HCT%XKE<$BbwsWLa$1T;U(*k-0a>a^0Am$BtY-8M0LMVDIVzdSMIPksrM{wU zr5iIwM(xzRQCaq7rVU%$>_Jrwnt1+1^$5@tM+YGt@JY2ysZQ;oC}~c#pqtUlIn`9$ z>?8>=v4SU;WRbB^FAiqI&TKNmkxc#2y+OgD%mDfgLg$V`$4R0(>Wd$gSNfQPz4uO0@GBffM(Zhr8Ce+CWrlRtSqrN8fee+U_O z?X}lVLU&rx9oja8)T|d-*~>73W;ucVD%(J;-GVb^%?TFvA8?YG+0W)| zPFt3565v9nias=XF%&SMq^}4J!m3?|^`!u0G1B(wKAHp19h^45ArsFAT7bD=j#$C+ z+?}>Asx`Ecn6(jNrU)_ujt$on$$RQ*pTsX}rf3LD$ywS0EGE9oh{u?&T3XO0q&R5h zL306v&I23Lv7%|rW?Q(;Y>-Nw8p-EJ$`RJF4pK3~?NVWR%$Wv4SAiqYEodiG&7JYM z*5p@T071l8vVN7~Q5+jaW3@9^?wDyjI5@WmHBuazSl~lh_(>AGyUw_`RzkgEN*t8= z0^0XDTQPgCPB#_=|6bWI<9spdHgE{Qyaywx{z#e_sB1$mQ(MDfgPm28lvpvOO1q~e z6SV^E9V7#~sMFc8(N~;192K!^CWN>N3pzuK5pprgC<8uskgXK7F1%U5QUxt7Zrap@%E9mR-?XgriOaArnMpthBg z)3Sk^Hhh6vj?WEivwYK>T2u???mK|ekQi1ny1+_^(K!e8J4-?WFCQvKE$b!jpwJ57 z7emBVuVKv0j~05eMLGKMYq{@ zaNnOL#V&i;;D3g93xtBc&0&n4M$iuTK_kf=3K_nw$5sNLd zPb>5c$;F3(*I5Pp{zB(a4H-M|%e#qb8YGSj&ZZ-9^v8MGyL%4)U+9_+GJPRU0uz)l zNhp7ju0xCBH3>b3Rj5=z%=1|iML|ddc_N{tN!FTMVmf~BuB^4-BXIy0By!XcHUKg> zV}=3{s#@{|Vvf%y1IeCDGNNhR;V4NMD2u&TjfjyEz)ayw?w#wT%^%$6z<<^grw$rv zI=Ejb_`)oyx29n<*Uh8G!N5hxfq*;Hs*Lmv&lky7M8-=TAFwPoc$twen2WzP)z&IX zVx7i_7#&5^4_h@VgNT;te&hkFK+q;VSY1y#nH&|hR2pHo*c;B)d*8l&d-m+HWE|_t z1gVajhA9YfRz(7P4!i~0X>_Sz8?PC4nq?CWC_uwVw;5^Ww;em>QNj)*c@Yr-J~kLs z`W@`=MMi<&<#a*8j8rpX9}Lt-zJCkxMsWS!u}3QogA7Atk3VSGkhRLu3SUhKym3+C&096=YmFPZ{+MoGQd;rPkZc%aHgS@ z!)W!K)xkzvL;{IIEtXIzx}l@cs4A(Vl_-Z#`9WQ9-~14tV05YTeDOR03Q;)aQLQ!< zm6mYwR@N#I(i(Ztg7M;~N5CJbu_#i6+$?Dgg13S45Z|{vtivR@9ofq%3&t?NstYVy zgpKeNQ-k{?k#V9fkxl1K+yTT241;cMM^|Hnz_)bj0uY)O^5#v-<9#Y%#y+QW+7*2Q zd9~dDQWXF)7V@kd>SFUdT`_=Y11p))czduaoHc8$MW@lsy&Y3R-u4lz7`!W!T#7A0 z1o@BVdJ~U6Qw3@oO3`dou$Wkyo6{ePMHEkX9Wy|%@=&tG|k8o7Fw_(2<~Gz0L}DS@Q<^sZU-Mipt{bIzf;>>i_5u`qIR8dwU{s{wmh%eWxy+*WCvM3ij{@aJqE>;J9U+1HMKlwFs+xsh|J2Cg&LCMFu{Gd9t(@C$VrSlJ@%(R_m85*6Tvt&l)upMv*Tto7BjNBQ*I*{H>X-LzO2b%%&Cz?)oo z&QWp{&dAriuQ#Sku=0rX_^3wwB)dAB`~_GlY6hNivOqISdUb9fKfXb8@$;2MAB&u9 z$55xB@TNjm25@wkZ+&VpgjtVG(+D~(9}WvY+a~+EJ?9<0Yk%gJB3}v385o<{!PtZ# z!+Zc6d^6PTv#E~06Nk0{+l7>eyx53|006~OH8i*Zeiz0jA;1E9mP8mK^c{SGjDfs$ zVmsJE*$!_5{;JjH57=i65gG?C zyb@`?84t=JWR$4Av%??RfOz2)_CpqEVxpN)4IzAfG*i?9avh3-!lpXlPX>_j6;+Rc zBxIN}hZP^M3*^+IieJdvw*J%O9|4<%fNWU>JpD8XbVtpkM-a6U6OnR&1XyT^+EOq{ zN8VQA3-y$_FTpb>{1_axM!hmi(ucH71!~c?m4RulQ1h0o>?E<-Pkl5_BUM#xUR85U zE2`YLRvAsRI9QbYfJBy9inS>jrkT|NVNGFtk(NmeL#v?;cxQU}$MrUO@u^MZtQiDd zEZ)uD!;V7Wt;U`zq5|NyWIt;TO%+7I0WSkV^4{7Qo1u<2bo1nzZn|Gjk;loE z5WE2set586ef8DvdCz-j!xdLtfu5MEi*y?&^@wJ^y#_LXZv2$L07*ywYxbfEfL+w9 z_Lb?61h2FoF&)9}TvUmYO+dz2{VmfF)Ig;g~u&W^u#{q zsKISNp8lu|Krl!)pxTYDSsAj1kdvxv=~=yHk}|d>t1y#PAEuTBs6z8x*%u7V?fV)< zb}Sw1NLJ(Q)gY6H+GG(aw>aMfyeel~Zwj8`Kno{mrFvaeH2IpETk3A}k!{7eR^wMG z6zp!16jibdh#~C_y{lT1raJPKO?sREWIMj24&A&kcdM7|tLAOf?Yw9tFnX*d+dN@4 z6snyP^0tpmo$WMaRE(RT+)?DJ7k~%rCvySQ1Qlj`OJ`FhKxXQA|8^bwV?MK^K`o_* zmC%D)0U}csFbP6;tO3v z)yF*NysGeFuOIz_R+&5`WijNdEmVXGi7Jh=)B>;i%{=~rIC+XTh8s~Eb&jEn5wy}9 zEDE+TQxho2s!b}GCBq!KRr$>5d)+j3HGgM=KaQ)+88weeQ3P$8z)dR@H-zZyL@;Fw z3v6P%s&cA@#U#}Yn~7>Wds0gc-a=MO62)fH81$xx4DccG?L%AYl*dUvRlZyPY|Wv% zE$t>H@&pV3UZl&^e+KOs;~K$D4-5p9!7SrHHYJ+OHM)>sYM84p#Fng}4skd%uxAk=<=F0*6C0Q0er>XpoE9si-WM?S zh6vw^VbUcO0Kn#iQDzo?&Wtu{>LYS-LD)z`kj#&vtvo5y(B2vpT$-cw^1L(c5j?GG z@keTO^C^g9G8w8{CTEH$oW3DuY$cc+3?@{M{KFNB=VsJOIK+&xxOKuCD5ois3_omA z12>axUvms#NpUJ~zVX0=vrjohvD$JCHF8M@?en<6+(O0H_QwNQJ@ zzQM72b-dnhC93SKGw>G3W=w`?CVRf;XEj)~S=F;`=0aoy9f}0Jq8?`|LtSuxP}>a5 z-K{{VmP-)Rcs(~yZQaO3-MN{S`U_4$FHe26l*DHJtK1BH(d3#3w%rz>O;Okaj0$F0 zrZQFH7S>9#yH}ExA;1%@Gb{YTr1Th+h`u#Ak<8Ub{_vx6nm)>1m;9`UyULp?p&+pn zVBE}Od_^rt)yS_Z!2DySXXNt$D^>22B}Z(@;U~|W8uGS}_^?XjWl)!tYiq~^Hz&Mi z>Vn3TYks!lwYnRCFc9>fX`ETcY`#L3vm^*2oA}T!+{k{(Lp?I7sjWcuruzCQg^bBB z3nGrDhErDwnq5l8af##FZT+4KFg_upIiZS+Gr?a>fij0&}tKMelYN!YtqW|=Kf^WB0`N7ZE<(OIFb_${JYV08uqd!c~5@W^) z#gq$Jf!8QkZ|a7sLAj>-736p%$-Qb-y=2a&I{;MMSdR9@erM0sLP}&t@H3VX&S^8j zAJ82(LIf@{93RPC16y|tLYDYbFYu>okD5*`xUHeC!6;;8>3VL++G>^r{-I?Sm^$(~ zKNV_{1%he2jBY3tp_EO!q*mjaS9kei#(}*gE1n-$((-dsB{uq{KB!Rfec~h{u%U}X zO+48Kds16C#fnXrH%ryG085`)N=Th#!2|?3!+wZ|V{;7yXiH%n^=M}Sqrse*uT+BS zVdtXGl*}zNutiicg{C`fy&8WO6HbiC|6cf zI@5J5c2`DIr3&NU?_IlEn44Qs&$ymEs3wli7`qnF3EBp2nWwrBOsWVzM}>Js`I&cQ zTk>E@<=Z+`HKB-crwDd-7TKe{G+?w}@ri*iS2HAr62t`G8l=gGRmo&kNqy}t3T3UE z6g_PTIMnLZD$^||4_4LjoBUO;S6>*&n>$s%Z25d@$g3CNf5s&!|H-*!KJ2tJc~i1& zEE+(?9H8~)b9Y-;gxTaXV<4A|M zaq0{xY!kv(BMe2NPc3k?lFS5u^SRnbvVX{k6MNipyLzX3i7QIxCu9NS8MU?o zJh#lR0X%gce`m5f_QAG?_rM5JJA-g04d&J8sksF+!ygjc zAYX1w?r|cCn(|aWX8O$50qA(gLw@`rbx{A3p^73;t)Ns+Jf;_$e&<5>#!2t#4LNF% zI44g<##4J{BMDGsDya@t8Rm36tPI^_l+L_Iy(nt7@ecvWS2WeMnz5@KI5#!dIfiZ= zS^M;d2oGAG>`&J=Q6*KAZ%jsf&mCLO@hj8*)s_u5S!kIS85PkMPRUeG^Fk%k0!p1E zJASPH1~*wnOpdBHC-<-U?4!*Lj(+A6+S}&1rRT>zGoC-@kr}THEAjM{G4)YX-U9kQwlap3`!{%HA+lL)^v+kxMbK60~rCC2CHr9Z0+SUrA;zroQZ|A z9dMRV=1tYl)ZctGKJW=MjiVYu+Ct=!i)D-KlVcwz%q6c@*_n@QHLKdHEPkpY?m_@c zC01{n`+8gVxF89sb778l0Wr6yI#d|Jd|NHkWW>m_x!Y`cO6jHo<6f95bG<|kHjTTW zx)Q=7N5Pxb#lFQYnYoOUEw;KcZ&Z+<uLu9=r$g|#Ir%b32vYJ}ap>}xdIIfu`P3%=1O&TmA zk~*Sl5V9OlhfL+QZwzXT)e4f2)?7qU6k^k#B!gK%F`kRn9p6w;(SKDk=|oCi0FvZCyF4JT2U& zz15^(U^CM!c}2C%$e>0td*?-=5fY2tj16unSG!E`xyiMvhU#WAHAfC+1}Y}6dSRTX zrMgV@F?F*<5|{RKli~Q?Y5tZn6Qp1jO5P+^Dl_f04wBGHXeWCyGkOUh8FPEugAu+c zXWL49V(V(N;EriGYXYe(Z*l2s>|FBk*-401GiT7L)XrYmrf zszvo}?Z#zHY9hds#x%C*In$KNE@1rr)PGMpN{-0ng#1OvilH$UatsFoX^|%JNABM= z3n&9W)4}7OWu{G_R*iTIEUmvhnew=q316v;Og|k~39{A7YQ~&wDR^jS&NN3;XIqH$ zaApvweh?(eQi-GTv=>YWAt}2yaGMYG&=zTs3s3o+K}M@Nw@=M0(;xWGB-&+HWOG9Y zQyESSD7G#kdcV3aYTVY=WTGmV*Qt->|D4+wr#b3XlKFm2g@8Cowqdp$d(>?J!xzwM z)feM$xyef=GtkQT>E=tyajG==WOBl1oZH+@0Va1!5fJ32d0fd>*nFzi85rus3Rfu5 z4gvFmJ@C|a7VjG1*vjaW@muDEgN48}M}28NFcqjEsGF0_HQ11In-cYah^g`{w(qO) zj1oiKuxQEm0lt|7J;Y0sis{2C6IvwRG|RIQ_>3!}=G!bFUjPn2UffEx+lCzDIL>a$ z+KgXgeWJ7p5XSJa4diI6W-{lBztbOW5z5JaKF&nlkRB08I@s{RqX7#}Fzxf)UA6G3lb2l1KQBtg<^t-Baltf{( z;veG5BiyRu&671tE{;U_G>_L8+F&BI0S!Q)g;;ksD*=4CXKKb>Ga=+|a`cLBet!S6 z3B1VFo~t-1nhbSm{#wkh6;P;>qLtL~CRrdDdTxRyO?{2Qw>cZXdhEZ@tK>%!Hhc56 zuB_ZT zL_Tx|Xp_Fv9jjZDy~s?7%^O6ajQf?(ErO(~Tc)bXU-?!|*izl26IpCHOW-M!n3+_*q!MGNQgQIy#x zz$djG7ey&H0Hy&5bOmsyQSj+R{lckC70tBk0B%|)0-^nI%A>Y-Wf0MA#_ov$LEeD4 z5{Uo@Md%eBqaPzzd#{+KWV)lWn6OXv$5MR8Vl+@4d%D+ zs0Y>327HKV3NS|sNMITXnUwUDV7-9X1n^)_I@4LIz{H?U^p)r;RmsdAva&sCUel}7 zCdtCd#msIy3`|DI5x_Q7L&KiDOhJ?(ii#KP&orQV1hja^RJAJFzN4Enq*^ITi1tCS zn(TE@1*|u#PJ|n|W2#A+v*91;AdMnhS+XG6qdBFIR2FG68nvU7842c&p>ORc%7#wV z*QO(oxd=Vz;~CP(0G>kmSvuH&U7Qj>GUrF0MxaA?0~1jWn`u&BHJqx}7|`i0qZ!|P zefqN*mzn7^{vqD9@#KI3G#qAg{7|#rPbLRpN8)%Gn5sf8SU6U_d{|w~hZX#$IY(Ec zRhoIrk(ol;lCyHq{4@eNGu3UL=Dn#7ldfe1@a?4T+&6l(Die!+w&sYmaxA1`b;Poo z))I^nrIM}Xks81C`y-#*@@*r=gU#Hl+GYQ6XDi^%dG!L>^0a#4e($MB``Yx!3O=^2 zwq@k9we&5QhWXch!&X^V)`!dVDkW!XQr05ediE0z8s z(~$v4Sw~L1&X2^H9Ah-KT@_)ya%tTLy8p(^52$1tA zy#4wdoCPF5j&@ZNjq~L#Zt=-;0U=)`{bu=v6oj}qXqxDtBe}|?odNNd>`S1kk-t;8P`I^!F zww@?vZN7?PLo}r(jZ+dJ0r|sBE=acMoii%hBLl-vd@?iQYvrqj#v^hp(aAYDT?TQJ ztM)##UUws8MWDj}+^a!&tC`m<@y1@s|Mi^kVt@YCD&w@G*@vuoo@Om38sSg6OdWb- zrOf-zJC*@25w%Y2-T3t6^seaks8%l3MCW=j_hL#r`2i&($-(uOaN}14nTF+6rc=-L z7t?uW_rA}y;!3SQz0B$PaaFdX$;6urZnT}mWv5Avyz+v)cui$O zwUAheTo|rua%vS!OR8zBQt*6KRPlcG`W>L$?(i`gj` zHM+-xX3Tp(Vigk*HPqB_cIq1GDsONn)wQfm^L}KR)s$B@Z`hYDRdJh>rq*dURIf*p zN#!m;*Vue;aSW5spV!-l9-v%Umwc1rI^yQY@y7=1$C0NJCw|IElK=zkcv^6#?dmT9Km;X zJE`forN31Mfs_+D#O4Ww!|UCxfk~~aDz2PqYAG_Ohj*3h=a1s0!VuCl6L3tff@U5y zj7^Bj{0giv31f*+M`uW`$^wB1uG%7`)2yOyE208Rta=frGv9oy(D-TXWW37Ws@>){ z&$SO{R2^4@#M;6h)~>Ex1@-%?bO znaRKlKr#mNc;q-!#=8m2tg|q+YOSx&52gE;Y#m)~sc(9?COkAIwLw6f2lueCxt`pX z4*0<&HBmi)db@~y%8XWjXoOqjzj3wVj4*VQ-C{tx_57($n|H%L8x4Of1}(UZa$pf# zoUpD@o3O4F-kW6yQ#5Fp?+Er>#>NmmOslFqD}+k<1z|bc-dWmet!3RK@J+$;@G}@) zQ7&eW#O2Xg*{0>z$g?=sn?_x|*0y!LXPpLhId;4%)o5yvMU>3hFP3GG1rk3{8yOT` zD#Hv^RyPNp4jEVLV~(^+oqLmNF9wtCg)+x6W0j}!M65<~LFRDcj$Jn@ z>AIH6K|QG|-8NNCvd`3_L)10uWTW_6cn(y`73$ps2on-|9kt9n7u7ir*Dsi*Bz|&O zDeCKhdM;JfROdSsT~X4EF-yd9xTK5oSF|ue9%WnWfYfTY7Fr)!QwS9wc`adJoMHIN z6}lcH2z>3-0Zn6jL{PCgnZ=skP4`oN)H2eJ*^u34OTB{FhfTf*U4cp54z`Xocef)p9;PlPq zf=PiIjt4`XF46jLlt9(?M`$<6hW~ZMb#IxHapYI|g#OsNjeEqZCc`flcnH}uZnOa= zTnr4g>=VorvY!*{T0z)9)31)04}TOVRqsDgL77x_9Q>HHZFM7Y8jsBR5r|{7S6OMs z&XTGrsE!KHsD`m;b?HQl#=vcqdPHrwg5hepOncR9U9#I~HnC{>dVfSlMs}^c^lO-= zjuNZ9y!v*ftN8UEgG%3vqJ&+)?x)nxNPU~_cTbWh+|ErpW{hf(ref zvM*a%aEQRJMCPHs@$Tlu>$wz4u#Htw6s2ZK=8*F0YIbZ&nyOh<*yJOW)rTs?Obj-a z+4=brAvOrO??ED@sdn^|1@fW_p+WTceevAoP~TvnY>zRImF;!*vmT@J9Oe1osO;uY zGrF$ZZRF6<>P~%iT5QP>6US71d<{C*E4(7M({TKp2msM&cwxO0^d#-PQ!nW5>9zWkzPO-^h(H9!5t{ zX(OX7q5o78QU7Lli}nfXP5OC$)6OBPGnb)cl^zDBjjUY~7E&LrQJae34`v`;GwV|W z!hA0T4knXIPC7Tl=P(dW4GiDMa~bxG}-s+>gq@Op?!D(c1?>F!?DDB5^ajRtrL;Q z$0EO4(YI=0D}>nDQbibPl~+^lx_s=Ub@VX7+jR8xa=SBIiAA5sL zDXS<~R6y_gktl=o$L|YOvdCnv-1>afb2D17F(TXN!1uu*V!0!#Z)A#sOtyg3j*WC( z&koJv_;+SG2J5?6j6?ywuufDs`)5p8^rPyhHWrf;V+R&USSe)wFc_m#zSu$dvii&_;^TTku?UNrITwu89j@m9jyR zYFtH|k_rm3`>t@w;?{E{h#iiq*BFl$G!qkKhJ7_oiufh@&hxNm)PtvOF1MfQ5u#3> z>yoQ$XB854<&xcLql1L{@iR*T^oBq4z#a zuVNYaXn7pRtkv~?dSP1hCa!r$-ZJCN}e0 zL9{@K6Nf9bau?O@=3iia2HI@v6s5qPe3u*{%*2YwXEl3#)aAmJSc%C+7kzTeLxduP05S+65&d6{);m3P zs~L0mNZNu$?OKf*`Qz*QjiN}&wqC6oMAfum`Pi(eIRoFA707Y*Dku~Y6|fle;S@w^ z5jqzG$0)U?1*F;>7fPql>Egd2dgf`O{Ttn3IXzPBVD`TaDB_Y0$=^1_&|UEvB+%Jj zMNxwZ2o75ruebi>)WuCPJ2KR=+j>FH<+fjj%2$6$^k~@>4kUdojLl?ahp`GNt(e^m zpUUAeAb0%B5_M}Yzb2|ClUBN444qF*nUj>qt_xLw20uqTeY4EtH zqVO6y8^?PuX=Wz39RJo>9DV*o()A+fyBIs5qP0lnr)Xrft5~oe-EW;r#v{mSDElp6 z_v_mgk7fQS@-5iQoeJ)0AegaaB&OfFL>Gxd2hITHx{839c!6Po9lFLMt1Gk|E*==! z4-s5dG&In$kfgcXSP#cBT~3$wN5++q-fiC-iUod*uX&le$-G`g6D#kK__EP_2J+`5 zlOXa%%!?UF(Q}nX^wY}32GrDSL0^M@C){Xc>PrTACtPg}jS?o% z%VGiS(d)y>*R^}E&32dBOfH`wpYw5bG{YxbDIu%nZe!*V$Bxg#mix}f=f4|;T;2|Y z-p^P(o(cA`)u)x6$3Zj&+omZu%K^dHjU9rngI~V^VP^|`;_Hs*?SFju%%&pZcpH~@ zBYB@&H?^J@HEns`*By~)6kOu1b{nbp9oKOjzX*WC^?ZG}P9(^nyPo zO63+h5<1mN3{{>7?P$~m?go=mQZRy{kuuorG7H<*tuI`ON14 z;##-dqARI8djPYKt=IR58Lp_Q(Z6wkkXc1pZZ?w%R0Oxn>1wH*jyQ|WLH9F1VCV5A zS@`yJQI!~t1^C>pnKhfX+4E_)(wa}|`MqK!Q2>7m7)d@@K80d^zJ+4x`<#`eVOQs0 ztkmUA7I+=R`g^^dSxCgv`MzdDVoY z*mExw%j@xUu2v7P{x|aunQwuT-*~?sAzENE$=O1)uE3us7ibtLa}XnK`@{{0-UGk_ zk$}@sGW^?E(ff$a7pwIm&4z6gUGEuP+xt-<;C_8k!(rfaS7N`_>3%w!yREtbeAHK} zj(tMA_cK9ebqU9&BRH`zvnPX|w%-)niXk0zGl)dv!%VNF`{(-uL(ko?4sGPq3L%W= zA3$$*i`MUOG~wZR$~{8H({m^oiT`DiWoGxi!DNI)qD&sY4+{B6n%(Ph15iOVMz~P% zlo%g(ovZ+NBXiv;KFd^6p;n!P09_Z8Q_+yVdOJfsK+#@A>-w^5RZ~-gSXNUl9?|=| zYN8N>$B~Q9a!^TaQ};5Jg3s*&L_S|Ft@~Mujp;GPp$fvrrek2`HNGAX+#%e!&IX+Y z?0GJ+;wWAtSJmiE=VA)~0%Yo)s%lz=$(>mJ-o2l<+zGpd7^bqhoEPLc68vx=+kA$A zW%}E3?+@VT0s_6fh^<`{cAiKk!|?mksm|jOu#Q2(ZSHZxpg7s}AyejLRzep2sXCm} zs#SArE!H?3ht~_-;Gj7w2q#!~+puG!W_jl>?!$`xBdUuKxp|J3g5o?UhNa?#&)sT4 z2)y4xVEb51(6x?p4v+iwW{C_|M*-1o@Js?K zzN~w1|m%u;zqhGjt;fVZc!Zn^4E3q0!MdrI1|Vc@ujdA%!tBa zQsf&A?kLn3HPmejs>TGPAvA+F4coOVM1<_hT=#UQmPE$~!zlT09a{Ad-IN1@#;ece zEQd7un#H8)T()rEDHd&=rZ>Q6yx^?1=dFz*=dqiB&x@Vy*eZ&^a?|kh9a8>zDl-VK z99O^(tkA0%Yw=?yn@eV2fj?uKd3UZqNEO(Ux!%iK7SaB|-cO~i@Hj0PshBAG#r zz~dXo_m?!r`Iy#J5CIR6vi;oG@V#g4W9)Wh(`B%Ei3 zY;rA>%I9&J_(}allg}@a88@Fym&)k+_}eZPjR_cvQsjB;Keui=UASFw&(f}gb&@zk zAOb?{pB2)Lzz_j$`Db&%de8ek4a^{J1bI6e#o*f!!_21Vi5{}q^`_^;2}9ud?HfMQPh2x6f zwPn{@-5(k@dPeG-*Tbl#VF>|Zhh>piDtRy=-_gi@ZqLeC09j@OU7x$q--eL(Qz4YckSPHROCeNd_NLvN!6o?RjG0bw=iW)zVg$ zMDA<*Ee)*1x~}v59=LUtFPG)k2e`zoWE_MO`W*reEfblq3wib#0{#v9(DiW2BsO6+ znAeEz^w8$n+EkKr)W~vLoC3Ivp zFc#TM!*HB?7ej0XLCVDF0FhtpEBog#L#aaE&kZ~GJ&yREkH6TbINN=4ve)x28nOC* zp2tc#Lvzn9OV%qG4&1&TCY~6Z7;4bca`V2 zYZeXuF0Q+QCpqLuQ`1=8ABxW6P^dMK*SZ=y$IYrKoFQA3jBy!=ZG_}prq(@2utN;N zc6>Kw?JMBrscDH!RnLb2%*t8Z_)fAP-eF&;2+O@#f9{hm51-E6gk=13lm=y{ffsY6 zwI}4T5871pIh#M^cd0;~+-SACx~8xaQ%NYbIiAwV#{g`=_v(#y6lr=C01oC%g%MS! z&B{>0R1#0_qt2jfNG0jSDjF0ZIlh=X`2IBgwO+1nx84lKd`m-|XI`^J9Jy9&ABngpEv};9x#~MSQ$RJLxz0hLgEVkR7(eEu}I;+|W9+wqtxB*}p!)>`cp^Y0#5>4&o(?{{TDrg2%oWd3}; zMIoPUxg1HdZd13#SXfnR9-6M%{&%JK^C<-F*Io!EJ-d;0i;(nfKeUQV0+cL6*X7C8cvNEw~@j&vz_;24jO7|&WB{D^NKQDSRd9` zv?7I@d7hn*Yxjl~;HluZ?-WCH2NoXmOQ&`u{=O1wJ{;A|L%Qh#L)X*T2aDo zC!0a`D?F|=p+wJJ?x3LkebE#ni`fkBVInh2sYrZoRFN4fF|5wzru+GL!bsV+aZJ2A zlO%Fki?PoZCE4hh}QaFK^<@5Cz=RZKA_2X`Y@PY58 z0-9~GkhZ4w#pP{se7u=J43`TA*-4e%GK<4ud$&J0j^|o4u2lXb*;|SVpT`ZM6xGwd zC)vK;bCPL9^V`cH+RW&l&d&M+keVgh{=Hn2WcPV-`~=UHq*CDA3%TCn_H6sCwT#wE zug0PKYJ=*5g}woWzU2A24~-p+FT0{5mN_OMM0Qm_(Atc#sp)~O;1C`}tx!Zf2;^B? zRT>PDOE~hQOvZySDpvm`iv3XFV*9c=mKV!pS$3^V4)Nr@f7fleU%-)gksEdqkjLr= z6C?=Mdi@>bmk)mNoCAX1!DU0q&0S@d=e3W`omP-MzsGoPV(_?}NM}u5@5cmAXA7PH z-)ngPZ+b4TfBVDrJ^GKd`$>uIf8X@*2$5I9V@|RNbM8kBct4-f57d2v6^(P)t!vMW z?~g!@!A(dHuyUNifK6pIy-f4mEO!3N*)W;5-{Sr_*Eo^(0o0kFFIF68^Mw@nUrAR- zVsL`LTV($Xlfbq4OY}tC(gLIixjyZDgWf-ZpO)uq^?nohy|l%Ce|e8U$S-^??_JUN zHZo)%6vwkeuMEI_d0bF1DzT679wmmB<~905YR_KSY1khGWK;2nJ)dtEkK6J*SItA1 zS@L=O(Lct2TsEBKdVeGA83W3KfVQQO&wlM{%tl+#bc8Y2?w~w4>(%&fFuSnmFa#WC z2tsWF2auw$${Mh#EG)i<$p`GjKbwZZh~Ib}cY5EC7;rsS^xWsffcQTTWa(h^e`;Zi z+YNg4w>uxEz02Y0_B}rTwjs6uUB~19cgf;$Zv>o=k;p`E%n+Ya6ge(HW=5x_yj-hS z1&c-=5QJIi`FGpf{WV{$-L81@O|EU8^$~B=5@FM6%awyf6u8xo#C3?wFF@kBq^1F? zsP5Fq2;&79+n{l8^~-T=LA>?{L!-`&5O`lUFZi(@=RCI;-KJ+C^Vi3+ukQluTkp4< z-Set52fsjUVkA7jm25KiU_^PHH|CxxC~+Wpvt`0@vJs?dESuRQ;CNhF-_zo-(*x{~ z*_yWP>oYBjoKT;|C!msxr6=@zF{)Cp`30RJtBL+w$oDIgg%0qQgI@Q>vil}xO_)ppenV`B2QU-i;h^gGnvgg z{U9DBzXk|D9Y3D_WeI0-yIT9P9tFYEw|~9~WGn|0&tC*R@Awl|egaQX(&DybMFWaKwa)yZ_r!NnyS!RoixXKqjNkX|)DLsAj#2v9*J!)oh+W4) zLtouSM5f^^p1B2tCA_~Pn3G@T{?2_(1=kYUKgkxQEy!q3^H5f_Y%**2idj6EcU}O{=qb9V~e-%VDi+a0R0RiU5Z9mEVheb~-M&@OJYmc!E^&Dj#NBF#*J%b6` zW;@5NWYG=fd_o&|8-|B)wN~`p?oQ#@Mid*0#loM%cD;r4>2smC%=dV{2EEH;D?n0X zlK(Nk%F1QV?S9@l7dC)ybO}$H6x`+8aG2+JfT-s`i%ibEcUa3fda+mmze5+teLEGW z>-So+Nzu0BXJJ;7@%ZlrrmpM0af*GTORZ}8@wiaT#oe0)(x(9)i=KMyVp)y`z#l|u znw8bQ&s#H#tssf`PNN0KXcV#@GYekb#%gdm7(w|0!MqPLNfdH#z(QF;I_NRj0livp z`L|Rz?FZO8%l%=_7m=M6iyoh#nLLxte(AYUZubzvx~eQM*Kc-&-F%P5XZg1H+39*E zM8A#Cb2g0kNn8JFo+bC{fm_JIU=^k5LdLG&)&AYS+v{mK)qHXbNFksRe;o=2Z?tH! zRH4t8YwXY}PXXoJj`K7Z7TsgMvFYbH00SN=$_VeT2JC-dv187VrRmQ0=#B7q|7)>Z z_$+`XBnI3k&(2Ek>waIqmFsj1f0SfELmw+ZvKC1F{5dv4sL*ln%iF%^wH1i8;YASw zPG|Gh>^QD2dypq?SL;)%_z}Ojjx{e3)tyH0%x{6k5ra@KLEt^pZAAx_2tx&pa-Q{_N6A&G*y2`s2Tf7 zIRelivUia>ka%y24ts9@$jH@s&v8c41ACIX^ke(EM*yg$?~^_A_>ZFnd5)g zzXe3^d;Sr4o#Zen@_oMC0JQ_pO&7~25G^>;iQ|lqC$&IMVmUsCXZ%0mUyz-h85H=Q zvT;BV|06Lk=?4BES2zA&|LWN^{sGPxn_YB~vxi;t11}vJc_e`*4YFUKP$vUVT>YzS zd4r1bc$q|xFL!9kl$RK$#*dngBqsovYB@sG&~s($Oh+d5x)pdkHJQ9Z7BrK~`Wwle zl&RHZen8p$z-xLmE-Wmz#jN;5L)xigPH{c6N(sdW|p? z6(f0Z{BNsyUp^-{{{A_zNsWz8ElU=lpgRO!r$&O_2eIxDmRUd$y9F>~0jP`H<9UX~ z=lJ^)+sb?Y?#TcW1#n_tNZ`9ROTvdM9s=o^!(8DJ|34p@ya?q!+qK3Jh`osEoDFEC zg@-q3)|UD}qFDj--{d0L*s(Qxqu{ZZSeiiXs}b{&bCVr#dW&SHrmZb5k>mGZ4SKvJL$UV`N#WuD2W;dI7%iB`Q-JP%5PMhxJwI=@vLU z9m{_%oz4|;Y%QLbV_tS`7BzS8M@1v58F(J0*wl7}iBr;-VhOLYU2e41%?%VwsUZA_ znQ4d4WS7eYr|z$}I-blx@NZUy5gQ&cB(+nt;dFYxlSocWYXhL2^l<;-ccd2kt;yY) zfc(+V+%+Xa6N%8o2FEX_t&d9!fenv?Sj67N@U}RO;NE@`*l)aa+@CK+u1?12u_$fW zJ$pf~@HZVNd#?{2*_f%xtgK!E0sETX#A8yPS-Lt8-D-b|x9Q2i^`gZ}}As-D+b$+FUrhh!C-v_d|(I9QQItu*qHkm)@L=<;lyb400m**#eS zm;+)Ol*HLtS>7}k1#~S@51it^A~{T2#c2}GKF_I?r)1Lm?@!I-_#=0Jvagqs+NE@%&Ww4eI<;cb;{ zi%1Xro5vs%>5+7lGrsC9;JQvJWI+jdc&qg27!Y2{i*l@z(FHNkM*?~N# z2WZv+k>St?bDb_@%U|J$zh7&d<0jpWji*Caq?_<^!-)=JWI#}{h6#n1++@qU(H zj9!sKjQZCZ8We?qF#=Ay-3(49n3)_w66kretq$|Py3ZcM|1jwsEGXH#X1*h6Ao$?x zcD|Sz7qTpz09+})jlBQ$kO1*;F#r?9i$US)!Q(&(nIg z^&1*B{G2?v{09fg4`(w_U9>K!sbZytxUclbA}CFD>jcZwsk{6oTnVDrH|LJ^!+)zw=O35hX7{{Y zww5JhufONbK4P(cVvsQ)n^cN=JTn6#9(ulN^VcEdZtNNZyWd$Yb zHK1#oc7;)8N!LUjZVUJiicxmM+G14>Txk1RC8DU8}>G)7#%Gk#88%W%(@-olNuPBw6;{&7HvXeEyGE z#zi*!)-`jVvtn-A;Oc@zuJnowFk@17aA8Q5O^^{))kHx8-r_%Gf|mRgd|soTQ8jJw zkqhS;iwmUu{UC!v;Xobif3@|ytnz$AT6(!@AeWUpplSNev(0|1Ts3xo2LlH#`6E0h zMsw0K!gQh&Z${92^QA$h)GvfyrOY=*oIjQC6?*3(oPKG7dI*7l8K&eiH=@0h5MLh% z3!uW!4JHwnk^^i)0g{Qpp!d`26qA)aTsM3^enGeR&W;vgnzt>`{b z@DWZwCnr87>yEM;i~2oj#30>azq6s8oQ6rnth$Zhw*mcv*m~$~MVgkS;nTCB^fiWcdUSL_X5Zkl_7xiDw6gDT>Y?tixIV=*udM0(zTCV8Q^oP%d}#RdA+Us_ z#O_=%AW$wU!yf=I4d#;T^}-sE+%2>gg^P#OqDpoj7yjnduQ3iaZ&N8KG%%`ydt=;t zUrUb0PJSgd!6{S>*+I2M8O18B>P*tO!=H!52&=+hK7L6?GP_4&O;l(&}Er^H!X z6Nhn(|4o7s9U_|u(`LQ_^C!=dU``-dFV>gCj$TfTO2{1@?N1MP3ajrTGgNkH7$Pr^ zvC7b*CKBhD9zjJmC*OAl7rueQ^QdiIbCH=4_*48K=^JpV)ZaJ=%Ni4OT05)C+^;(9 z#6*K(mp}{l*S{mF!L2o}HU_PDmCo_pti(|D%&M~OA=C&U-_kcoDakj)S zpU6s~h~RJBk}X3auDMROvw~QLpko*s@5_?`*E%t+?Us1pcu8C5$gb*yp3; z|Kges12p=WxIt{eo-;GerZP*^etLB4{U-6@w^>Gzek#rS%&ypu{~4PkDIW+I5@^fv z4^TRQPaY1vKcUFypsRcVL~uRkr6{QEw|Or@Xf4Z&dl1P@pU$fsq^$P9SZshD#5xR| zI)M2!mf617X#vI0?Z>mGJ1rm+Wh{)uuCk{6`#xcCy&Fg|kKpekrF#ecH${KQrStec zMTEwPXxyd$&nxuz_TC^&`fNgDhj^?f+qJ}GLLrMZU@7 z6fn)SpcpgQ;BmLNP#wu(cfTf)j#nm+*Yd3Kx*sd!|7ROGom&)9H7}g>@5Wz}>b}j1 zEME1e?p&Dz!*MAwr&aFhXH(1cEA5Yd23r>LCh1sNr1 zOr2P`G6`T4h_Q0gI5=r(A+aYHnEH#j-4#{0IPn`6~#3g{y^PQLf z*dxrbRC{vjDj?bl@JrprK2Mqg%moJt_}10cY6v*vk?Qv3dE#cxoWMu_+H^dkw67Ds zx8y*6&kpF<)s)oO^R}4szkXH9H?#`GLi>BZl+trAmYkefy+U#0Xkz$ZBrgxZ1UmG^ zYx7Jd!zlE|V9tsBcI&bTV@X2Q03w2QGTlZy2H{W;4QfwP;5zaac|U}!P&Fmu&3qi+QA(|p$%vEf!pdXcGt z>^q^+BF9U?VFOP2T6c34xAQ<+D!Ds+DyF(f-@Uw2dLOIq+{geN2Bp+ke%bM1n!4xP zS(%pA>44Rsk{oc2lPm4t8VxOo8=q)q&eZ-j$ zZ54{lzEM$$dYbx5l}oI}#}|tC)Y0e+LW=77`V7@%{-Ur+l8ByNk1^{sx#smCbLY4M zS{lW8FqvT+pajDi>tI(s|E@)HfTp&}!P-;?zrtLwAS}iy96jsk?bX=wAMhTZ`WJ3} z|Mf;QXDX_0GOQu*Gy1pWpFs?PAl{b7e zj1Ww;{Xq-#fBfTu<+)~6hF^iZ^o<_ucTkrOA&kX`5+TkP-U*IyZBTX&RHtYtLChP0 z?_+{na=D&;@W}eL-hAfi=f}Ub@_m3G3WA`on`hM|jHA;3891h=X*&IV7U%!|gQmq| zI=ji^js&4?+bfnG=pKU|JT2GgXz9NJD#7{L3k@dFMa|-|(qTwD? z_5p0ys`Ki-cs?tFgwNZ)1572@OdX%jIT%vi0L#dzLFYSzk`$Uw4%>fy!qs1@hrm;q zwua6LN*N(tTq8=XH(S|7)i)>eYE+5T>GJ{QI=R-XEyzWi?V86xY?x+n@4_@sHIKgQ5=Ju@&oJ1#e9k%(3MCj6 z2lWEOTfmQByfKfsb8K|hvPkUKizYw{Y!b@*w3x)*3iOp708GB-Jj3TwelXlE6AS=- zSZ{3IzgHX3(HYfe+ua*L`dMzV)bJC^BNM+|DmRUxZplZEaUjO7)9s3+F}F^QmDs#> zlhMC(WZTmt`#;r#ELUrVuP!R|Z09Na4@E9jXX%=r$K ziAT~I_pQhfoCj_X1&L>*09a&u4cc6P>}F%)0x50{k%IpFuCaeS^(3$53&wpd5h_GBVU&N z@aa%cRw9Nx#V!^S%sE^^q{bpZXS%gpGBytuHy>WI?v$&jZA(24bxF#nyl;7{I2neLKNcII6! zyEpR<38N4E0FM$r6ayOh()AVGL1m#cND zR)BkNu=!h^xZ39cb1Rx*hc3XV10J`_kMi7~!>c{M|0>JcPR}z~EGToxH3EU2!G0KC zEyW4~?7^zYHz~q?UH=c)`MW*sqJ6O+I%kN)&G1(tD|e8~U-c6j0+-J&XIeiDXUXb~ zJBrdH#NTKbs@&?T%b#ra3g^$ws0=sA{af6;s%28piEAR-S;g^-3(Po&Y9t{V2^!)OrYJo^(;z4!xq=%>CI7 za{rXVW6{w+DA$bF>326bGlTuRot7L`U7@F>ovV{9k0Pn2-6Nc%5~{REsLxVuX}275Y_PhniR*cu*s@mi=I z?7k9<2@En!tE;%H=g#1$yo_61es$WQesIfx%)tCxxf?{Y+1NaD&G47;G8A!5t$wt{ zIj3Dq%?a9cp!i{tYrmmTMdSUrhlc$hR2fXJk!Wmen9#{?{cc*6Ii0}7wq5bk`Vg$5 zQ$ylNXRo;`hR-L5;TYeU=N15rI}e}rdOgItH;NNx6vPmIX2Lus^m zI%9Q&Ln$ikkww#FSEDFerUit}DCao^+cS?dHsH@^-z^ayUs3=gD6s1Q&~6vW|JcY z6k3Lk8T@FqHK`HaOJpXNss0GHwA@g6o9iYg@{Lk)+ALdKTVuo&)6K1xmw6KrAzQH` zCt~TlSP>x;LG*S=41mXvO1$+AUYy=9pEjIid+#0aYbZorIE$eRMp7kzn;%s-4a_GA zu9}-xaTxj`kfQ8C?fmLNEzTMBOWSknM~;k|*>SE~l^v^Inq|w);jz#UMP_8<$r|0U z+nNg;H2o)MF|sZAObN{+@Kle=k~@}WGX53Q*f-m0uz2Yo$h+v%D$!+S zs%_J4jNSkn<37=eNcj>8e%HUiHmnf${JmNdJ_YLQXBn%c!>MZF3n{3J!AuDSiwK*8Kq`)hOm}q$Juav6?@?*t7cu!9y5F`1ckj&>#_?1w+K$3!_mO%)uD9Y*jb0PeZpa+ekTkR)vk> z9gO8t|1^4ezS>~&cg;efu{y7;EZ-tSv`T%7FGXNZ97_u{++ws0>XO2yE0vyZ%QVS| z8)N$Hc_+_uf5v?Thl8P`dZ9jl;UwM)l1`g40gY9XC#1p!n#RWfd{M5;P>gNR_B^S~ zONHew4qwo*N4;D}{aX*C-aYUS-ft*XuFD$;DIZ4&y$eB5+97=xrtFdlA1L;Iu6-Oo z93y-aPFkt){{l^)pjsT~Gwn7*Kqb`4Iq9SYX2K{dON>B^IeESX|f z9uFc!pqg}p`n4vz#uslx4pcsVE6{~cWM6B14eLg@xAK=!l64r%ch@+lwyVN5R z^rI=M>n7F|7IK(Aw&lg#z&9FRWA4Go@3ia+_>Y()KQK{?dn%2% zNhOG~o0&O!cOkIDDb|bmmA$%+^jzmL^WCORkJ4q9V&}k!*yqs2=in37GM zn3dcSq?YttB%7443xmN?+7;Q*d0Ct1OzO}GMxmo5a6PzOqB@t~8v8;l3RI>oX-T2R zu^CcNbHHiCDYu7qnCtlncDtE1VkBxmKHrr3Lesv`bkPV(txYjY zMp{{9m0UlrhmmahyPBnW0Fv_5NNus;gB86^*(AdvlTj+}jqvKGf~l~8^4h&pv&JOL zIL@muZ?F=0Ur;-76@W%!K!7ueR zxQou_Wb}){rKwTkx-;8&q{p}4<3XdSlml;|;82o;F%+Ttv*)l|I*$olNrP69OmSRt zW=!^vBKNQ&XMmMGx~79ORzT3eE&H zqEn|xw4(H&SFi;7WI>Dtx(X`OrF|KtlxbseUkB6fW1yrl?S7^aeD<4L|1uc>@dYL- zr=p~B%7sBRY>KeGmXPC#|LiuQ)&}qvtYSsSO$N~zEpJ2oF@TyHn#inXAzVr@=x#C=FIW{v&( zMy11~3;yR^g-|>!8y?FZehEbu)#!<%9(If5JHA%DI0rF5H@tZU+4K+bnEY_Faxr?+ zfPkt1w*m{;Z;zCXsPP@L0TnQH;csft2LlnbrpoH-MynNxVSF%qTVOMiNlnrcHOSl( zFS%alR1?e4h`6T8-Y>V!475?d4>C>@;VpmI2wt6ac7Gnwc7T# zO(<`T#^5E9$b#k)oDEn;$MJO^nFxNUG>+*a;tng^X+6#-%tNvW7ZXf(IQ?{YtXl$M zTN(~Z^&zJrUM&~2HB(bBRj}+nWu8(1i_T05X(2|lW_n{(C`vw+SfaBkxop4hH@vRr z+DL>PNH$t4zHB@gzkuu}f-1rjI0=7pJ=m0@wxatn*~k^o9mJjYL#r(9X?Z;}EB=xB z0V}pkG8B}N8p}>=gZbA>A2bYVv{Gt4jnxU;H1C~FKI5~VMT7->skm7mj887t*x~VX z<|F!UKx~}|Blu%+aq#!au3-pSmo!@HNO54ECsE9-tHXE=^9BY8>q7^4KjJ3>^qqX5Njg08>@L1`W(lglNF z#-Vl?)p1cU=XBobGg1V&Px_Oykiss3Og?3~RhqVbb_v9v}N`mCS3m}GEBk1Hm2E|GN}g}t5m_B2Zn=< z>**>jyz3?^t%joL9pyEohf&sWAZ)^;`rZ?EcZe{};jP$*`Y2*Y(MTkD2yHc;u_g&? zA_P1py`iXa+^kh2omJ{DtO@h-Zn1@BQu-DQD3wOIlo4o5$;@Eh%!Z`3KMA;V-sX861Yt8<9vehXy^6pD1q8$ysK@jso}yq&R`L;Q^#D>4-u z(hND3?kbh8Am>ccLR&g$L7_~JSZSzGRu28%kxnND3l|kkR(61%L`Km;4`)qDU11PM zY##GJ1}QaL1h=dcj(XKc{S+l(X~J3J~TE!V~<$zJ3VyXdZr*SWYZPW+ID<%cX_xY1B!yq*IPPY={v=t>o}T7)6%Y@nJSTPRyPu=6UTTs=uksM zG2}CH3=En8Jt0xx6ry+n>7q;Rn7Tx?5e&Re3E8cK#cjF9Up(bHtJElAg}NH*PvbMEvE2uFC4vWmaDEgfsFBE!Jj(za_MTZzo_E z(bJZeUq1)RgUk+OTg$|T4+SCUs#TbEap8rGwLH8y)2+SHy2t~Bg4?N09q69)tP@2j zjzhA-)J-%!o1ELx*-x{+E7jYNucyR5)NcP;rA1#Vq5g~uaMWg5UYiwXUx8!{SH08b zg9L9AQt{6Mqiy2D_y|I-qJR^Payl<~AOzdnrDOSYy+GaxYdBqAjFPqqt06L_A=jQW z;C9_jGX9(yF+15G8kvsL)&#febW1!y2X5eNT{3%<_OSTqlT(1vwr3SE- z8oe3usi*^=x9SogUw_R&Mwn0=2t!%3%R-mAsYiB${}jMi80dOr2$X(UML!6XcH}_~ zoX;^8!(LY{)(_1@W2*^h+!FDD=y{D+*a$-K<-spZQ=6>%1A3xAh%+XF&y$FCQ`(Jj zZT6UPs-&vMnH8oFHs1>K3{!-bY)5V<%6=k3xpZUnd6E9SrjvwknJbv)B)ugZHy1?n;Sk zc7N*Gfw}ZT6!RQOc~4l%vz%{aGfCqmH|ljI-)U(lvb2J%%!_}5lv)HJJFy5)ZJ$>sELS{xvt6{Q_!8|~nAcNf#6vlj& zFqO>2Oh!n_1p^zG9_vE9CXy%&<0713`q280_Z;$xRwgh{VwqLdT1JRg+e^#6PWJaj z`DKUs{oAIXV3r}uK=u%!N$u*|Po2=@+K4sz9*dJ`bTiV5h)#q!UM?Zv(3^&#JOGTBS&KP!dvCgYT2XP6(_J zA_2nVc)&?m!j;wsd`D^B7eto}}wMHzbiK3l|EfDf$Q zF&s+tWQ5Iz?~&uk)bRBCfF09(lPGc;lZD0=ze#~jfQLdATTi+(B;7Ss91W2n-9C*{ z4d-Poth&9&_>55ASKr|}aYAWHE)Vcwqm-dWoi|p716%l(df6XD!kS;Im0c&E!M-1j ziRCufLvu~F!H^JhI`vc;Xk%^$X2cA(QL%x8I}_7^Vb*P4a;A4n0iuRAFhSdqspx~0 z&}SATh3MtwiPIr94x@lr%9vpZHb2`>gEC1&J26I@iJOpKE-Okc=EZjF1e4yI;V4-j z#db$KpXrAF7XD?Owu;8n&z9AS)4*=I>VL+5_q z7#W6X;-nU2-(E+7If#+8K@f197Nj5OS>Ms>h;w+qsIrWNL@gH)%wvYh7!}?Q4D_hqAQp{TSSBg@-8_L}rTMpIUKz4VyXO??d$aConO+r5f}8NEUs0(w zqMC0`C1Kg&Y&ofYmmf(Oy;(B1Me%- zP*e&iwKf!MI5JUgfg~&Wkkp2GE3QSlV6iV zJ@dk_zJD+=hQAl4(HrlILQt5qMo8ueI~;v95EE(bjq`T&c^E@`7!G5=*dkLsw}2oE z3YMfg3QFAEM4^cA3`e7gh6RSV1`(Dh8Rs%gxEv70vcaM#LIr}EA(Kau2CyjU?xWVd zXK%sDg+K$-uf+`|I|b;Yg)css#S&} zooGQMju(JUc1Pj^ox&DDB4X7S0dcb!&2od5K|>T+?#r( z5PdUtDkWg0N+CMsURuj(5SlhJ>*o9w|Mv4?QZcx$QsHmOu~{_zZZpZ+6Zncvf+_sc>hT8ueN=k47^>hw351z2`DazAxZxT z%LJxC`6lIvt1}%do#Y?v%fnUgx@x#1IM?%@qV5vY~9eh#|D@t^I&a!yXvbA@J0T-z&#t z&vPwI!3(IPV}=zbEtT0(P@nN=#zZQpL3~o-H%yNC6@VeCXO>vk{#`e_L_t@F%^_}q zB{FqMJ3WqSjAIdJ!GGDX`Wki;21gl?6;9+r7-n^*IT@V&V~i=#Q)ImlsTzl|E#3%l zF3Xg_PIXNW^I`hz0`q--2Ak_Bf>1U76gL@-^FF#@4*s^@&4+KwfISkNCrtd0)P4#^ za3vD~kWz7g8_ua}SWmNXo59I)#U9IMrlhK;pp`Sqr=*)UIlU@F$OtMQ+y$8t3>lYF zIgsnx!6wH875}c8DSPVgH`c9~??7j%-D@;1R_tHo>JD zVh_^_POpDIRw}I?HGy#Mr{zWE%f$PI?yg@cS zp;$6)YpuOJ0mZEzs-3AqOC-Gm8!ORpwpkkP0-fx~gRc6;do5XZ;%4ux9xmu`Y+c}M zv7qcjfl$6BK67|vo!q>b%6U&Ti56Akbz^su3}-22f5Eacr^zsoTP~M)x-{6BEmN94 zElEcU2{SBA1E(?_*sT`^ZzE3jSzHqRB;&QY+6TQcJ}uB7=Nec`6l~qIP%G{C=iXLn zsVY=DHBk+=h}jG+);*<)_dD}B$|M;#eSbUim#q70+CMeS0dS$FS^lR!@bC7$ABc{t zYCezaT|l;fBazhLI`?a*AH06LM#}|QKvvhueDnHk&q1i=F{lUh73WvE_bRu6E;hppE%#aw;roNrua&~ z)1&H$D=?y1?PD{dNHJxZqcG0O;%8y@MP77^$%E2D30SGUE{zO-=XY4>Ql z`;*00kn`>*3VrVj!q!cD@3a%(!@dI+@d25K_&Ex6xZj+h!*mwA4FKT-i6HQ_U@4-= zYQ7+aZ!`#oXb7}qs0@;zu?H3{csLUX&zYggq7h5DUX`;z zAblEf)8XGQ-U&TM!pDSf)%=mEeh4ytd|x)rK|)YcvXJ*ji^x3mo6RUIa7>O{E_}0%0lXJck^~a7N zwc+dlNnzBI7M3_?2y}@pT;`5)Jep`~Xx4;tqMvj?oDgPRLL06_SEcV8BA&)NjhtMt z?SCp>R=7^kT<7KHGQ1Qo!CFWm`+%IyLSNDt(p@vPrI9{}#p` z(ta6iu{);X)Fv8cgnr{=Sf);|qpHRLEDNKx418yuzkFUsrxQy*m_FzwU}Xqabyu-F zbsLGnPNr1A7_MI&NuytcIsVCm3ZzDov;l1gWsjIaSKAcrCzc_~o&ImD#oDT;lR;F2*HFsolqu8i57y|I7&6F0z$%bYreMGBA1=Lvss{;)vL|;5a+YK*MBgp;%WRw)J-D%bE~Q9 zz)srp+Yyui5~|F(30s$8e^Jz}^&%<#xqCc*&Zb_9dYiGIgYWLE^YnG7>6dZ{qs-~H z!Ji11Gl2I77(#FvklolH@P>Jt7Woc#?FZ^lTP}5}N~k0LeN{ZC6($|-S1-Vje?lW@ zQD`J?HN5E%zwUkIu)Ywq(5K{u_=MH!NO5qWXw^wKml)jzF1g2N%h>HxfFKe$mEgw5 zk1EvFOO&=XRkbK^8q?^pO{?Tz5*dKE17MaVeZ*?- z?HgugjENocLGm$6H4`#qzT@S}N{x^Ay(ay_H9!_>%ZPjrkPqE0M$ZaEda<~LJwkQ{ zQz_9Xi7d0~k+iKe2q1tDRhm#3zpf$)p!?CO2ThuW$9Y1Q2E&!4*|oJ(qgx`1oGFUT zGVe6!txOV_Uznv>B3N88-aDw*1Q-C8f9b>nUI-ME@Aw@J+?|qaRG+V_-au?*7JvXk zy!;=Qf8^Q>{N=Dab%iRQII5R0W{~ag@U1o!0Z0J(Fv3A@I%l`g7$caY{Fz`@?~{wB>IPlveiTF&pAH1NTks zkwAI?8*@EO5)z3g(0iGUp#gAB3EtPwr{1eX?fO%CVX=B3-j*w73>k)H&MD85gu(?} zY#8)9a=cC}`$ZVJ^bUqGwe)$VfUxE$B<8y|weQP zELL-Pd&iPgm4V~Yw@dGYKO(8{c%0YSf8Wdg?E+taNFEQ*Qj4c;M1Gc6|Eq2(?kgbQ z#D7-P1Z+#iH1OJ%sKd2oTCLgK|wns6ne!yzQ z!F5hh+S7!FPiEZC%F^_xu7aIiRt7HrJ8o5fmOIJy>GH*eNkQp`W&F3U`ErHT@;qz% zAEIbumB9L2ipw6WW zr?i;P3WdQhJ)jN&PEw8d+P1wY!uK7hvDdf1U&t`^pe>`n0o3UUAfbM2SQY~+4QUN7 z=ly8iFlRzukNV_a%exb_b)skwzZE7RF>_e$w?xY|&@tJ2_md{^s>J2-b0f!Lt^o7O z0!{u#Nn73Xz|oXhXd}P&%<@6F>H^S`Jpj)CsZ%MP1fKi`;J z;vKy{V`p9Wen?Cp^u>T{-}afc4cy#;$M1cJGS34H`GkBrBqUiQ5hiTJ7O{C~7HM!6 zh&s|@gmT3la0AzqnLO`HspGb?h(3V$bZTxSip=LLD#$@P-roZMzGd(sR>Yz}eC_`{ zcvU^<;QywhyaO*S+X#@Pv~c)p{|4X~w}4A+MUBAM%d9Hiz{l}u`B0d(0rRbaHlF4} zj&?D{D1pm6Af_06Bd8WepxkJ^;tl%Esr7PEQ3vAc)zCE5Tv(k2ikeERPH94rBx3?$NmJV!aKlnQxKb5VlO z@K;1iWQzs`j0R}EGXQ8j5+90Jp5qMLkmS)H1S#ErIwSobLk*kz69D_G;H`vlqJLkH z!%*fhISS@8+C8tLTNOmCTYiI*w1)EsL&Z206>L4@WpGLe0k``P=ooI$1L1mHBh~kN zVefKS&8X^$>7Sah$}C9x?K3wDheuy}NPWPt&!_uJ#r~_!n(3&?6NsJ6 zY#AlF98J~LR^6=57!_3Gx}I&A+=dhKXcZK7O+*B4*sT}2z5V=Ev-$J9wpp>l@9Y=b zs+UFHOLelsCun?aC>quipbPpJ^?cz_hs9zVod5NXsmuOvwD$>Ml?dFH<}vkLg#iA; zX5of++((Jr<@8Vl+xj+>;{tWD=RfEeXm`z=ujl7FPkeuQAq3#XaRQH>dC$AUujkS7 z%ZAZBAlIm%p#e4!P~^VpN(^STeg zZ8woJP>6INc19Tu&DAq`PmU1hza!Ki-kLy|m4eI%spopBC%>22|I+UZ^N^n?nuh4v|OJFSrpSk-z?5e-Sx%1jAH;u8SjMP|-lsY#b25 zUHlLBt;Wjtd-Z$~xE(8wk-XpVzVgkf>+rZiVVIhm>`N(UgTtW9;e9E-@_Q0x7_6@C z@EXOJ$npP#{5PslKfAM3$(pe=O8A8z#AFr~=XlQYHjL&X%Fg#}DBW_$zN-5b>tOpE zKw6(wU%FiU+jqZ-;2qD`1%iV$SE%2{P4w3HwqK@Kw=o?9nuX*cii(%_Hw-{ zRmbNkA@wR($RC_963b&rY5&R{vzLy$%Ew_olORSyx+Q=ypE4BR>IIII5~D~sVlnL0 zY|q_HTe^YY0Wb>|*iR547DxL1ktD-Ft_WDrGRNZZd3JrU8oMuK`VUfpH}d@UCjSDckRH}U79rbbjEFQzoo0A(FV+N&*PL5 zK!6pMGUxiwBx5h0k|ozb(9dX$RG@OpHXgkj+W*xmu3`ZEbL}Jr;ri0=`^hH<2a`aJ zM==f$2h@FUK(*zbR;?UBfbTljKlh(YPHBJ5B;IN)3wER_r33LNHnhr#-fjkO9>!ZL%tFB}+KN4J-j0ziXZ8;^sKn5HK67pRdhqDzM2(Q43z^j)UK7TkP zm!>1p>ljVn!o>T!6KusV;~2zaGo>@4d3@yKG`LqLmyI)@MO`kDvlQnCRD!sRej_nl zc)TBPPk$W;U3*5$vJveTLUj1;jA&x_;Ej;=AR#!`O-aKoCIbzseBac8+POF}#FSSS z0EM}=q|BKLtMoPY00#4p%PyVa&n;efZINW8t=#0b2LynX|CO1N6Q&&&&6d}WULkh) zP|*tQ+;kOuR3KBUUN$6KrU9fs9MW&c#a2j6+DU!1D3T5d<+pR{Bz5r+V7l(y-_u}G zuE({Y%oCzyeR6;xl#ozLP!%%4KOXmLxBQyuE%R(U4^GpHfL|>eH9*MtcbCy603kwd zQ>&C+cQCUJA3I@-kK(y_9L-#`)y2%3mYyZSGzX=}OmflWd0CTPgCoHVEPg8NHr1kJ z>qpnfLL7Ic(^Xz$hj$ne2b^gLp4KM0;*|~Oj5MMcdRu5)GHQU9E&ss%;o=G!NQ<-o z>4>(%lA{IzjtmZmrGT3>Q0{(ZUm?N(h@N6K0J7+9D&1%>Y{$nLO5E`P_IUfOh?J5hOm^qh(pVq@6&IjTcyrYns{jaPP*MaVe0R7- ziNVf+#p&n^eknnSJ??-B?L+@G5Q3R5U^S@GUz_&fC=fyh21*Z2=W%1P0zzGCJ^=Ag zsxlVikVM_dNjuw2bx^>xM!VM*-)8Y28V;ml2e zIt}SU%yIyzZ^RLhy#E5qwSwXzSyX@eoiC9>PIZ~P9}y=ofd`bR^-xXB*@-dCYk+eu z>We)}^q<8E@yIQffM3@-eBVOmTmbRo;9w4)m{@E?KuwH4g+d;@KS7%!q-y4U2RtmE zeg7TPuHYZQ81s&1WnN~pinzTuWwXAZyAMAQ6g6uCaG0fh zP9liWUx*!Mc`2<6Gg#s?5A(Brc6?8(Gs<6zpteJFa1lycP_k0HG~3KNCEU0TdT$($ zzYgfA7_8WA9^no0URKr!s=&ViGxRUM)(x-d&dVNBFR3WyLrGd9olGKvl|s-aZppGd z302}$%FMAG{z{WbF>!eN1UloOJ5^b-!64y&{h>VcJ!KcV17vWe>6?xW%K|pGs3uKb z^a8hT=M^mhN|f^?;CN}X3?EeCnEKeg7l_qP9aQqf+Y81}RM+R5#*ptqQ)Drm+R~$u z5uZ~tFG@U{)FR6SQPSF_vzV%_GJI2a9IG4cNd9e99E~Ls({oMbwQ?}2k-2{xwYm%{ z+M%W(Ma|3+Zc_^pMlx&~R8Ek1{Hm3JB(Q>24z<>L0B6sR&gn3WiE41L7EhRM6Q?rE^X* zk|18vE?|xTCWUcwK#{Ykqj^%ZKxQC8(B4~{j&3gvCpYXvk0}upX($HXpQ{T_*;HMc zHg)j3Q5|=<(-Ez-e_+l_jw2yC04)bl6#;#!3`D>CWB1b0YNqhbuhs4Dkz=(ScaWn! zI73`x6Y!sUCZfpaUxWq z4FXgJ{|#jXpl(D63JXE1hzNphV)^u+52rV;HotivuIKh}mdTO2uinngpPqNQm>*8P z=b!1$$Z{CK3zu+!LncKsV+1M92UUtd{=v$JxaHcZiO+H@g_mAgM0M%m1>aA|ZMcU< zW(t6ZEpL_#&l2?8?()Qvf}x?)vXXC`UjPxv<^tF|fQOR$Pr#`qxqkw!uLcr(CQ{?5 zt_lTbrFw!NP6F0k=2VV#-ZaCFpM+CU$}sw?L%EwuuB&Tr*CYyw{ed7gW?77a4$KmQ zFo2Ar5f$S``mq+wjfsA13oaIZ8=7DU@`l^;j^uI{bj;yOI+)k!aVF(rS%3XUC{X6(mMmVtK}=h(a7k+3{v>trBu5-t zU|Dbf2REd!jxp>2Xan#qmJ@cmp7auB`2FHvIbYwUq1St*YMAf+jWk3G#%r@e=ZWxD zu%1Ld$porzv;h=-v5y#_Sb+)iDzX7nz)XzP)Ne16vL2$97Cv-`uog?%~Gm zN10_-3qQ)!@thaCle@5SaCf&vMdXuPPoF4XOyBH9qE0tNN-j<9w;uDs(de8=gN1$~RbPL_{Ea2{@~ zV;YQyqp!UGN__)c4)0+S?RWS+R))bd1)47e&T6pncNTryKK*;!uIF@lQ%7ejwzkP( zvv6puEi*0(Z?V;GNbD|FY%nf(U3V=R-b};c4+-Xc|4*@)C?67Lf;o8G{pWmQSZ7c+ z?wdfUw4FzGn-$`QUA}66q$f&&7wqdMFR;pDHBN9_VlBhR5)B=<=VJ**5-*p-47>R4 z#2AP<(hqYZ6Ul1zZta+JA)o>g)>zf@e_XF-hy8VjPU@^M(jpvjRytlkTm+#~4ASgm zLfA{#lo`iNVTxb=lka7paoEv_%LTr@K}y)j_3uRQk8R*E^($~IqCUq!?l&hcm*i3< zyPci)vYM|qn*EkT#9}n~tV4smKEbw%VMB#7I2)AP9#8gFu~l3jSXShmvZ#Gh1l0~= z*kfSwzB_cI*KOyHDJqr z$c4Uql_KehlDQ@6kM012F>$J>+r%|N&P*Jkil^7p%S(C+DocV6r3WJOXN<+yMc75m zo{uiTzo!&VGQ5TY4J`O%f{C{?SyTn->V*pgzAx0N-(ITM>3Ys6*K!+|$r8UUp zy3;)1eLIJh-iCnty{xYE>~gVzU?`#Tp7TN22=en^<2Jwlr!Ff8g=dKdGYd(;o@+7A zg5c#s^w9g+YVSj&!n;x&EVet)#~u z4E$Y-oz7?smnvW4XFn8S+s9)aHp$0uvI4e*YEbRnZ!29-q1|T|b5RB!Z`&to%X4a+ zH>PYMz}n_*ui2HMZp=D!#;xJL_x88<3ur~2pJsJ>J=nBM&F$aFZG;Tt7Qu-{ty z$eg9S{m{ryrQ<;w7$q4912j<)A6*MA2B&-WfplG`g$0A69(*xcrXGnAJ*K0zPzI7LNX{Koq# zRtym+-?`}NNR2|ifW{Y#+vz+@R3^9G#B{5VnnO5&|8YJcQ%;G9+k#Y6I7MbD&Weo; zmyy$5?iNz;<9E#@NYb;=UO|O?!8|YwdZE zJN!#xgJZZ%!^-J|(P=Z2^Ca-NR?OJb=KOl^GF-1bN+@s!wC5n9t>yFZ;0C2Ymv6eS z{!9n=ui-H!uKXc=9sJG+8Hy~&E9+BVEm7aBrQl%<&SN-J@=Ncnqke!-3|d49l`>A? z>zzSm6KH{=y3Pba!X3^E#G@e*S^=WzI=0WbwFZhV3P4^zy|2M*5l40lhb_ZLXrDZ` z+sjm#Ue&^r`>)6A;jGu=)Jo6htQC-bfx1+NeW#S?4jQdQ*D(07YRO#N8_CPN3=I66 z3W)pmULTCPVus`m7ZXNO4;C}!P-PFF+_VY%87MrM-ygi!{ zeXP3fN1ATe$Ms%?>bJ#g)Vk9^5AUyrzURwHx&QdnL728knO9Y;l>(>H zV-mSQft5p-Q^RH8^M87;KKu2Br~XBS9@E`npz-Fn5?Ijta_E%>!TG$11M>3`=qJ+Q zeeztHTe!Vwj^o-t+m!SN0z%6@KqL{R{JQJ+t#dRlYtyk4+okRs-DT+>@A`R0Yy;rT zzAwk^Xp6s*3kS{3_Sko z8x1_!MZWo3nc*qRH$Mel8vqinV=;v$Bp&sH73FAeXuG^&RCGjh<D z)kXQ|&==0~eQES*pa=}P9BaiohPhEgzP{g72O9U}aa|S9l*DD*ViH5AIO`1u!(ve> zH*`%jyd2IMV#%V*Qr{f^(#h_sJUT?{jVg;~+O?%3Ai0|UJ06+gmJ&%nNw#9_)#{Yp zdw9(HlOfn;1g)&S>7g?%vtQKrXM{Rfvzd+XH-Wn@7BOL(@ZfQfJGoWVKztA8{r)sc z$^MWD`qi{`+nIBro-&zCn&!yh=;no(*uaHX-&$+s=u1BA5#UQmXy1K9(4b}NdwZfRelUr%8qyCk zBa1Df9X6oodM7i~7m56av`;G02Ej>c-|6Ed7oa?fmZA^AZRUd$vfKncos5*-0u-qN zCL_$!iu^FnrShnfT9R7Twjc`e0SYQ&))H_(b(D6Zyi}<-w`|fSaJg}=;_WuaR-Xsy z?snqhWy_t;)c-E7fj`!xo#OHDYAzoA%#$XbAVy>Y0YLzj5*1P%ZZ#{R2XPP4=BR&0 zB5u))YsyBFO6br&L(AZ%>72_Gq^fWeGVihavDSHqG zij<(lJK4P$vX3lojJhe&-@Nl(H*g-jyJVi|VVak7UMy%dZ1(NRFw5T35B$(P zUC`@Q``@xZS9uFd7GzM5|2EJoJR&)$wH^q{-}NzhVvdLWo9r?{X3h|wDoDhop~-bp z(0id8^t_6lhBp50>$5|WQ+dv9qZ~6=@YSE69D}teq$=gyk^z)d86$o1rX!y=#IU2G z&QelvA>t=(yn93NPMV{^M)2Glaqy}-!YH<*xm(|j&<~wh1hj6fzq8GeYfZGMTi~kk zx*(Xbz2c|HeOLyJ&uK$KBn>Zw^g@dlJ4wa)Xr#di3cTUxaL-U!FdtYr3%+8>(fQTK zB$Pv5Ck0raAAf&JoLYoCnGJQjRFoZzuASb45ZL<9C#}wI*|; zWd1nH-53(C;QciunBZhkBDu)V+W@!$lv$~ZNpSDh#9h|)443wP)Rq5XCp^4fTD*v)m zq-0^T1Y?o}k3-y#8tTwH=>`;WL~b(B*|FnCjH|iyJW;{5Oc_a}22$U|CnC{NQBe-L zX%!Fl^+L*r@>g|xw4Fpn9<`?^A!%(Yqrc!jSQy{Ud1*ArgmR-uNK^AcSuAxisx7|M zyB@IEScNqz(DyD(tCC>wBQ4CrYV5d{1Red`VR_&iF=P`!Ov>F;Yjvc3E!^5C*E?&c zu9oOjKwF+nB7}w*f6$~!H`Ab!yRujh>zbZEvJq;PNiHh}kq=ZZJdNpxlF+89R=D2_ zw`D<6s-Ere*2~*#AQZ=Jrydwq6bgQPAs%R%e60%J?SCUCF%N@6DyUdmL$EgtIJy*c zrNjwETW{-+^v-M4x78%$*;k}&KA7*4ZtzjhqN`o(m*_Iy7+i<&Qi9`6%O1AYv!K1Ih7Y}iw#X3RF1k!(`%3Y z==$4&J z9P@0E_`dzqr{(jUQw~z~azGQD`^}N6ChEO4x6yctwp_lYm^FH$X6~5jwruikk)ZJ< zGG5hiRiC+RTE4LA$G3QR3(5ffa#`{@`K_T4%)peIijBzBvZUfDT;q%5r}5BS_eOoE zR`nm$Z-12raremPJ)raGzS?n zNBbF*TFM+1M z-QFY3=qDWTBZ2E(SwZM(-d?S{!jfGkP^BVF4u^7=A*j`>&0;<2F+5LIym6xG0_QEO z1yf!^I^9_2UBa=0V>7FQ`+PG2(?k|&5Rn~=J&@gd_6lbyY zIF1b_wdizz7>ZJ+!yYy9CKEJWQpeIG$~STy?|NZ9{kwWri^u%2=?g+YUd@G}j0zr0 z6GPt7c>#;ztoKQVy)wx?T&Quyf?Z-YPD!KWQ+)02?X?}&E@nkCNZYI=zD@(KloUQn zXrB7fFPjD72-0}pQ6f`p+o0ihnaoz2{0pc>J2X)-Fds@g{?NfB^~ zt*AOc3;O4{)RI`X*V6FXB_|pF0r@&E|^PN1-wu(3F?T0bWQmyucjiXQhS*fTW9dY2+Q{0f*bV*A(iD@?NW!h{#+;{IVhZY zf~sNNPj#QJ*PUloY?lRenuHL+i5OXJO4kL4ZIr*Ih0W5i37l%=@x~!4D|1S1WZN}9 zi3?=(Ew*mXw4}JEMP%z*5)fRly2r{*dCFBq#pCzqn29qrOP*sq&c+%ONe?TA zy|U_1J!}?rBowf9qf)F)(|-(3#MJ6@g>jiQ3QEXSj3{PekPWM5E$EiIAZ>^$sGjlS zSo?YMp0^%n#>N+CE|Fo@K=Z^1vYKo>MZmxmdzJQ6rOo~z_I?!SJt*iC;8qa1mljTt znwH=Xt&dKoK=kuel)=caFh2`-gxgiHW#wO7(tW7ZxC?XXqET^vpr#{e1a&i^54+Zi z{|%LCm~kb~U>4zyZwgAzQNU(IfI7ie7QImDrOIiHc5P&MoFcPca7Y8M#|6bh0 z2JzYa0x~Zi$#;+=m(Bfa2;Nk7p!wuNPK83@1cEDyu2CH_s?F&@8I~rq3-P;Z6q!)+ z%B5^$n5(LDLeFmg=v;9Vor!!V^pR!S-lWZwX&%bUmj|nC+P2Gzz!_0Zm}z@lAG5q? zg8_^~kw#%Hr>UMzSN@#iQ<||yjs+o{@Q%m}6$GN2Bq=K8C`xQ3Y7rd7yX@S{wzNbj zrGRAR+A(nUSh#FSnVHYR0=H#>LmgB=xQwdKN!1W0b)1~BR_>(|gh$fR!{S|GUBj~(`7EBKH5UjEaFH-w<+eEcx2Wu`v2KS3MNlYvD;kvngB(7Ma_~{Lc>l de|xjfjF)=!3PO_faUj5tl$g9|t*~Lx{{u56iiQ9H literal 0 HcmV?d00001 diff --git a/vectordb_bench/frontend/components/check_results/filters.py b/vectordb_bench/frontend/components/check_results/filters.py index 6016c0040..e42a63fd6 100644 --- a/vectordb_bench/frontend/components/check_results/filters.py +++ b/vectordb_bench/frontend/components/check_results/filters.py @@ -1,4 +1,4 @@ -from vectordb_bench.backend.cases import Case +from vectordb_bench.backend.cases import Case, CaseLabel from vectordb_bench.backend.dataset import DatasetWithSizeType from vectordb_bench.backend.filter import FilterOp from vectordb_bench.frontend.components.check_results.data import getChartData @@ -75,6 +75,10 @@ def getShowDbsAndCases(st, result: list[CaseResult], filter_type: FilterOp) -> t ) showCaseNames = [] + # Handle FTS cases separately + fts_cases = [case for case in allCases if case.label == CaseLabel.FullTextSearchPerformance] + non_fts_cases = [case for case in allCases if case.label != CaseLabel.FullTextSearchPerformance] + if filter_type == FilterOp.NonFilter: allCaseNameSet = set({case.name for case in allCases}) allCaseNames = [case_name for case_name in CASE_NAME_ORDER if case_name in allCaseNameSet] + [ @@ -101,7 +105,10 @@ def getShowDbsAndCases(st, result: list[CaseResult], filter_type: FilterOp) -> t optionLables=[v.value for v in datasetWithSizeTypes], ) datasets = [dataset_with_size_type.get_manager() for dataset_with_size_type in showDatasetWithSizeTypes] - showCaseNames = list(set([case.name for case in allCases if case.dataset in datasets])) + showCaseNames = list(set([case.name for case in non_fts_cases if case.dataset in datasets])) + # Add FTS cases + fts_case_names = [case.name for case in fts_cases] + showCaseNames.extend(fts_case_names) return showDBNames, showCaseNames diff --git a/vectordb_bench/frontend/components/check_results/nav.py b/vectordb_bench/frontend/components/check_results/nav.py index 2267024bb..a6515ac24 100644 --- a/vectordb_bench/frontend/components/check_results/nav.py +++ b/vectordb_bench/frontend/components/check_results/nav.py @@ -24,6 +24,7 @@ def NavToPages(st): {"name": "Run Test", "link": "run_test"}, {"name": "Results", "link": "results"}, {"name": "Qps & Recall", "link": "qps_recall"}, + {"name": "Full Text Search", "link": "full_text_search"}, {"name": "Quries Per Dollar", "link": "quries_per_dollar"}, {"name": "Concurrent", "link": "concurrent"}, {"name": "Label Filter", "link": "label_filter"}, diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index 2e104ce54..685a38ec2 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -8,6 +8,7 @@ get_case_config_inputs, get_custom_case_cluter, get_custom_streaming_case_cluster, + get_selectable_case_items, ) from vectordb_bench.frontend.config.styles import ( CASE_CONFIG_SETTING_COLUMNS, @@ -47,7 +48,7 @@ def caseSelector(st, activedDbList: list[DB]): def caseClusterExpander(st, caseCluster: UICaseItemCluster, dbToCaseClusterConfigs, activedDbList: list[DB]): expander = st.expander(caseCluster.label, False) activedCases: list[CaseConfig] = [] - for uiCaseItem in caseCluster.uiCaseItems: + for uiCaseItem in get_selectable_case_items(caseCluster, activedDbList): if uiCaseItem.isLine: addHorizontalLine(expander) else: diff --git a/vectordb_bench/frontend/components/run_test/generateTasks.py b/vectordb_bench/frontend/components/run_test/generateTasks.py index 725dea769..5d848bb94 100644 --- a/vectordb_bench/frontend/components/run_test/generateTasks.py +++ b/vectordb_bench/frontend/components/run_test/generateTasks.py @@ -1,4 +1,5 @@ from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.cases import CaseLabel from vectordb_bench.models import CaseConfig, CaseConfigParamType, TaskConfig @@ -6,18 +7,34 @@ def generate_tasks(activedDbList: list[DB], dbConfigs, activedCaseList: list[Cas tasks = [] for db in activedDbList: for case in activedCaseList: + # Get the index type for this case + index_type = allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None) + + # Special handling for FTS cases + if case.case.label == CaseLabel.FullTextSearchPerformance: + from vectordb_bench.backend.clients.api import IndexType + + index_type = IndexType.FTS + elif index_type is None: + # Default to AUTOINDEX for cases without specific index type + from vectordb_bench.backend.clients.api import IndexType + + index_type = IndexType.AUTOINDEX + + # Create the database case config cfg = {key.value: value for key, value in allCaseConfigs[db][case].items()} # Many DBCaseConfig models require an `index` field, while the UI stores the selection under `IndexType`. # Passing both keeps backwards-compatibility (extra fields are ignored) and enables strict models (e.g. OceanBase). if CaseConfigParamType.IndexType in allCaseConfigs[db][case] and "index" not in cfg: cfg["index"] = allCaseConfigs[db][case][CaseConfigParamType.IndexType] + + db_case_config = db.case_config_cls(index_type)(**cfg) + task = TaskConfig( db=db.value, db_config=dbConfigs[db], case_config=case, - db_case_config=db.case_config_cls(allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None))( - **cfg - ), + db_case_config=db_case_config, ) tasks.append(task) diff --git a/vectordb_bench/frontend/components/welcome/welcomePrams.py b/vectordb_bench/frontend/components/welcome/welcomePrams.py index 48bf5995c..6a7cf48e0 100644 --- a/vectordb_bench/frontend/components/welcome/welcomePrams.py +++ b/vectordb_bench/frontend/components/welcome/welcomePrams.py @@ -80,6 +80,16 @@ def welcomePrams(st): "image": "fig/homepage/table.png", "link": "tables", }, + { + "title": "FullTextSearch Performance", + "description": ( + "" + "To view BM25 full text search performance across datasets, payload modes, and backends." + "" + ), + "image": "fig/homepage/full_text_search.png", + "link": "full_text_search", + }, { "title": "Concurrent Performance", "description": ( @@ -148,7 +158,7 @@ def welcomePrams(st): for option in options: option["image"] = get_image_as_base64(option["image"]) - for option in options[:7]: + for option in options[:8]: html_content += f"""
@@ -167,7 +177,7 @@ def welcomePrams(st):
""" - for option in options[7:9]: + for option in options[8:10]: html_content += f"""
diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index d15c4e7ee..764dad2bc 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -4,7 +4,7 @@ from vectordb_bench.backend.cases import CaseLabel, CaseType from vectordb_bench.backend.clients import DB from vectordb_bench.backend.clients.api import IndexType, MetricType, SQType -from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.dataset import DatasetWithSizeType, FtsDatasetWithSizeType from vectordb_bench.frontend.components.custom.getCustomConfig import get_custom_configs from vectordb_bench.models import CaseConfig, CaseConfigParamType @@ -12,6 +12,7 @@ MAX_STREAMLIT_INT = (1 << 53) - 1 DB_LIST = [d for d in DB if d != DB.Test] +FTS_SUPPORTED_DBS = {DB.Milvus, DB.ZillizCloud, DB.ElasticCloud, DB.Vespa, DB.TurboPuffer} class Delimiter(Enum): @@ -52,6 +53,7 @@ class UICaseItem(BaseModel): description: str = "" cases: list[CaseConfig] = [] caseLabel: CaseLabel = CaseLabel.Performance + supportedDbs: list[DB] | None = None extra_custom_case_config_inputs: list[ConfigInput] = [] tmp_custom_config: dict = dict() @@ -104,12 +106,21 @@ def get_cases(self) -> list[CaseConfig]: ] return cases + def supports_dbs(self, dbs: list[DB]) -> bool: + if self.supportedDbs is None: + return True + return all(db in self.supportedDbs for db in dbs) + class UICaseItemCluster(BaseModel): label: str = "" uiCaseItems: list[UICaseItem] = [] +def get_selectable_case_items(caseCluster: UICaseItemCluster, activedDbList: list[DB]) -> list[UICaseItem]: + return [uiCaseItem for uiCaseItem in caseCluster.uiCaseItems if uiCaseItem.supports_dbs(activedDbList)] + + def get_custom_case_items() -> list[UICaseItem]: custom_configs = get_custom_configs() return [ @@ -157,6 +168,30 @@ def generate_normal_cases(case_id: CaseType, custom_case: dict | None = None) -> return [CaseConfig(case_id=case_id, custom_case=custom_case)] +def generate_fts_case(dataset_with_size_type: FtsDatasetWithSizeType) -> CaseConfig: + return CaseConfig( + case_id=CaseType.FTSBm25Performance, + custom_case={"dataset_with_size_type": dataset_with_size_type.value}, + ) + + +def get_fts_case_items() -> list[UICaseItem]: + dataset_with_size_types = list(FtsDatasetWithSizeType) + return [ + UICaseItem( + label=f"FTS BM25 Performance - {dataset_with_size_type.value}", + description=( + f"This case tests native BM25 full-text search performance on {dataset_with_size_type.value}. " + "It measures index building time, recall, serial latency, and search QPS." + ), + cases=[generate_fts_case(dataset_with_size_type)], + caseLabel=CaseLabel.FullTextSearchPerformance, + supportedDbs=list(FTS_SUPPORTED_DBS), + ) + for dataset_with_size_type in dataset_with_size_types + ] + + def get_custom_case_cluter() -> UICaseItemCluster: return UICaseItemCluster(label="Custom Search Performance Test", uiCaseItems=get_custom_case_items()) @@ -358,6 +393,10 @@ def generate_int_filter_cases(dataset_with_size_type: DatasetWithSizeType) -> li ) ], ), + UICaseItemCluster( + label="Full-Text Search (FTS) Test", + uiCaseItems=get_fts_case_items(), + ), ] # DIVIDER = "DIVIDER" @@ -372,6 +411,7 @@ def generate_int_filter_cases(dataset_with_size_type: DatasetWithSizeType) -> li CaseType.Performance1024D10M, CaseType.CapacityDim960, CaseType.CapacityDim128, + CaseType.FTSBm25Performance, ] CASE_NAME_ORDER = [case.case_cls().name for case in DISPLAY_CASE_ORDER] @@ -379,15 +419,6 @@ def generate_int_filter_cases(dataset_with_size_type: DatasetWithSizeType) -> li # item for item in CASE_LIST_WITH_DIVIDER if isinstance(item, CaseType)] -class InputType(IntEnum): - Text = 20001 - Number = 20002 - Option = 20003 - Float = 20004 - Bool = 20005 - Select = 20006 - - class CaseConfigInput(BaseModel): label: CaseConfigParamType inputType: InputType = InputType.Text @@ -517,7 +548,7 @@ class CaseConfigInput(BaseModel): displayLabel="Reranking Metric", inputType=InputType.Option, inputConfig={ - "options": [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD", "DP"]], + "options": [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD", "DP", "BM25"]], }, isDisplayed=lambda config: config.get(CaseConfigParamType.reranking, False), ) @@ -1467,7 +1498,7 @@ class CaseConfigInput(BaseModel): label=CaseConfigParamType.rerankingMetric, inputType=InputType.Option, inputConfig={ - "options": [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD"]], + "options": [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD", "BM25"]], }, isDisplayed=lambda config: config.get(CaseConfigParamType.quantizationType, None) == "bit" and config.get(CaseConfigParamType.reranking, False), @@ -2119,6 +2150,113 @@ class CaseConfigInput(BaseModel): ), ) +CaseConfigParamInput_IndexType_FTS = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="FTS Index Type (currently only AUTOINDEX supported)", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.FTS.value, + ], + }, + isDisplayed=lambda config: False, # Hidden since FTS only has one index type currently +) + +CaseConfigParamInput_FTS_inverted_index_algo = CaseConfigInput( + label=CaseConfigParamType.inverted_index_algo, + displayLabel="Inverted Index Algorithm", + inputHelp="Algorithm for building and querying sparse inverted index.", + inputType=InputType.Option, + inputConfig={ + "value": "DAAT_MAXSCORE", + "options": ["DAAT_MAXSCORE", "DAAT_WAND", "TAAT_NAIVE"], + }, +) + +CaseConfigParamInput_FTS_bm25_k1 = CaseConfigInput( + label=CaseConfigParamType.bm25_k1, + displayLabel="BM25 k1", + inputHelp="BM25 k1 parameter controls term frequency saturation [1.2, 2.0]. Higher values emphasize term frequency.", + inputType=InputType.Float, + inputConfig={ + "value": 1.5, + "step": 0.1, + "min": 1.2, + "max": 2.0, + }, +) + +CaseConfigParamInput_FTS_bm25_b = CaseConfigInput( + label=CaseConfigParamType.bm25_b, + displayLabel="BM25 b", + inputHelp="BM25 b parameter controls document length normalization [0.0, 1.0]. 0.75 is commonly used.", + inputType=InputType.Float, + inputConfig={ + "value": 0.75, + "step": 0.05, + "min": 0.0, + "max": 1.0, + }, +) + +CaseConfigParamInput_FTS_analyzer_tokenizer = CaseConfigInput( + label=CaseConfigParamType.analyzer_tokenizer, + displayLabel="Analyzer Tokenizer", + inputHelp="Text tokenizer type for BM25 analysis.", + inputType=InputType.Option, + inputConfig={ + "value": "standard", + "options": ["standard", "whitespace", "keyword"], + }, +) + +CaseConfigParamInput_FTS_analyzer_enable_lowercase = CaseConfigInput( + label=CaseConfigParamType.analyzer_enable_lowercase, + displayLabel="Enable Lowercase Filter", + inputHelp="Convert text to lowercase during analysis.", + inputType=InputType.Bool, + inputConfig={ + "value": True, + }, +) + +CaseConfigParamInput_FTS_analyzer_max_token_length = CaseConfigInput( + label=CaseConfigParamType.analyzer_max_token_length, + displayLabel="Max Token Length", + inputHelp="Maximum length of individual tokens (optional, leave empty to disable).", + inputType=InputType.Number, + inputConfig={ + "value": None, + "min": 1, + "max": 100, + }, +) + +CaseConfigParamInput_FTS_analyzer_stop_words = CaseConfigInput( + label=CaseConfigParamType.analyzer_stop_words, + displayLabel="Stop Words", + inputHelp="Comma-separated list of stop words to filter out.", + inputType=InputType.Text, + inputConfig={ + "value": "", + "placeholder": "of,to,the,and,or", + }, +) + +CaseConfigParamInput_FTS_drop_ratio_search = CaseConfigInput( + label=CaseConfigParamType.drop_ratio_search, + displayLabel="Drop Ratio (search)", + inputHelp="Search-time sparsification ratio. 0.0 keeps best recall; larger is faster.", + inputType=InputType.Float, + inputConfig={ + "value": 0.0, + "step": 0.1, + "min": 0.0, + "max": 0.99, + }, +) + + MilvusLoadConfig = [ CaseConfigParamInput_IndexType, CaseConfigParamInput_M, @@ -2168,6 +2306,28 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_Milvus_use_partition_key, ] + +MilvusFtsConfig = [ + CaseConfigParamInput_IndexType_FTS, + CaseConfigParamInput_FTS_inverted_index_algo, + CaseConfigParamInput_FTS_bm25_k1, + CaseConfigParamInput_FTS_bm25_b, + CaseConfigParamInput_FTS_analyzer_tokenizer, + CaseConfigParamInput_FTS_analyzer_enable_lowercase, + CaseConfigParamInput_FTS_analyzer_max_token_length, + CaseConfigParamInput_FTS_analyzer_stop_words, + CaseConfigParamInput_FTS_drop_ratio_search, +] + +ZillizCloudFtsConfig = [ + CaseConfigParamInput_IndexType_FTS, + CaseConfigParamInput_ZillizLevel, +] + +ElasticCloudFtsConfig = [] +VespaFtsConfig = [] +TurboPufferFtsConfig = [] + WeaviateLoadConfig = [ CaseConfigParamInput_MaxConnections, CaseConfigParamInput_EFConstruction_Weaviate, @@ -2976,9 +3136,11 @@ class FilterType(Enum): CaseLabel.Load: MilvusLoadConfig, CaseLabel.Performance: MilvusPerformanceConfig, CaseLabel.Streaming: MilvusPerformanceConfig, + CaseLabel.FullTextSearchPerformance: MilvusFtsConfig, }, DB.ZillizCloud: { CaseLabel.Performance: ZillizCloudPerformanceConfig, + CaseLabel.FullTextSearchPerformance: ZillizCloudFtsConfig, }, DB.WeaviateCloud: { CaseLabel.Load: WeaviateLoadConfig, @@ -2987,6 +3149,7 @@ class FilterType(Enum): DB.ElasticCloud: { CaseLabel.Load: ESLoadingConfig, CaseLabel.Performance: ESPerformanceConfig, + CaseLabel.FullTextSearchPerformance: ElasticCloudFtsConfig, }, DB.AWSOpenSearch: { CaseLabel.Load: AWSOpensearchLoadingConfig, @@ -3035,6 +3198,7 @@ class FilterType(Enum): DB.Vespa: { CaseLabel.Load: VespaLoadingConfig, CaseLabel.Performance: VespaPerformanceConfig, + CaseLabel.FullTextSearchPerformance: VespaFtsConfig, }, DB.LanceDB: { CaseLabel.Load: LanceDBLoadConfig, @@ -3064,14 +3228,18 @@ class FilterType(Enum): CaseLabel.Load: PolarDBConfig, CaseLabel.Performance: PolarDBConfig, }, + DB.TurboPuffer: { + CaseLabel.FullTextSearchPerformance: TurboPufferFtsConfig, + }, } def get_case_config_inputs(db: DB, case_label: CaseLabel) -> list[CaseConfigInput]: - if db not in CASE_CONFIG_MAP: - return [] + db_case_config_map = CASE_CONFIG_MAP.get(db, {}) if case_label == CaseLabel.Load: - return CASE_CONFIG_MAP[db][CaseLabel.Load] - elif case_label == CaseLabel.Performance or case_label == CaseLabel.Streaming: - return CASE_CONFIG_MAP[db][CaseLabel.Performance] + return db_case_config_map.get(CaseLabel.Load, []) + if case_label == CaseLabel.Performance or case_label == CaseLabel.Streaming: + return db_case_config_map.get(CaseLabel.Performance, []) + elif case_label == CaseLabel.FullTextSearchPerformance: + return db_case_config_map.get(CaseLabel.FullTextSearchPerformance, []) return [] diff --git a/vectordb_bench/frontend/pages/full_text_search.py b/vectordb_bench/frontend/pages/full_text_search.py new file mode 100644 index 000000000..ea20076c3 --- /dev/null +++ b/vectordb_bench/frontend/pages/full_text_search.py @@ -0,0 +1,365 @@ +import json +import re +from pathlib import Path +from typing import Any + +import pandas as pd +import plotly.express as px +import streamlit as st + +from vectordb_bench import config +from vectordb_bench.frontend.components.check_results.footer import footer +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import NavToPages +from vectordb_bench.frontend.config.styles import FAVICON + +RESULT_DIR = config.RESULTS_LOCAL_DIR / "FullTextSearch" +DATASET_ORDER = [ + "MS MARCO Small", + "MS MARCO Medium", + "MS MARCO Large", + "HotpotQA Small", + "HotpotQA Medium", + "HotpotQA Large", +] +# Published FTS results currently cover this cloud/service backend subset. +BACKEND_ORDER = ["ZillizCloud", "ElasticSearch", "Vespa", "TurboPuffer"] +BACKEND_COLORS = { + "ZillizCloud": "#0D6EFD", + "ElasticSearch": "#04D6C8", + "Vespa": "#61D790", + "TurboPuffer": "#FF6B2C", +} +SIZE_ORDER = ["Small", "Medium", "Large"] + + +def _normalize_backend(db: str, result_file: Path) -> str: + if db == "ElasticCloud": + return "ElasticSearch" + if db: + return db + return result_file.parent.name + + +def _dataset_parts(dataset_label: str) -> tuple[str, str, str]: + if dataset_label.startswith("MS MARCO"): + family = "MS MARCO" + elif dataset_label.startswith("HotpotQA"): + family = "HotpotQA" + else: + family = dataset_label.split(" ", 1)[0] + + size = next((name for name in SIZE_ORDER if name in dataset_label), "") + return family, size, f"{family} {size}".strip() + + +def _dataset_doc_count(dataset_label: str) -> str: + match = re.search(r"\(([^)]+)\)", dataset_label) + return match.group(1) if match else "" + + +def _dataset_axis_label(dataset: str, doc_count: str) -> str: + return f"{dataset}
{doc_count}" if doc_count else dataset + + +def _dataset_axis_order(data: pd.DataFrame) -> list[str]: + labels = [] + for dataset in DATASET_ORDER: + matches = data[data["dataset"].astype(str) == dataset] + if not matches.empty: + labels.append(matches["dataset_axis_label"].iloc[0]) + return labels + + +def _backend_metric_order(data: pd.DataFrame, metric: str, ascending: bool) -> list[str]: + if data.empty or metric not in data: + return BACKEND_ORDER + + metric_data = data[["backend", metric]].dropna().copy() + if metric_data.empty: + return BACKEND_ORDER + + metric_data["backend"] = metric_data["backend"].astype(str) + scores = metric_data.groupby("backend")[metric].mean() + ordered = scores.sort_values(ascending=ascending).index.tolist() + return ordered + [backend for backend in BACKEND_ORDER if backend not in ordered] + + +def _run_context(task_label: str) -> str: + if "mathgt" in task_label: + return "Math GT" + return "Recorded" + + +def _parse_result_file(result_file: Path) -> list[dict[str, Any]]: + with result_file.open() as f: + test_result = json.load(f) + + task_label = test_result.get("task_label") or result_file.stem + rows = [] + for case_result in test_result.get("results", []): + metrics = case_result.get("metrics", {}) + task_config = case_result.get("task_config", {}) + case_config = task_config.get("case_config", {}) + custom_case = case_config.get("custom_case") or {} + dataset_label = custom_case.get("dataset_with_size_type", "") + dataset_family, dataset_size, dataset_key = _dataset_parts(dataset_label) + dataset_doc_count = _dataset_doc_count(dataset_label) + dataset_axis_label = _dataset_axis_label(dataset_key, dataset_doc_count) + backend = _normalize_backend(task_config.get("db", ""), result_file) + payload = metrics.get("payload_profile") or custom_case.get("payload_profile") or "ids_only" + + rows.append( + { + "backend": backend, + "dataset_family": dataset_family, + "dataset_size": dataset_size, + "dataset": dataset_key, + "dataset_doc_count": dataset_doc_count, + "dataset_axis_label": dataset_axis_label, + "payload": payload, + "context": _run_context(task_label), + "task_label": task_label, + "load_s": metrics.get("load_duration", 0.0), + "qps": metrics.get("qps", 0.0), + "recall": metrics.get("recall", 0.0), + "p95_s": metrics.get("serial_latency_p95", 0.0), + "p99_s": metrics.get("serial_latency_p99", 0.0), + "concurrency": metrics.get("conc_num_list") or [], + "concurrent_qps": metrics.get("conc_qps_list") or [], + } + ) + + return rows + + +def _latest_backend_result_files(result_dir: Path) -> list[Path]: + result_files = [] + backend_dirs = sorted(path for path in result_dir.iterdir() if path.is_dir()) + for backend_dir in backend_dirs: + backend_files = sorted(backend_dir.glob("result_*.json")) + if backend_files: + result_files.append(backend_files[-1]) + return result_files + + +def load_full_text_search_rows(result_dir: Path = RESULT_DIR) -> pd.DataFrame: + if not result_dir.exists(): + return pd.DataFrame() + + rows = [] + for result_file in _latest_backend_result_files(result_dir): + rows.extend(_parse_result_file(result_file)) + + data = pd.DataFrame(rows) + if data.empty: + return data + + data = data[data["backend"] != "Milvus"].copy() + if data.empty: + return data + + data["dataset"] = pd.Categorical(data["dataset"], DATASET_ORDER, ordered=True) + data["backend"] = pd.Categorical(data["backend"], BACKEND_ORDER, ordered=True) + data["dataset_size"] = pd.Categorical(data["dataset_size"], SIZE_ORDER, ordered=True) + return data.sort_values(["dataset", "backend", "payload"]).reset_index(drop=True) + + +def _filter_data(st: Any, data: pd.DataFrame) -> pd.DataFrame: + with st.sidebar: + st.header("Filters") + selected_datasets = st.multiselect( + "Dataset", + [dataset for dataset in DATASET_ORDER if dataset in set(data["dataset"].astype(str))], + default=[dataset for dataset in DATASET_ORDER if dataset in set(data["dataset"].astype(str))], + ) + backend_options = [backend for backend in BACKEND_ORDER if backend in set(data["backend"].astype(str))] + selected_backends = st.multiselect("Backend", backend_options, default=backend_options) + payloads = sorted(data["payload"].dropna().unique().tolist()) + default_payloads = ["ids_only"] if "ids_only" in payloads else payloads + selected_payloads = st.multiselect("Payload", payloads, default=default_payloads) + + filters = ( + data["dataset"].astype(str).isin(selected_datasets) + & data["backend"].astype(str).isin(selected_backends) + & data["payload"].isin(selected_payloads) + ) + + return data[filters].copy() + + +def _draw_summary_table(st: Any, data: pd.DataFrame) -> None: + columns = [ + "dataset", + "backend", + "payload", + "load_s", + "qps", + "recall", + "p95_s", + "p99_s", + ] + st.dataframe( + data[columns], + hide_index=True, + width="stretch", + column_config={ + "load_s": st.column_config.NumberColumn("Load s", format="%.4f"), + "qps": st.column_config.NumberColumn("QPS", format="%.4f"), + "recall": st.column_config.NumberColumn("Recall", format="%.4f"), + "p95_s": st.column_config.NumberColumn("p95 s", format="%.4f"), + "p99_s": st.column_config.NumberColumn("p99 s", format="%.4f"), + }, + ) + + +def _draw_metric_chart( + st: Any, + data: pd.DataFrame, + metric: str, + title: str, + backend_order: list[str] | None = None, +) -> None: + if backend_order is None: + backend_order = BACKEND_ORDER + + show_text = data["payload"].nunique() <= 1 + fig = px.bar( + data, + x="dataset_axis_label", + y=metric, + color="backend", + pattern_shape="payload", + barmode="group", + category_orders={"dataset_axis_label": _dataset_axis_order(data), "backend": backend_order}, + color_discrete_map=BACKEND_COLORS, + hover_data=["dataset_doc_count", "payload", "context", "task_label"], + text_auto=".4g" if show_text else False, + title=title, + ) + if show_text: + text_template = "%{y:.4f}" if metric == "recall" else "%{y:.1f}" + fig.update_traces( + texttemplate=text_template, + textposition="outside", + textangle=0, + textfont={"size": 11}, + cliponaxis=False, + ) + fig.update_layout( + margin={"l": 0, "r": 0, "t": 56, "b": 12, "pad": 8}, + legend={"orientation": "h", "yanchor": "bottom", "y": 1, "xanchor": "right", "x": 1, "title": ""}, + xaxis_title="", + xaxis={"tickfont": {"size": 12}}, + uniformtext={"minsize": 10, "mode": "show"}, + ) + st.plotly_chart(fig, width="stretch", key=f"fts-{metric}") + + +def _concurrency_rows(data: pd.DataFrame) -> pd.DataFrame: + rows = [] + for row in data.to_dict("records"): + for concurrency, qps in zip(row["concurrency"], row["concurrent_qps"], strict=True): + rows.append( + { + "dataset": row["dataset"], + "dataset_axis_label": row["dataset_axis_label"], + "dataset_doc_count": row["dataset_doc_count"], + "backend": row["backend"], + "payload": row["payload"], + "context": row["context"], + "concurrency": concurrency, + "qps": qps, + "task_label": row["task_label"], + } + ) + return pd.DataFrame(rows) + + +def _draw_concurrency_chart(st: Any, data: pd.DataFrame) -> None: + concurrency_data = _concurrency_rows(data) + if concurrency_data.empty: + return + + fig = px.line( + concurrency_data, + x="concurrency", + y="qps", + color="backend", + line_dash="dataset_axis_label", + symbol="payload", + markers=True, + category_orders={"dataset_axis_label": _dataset_axis_order(data), "backend": BACKEND_ORDER}, + color_discrete_map=BACKEND_COLORS, + hover_data=["dataset", "dataset_doc_count", "payload", "context", "task_label"], + title="Concurrent Search QPS", + ) + fig.update_layout( + margin={"l": 0, "r": 0, "t": 48, "b": 12, "pad": 8}, + legend={"orientation": "h", "yanchor": "bottom", "y": 1, "xanchor": "right", "x": 1, "title": ""}, + ) + fig.update_xaxes(title_text="Concurrency") + fig.update_yaxes(title_text="QPS") + st.plotly_chart(fig, width="stretch", key="fts-concurrency-qps") + + +def main(): + st.set_page_config( + page_title="Full Text Search Cloud Results", + page_icon=FAVICON, + layout="wide", + ) + + drawHeaderIcon(st) + NavToPages(st) + + st.title("Full Text Search Cloud Results") + st.caption("Published FTS results for Zilliz Cloud, ElasticSearch, Vespa, and TurboPuffer.") + + data = load_full_text_search_rows() + if data.empty: + st.warning("No FullTextSearch result JSONs found.") + footer(st.container()) + return + + shown_data = _filter_data(st, data) + if shown_data.empty: + st.warning("No rows match the selected filters.") + footer(st.container()) + return + + _draw_summary_table(st, shown_data) + chart_tabs = st.tabs(["QPS", "Recall", "Load"]) + with chart_tabs[0]: + qps_data = shown_data + _draw_metric_chart( + st, + qps_data, + "qps", + "Search QPS", + _backend_metric_order(qps_data, "qps", ascending=False), + ) + with chart_tabs[1]: + recall_data = shown_data[shown_data["payload"] == "ids_only"] + _draw_metric_chart( + st, + recall_data, + "recall", + "Math-GT Recall", + _backend_metric_order(recall_data, "recall", ascending=False), + ) + with chart_tabs[2]: + load_data = shown_data[shown_data["payload"] == "ids_only"] + _draw_metric_chart( + st, + load_data, + "load_s", + "Load Duration", + _backend_metric_order(load_data, "load_s", ascending=True), + ) + + footer(st.container()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/pages/qps_recall.py b/vectordb_bench/frontend/pages/qps_recall.py index fb8f680c5..75dcd570b 100644 --- a/vectordb_bench/frontend/pages/qps_recall.py +++ b/vectordb_bench/frontend/pages/qps_recall.py @@ -41,7 +41,10 @@ def main(): def case_results_filter(case_result: CaseResult) -> bool: case = case_result.task_config.case_config.case - return case.label == CaseLabel.Performance and case.filters.type == FilterOp.NonFilter + # Include both vector performance cases and FTS cases + return ( + case.label == CaseLabel.Performance and case.filters.type == FilterOp.NonFilter + ) or case.label == CaseLabel.FullTextSearchPerformance default_selected_task_labels = ["standard_20260403", "standard_20250519"] # Filter defaults to only include labels that exist in results diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 8b603be24..78e6958a2 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -43,14 +43,18 @@ def __init__(self): # set default data source by ENV if config.DATASET_SOURCE.upper() == "ALIYUNOSS": self.dataset_source: DatasetSource = DatasetSource.AliyunOSS + elif config.DATASET_SOURCE.upper() == "IR_DATASETS": + self.dataset_source: DatasetSource = DatasetSource.IR_DATASETS else: self.dataset_source: DatasetSource = DatasetSource.S3 def set_drop_old(self, drop_old: bool): self.drop_old = drop_old - def set_download_address(self, use_aliyun: bool): - if use_aliyun: + def set_download_address(self, use_aliyun: bool, use_ir_datasets: bool = False): + if use_ir_datasets: + self.dataset_source = DatasetSource.IR_DATASETS + elif use_aliyun: self.dataset_source = DatasetSource.AliyunOSS else: self.dataset_source = DatasetSource.S3 diff --git a/vectordb_bench/metric.py b/vectordb_bench/metric.py index 5a7c14e82..f442b2e97 100644 --- a/vectordb_bench/metric.py +++ b/vectordb_bench/metric.py @@ -119,3 +119,11 @@ def calc_ndcg(ground_truth: list[int], got: list[int], ideal_dcg: float) -> floa idx = ground_truth.index(got_id) dcg += 1 / np.log2(idx + 2) return dcg / ideal_dcg + + +def calc_recall_fts(k: int, ground_truth: list[int], got: list[int]) -> float: + if not ground_truth or k <= 0: + return 0.0 + gt_set = set(ground_truth) + hits = gt_set & set(got[:k]) + return calc_recall(len(gt_set), gt_set, hits) diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index dc1709cc0..855268cd1 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -18,6 +18,7 @@ DBConfig, EmptyDBCaseConfig, ) +from .backend.clients.api import IndexType from .base import BaseModel from .metric import Metric @@ -45,6 +46,16 @@ class CaseConfigParamType(Enum): """ IndexType = "IndexType" + drop_ratio_search = "drop_ratio_search" + drop_ratio_build = "drop_ratio_build" + bm25_k1 = "bm25_k1" + bm25_b = "bm25_b" + inverted_index_algo = "inverted_index_algo" + analyzer_tokenizer = "analyzer_tokenizer" + analyzer_enable_lowercase = "analyzer_enable_lowercase" + analyzer_max_len = "analyzer_max_len" + analyzer_max_token_length = "analyzer_max_token_length" # noqa: S105 + analyzer_stop_words = "analyzer_stop_words" index = "index" M = "M" EFConstruction = "efConstruction" @@ -416,6 +427,10 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: # Safely instantiate DBCaseConfig (fallback to EmptyDBCaseConfig on None) raw_case_cfg = task_config.get("db_case_config") or {} index_value = raw_case_cfg.get("index", None) + + # Handle FTS cases + if case_config.get("case_id") == CaseType.FTSBm25Performance.value: + index_value = IndexType.FTS try: task_config["db_case_config"] = db.case_config_cls(index_type=index_value)(**raw_case_cfg) except Exception: diff --git a/vectordb_bench/restful/format_res.py b/vectordb_bench/restful/format_res.py index 326986319..a7f8cfe9d 100644 --- a/vectordb_bench/restful/format_res.py +++ b/vectordb_bench/restful/format_res.py @@ -31,11 +31,13 @@ class FormatResult(BaseModel): load_duration: int = 0 qps: float = 0 serial_latency_p99: float = 0 + serial_latency_p95: float = 0 recall: float = 0 ndcg: float = 0 conc_num_list: list[int] = [] conc_qps_list: list[float] = [] conc_latency_p99_list: list[float] = [] + conc_latency_p95_list: list[float] = [] conc_latency_avg_list: list[float] = [] @@ -66,7 +68,7 @@ def format_results(test_results: list[TestResult], task_label: str) -> list[dict params=task_config.db_case_config.model_dump(), case_name=case.name, dataset=dataset.full_name, - dim=dataset.dim, + dim=getattr(dataset, "dim", 0), filter_type=filter_.type.name, filter_rate=filter_.filter_rate, k=task_config.case_config.k, diff --git a/vectordb_bench/results/FullTextSearch/ElasticCloud/result_20260626_fts_standard_elasticcloud.json b/vectordb_bench/results/FullTextSearch/ElasticCloud/result_20260626_fts_standard_elasticcloud.json new file mode 100644 index 000000000..f93240e37 --- /dev/null +++ b/vectordb_bench/results/FullTextSearch/ElasticCloud/result_20260626_fts_standard_elasticcloud.json @@ -0,0 +1,1622 @@ +{ + "run_id": "fts_standard_elasticcloud", + "task_label": "fts_standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 92.7146, + "optimize_duration": 90.3815, + "load_duration": 183.0961, + "qps": 674.5161, + "serial_latency_p99": 0.0807, + "serial_latency_p95": 0.054, + "recall": 0.9191, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 646.8905, + 674.5161 + ], + "conc_latency_p99_list": [ + 0.18089975638085887, + 0.2406859093923414 + ], + "conc_latency_p95_list": [ + 0.12103873724954597, + 0.17825506660083193 + ], + "conc_latency_avg_list": [ + 0.06169217704263551, + 0.11838266990017098 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 47.43239876568051 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:49:51.080771", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 92.7146, + "optimize_duration": 90.3815, + "load_duration": 183.0961, + "qps": 593.5741, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 550.5964, + 593.5741 + ], + "conc_latency_p99_list": [ + 0.2036623409306056, + 0.24277012127848122 + ], + "conc_latency_p95_list": [ + 0.13283362570000462, + 0.19686596379851826 + ], + "conc_latency_avg_list": [ + 0.07247916053391205, + 0.13447533944632156 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 47.43239876568051 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:57:42.938491", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 17.887, + "optimize_duration": 35.1281, + "load_duration": 53.015, + "qps": 1985.7794, + "serial_latency_p99": 0.0245, + "serial_latency_p95": 0.0172, + "recall": 0.9241, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 1866.3556, + 1985.7794 + ], + "conc_latency_p99_list": [ + 0.06111374244974294, + 0.07737018370979049 + ], + "conc_latency_p95_list": [ + 0.04278086074828025, + 0.060899814298318235 + ], + "conc_latency_avg_list": [ + 0.021414252878386827, + 0.04022542792055797 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 55.803761 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:43:25.431349", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 17.887, + "optimize_duration": 35.1281, + "load_duration": 53.015, + "qps": 1674.5226, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 1605.1991, + 1674.5226 + ], + "conc_latency_p99_list": [ + 0.06576651665181996, + 0.08476530473002608 + ], + "conc_latency_p95_list": [ + 0.04643557349754701, + 0.0706571103990427 + ], + "conc_latency_avg_list": [ + 0.024898764599791963, + 0.047717378295758305 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 55.803761 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:47:10.387803", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1.9662, + "optimize_duration": 31.09, + "load_duration": 33.0562, + "qps": 6719.6031, + "serial_latency_p99": 0.0053, + "serial_latency_p95": 0.0043, + "recall": 0.9367, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 6293.0808, + 6719.6031 + ], + "conc_latency_p99_list": [ + 0.021160466300316292, + 0.033499905739445265 + ], + "conc_latency_p95_list": [ + 0.014183212001444187, + 0.024311021799803705 + ], + "conc_latency_avg_list": [ + 0.0063481626425471735, + 0.011883786454976971 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 56.88413 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:38:23.654768", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1.9662, + "optimize_duration": 31.09, + "load_duration": 33.0562, + "qps": 3471.0893, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 3245.7069, + 3471.0893 + ], + "conc_latency_p99_list": [ + 0.03633042563960771, + 0.04837336179916747 + ], + "conc_latency_p95_list": [ + 0.025955834001797476, + 0.038765777499065734 + ], + "conc_latency_avg_list": [ + 0.01231323081025481, + 0.023020162052078588 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 56.88413 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:41:05.239861", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 160.1365, + "optimize_duration": 92.5894, + "load_duration": 252.7259, + "qps": 1582.6793, + "serial_latency_p99": 0.0502, + "serial_latency_p95": 0.0306, + "recall": 0.9352, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 1485.1078, + 1582.6793 + ], + "conc_latency_p99_list": [ + 0.10130014419868526, + 0.1204227190394886 + ], + "conc_latency_p95_list": [ + 0.06413858909909301, + 0.08464620050290249 + ], + "conc_latency_avg_list": [ + 0.026891243904133805, + 0.05046188340033532 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 57.83363736188793 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:25:04.982818", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 160.1365, + "optimize_duration": 92.5894, + "load_duration": 252.7259, + "qps": 1204.033, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 1131.3448, + 1204.033 + ], + "conc_latency_p99_list": [ + 0.10636549504153667, + 0.13718578399857506 + ], + "conc_latency_p95_list": [ + 0.0707971137984714, + 0.10115620599754038 + ], + "conc_latency_avg_list": [ + 0.03531235997717238, + 0.06635058395138439 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 57.83363736188793 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:32:21.485921", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 17.8893, + "optimize_duration": 34.2775, + "load_duration": 52.1668, + "qps": 5702.2528, + "serial_latency_p99": 0.0092, + "serial_latency_p95": 0.0065, + "recall": 0.9437, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 5383.2674, + 5702.2528 + ], + "conc_latency_p99_list": [ + 0.02630764468107369, + 0.03738318958105083 + ], + "conc_latency_p95_list": [ + 0.017680951600959793, + 0.02877839460124961 + ], + "conc_latency_avg_list": [ + 0.007422129557916056, + 0.014006341815361298 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 57.242324 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:19:23.319688", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 17.8893, + "optimize_duration": 34.2775, + "load_duration": 52.1668, + "qps": 3677.9597, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 3473.3114, + 3677.9597 + ], + "conc_latency_p99_list": [ + 0.0357281443210376, + 0.04769462314165139 + ], + "conc_latency_p95_list": [ + 0.025091530600184316, + 0.03831459170251036 + ], + "conc_latency_avg_list": [ + 0.011505997403950962, + 0.021721806785628715 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 57.242324 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:22:25.552010", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1.8862, + "optimize_duration": 31.0352, + "load_duration": 32.9214, + "qps": 12479.8332, + "serial_latency_p99": 0.0029, + "serial_latency_p95": 0.0023, + "recall": 0.9497, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 11315.3177, + 12479.8332 + ], + "conc_latency_p99_list": [ + 0.012597403119434608, + 0.024557725561316985 + ], + "conc_latency_p95_list": [ + 0.00729868780035758, + 0.01442482639795344 + ], + "conc_latency_avg_list": [ + 0.003526799491246593, + 0.00638580620427849 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 55.01342 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:14:30.502248", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1.8862, + "optimize_duration": 31.0352, + "load_duration": 32.9214, + "qps": 5037.2489, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 4701.0004, + 5037.2489 + ], + "conc_latency_p99_list": [ + 0.026061404299944115, + 0.03860565954051707 + ], + "conc_latency_p95_list": [ + 0.017935610502172503, + 0.030578274452818733 + ], + "conc_latency_avg_list": [ + 0.008499634080099454, + 0.01585666237961175 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75 + }, + "unapplied_bm25_params": { + "avgdl": 55.01342 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-06-23T17:17:03.106278", + "version": "", + "note": "", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1782172800.0 +} diff --git a/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json b/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json new file mode 100644 index 000000000..a063a9b83 --- /dev/null +++ b/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json @@ -0,0 +1,1646 @@ +{ + "run_id": "fts_standard_turbopuffer", + "task_label": "fts_standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1464.6515, + "optimize_duration": 60.0477, + "load_duration": 1524.6993, + "qps": 1005.2819, + "serial_latency_p99": 0.044, + "serial_latency_p95": 0.0329, + "recall": 0.8665, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 989.8197, + 982.1386, + 1005.2819 + ], + "conc_latency_p99_list": [ + 1.0339552274506423, + 1.0614141076010128, + 1.067533793257171 + ], + "conc_latency_p95_list": [ + 0.047041563849779776, + 0.05456624450198433, + 0.1361412918016783 + ], + "conc_latency_avg_list": [ + 0.03993696978478835, + 0.06008994437867671, + 0.0777533349887082 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_large\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-large-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:21:56.394316", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-hotpotqa-large-c20-b1000-20260624T105757Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1464.6515, + "optimize_duration": 60.0477, + "load_duration": 1524.6993, + "qps": 979.5781, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 979.5781, + 969.5597, + 964.3595 + ], + "conc_latency_p99_list": [ + 1.035232208402449, + 1.0622234767588816, + 1.068804296057642 + ], + "conc_latency_p95_list": [ + 0.047188996000477344, + 0.054698529998131545, + 1.008040709995839 + ], + "conc_latency_avg_list": [ + 0.04051427541809371, + 0.06074978477290257, + 0.08095703978218151 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_large\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-large-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:43:28.084741", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-hotpotqa-large-c20-b1000-20260624T105757Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 72.6864, + "optimize_duration": 60.0531, + "load_duration": 132.7395, + "qps": 1470.4418, + "serial_latency_p99": 0.0481, + "serial_latency_p95": 0.0263, + "recall": 0.9224, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1470.4418, + 1430.4126, + 1302.5586 + ], + "conc_latency_p99_list": [ + 0.196402730199046, + 1.0448004564788427, + 1.0593402567028534 + ], + "conc_latency_p95_list": [ + 0.030726359249456436, + 0.03609789424808695, + 0.04879993350186851 + ], + "conc_latency_avg_list": [ + 0.026958557542264637, + 0.04139604535845175, + 0.05992647821000913 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_medium\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-medium-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:15:28.328411", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-hotpotqa-medium-c20-b1000-20260624T105757Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 72.6864, + "optimize_duration": 60.0531, + "load_duration": 132.7395, + "qps": 1540.7809, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1540.7809, + 1473.7761, + 1361.0409 + ], + "conc_latency_p99_list": [ + 0.1373698992803109, + 1.0427753794664023, + 1.0594265741333946 + ], + "conc_latency_p95_list": [ + 0.02906204179889753, + 0.03409058889992593, + 0.04557989979657569 + ], + "conc_latency_avg_list": [ + 0.025724693255668438, + 0.04015530633661177, + 0.05740395977107171 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_medium\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-medium-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:39:57.159706", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-hotpotqa-medium-c20-b1000-20260624T105757Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 7.1608, + "optimize_duration": 60.0462, + "load_duration": 67.207, + "qps": 1589.0248, + "serial_latency_p99": 0.0413, + "serial_latency_p95": 0.0222, + "recall": 0.9382, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1298.648, + 1556.9237, + 1589.0248 + ], + "conc_latency_p99_list": [ + 1.0122341442467582, + 1.040216220889779, + 1.0511543367576086 + ], + "conc_latency_p95_list": [ + 0.0409361274978437, + 0.03276492429940844, + 0.03599922600260471 + ], + "conc_latency_avg_list": [ + 0.030516342558019813, + 0.03799870032145427, + 0.0491667296798954 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_small\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-small-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:09:49.389682", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-hotpotqa-small-c20-b1000-20260624T105757Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 7.1608, + "optimize_duration": 60.0462, + "load_duration": 67.207, + "qps": 1732.5275, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1732.5275, + 1573.2425, + 1644.7612 + ], + "conc_latency_p99_list": [ + 0.10047229536954584, + 1.0371929118002297, + 1.050296281276096 + ], + "conc_latency_p95_list": [ + 0.02665470994543283, + 0.032170177746593254, + 0.03446190499817008 + ], + "conc_latency_avg_list": [ + 0.022886237167794053, + 0.037611734150211816, + 0.04739313356214065 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_small\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-small-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:36:19.214107", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-hotpotqa-small-c20-b1000-20260624T105757Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 832.0622, + "optimize_duration": 60.0494, + "load_duration": 892.1117, + "qps": 1316.9277, + "serial_latency_p99": 0.042, + "serial_latency_p95": 0.026, + "recall": 0.9395, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1316.9277, + 1297.3733, + 1284.6661 + ], + "conc_latency_p99_list": [ + 1.0073482914027407, + 1.0495113781692633, + 1.0606270659984147 + ], + "conc_latency_p95_list": [ + 0.035672420999617316, + 0.04159055870040899, + 0.04908532199988258 + ], + "conc_latency_avg_list": [ + 0.03006028460855096, + 0.045569930162918985, + 0.061195468189531976 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 10626.3967, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_large_c20_b1000_cli_sdkretry_20260624T094413Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-large-c20-b1000-sdkretry-20260624T094413Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:05:15.433938", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-tpuf-msmarco-large-c20-b1000-sdkretry-20260624T094413Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 832.0622, + "optimize_duration": 60.0494, + "load_duration": 892.1117, + "qps": 1302.7749, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1302.7749, + 1164.7122, + 1273.9904 + ], + "conc_latency_p99_list": [ + 1.0048391348012942, + 1.0523817389193573, + 1.0604548653973325 + ], + "conc_latency_p95_list": [ + 0.03545594900060678, + 0.04788232199934996, + 0.04981473534498943 + ], + "conc_latency_avg_list": [ + 0.030413566662682186, + 0.0503768273187017, + 0.061305946983837616 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 8841823, + "insert_rows_per_second": 10626.3967, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_large_c20_b1000_cli_sdkretry_20260624T094413Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-large-c20-b1000-sdkretry-20260624T094413Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:33:35.259663", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-tpuf-msmarco-large-c20-b1000-sdkretry-20260624T094413Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 70.9455, + "optimize_duration": 60.0465, + "load_duration": 130.9921, + "qps": 1506.0323, + "serial_latency_p99": 0.0363, + "serial_latency_p95": 0.0227, + "recall": 0.9488, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1506.0323, + 1501.8061, + 1494.903 + ], + "conc_latency_p99_list": [ + 0.1576661457991571, + 1.040742779001448, + 1.0539034581720625 + ], + "conc_latency_p95_list": [ + 0.0298914346043603, + 0.03358310699695721, + 0.03905020675083514 + ], + "conc_latency_avg_list": [ + 0.026345458167461474, + 0.03912392624068874, + 0.05247619187674705 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 14095.3267, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_medium_c20_b1000_cli_20260624T085600Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-medium-c20-b1000-cli-20260624T085600Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:00:41.474589", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-tpuf-msmarco-medium-c20-b1000-cli-20260624T085600Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 70.9455, + "optimize_duration": 60.0465, + "load_duration": 130.9921, + "qps": 1573.3805, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1573.3805, + 1444.0492, + 1421.5596 + ], + "conc_latency_p99_list": [ + 0.12313141820966869, + 1.0423146819185058, + 1.0560591126610959 + ], + "conc_latency_p95_list": [ + 0.02848500000254715, + 0.0345763255034399, + 0.04240300605015367 + ], + "conc_latency_avg_list": [ + 0.025195208364014388, + 0.04078606566304797, + 0.05524199400874184 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 1000000, + "insert_rows_per_second": 14095.3267, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_medium_c20_b1000_cli_20260624T085600Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-medium-c20-b1000-cli-20260624T085600Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:30:55.307994", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-tpuf-msmarco-medium-c20-b1000-cli-20260624T085600Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 7.6649, + "optimize_duration": 60.0529, + "load_duration": 67.7177, + "qps": 1588.1696, + "serial_latency_p99": 0.0379, + "serial_latency_p95": 0.0204, + "recall": 0.9537, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1588.1696, + 1437.2379, + 1542.8857 + ], + "conc_latency_p99_list": [ + 0.14128930038219492, + 1.043362593751226, + 1.0526617984978657 + ], + "conc_latency_p95_list": [ + 0.03021625500296065, + 0.03545568099798402, + 0.03865519299870357 + ], + "conc_latency_avg_list": [ + 0.024957518324872457, + 0.041209095071141, + 0.050988707258252025 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 13046.4846, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_small_c20_b1000_cli_20260624T084228Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-small-c20-b1000-cli-20260624T084228Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T11:56:16.498129", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-tpuf-msmarco-small-c20-b1000-cli-20260624T084228Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 7.6649, + "optimize_duration": 60.0529, + "load_duration": 67.7177, + "qps": 1565.7761, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1565.7761, + 1296.9044, + 1395.2693 + ], + "conc_latency_p99_list": [ + 0.1240674191407033, + 1.049226711880474, + 1.055339480610637 + ], + "conc_latency_p95_list": [ + 0.028216213650011923, + 0.04351905840157994, + 0.04549482634683953 + ], + "conc_latency_avg_list": [ + 0.0253025184556393, + 0.0456011056513432, + 0.056052650601209515 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 100000, + "insert_rows_per_second": 13046.4846, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + }, + "num_per_batch": 1000, + "load_concurrency": 0, + "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_small_c20_b1000_cli_20260624T084228Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-small-c20-b1000-cli-20260624T084228Z_turbopuffer.json" + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "2026-06-24T12:28:10.345065", + "version": "", + "note": "", + "api_key": "**********", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-tpuf-msmarco-small-c20-b1000-cli-20260624T084228Z", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1782259200.0 +} diff --git a/vectordb_bench/results/FullTextSearch/Vespa/result_20260626_fts_standard_vespa.json b/vectordb_bench/results/FullTextSearch/Vespa/result_20260626_fts_standard_vespa.json new file mode 100644 index 000000000..eb7160a3e --- /dev/null +++ b/vectordb_bench/results/FullTextSearch/Vespa/result_20260626_fts_standard_vespa.json @@ -0,0 +1,1514 @@ +{ + "run_id": "fts_standard_vespa", + "task_label": "fts_standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 326.0195, + "optimize_duration": 0.0467, + "load_duration": 326.0662, + "qps": 178.292, + "serial_latency_p99": 0.445, + "serial_latency_p95": 0.4445, + "recall": 0.7532, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 92.1252, + 178.292 + ], + "conc_latency_p99_list": [ + 0.4649717587499981, + 0.47357734852928834 + ], + "conc_latency_p95_list": [ + 0.45766577114909524, + 0.4641533440004423 + ], + "conc_latency_avg_list": [ + 0.4314541248062237, + 0.4440924808065595 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 47.43239876568051, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 326.0195, + "optimize_duration": 0.0467, + "load_duration": 326.0662, + "qps": 172.1563, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 88.631, + 172.1563 + ], + "conc_latency_p99_list": [ + 0.4895644657585399, + 0.4961875680503364 + ], + "conc_latency_p95_list": [ + 0.4765562847500405, + 0.4800053587003276 + ], + "conc_latency_avg_list": [ + 0.44739083606485935, + 0.45469620353127227 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 47.43239876568051, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 87.4509, + "optimize_duration": 0.0146, + "load_duration": 87.4655, + "qps": 188.1674, + "serial_latency_p99": 0.2691, + "serial_latency_p95": 0.2192, + "recall": 0.9134, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 129.2045, + 188.1674 + ], + "conc_latency_p99_list": [ + 0.4561221865990228, + 0.47219445047965564 + ], + "conc_latency_p95_list": [ + 0.44640964640020686, + 0.46283690039963404 + ], + "conc_latency_avg_list": [ + 0.3080734912494232, + 0.4224045833953113 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 55.803761, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 87.4509, + "optimize_duration": 0.0146, + "load_duration": 87.4655, + "qps": 184.839, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 124.568, + 184.839 + ], + "conc_latency_p99_list": [ + 0.470236903199766, + 0.4859163774794434 + ], + "conc_latency_p95_list": [ + 0.45786149219966316, + 0.47220526239943866 + ], + "conc_latency_avg_list": [ + 0.3197571245104845, + 0.42871789511037967 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 55.803761, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 9.4358, + "optimize_duration": 0.0081, + "load_duration": 9.4439, + "qps": 903.9248, + "serial_latency_p99": 0.0311, + "serial_latency_p95": 0.0264, + "recall": 0.9262, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 903.9248, + 210.3829 + ], + "conc_latency_p99_list": [ + 0.10201139999971932, + 23.7793526906097 + ], + "conc_latency_p95_list": [ + 0.07956749100012528, + 0.1337348711004779 + ], + "conc_latency_avg_list": [ + 0.04422038388924561, + 0.37968395195953486 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 56.88413, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 9.4358, + "optimize_duration": 0.0081, + "load_duration": 9.4439, + "qps": 788.0643, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 788.0643, + 342.891 + ], + "conc_latency_p99_list": [ + 0.116296334949675, + 1.4236666194401957 + ], + "conc_latency_p95_list": [ + 0.089655852249507, + 0.14479435460016243 + ], + "conc_latency_avg_list": [ + 0.0506907555806803, + 0.22207524820958327 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 56.88413, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 702.0398, + "optimize_duration": 0.074, + "load_duration": 702.1138, + "qps": 201.2276, + "serial_latency_p99": 0.4449, + "serial_latency_p95": 0.4444, + "recall": 0.9057, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 108.7747, + 201.2276 + ], + "conc_latency_p99_list": [ + 0.46301382789988565, + 0.47322508225033744 + ], + "conc_latency_p95_list": [ + 0.45525903649968313, + 0.4619954339998458 + ], + "conc_latency_avg_list": [ + 0.36484632344246043, + 0.39432222691413743 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 57.83363736188793, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 702.0398, + "optimize_duration": 0.074, + "load_duration": 702.1138, + "qps": 193.9787, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 108.0544, + 193.9787 + ], + "conc_latency_p99_list": [ + 0.48343497323963674, + 0.4929514833995654 + ], + "conc_latency_p95_list": [ + 0.4710945042004823, + 0.4792746837499635 + ], + "conc_latency_avg_list": [ + 0.36689615236970297, + 0.4052692454002725 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 57.83363736188793, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 81.6544, + "optimize_duration": 0.0155, + "load_duration": 81.67, + "qps": 317.5358, + "serial_latency_p99": 0.1402, + "serial_latency_p95": 0.1062, + "recall": 0.9859, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 283.1602, + 317.5358 + ], + "conc_latency_p99_list": [ + 0.38076332676982494, + 0.4561359607801206 + ], + "conc_latency_p95_list": [ + 0.3054511334998549, + 0.44553361209977993 + ], + "conc_latency_avg_list": [ + 0.1408606566523405, + 0.2508771823668165 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 57.242324, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 81.6544, + "optimize_duration": 0.0155, + "load_duration": 81.67, + "qps": 299.9899, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 276.9879, + 299.9899 + ], + "conc_latency_p99_list": [ + 0.38208317819017773, + 0.4668930329600153 + ], + "conc_latency_p95_list": [ + 0.3130642607502523, + 0.4508931028001825 + ], + "conc_latency_avg_list": [ + 0.14397322018103537, + 0.2652584571078699 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 57.242324, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 8.7233, + "optimize_duration": 0.0095, + "load_duration": 8.7328, + "qps": 643.7175, + "serial_latency_p99": 0.019, + "serial_latency_p95": 0.0155, + "recall": 0.9857, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 643.7175, + 553.2639 + ], + "conc_latency_p99_list": [ + 0.06595422097017949, + 0.5136447451604238 + ], + "conc_latency_p95_list": [ + 0.049367965999931575, + 0.0805305323999846 + ], + "conc_latency_avg_list": [ + 0.02392549430111337, + 0.07046851805299408 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 11463.5516, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 55.01342, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 8.7233, + "optimize_duration": 0.0095, + "load_duration": 8.7328, + "qps": 601.592, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 601.592, + 551.9681 + ], + "conc_latency_p99_list": [ + 0.07571082960002969, + 0.7508284560699189 + ], + "conc_latency_p95_list": [ + 0.053776488000039535, + 0.09836964239973439 + ], + "conc_latency_avg_list": [ + 0.030178424329577746, + 0.10644925614934629 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 100000, + "insert_rows_per_second": 11463.5516, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "unapplied_bm25_params": {}, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "Vespa", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "url": "**********", + "port": 8080 + }, + "db_case_config": { + "metric_type": "BM25", + "bm25_k1": 1.2, + "bm25_b": 0.75, + "bm25_avgdl": 55.01342, + "feed_client_command": "vespa", + "feed_client_connections": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1782259200.0 +} diff --git a/vectordb_bench/results/FullTextSearch/ZillizCloud/result_20260626_fts_standard_zillizcloud.json b/vectordb_bench/results/FullTextSearch/ZillizCloud/result_20260626_fts_standard_zillizcloud.json new file mode 100644 index 000000000..032a97292 --- /dev/null +++ b/vectordb_bench/results/FullTextSearch/ZillizCloud/result_20260626_fts_standard_zillizcloud.json @@ -0,0 +1,1622 @@ +{ + "run_id": "fts_standard_zillizcloud", + "task_label": "fts_standard", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 92.5443, + "optimize_duration": 81.9515, + "load_duration": 174.4958, + "qps": 1291.365, + "serial_latency_p99": 0.02, + "serial_latency_p95": 0.0136, + "recall": 0.9935, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 1291.365, + 1163.3626 + ], + "conc_latency_p99_list": [ + 0.04671075918056886, + 0.09800235276106833 + ], + "conc_latency_p95_list": [ + 0.038866160500401745, + 0.08286118600008194 + ], + "conc_latency_avg_list": [ + 0.027654020284034526, + 0.06254546518180665 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T16:12:30.162495", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 92.5443, + "optimize_duration": 81.9515, + "load_duration": 174.4958, + "qps": 1311.121, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 1252.5852, + 1311.121 + ], + "conc_latency_p99_list": [ + 0.04848522670017702, + 0.08333016816090094 + ], + "conc_latency_p95_list": [ + 0.04233008824876378, + 0.0747867592010152 + ], + "conc_latency_avg_list": [ + 0.03172382021377068, + 0.05988630893483264 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 47.43239876568051 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T16:21:46.167469", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 19.2043, + "optimize_duration": 84.4117, + "load_duration": 103.616, + "qps": 5024.8673, + "serial_latency_p99": 0.0099, + "serial_latency_p95": 0.0074, + "recall": 0.9348, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 4539.6507, + 5024.8673 + ], + "conc_latency_p99_list": [ + 0.02007044190080706, + 0.03161907671967124 + ], + "conc_latency_p95_list": [ + 0.015511998999681963, + 0.02536790084959648 + ], + "conc_latency_avg_list": [ + 0.009547202163125637, + 0.01752803334968866 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T16:04:10.277433", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 19.2043, + "optimize_duration": 84.4117, + "load_duration": 103.616, + "qps": 4094.1935, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 3772.8914, + 4094.1935 + ], + "conc_latency_p99_list": [ + 0.021262989360184278, + 0.03570330930069758 + ], + "conc_latency_p95_list": [ + 0.01692326859993045, + 0.028539174500110676 + ], + "conc_latency_avg_list": [ + 0.010518937966150272, + 0.01910471804373547 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.803761 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T16:09:07.352737", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1.9379, + "optimize_duration": 25.6039, + "load_duration": 27.5418, + "qps": 11840.6594, + "serial_latency_p99": 0.0036, + "serial_latency_p95": 0.0031, + "recall": 0.9987, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 9511.1365, + 11840.6594 + ], + "conc_latency_p99_list": [ + 0.00816284980028285, + 0.014502515860076526 + ], + "conc_latency_p95_list": [ + 0.006357606800520441, + 0.01137593650018971 + ], + "conc_latency_avg_list": [ + 0.004235700703938657, + 0.007192490032664778 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T15:59:30.355088", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1.9379, + "optimize_duration": 25.6039, + "load_duration": 27.5418, + "qps": 8475.056, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 7722.3517, + 8475.056 + ], + "conc_latency_p99_list": [ + 0.009759001410220668, + 0.018094878880474424 + ], + "conc_latency_p95_list": [ + 0.007649297249099612, + 0.013853047499105738 + ], + "conc_latency_avg_list": [ + 0.005133058755745676, + 0.009210371172970354 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 56.88413 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T16:02:03.206355", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 183.6571, + "optimize_duration": 98.8113, + "load_duration": 282.4684, + "qps": 2859.4474, + "serial_latency_p99": 0.008, + "serial_latency_p95": 0.0058, + "recall": 0.9636, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 2397.9818, + 2859.4474 + ], + "conc_latency_p99_list": [ + 0.025391751239949356, + 0.038862020799206254 + ], + "conc_latency_p95_list": [ + 0.02053890500028501, + 0.032976875500025926 + ], + "conc_latency_avg_list": [ + 0.013524379689949012, + 0.02457546268064057 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T15:37:30.224815", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 183.6571, + "optimize_duration": 98.8113, + "load_duration": 282.4684, + "qps": 2997.5483, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 2764.3614, + 2997.5483 + ], + "conc_latency_p99_list": [ + 0.027363914300149103, + 0.0428042954213015 + ], + "conc_latency_p95_list": [ + 0.022107690500342867, + 0.03582373630015354 + ], + "conc_latency_avg_list": [ + 0.014345869193528543, + 0.026168574204900128 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.83363736188793 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T15:50:48.791703", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 20.5452, + "optimize_duration": 119.3876, + "load_duration": 139.9328, + "qps": 10440.9225, + "serial_latency_p99": 0.0045, + "serial_latency_p95": 0.0037, + "recall": 0.9194, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 8403.6721, + 10440.9225 + ], + "conc_latency_p99_list": [ + 0.010635902699323196, + 0.01738944091935992 + ], + "conc_latency_p95_list": [ + 0.008483375000650994, + 0.01400671859992144 + ], + "conc_latency_avg_list": [ + 0.005399416001780447, + 0.00889505466866669 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T15:28:10.214232", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 20.5452, + "optimize_duration": 119.3876, + "load_duration": 139.9328, + "qps": 7477.8788, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 6408.2792, + 7477.8788 + ], + "conc_latency_p99_list": [ + 0.012072665600589968, + 0.02024130038942528 + ], + "conc_latency_p95_list": [ + 0.009671015750882361, + 0.01634627664943764 + ], + "conc_latency_avg_list": [ + 0.006184244148888284, + 0.01044489949882264 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 57.242324 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T15:31:42.678098", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1.8223, + "optimize_duration": 18.0832, + "load_duration": 19.9055, + "qps": 14523.9158, + "serial_latency_p99": 0.003, + "serial_latency_p95": 0.0028, + "recall": 0.9854, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 11465.34, + 14523.9158 + ], + "conc_latency_p99_list": [ + 0.006605165030141507, + 0.013721053038825629 + ], + "conc_latency_p95_list": [ + 0.005089374199815211, + 0.009782408600040073 + ], + "conc_latency_avg_list": [ + 0.003510005176568982, + 0.006679762620736736 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T15:23:49.914938", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1.8223, + "optimize_duration": 18.0832, + "load_duration": 19.9055, + "qps": 9382.1707, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [ + 40, + 80 + ], + "conc_qps_list": [ + 9382.1707, + 9299.0682 + ], + "conc_latency_p99_list": [ + 0.007831546979778047, + 0.014077205400753908 + ], + "conc_latency_p95_list": [ + 0.006045342150900976, + 0.011566458601373598 + ], + "conc_latency_avg_list": [ + 0.004228128073816301, + 0.008412116105405435 + ], + "payload_profile": "text", + "payload_estimated_bytes_per_query": 53200, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0, + "fts_manifest": { + "bm25": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "analyzer": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "applied_bm25_params": {}, + "unapplied_bm25_params": { + "k1": 1.2, + "b": 0.75, + "avgdl": 55.01342 + }, + "applied_analyzer_params": {}, + "unapplied_analyzer_params": { + "filter": [ + "lowercase" + ], + "tokenizer": "standard" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "2026-06-23T15:26:11.226516", + "version": "", + "note": "", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "ZillizCloudFTSBench" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "text" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1782172800.0 +} From 215eb89c83c9eec587734179b9b87f90425bf521 Mon Sep 17 00:00:00 2001 From: Zihao Wang <47910959+JoeJRW@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:20:54 +0800 Subject: [PATCH 35/49] feat(alisql): switch to mysqlclient and add shards/quantization params (#808) - Replace mysql-connector-python with mysqlclient (C extension) - Use array.array for vector serialization instead of numpy - Add --shards and --quantization CLI options for vector index building - Enable concurrent insert (thread_safe=True) with a per-worker connection pool Co-authored-by: Claude Opus 4.6 --- README.md | 2 +- pyproject.toml | 2 +- .../backend/clients/alisql/alisql.py | 101 ++++++++++++------ vectordb_bench/backend/clients/alisql/cli.py | 22 ++++ .../backend/clients/alisql/config.py | 4 + 5 files changed, 99 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 72e7f8ca7..3c9b985a5 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ All the database client supported | oceanbase | `pip install vectordb-bench[oceanbase]` | | hologres | `pip install vectordb-bench[hologres]` | | tencent_es | `pip install vectordb-bench[tencent_es]` | -| alisql | `pip install 'vectordb-bench[alisql]'` | +| alisql | `pip install vectordb-bench[alisql]` | | polardb | `pip install vectordb-bench[polardb]` | | doris | `pip install vectordb-bench[doris]` | | zvec | `pip install vectordb-bench[zvec]` | diff --git a/pyproject.toml b/pyproject.toml index bf9552cc3..9740b7228 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ clickhouse = [ "clickhouse-connect" ] vespa = [ "pyvespa" ] lancedb = [ "lancedb" ] oceanbase = [ "mysql-connector-python" ] -alisql = [ "mysql-connector-python" ] +alisql = [ "mysqlclient" ] polardb = [ "PyMySQL" ] doris = [ "doris-vector-search" ] turbopuffer = [ "turbopuffer" ] diff --git a/vectordb_bench/backend/clients/alisql/alisql.py b/vectordb_bench/backend/clients/alisql/alisql.py index 6d1fbaefe..196d88731 100644 --- a/vectordb_bench/backend/clients/alisql/alisql.py +++ b/vectordb_bench/backend/clients/alisql/alisql.py @@ -1,7 +1,9 @@ +import array import logging -from contextlib import contextmanager +import queue +from contextlib import contextmanager, suppress -import mysql.connector as mysql +import MySQLdb import numpy as np from ..api import VectorDB @@ -11,6 +13,8 @@ class AliSQL(VectorDB): + thread_safe = True + def __init__( self, dim: int, @@ -25,8 +29,9 @@ def __init__( self.case_config = db_case_config self.table_name = collection_name self.dim = dim + # Pool of extra connections used to parallelize inserts; built in init(). + self._insert_pool: queue.SimpleQueue | None = None - # construct basic units self.conn, self.cursor = self._create_connection() if drop_old: @@ -39,12 +44,11 @@ def __init__( self.conn = None def _create_connection(self): - conn = mysql.connect( + conn = MySQLdb.connect( host=self.db_config["host"], user=self.db_config["user"], port=self.db_config["port"], password=self.db_config["password"], - buffered=True, ) cursor = conn.cursor() @@ -53,13 +57,40 @@ def _create_connection(self): return conn, cursor + def _acquire_insert_conn(self): + """Borrow a connection from the insert pool, opening a new one if empty. + + The pool grows lazily to the number of concurrent insert workers: with N + worker threads at most N connections are checked out at once. + """ + try: + return self._insert_pool.get_nowait() + except queue.Empty: + conn, cursor = self._create_connection() + cursor.execute("SET sql_mode = ''") + return conn, cursor + + def _drain_insert_pool(self): + """Close every pooled insert connection. Called from init()'s finally, after + all insert workers have joined, so nothing is checked out at this point. + """ + pool, self._insert_pool = self._insert_pool, None + if pool is None: + return + while True: + try: + conn, cursor = pool.get_nowait() + except queue.Empty: + break + with suppress(Exception): + cursor.close() + conn.close() + def _drop_db(self): assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" log.info(f'{self.name} client drop db : {self.db_config["database"]}') - # flush tables before dropping database to avoid some locking issue - self.cursor.execute("FLUSH TABLES") self.cursor.execute(f'DROP DATABASE IF EXISTS {self.db_config["database"]}') self.cursor.execute("COMMIT") self.cursor.execute("FLUSH TABLES") @@ -117,13 +148,18 @@ def init(self): f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %s" ) + self._search_cursor = self.cursor + self._insert_pool = queue.SimpleQueue() + try: yield finally: + self._drain_insert_pool() self.cursor.close() self.conn.close() self.cursor = None self.conn = None + self._search_cursor = None def ready_to_load(self) -> bool: pass @@ -138,6 +174,10 @@ def optimize(self, data_size: int) -> None: index_options = f"DISTANCE={index_param['metric_type']}" if index_param["index_type"] == "HNSW" and index_param["M"] is not None: index_options += f" M={index_param['M']}" + if index_param.get("shards") is not None: + index_options += f" SHARDS={index_param['shards']}" + if index_param.get("quantization") is not None: + index_options += f" QUANTIZATION={index_param['quantization']}" self.cursor.execute(f""" ALTER TABLE {self.db_config["database"]}.{self.table_name} @@ -151,7 +191,7 @@ def optimize(self, data_size: int) -> None: @staticmethod def vector_to_hex(v): # noqa: ANN001 - return np.array(v, "float32").tobytes() + return array.array("f", v).tobytes() def insert_embeddings( self, @@ -159,28 +199,29 @@ def insert_embeddings( metadata: list[int], **kwargs, ) -> tuple[int, Exception]: - """Insert embeddings into the database. - Should call self.init() first. - """ - assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" + """Insert one batch of embeddings. Should call self.init() first. + thread_safe=True: the concurrent insert runner may call this from several + worker threads sharing this instance. Each call borrows its own connection + from self._insert_pool, so the workers drive parallel insert streams. + """ + conn, cursor = self._acquire_insert_conn() try: - metadata_arr = np.array(metadata) - embeddings_arr = np.array(embeddings) + embeddings_f32 = np.asarray(embeddings, dtype=np.float32) + batch_data = [(int(metadata[i]), embeddings_f32[i].tobytes()) for i in range(len(metadata))] - batch_data = [] - for i, row in enumerate(metadata_arr): - batch_data.append((int(row), self.vector_to_hex(embeddings_arr[i]))) - - self.cursor.executemany(self.insert_sql, batch_data) - self.cursor.execute("COMMIT") - self.cursor.execute("FLUSH TABLES") - - return len(metadata), None + cursor.executemany(self.insert_sql, batch_data) + cursor.execute("COMMIT") except Exception as e: log.warning(f"Failed to insert data into Vector table ({self.table_name}), error: {e}") + # the connection may be left in a bad state; drop it instead of reusing + with suppress(Exception): + cursor.close() + conn.close() return 0, e + else: + self._insert_pool.put((conn, cursor)) + return len(metadata), None def search_embedding( self, @@ -191,17 +232,17 @@ def search_embedding( **kwargs, ) -> list[int]: assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" + assert self._search_cursor is not None, "Cursor is not initialized" - search_param = self.case_config.search_param() # noqa: F841 + query_bytes = self.vector_to_hex(query) try: if filters: - self.cursor.execute(self.select_sql_with_filter, (filters.get("id"), self.vector_to_hex(query), k)) + self._search_cursor.execute(self.select_sql_with_filter, (filters.get("id"), query_bytes, k)) else: - self.cursor.execute(self.select_sql, (self.vector_to_hex(query), k)) - return [row[0] for row in self.cursor.fetchall()] + self._search_cursor.execute(self.select_sql, (query_bytes, k)) + return [row[0] for row in self._search_cursor.fetchall()] - except mysql.Error: + except MySQLdb.Error: log.exception("Failed to execute search query") raise diff --git a/vectordb_bench/backend/clients/alisql/cli.py b/vectordb_bench/backend/clients/alisql/cli.py index aee3ec2b2..55eedaf97 100644 --- a/vectordb_bench/backend/clients/alisql/cli.py +++ b/vectordb_bench/backend/clients/alisql/cli.py @@ -85,6 +85,26 @@ class AliSQLHNSWTypedDict(AliSQLTypedDict): ), ] + shards: Annotated[ + int | None, + click.option( + "--shards", + type=int, + help="Number of shards for the vector index", + required=False, + ), + ] + + quantization: Annotated[ + str | None, + click.option( + "--quantization", + type=click.Choice(["SQ8", "SQ16"], case_sensitive=False), + help="Quantization algorithm for the vector index", + required=False, + ), + ] + @cli.command() @click_parameter_decorators_from_typed_dict(AliSQLHNSWTypedDict) @@ -106,6 +126,8 @@ def AliSQLHNSW( db_case_config=AliSQLHNSWConfig( M=parameters["m"], ef_search=parameters["ef_search"], + shards=parameters["shards"], + quantization=parameters["quantization"].upper() if parameters["quantization"] is not None else None, ), **parameters, ) diff --git a/vectordb_bench/backend/clients/alisql/config.py b/vectordb_bench/backend/clients/alisql/config.py index 16c56e90f..7d384c981 100644 --- a/vectordb_bench/backend/clients/alisql/config.py +++ b/vectordb_bench/backend/clients/alisql/config.py @@ -51,6 +51,8 @@ def parse_metric(self) -> str: class AliSQLHNSWConfig(AliSQLIndexConfig, DBCaseConfig): M: int | None = None ef_search: int | None = None + shards: int | None = None + quantization: str | None = None index: IndexType = IndexType.HNSW def index_param(self) -> dict: @@ -58,6 +60,8 @@ def index_param(self) -> dict: "metric_type": self.parse_metric(), "index_type": self.index.value, "M": self.M, + "shards": self.shards, + "quantization": self.quantization, } def search_param(self) -> dict: From 61478bc8712578e70f1ac185fef06db5c6dcb740 Mon Sep 17 00:00:00 2001 From: Poisky <71500147+Poisky@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:21:57 +0800 Subject: [PATCH 36/49] feat:add Lindorm HNSW reorder factor (#807) Co-authored-by: ningshuo --- README.md | 3 ++- vectordb_bench/backend/clients/lindorm/cli.py | 6 +++++- vectordb_bench/backend/clients/lindorm/config.py | 7 ++++--- vectordb_bench/frontend/config/dbCaseConfigs.py | 5 +++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3c9b985a5..e197614be 100644 --- a/README.md +++ b/README.md @@ -519,7 +519,7 @@ Lindorm supports index types: hnsw, ivfpq, or ivfbq. ```shell vectordbbench lindormhnsw --case-type Performance768D10M --index-name --k 10 \ --host --port --user --password --m 32 \ ---ef-construction 400 --ef-search 150 +--ef-construction 400 --ef-search 150 --reorder-factor 2 ``` **Example: Run ivfpq index test** @@ -553,6 +553,7 @@ To list the options for Lindorm, execute `vectordbbench lindormhnsw --help`, The --m INTEGER hnsw m [required] --ef-construction INTEGER hnsw ef-construction [required] --ef-search INTEGER hnsw ef-search [required] + --reorder-factor INTEGER reorder factor ``` ### Run PolarDB from command line diff --git a/vectordb_bench/backend/clients/lindorm/cli.py b/vectordb_bench/backend/clients/lindorm/cli.py index df466e517..cd4f69ae6 100644 --- a/vectordb_bench/backend/clients/lindorm/cli.py +++ b/vectordb_bench/backend/clients/lindorm/cli.py @@ -34,7 +34,10 @@ class LindormTypedDict(CommonTypedDict): ] -class LindormHNSWTypedDict(CommonTypedDict, LindormTypedDict, HNSWFlavor3): ... +class LindormHNSWTypedDict(CommonTypedDict, LindormTypedDict, HNSWFlavor3): + reorder_factor: Annotated[ + int, click.option("--reorder-factor", type=int, help="reorder factor", required=False, default=2) + ] @cli.command() @@ -56,6 +59,7 @@ def LindormHNSW(**parameters: Unpack[LindormHNSWTypedDict]): efConstruction=parameters["ef_construction"], efSearch=parameters["ef_search"], filter_type=parameters["filter_type"], + reorder_factor=parameters["reorder_factor"], number_of_regions=parameters["number_of_regions"], ), **parameters, diff --git a/vectordb_bench/backend/clients/lindorm/config.py b/vectordb_bench/backend/clients/lindorm/config.py index 0e0d4aae6..f8976103d 100644 --- a/vectordb_bench/backend/clients/lindorm/config.py +++ b/vectordb_bench/backend/clients/lindorm/config.py @@ -48,6 +48,7 @@ class HNSWConfig(LindormIndexConfig, DBCaseConfig): efSearch: int | None = None filter_type: str | None = "efficient_filter" k_expand_scope: int | None = 1000 + reorder_factor: int | None = 2 def index_param(self, dim: int | None = None) -> dict: return { @@ -61,7 +62,7 @@ def index_param(self, dim: int | None = None) -> dict: } def search_param(self, do_filter: bool = False) -> dict: - search_ext_param = {"lvector": {"ef_search": str(self.efSearch)}} + search_ext_param = {"lvector": {"ef_search": str(self.efSearch), "reorder_factor": str(self.reorder_factor)}} if do_filter: search_ext_param["lvector"]["filter_type"] = self.filter_type if self.filter_type == "efficient_filter": @@ -80,7 +81,7 @@ class IVFPQConfig(LindormIndexConfig, DBCaseConfig): centroids_hnsw_efSearch: int | None = None filter_type: str | None = "efficient_filter" - reorder_factor: int | None = 10 + reorder_factor: int | None = 2 client_refactor: bool = False k_expand_scope: int | None = 1000 @@ -125,7 +126,7 @@ class IVFBQConfig(LindormIndexConfig, DBCaseConfig): centroids_hnsw_efSearch: int | None = None filter_type: str | None = "efficient_filter" - reorder_factor: int | None = 10 + reorder_factor: int | None = 2 client_refactor: bool = False k_expand_scope: int | None = 1000 diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 764dad2bc..d3305f47d 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -2851,9 +2851,10 @@ class FilterType(Enum): inputConfig={ "min": 1, "max": 200, - "value": 10, + "value": 2, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.IVFPQ.value + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value + or config[CaseConfigParamType.IndexType] == IndexType.IVFPQ.value or config[CaseConfigParamType.IndexType] == IndexType.IVFBQ.value, inputHelp="Reorder factor", ) From 6575cdcf9be1aeeed3c155ee4b2d2569973727b1 Mon Sep 17 00:00:00 2001 From: abner-ma <969023674@qq.com> Date: Thu, 2 Jul 2026 17:39:45 +0800 Subject: [PATCH 37/49] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E6=95=B0=E6=8D=AE=E9=9B=86=E5=8F=AA=E8=83=BD=E8=BE=93?= =?UTF-8?q?=E5=85=A5=E4=B8=80=E4=B8=AAtrain.parquet=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98=20(#805)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改后，对于自定义数据集，能输入多个train.parquet文件 --- vectordb_bench/backend/dataset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 6a6f262cc..f905f469a 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -130,6 +130,8 @@ def file_count(self) -> int: @property def train_files(self) -> list[str]: + if ("," not in self.train_file) and self.file_num > 1: + return utils.compose_train_files(self.file_num, self.use_shuffled) train_file = self.train_file prefix = f"{train_file}" train_files = [] From d4f180ba624324eab4f1bbd286341704b2d9ac1d Mon Sep 17 00:00:00 2001 From: Zijun Yang <37757768+zpatronus@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:52:18 +0800 Subject: [PATCH 38/49] feat(runner): add configurable --serial-cooldown between concurrent and serial search phases (#803) The serial search phase measures single-query latency. When it launches immediately after the concurrent phase, the end-to-end path may still be in a saturated state, affecting the accuracy of serial latency especially p99/p95. Add a --serial-cooldown parameter (default 0, in seconds, supports decimals) so backends that need it can opt in. --- README.md | 3 ++- vectordb_bench/__init__.py | 1 + vectordb_bench/backend/task_runner.py | 6 ++++++ vectordb_bench/cli/cli.py | 11 +++++++++++ vectordb_bench/models.py | 1 + 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e197614be..e4b4d7b0e 100644 --- a/README.md +++ b/README.md @@ -429,7 +429,8 @@ Execute tests for the index types: HGraph. NUM_PER_BATCH=10000 vectordbbench hologreshgraph --host Hologres_Endpoint --port 80 \ --user ACCESS_ID --password ACCESS_KEY --database DATABASE_NAME \ --m 64 --ef-construction 400 --case-type Performance768D10M \ ---index-type HGraph --ef-search 400 --k 10 --num-concurrency 1,60,70,75,80,90,95,100,110,120 +--index-type HGraph --ef-search 400 --k 10 --num-concurrency 1,60,70,75,80,90,95,100,105,110,115,120,125,130 \ +--serial-cooldown 3 ``` To list the options for Hologres, execute `vectordbbench hologreshgraph --help`, The following are some Hologres-specific command-line options. diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index 3e5c1e69e..1491630e0 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -35,6 +35,7 @@ class config: CONCURRENCY_DURATION = 30 CONCURRENCY_TIMEOUT = 3600 + SERIAL_COOLDOWN = 0 CLOUD_INSERT_READINESS_TIMEOUT = env.float("CLOUD_INSERT_READINESS_TIMEOUT", None) CLOUD_INSERT_READINESS_POLL_INTERVAL = env.float("CLOUD_INSERT_READINESS_POLL_INTERVAL", 5.0) diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 6f9025d4a..483e4a941 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -370,6 +370,12 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: m.conc_latency_avg_list, ) = search_results if TaskStage.SEARCH_SERIAL in self.config.stages: + cooldown = self.config.case_config.concurrency_search_config.serial_cooldown + if TaskStage.SEARCH_CONCURRENT in self.config.stages and cooldown > 0: + log.info( + f"Cooldown {cooldown}s before serial search to ensure a stable measurement environment" + ) + time.sleep(cooldown) search_results = self._serial_search() if self.is_fts: m.recall, m.serial_latency_p99, m.serial_latency_p95 = search_results diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index d7854be6c..a657effe9 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -412,6 +412,16 @@ class CommonTypedDict(TypedDict): "Set to a negative value to wait indefinitely.", ), ] + serial_cooldown: Annotated[ + float, + click.option( + "--serial-cooldown", + type=float, + default=config.SERIAL_COOLDOWN, + show_default=True, + help="Cooldown in seconds between concurrent and serial search phases", + ), + ] custom_case_name: Annotated[ str, click.option( @@ -830,6 +840,7 @@ def run( concurrency_duration=parameters["concurrency_duration"], num_concurrency=[int(s) for s in parameters["num_concurrency"]], concurrency_timeout=parameters["concurrency_timeout"], + serial_cooldown=parameters["serial_cooldown"], ), custom_case=get_custom_case_config(parameters), ), diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 855268cd1..1d1b88e6f 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -195,6 +195,7 @@ class ConcurrencySearchConfig(BaseModel): num_concurrency: list[int] = config.NUM_CONCURRENCY concurrency_duration: int = config.CONCURRENCY_DURATION concurrency_timeout: int = config.CONCURRENCY_TIMEOUT + serial_cooldown: float = config.SERIAL_COOLDOWN class CaseConfig(BaseModel): From 129c113e51c988894225f4012bd87872034b2a91 Mon Sep 17 00:00:00 2001 From: softhuafei <1148665215@qq.com> Date: Thu, 2 Jul 2026 21:35:27 +0800 Subject: [PATCH 39/49] feat: add support for Alibaba Cloud ADB-PG Nova (#786) Co-authored-by: linzhi.wzw --- README.md | 39 ++ pyproject.toml | 1 + tests/test_adbpg.py | 210 ++++++++++ vectordb_bench/backend/clients/__init__.py | 16 + .../backend/clients/adbpg/__init__.py | 0 vectordb_bench/backend/clients/adbpg/adbpg.py | 373 ++++++++++++++++++ vectordb_bench/backend/clients/adbpg/cli.py | 205 ++++++++++ .../backend/clients/adbpg/config.py | 137 +++++++ vectordb_bench/cli/vectordbbench.py | 2 + .../frontend/config/dbCaseConfigs.py | 120 ++++++ vectordb_bench/frontend/config/styles.py | 1 + vectordb_bench/models.py | 11 +- 12 files changed, 1113 insertions(+), 2 deletions(-) create mode 100644 tests/test_adbpg.py create mode 100644 vectordb_bench/backend/clients/adbpg/__init__.py create mode 100644 vectordb_bench/backend/clients/adbpg/adbpg.py create mode 100644 vectordb_bench/backend/clients/adbpg/cli.py create mode 100644 vectordb_bench/backend/clients/adbpg/config.py diff --git a/README.md b/README.md index e4b4d7b0e..0ac2f01ad 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ All the database client supported | zvec | `pip install vectordb-bench[zvec]` | | endee | `pip install vectordb-bench[endee]` | | lindorm | `pip install vectordb-bench[lindorm]` | +| adbpg | `pip install vectordb-bench[adbpg]` | ### Run @@ -557,6 +558,44 @@ To list the options for Lindorm, execute `vectordbbench lindormhnsw --help`, The --reorder-factor INTEGER reorder factor ``` +### Run ADBPG (Aliyun AnalyticDB for PostgreSQL) from command line + +ADBPG Nova uses the fastann/Nova vector index engine with `USING ann` syntax. + +**Example: Run novamr index benchmark (BioASQ 1M, 1024-dim)** + +```shell +vectordbbench adbpgnova --case-type Performance1024D1M --k 10 \ +--host --port 5432 --db-name postgres \ +--user-name --password \ +--algorithm novamr --hnsw-m 48 --ef-construction 600 \ +--ef-search 130 --max-scan-points 5000 --quantize-rescore-amp 2.0 +``` + +**Example: Run from config file** + +```shell +vectordbbench adbpgnova --config-file adbpg_bioasq1m_novamr.yml +``` + +To list the options for ADBPG, execute `vectordbbench adbpgnova --help`. The following are some ADBPG-specific command-line options. + +```text + --user-name TEXT Db username [required] + --password TEXT Postgres database password [$POSTGRES_PASSWORD] + --host TEXT Db host [required] + --port INTEGER Postgres database port [default: 5432] + --db-name TEXT Db name [required] + --algorithm TEXT algorithm [default: novamr] + --hnsw-m INTEGER hnsw_m [default: 16] + --ef-construction INTEGER ef_construction [default: 200] + --ef-search INTEGER ef_search [default: 100] + --max-scan-points INTEGER max scan points [default: 2000] + --quantize-rescore-amp FLOAT fastann.quantize_rescore_amp [default: 1.0] + --nova-adaptive-gamma FLOAT fastann.nova_adaptive_gamma [default: 0.0] + --auto-reduction/--no-auto-reduction Index WITH auto_reduction=on [default: False] +``` + ### Run PolarDB from command line PolarDB supports index types: faiss_hnsw_flat, faiss_hnsw_pq, and faiss_hnsw_sq. diff --git a/pyproject.toml b/pyproject.toml index 9740b7228..a5ef69112 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ endee = [ "endee==0.1.10" ] lindorm = [ "opensearch-py" ] seekdb = [ "mysql-connector-python" ] pinot = [ "requests" ] +adbpg = [ "psycopg", "psycopg-binary", "pgvector" ] [project.urls] Repository = "https://github.com/zilliztech/VectorDBBench" diff --git a/tests/test_adbpg.py b/tests/test_adbpg.py new file mode 100644 index 000000000..465acf762 --- /dev/null +++ b/tests/test_adbpg.py @@ -0,0 +1,210 @@ +"""Unit tests for the ADB-PG Nova client config layer. + +These tests do not require a live database — they only exercise: + - AdbpgConfig defaults and connection-string assembly + - AdbpgIndexConfig.index_param() WITH-clause options (incl. raw auto_reduction) + - AdbpgIndexConfig.session_param() fastann GUC emission + - TestResult.read_file() round-trip when password is absent in saved JSON + (regression for the result-loading failure caused by polymorphic + serialization stripping subclass fields from DBConfig) + +Usage: + pytest tests/test_adbpg.py -v +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.adbpg.config import AdbpgConfig, AdbpgIndexConfig +from vectordb_bench.backend.clients.api import MetricType +from vectordb_bench.models import TestResult + +if TYPE_CHECKING: + from pathlib import Path + + +def make_index_config(**overrides) -> AdbpgIndexConfig: + base = { + "metric_type": MetricType.COSINE, + "hnsw_m": 32, + "ef_search": 100, + "ef_construction": 200, + "nlist": 1024, + "algorithm": "novamr", + "rabitq_bits": 7, + "quantize_rescore_amp": 1.0, + "nova_adaptive_gamma": 0.0, + "max_scan_points": 2000, + "index_scan_mode": "snapshot", + "auto_reduction": False, + "nprobe": 5, + } + base.update(overrides) + return AdbpgIndexConfig(**base) + + +class TestAdbpgConfig: + def test_defaults_allow_construction_without_password(self): + # Regression: result JSON only contains DBConfig parent fields + # (db_label/version/note) because of pydantic polymorphic serialization. + # AdbpgConfig must therefore be constructible from that minimal dict. + cfg = AdbpgConfig(db_label="", version="", note="") + assert cfg.host == "localhost" + assert cfg.port == 5432 + assert cfg.db_name == "postgres" + assert cfg.password.get_secret_value() == "" + + def test_to_dict_carries_utility_session_option(self): + cfg = AdbpgConfig( + user_name=SecretStr("u"), + password=SecretStr("pw"), + host="h.example.com", + port=5432, + db_name="postgres", + ) + d = cfg.to_dict() + assert d["table_name"] == "vector" + cc = d["connect_config"] + assert cc["host"] == "h.example.com" + assert cc["user"] == "u" + assert cc["password"] == "pw" # noqa: S105 + assert cc["dbname"] == "postgres" + assert cc["options"] == "-c gp_session_role=utility" + + +class TestAdbpgIndexConfigBuild: + def test_parse_metric(self): + assert make_index_config(metric_type=MetricType.L2).parse_metric() == "l2" + assert make_index_config(metric_type=MetricType.COSINE).parse_metric() == "cosine" + assert make_index_config(metric_type=MetricType.IP).parse_metric() == "ip" + + def test_parse_metric_unsupported_raises(self): + with pytest.raises(ValueError, match="Metric type"): + make_index_config(metric_type=None).parse_metric() + + def test_index_param_options_default(self): + params = make_index_config().index_param() + names = {opt["option_name"]: opt for opt in params["index_creation_with_options"]} + assert names["algorithm"]["val"] == "novamr" + assert names["hnsw_m"]["val"] == 32 + assert names["hnsw_ef_construction"]["val"] == 200 + assert names["nlist"]["val"] == 1024 + assert names["rabitq_bits"]["val"] == 7 + assert names["max_key_len"]["val"] == 1 + # auto_reduction is omitted when False + assert "auto_reduction" not in names + + def test_index_param_auto_reduction_emits_raw(self): + params = make_index_config(auto_reduction=True).index_param() + opt = next(o for o in params["index_creation_with_options"] if o["option_name"] == "auto_reduction") + # `raw=True` so the value is rendered as a bare SQL identifier (`on`) + # rather than a quoted literal. + assert opt["val"] == "on" + assert opt.get("raw") is True + + def test_index_param_pca_dim_omitted_when_none(self): + params = make_index_config(pca_dim=None).index_param() + names = {opt["option_name"] for opt in params["index_creation_with_options"]} + assert "pca_dim" not in names + + def test_index_param_pca_dim_emitted_when_set(self): + params = make_index_config(pca_dim=448).index_param() + opt = next(o for o in params["index_creation_with_options"] if o["option_name"] == "pca_dim") + assert opt["val"] == 448 + + +class TestAdbpgIndexConfigSession: + def test_session_param_emits_all_search_gucs(self): + cfg = make_index_config( + quantize_rescore_amp=0.6, + nova_adaptive_gamma=0.0, + ef_search=50, + max_scan_points=16000, + index_scan_mode="snapshot", + nprobe=64, + ) + opts = cfg.session_param()["session_options"] + emitted = {o["parameter"]["setting_name"]: o["parameter"]["val"] for o in opts} + assert emitted["fastann.quantize_rescore_amp"] == "0.6" + assert emitted["fastann.nova_adaptive_gamma"] == "0.0" + assert emitted["fastann.hnsw_ef_search"] == "50" + assert emitted["fastann.hnsw_max_scan_points"] == "16000" + assert emitted["fastann.index_scan_mode"] == "snapshot" + # novad-specific GUC is always emitted (no-op for HNSW algorithms) + assert emitted["fastann.nova_nprobe"] == "64" + + def test_session_param_emits_zero_values(self): + # Forcing 0 / 0.0 must still produce a SET command — callers rely on + # being able to pin a GUC to zero. + cfg = make_index_config(quantize_rescore_amp=0.0, nova_adaptive_gamma=0.0, nprobe=0) + opts = cfg.session_param()["session_options"] + emitted = {o["parameter"]["setting_name"]: o["parameter"]["val"] for o in opts} + assert emitted["fastann.quantize_rescore_amp"] == "0.0" + assert emitted["fastann.nova_adaptive_gamma"] == "0.0" + assert emitted["fastann.nova_nprobe"] == "0" + + +class TestResultRoundTrip: + def test_read_file_with_minimal_db_config(self, tmp_path: Path): + """Saved result JSON keeps only DBConfig parent fields for db_config. + + TestResult.read_file must still rehydrate the AdbpgConfig instance + without raising a Field-required pydantic ValidationError. + """ + result_dir = tmp_path / "AnalyticDB for PostgreSQL" + result_dir.mkdir() + result_file = result_dir / "result_test_run.json" + payload = { + "run_id": "round-trip", + "task_label": "round-trip", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1.0, + "serial_latency_p99": 0.0, + "serial_latency_p95": 0.0, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_p95_list": [], + "conc_latency_avg_list": [], + }, + "task_config": { + "db": DB.Adbpg.value, + "db_config": {"db_label": "", "version": "", "note": ""}, + "db_case_config": { + "metric_type": "COSINE", + "algorithm": "novamr", + "hnsw_m": 16, + "ef_search": 100, + "ef_construction": 200, + "nlist": 1024, + }, + "case_config": {"case_id": 5, "custom_case": {}, "k": 10}, + "stages": ["search_serial"], + "load_concurrency": 0, + }, + "label": ":)", + } + ], + "timestamp": 0.0, + } + result_file.write_text(json.dumps(payload)) + + tr = TestResult.read_file(result_file, trans_unit=False) + assert len(tr.results) == 1 + rehydrated = tr.results[0].task_config.db_config + assert isinstance(rehydrated, AdbpgConfig) + assert rehydrated.host == "localhost" # came from default diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 4fd50871c..66cebce65 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -63,6 +63,7 @@ class DB(Enum): PolarDB = "PolarDB" Pinot = "Pinot" SeekDB = "SeekDB" + Adbpg = "AnalyticDB for PostgreSQL" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -269,6 +270,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return SeekDB + if self == DB.Adbpg: + from .adbpg.adbpg import Adbpg + + return Adbpg + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -477,6 +483,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return SeekDBConfig + if self == DB.Adbpg: + from .adbpg.config import AdbpgConfig + + return AdbpgConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -683,6 +694,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _seekdb_case_config.get(index_type) + if self == DB.Adbpg: + from .adbpg.config import AdbpgIndexConfig + + return AdbpgIndexConfig + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/adbpg/__init__.py b/vectordb_bench/backend/clients/adbpg/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/adbpg/adbpg.py b/vectordb_bench/backend/clients/adbpg/adbpg.py new file mode 100644 index 000000000..bc5ac3486 --- /dev/null +++ b/vectordb_bench/backend/clients/adbpg/adbpg.py @@ -0,0 +1,373 @@ +"""Wrapper around the Aliyun ADBPG (AnalyticDB for PostgreSQL) vector database.""" + +import logging +from collections.abc import Generator, Sequence +from contextlib import contextmanager +from typing import Any + +import numpy as np +import psycopg +from pgvector.psycopg import register_vector +from psycopg import Connection, Cursor, sql + +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import VectorDB +from .config import AdbpgConfigDict, AdbpgIndexConfig + +log = logging.getLogger(__name__) + + +class Adbpg(VectorDB): + """ADBPG vector database client, using psycopg.""" + + # psycopg Cursor is not thread-safe and the COPY protocol cannot be + # interleaved on a shared connection. Match PgVector/VectorChord and + # let ConcurrentInsertRunner clamp max_workers=1. + thread_safe: bool = False + + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + conn: psycopg.Connection[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None + + _search: sql.Composed + + def __init__( + self, + dim: int, + db_config: AdbpgConfigDict, + db_case_config: AdbpgIndexConfig, + drop_old: bool = False, + with_scalar_labels: bool = False, + **kwargs, + ): + self.name = "Adbpg" + self.case_config = db_case_config + # Allow the framework layer (task_runner) to inject a case-specific table + # name via the `collection_name` kwarg (see Doris for the same pattern). + override_name = kwargs.get("collection_name") + self.table_name = override_name if override_name else db_config["table_name"] + self.connect_config = db_config["connect_config"] + self.dim = dim + self.with_scalar_labels = with_scalar_labels + + self._primary_field = "id" + self._vector_field = "embedding" + self._scalar_label_field = "label" + # Index name derives from the table name + algorithm, e.g. vector_1024d_10m_novamr_index. + self._index_name = f"{self.table_name}_{self.case_config.algorithm}_index" + + self.where_clause = "" + + # construct basic units + self.conn, self.cursor = self._create_connection(**self.connect_config) + + log.info(f"{self.name} config values: {self.connect_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ), + ): + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + f"{self.name} config values: {self.connect_config}\n{self.case_config}" + ) + log.error(msg) + raise RuntimeError(msg) + + if drop_old: + self._drop_index() + self._drop_table() + self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + @staticmethod + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + register_vector(conn) + conn.autocommit = False + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + + def _generate_search_query(self) -> sql.Composed: + search_param = self.case_config.search_param() + distance_operator = { + "l2": "<->", + "ip": "<#>", + "cosine": "<=>", + }.get(search_param["metric"], "<->") + + where_clause = sql.SQL(self.where_clause) if self.where_clause else sql.SQL("") + + return sql.Composed( + [ + sql.SQL( + "SELECT {primary_field} FROM public.{table_name} {where_clause} ORDER BY {vector_field} ", + ).format( + table_name=sql.Identifier(self.table_name), + primary_field=sql.Identifier(self._primary_field), + where_clause=where_clause, + vector_field=sql.Identifier(self._vector_field), + ), + sql.SQL(distance_operator), + sql.SQL(" {search_vector}::vector({dim}) LIMIT %s::int").format( + search_vector=sql.Placeholder(), + dim=self.dim, + ), + ], + ) + + @contextmanager + def init(self) -> Generator[None, None, None]: + """Open a session, apply GUCs, yield, then close.""" + self.conn, self.cursor = self._create_connection(**self.connect_config) + + session_options: Sequence[dict[str, Any]] = self.case_config.session_param()["session_options"] + + if len(session_options) > 0: + for setting in session_options: + command = sql.SQL("SET {setting_name} = {val};").format( + setting_name=sql.Identifier(setting["parameter"]["setting_name"]), + val=sql.Identifier(str(setting["parameter"]["val"])), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + self.conn.commit() + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop table : {self.table_name}") + + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS public.{table_name}").format( + table_name=sql.Identifier(self.table_name), + ), + ) + self.conn.commit() + + def optimize(self, data_size: int | None = None): + self._post_insert() + + def _post_insert(self): + log.info(f"{self.name} post insert before optimize") + if self.case_config.create_index_after_load: + self._drop_index() + self._create_index() + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop index : {self._index_name}") + + drop_index_sql = sql.SQL("DROP INDEX IF EXISTS {index_name}").format( + index_name=sql.Identifier(self._index_name), + ) + log.debug(drop_index_sql.as_string(self.cursor)) + self.cursor.execute(drop_index_sql) + self.conn.commit() + + def _set_parallel_index_build_param(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + if index_param["build_parallel_processes"] is not None: + self.cursor.execute( + sql.SQL("SET fastann.build_parallel_processes TO {};").format( + index_param["build_parallel_processes"], + ), + ) + self.conn.commit() + + results = self.cursor.execute(sql.SQL("SHOW fastann.build_parallel_processes;")).fetchall() + log.info(f"{self.name} parallel index creation parameters: {results}") + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client create index : {self._index_name}") + + index_param = self.case_config.index_param() + self._set_parallel_index_build_param() + + # Pre-build GUC: raise optimizer level before creating the ANN index. + self.cursor.execute(sql.SQL("SET fastann.nova_build_optimize_level = 3;")) + self.conn.commit() + + options = [] + options.append(sql.SQL("dim = {dim}").format(dim=sql.Literal(self.dim))) + options.append( + sql.SQL("distancemeasure = {measure}").format( + measure=sql.Identifier(index_param["metric"]), + ), + ) + + for option in index_param["index_creation_with_options"]: + if option["val"] is not None: + # When `raw` is set, emit the value as a bare SQL token + # (e.g. auto_reduction=on) instead of a quoted literal. + rendered_val = sql.SQL(str(option["val"])) if option.get("raw") else sql.Literal(option["val"]) + options.append( + sql.SQL("{option_name} = {val}").format( + option_name=sql.Identifier(option["option_name"]), + val=rendered_val, + ), + ) + + with_clause = sql.SQL("WITH ({});").format(sql.SQL(", ").join(options)) if options else sql.Composed(()) + + # Covering index: always INCLUDE the primary field (e.g. id). + index_create_sql = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index_name} ON public.{table_name} + USING ann ({vector_field}) INCLUDE ({primary_field}) + """, + ).format( + index_name=sql.Identifier(self._index_name), + table_name=sql.Identifier(self.table_name), + vector_field=sql.Identifier(self._vector_field), + primary_field=sql.Identifier(self._primary_field), + ) + + full_sql = (index_create_sql + with_clause).join(" ") + log.debug(full_sql.as_string(self.cursor)) + self.cursor.execute(full_sql) + self.conn.commit() + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + log.info(f"{self.name} client create table : {self.table_name}") + + if self.with_scalar_labels: + self.cursor.execute( + sql.SQL( + """ + CREATE TABLE IF NOT EXISTS public.{table_name} + ({primary_field} BIGINT PRIMARY KEY, embedding vector({dim}), {label_field} VARCHAR(64)); + """, + ).format( + table_name=sql.Identifier(self.table_name), + primary_field=sql.Identifier(self._primary_field), + dim=dim, + label_field=sql.Identifier(self._scalar_label_field), + ), + ) + else: + self.cursor.execute( + sql.SQL( + """ + CREATE TABLE IF NOT EXISTS public.{table_name} + ({primary_field} BIGINT PRIMARY KEY, embedding vector({dim})); + """, + ).format( + table_name=sql.Identifier(self.table_name), + primary_field=sql.Identifier(self._primary_field), + dim=dim, + ), + ) + + self.cursor.execute( + sql.SQL( + "ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;", + ).format(table_name=sql.Identifier(self.table_name)), + ) + self.conn.commit() + except Exception as e: + log.warning(f"Failed to create adbpg table: {self.table_name} error: {e}") + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs: Any, + ) -> tuple[int, Exception | None]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + if self.with_scalar_labels: + assert labels_data is not None, "labels_data should be provided if with_scalar_labels is set to True" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + with self.cursor.copy( + sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( + table_name=sql.Identifier(self.table_name), + ), + ) as copy: + for i, row in enumerate(metadata_arr): + if self.with_scalar_labels: + copy.set_types(["bigint", "vector", "varchar"]) + copy.write_row((row, embeddings_arr[i], labels_data[i])) + else: + copy.set_types(["bigint", "vector"]) + copy.write_row((row, embeddings_arr[i])) + self.conn.commit() + + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into adbpg table ({self.table_name}), error: {e}") + return 0, e + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.where_clause = "" + elif filters.type == FilterOp.NumGE: + self.where_clause = f"WHERE {self._primary_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self.where_clause = f"WHERE {self._scalar_label_field} = '{filters.label_value}'" + else: + msg = f"Not support Filter for Adbpg - {filters}" + raise ValueError(msg) + + self._search = self._generate_search_query() + + def search_embedding( + self, + query: list[float], + k: int = 100, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + q = np.asarray(query) + result = self.cursor.execute( + self._search, + (q, k), + prepare=True, + binary=True, + ) + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/adbpg/cli.py b/vectordb_bench/backend/clients/adbpg/cli.py new file mode 100644 index 000000000..e579f9028 --- /dev/null +++ b/vectordb_bench/backend/clients/adbpg/cli.py @@ -0,0 +1,205 @@ +import os +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + get_custom_case_config, + run, +) + + +class AdbpgTypedDict(CommonTypedDict): + user_name: Annotated[ + str, + click.option("--user-name", type=str, help="Db username", required=True), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="Postgres database password", + default=lambda: os.environ.get("POSTGRES_PASSWORD", ""), + show_default="$POSTGRES_PASSWORD", + ), + ] + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + port: Annotated[ + int, + click.option( + "--port", + type=int, + help="Postgres database port", + default=5432, + show_default=True, + required=False, + ), + ] + db_name: Annotated[str, click.option("--db-name", type=str, help="Db name", required=True)] + hnsw_m: Annotated[ + int, + click.option("--hnsw-m", type=int, help="hnsw_m", default=48, show_default=True, required=False), + ] + ef_search: Annotated[ + int, + click.option("--ef-search", type=int, help="ef_search", default=150, show_default=True, required=False), + ] + ef_construction: Annotated[ + int, + click.option( + "--ef-construction", + type=int, + help="ef_construction", + default=600, + show_default=True, + required=False, + ), + ] + nlist: Annotated[ + int, + click.option("--nlist", type=int, help="nlist", default=1024, show_default=True, required=False), + ] + rabitq_bits: Annotated[ + int, + click.option("--rabitq-bits", type=int, help="rabitq_bits", default=7, show_default=True, required=False), + ] + quantize_rescore_amp: Annotated[ + float, + click.option( + "--quantize-rescore-amp", + type=float, + help="fastann.quantize_rescore_amp", + default=0.0, + show_default=True, + required=False, + ), + ] + nova_adaptive_gamma: Annotated[ + float, + click.option( + "--nova-adaptive-gamma", + type=float, + help="fastann.nova_adaptive_gamma", + default=0.0, + show_default=True, + required=False, + ), + ] + auto_reduction: Annotated[ + bool, + click.option( + "--auto-reduction/--no-auto-reduction", + "auto_reduction", + type=bool, + help="Index WITH auto_reduction=on when enabled", + default=False, + show_default=True, + required=False, + ), + ] + max_scan_points: Annotated[ + int, + click.option( + "--max-scan-points", + type=int, + help="max_scan_points", + default=20000, + show_default=True, + required=False, + ), + ] + index_scan_mode: Annotated[ + str, + click.option( + "--index-scan-mode", + type=str, + help="fastann.index_scan_mode", + default="snapshot", + show_default=True, + required=False, + ), + ] + algorithm: Annotated[ + str, + click.option( + "--algorithm", + type=str, + help="algorithm", + default="novamr", + show_default=True, + required=False, + ), + ] + build_parallel_processes: Annotated[ + int, + click.option( + "--build-parallel-processes", + type=int, + help="Sets the maximum process to build index", + required=False, + ), + ] + pca_dim: Annotated[ + int | None, + click.option( + "--pca-dim", + type=int, + help="PCA dimension for index dimensionality reduction", + default=None, + show_default=True, + required=False, + ), + ] + nprobe: Annotated[ + int, + click.option( + "--nprobe", + type=int, + help="fastann.nova_nprobe (novad search)", + default=5, + show_default=True, + required=False, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(AdbpgTypedDict) +def AdbpgNova(**parameters: Unpack[AdbpgTypedDict]): + from .config import AdbpgConfig, AdbpgIndexConfig + + parameters["custom_case"] = get_custom_case_config(parameters) + run( + db=DB.Adbpg, + db_config=AdbpgConfig( + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + db_name=parameters["db_name"], + ), + db_case_config=AdbpgIndexConfig( + hnsw_m=parameters["hnsw_m"], + ef_search=parameters["ef_search"], + ef_construction=parameters["ef_construction"], + nlist=parameters["nlist"], + algorithm=parameters["algorithm"], + build_parallel_processes=parameters["build_parallel_processes"], + rabitq_bits=parameters["rabitq_bits"], + quantize_rescore_amp=parameters["quantize_rescore_amp"], + nova_adaptive_gamma=parameters["nova_adaptive_gamma"], + auto_reduction=parameters["auto_reduction"], + pca_dim=parameters["pca_dim"], + max_scan_points=parameters["max_scan_points"], + index_scan_mode=parameters["index_scan_mode"], + nprobe=parameters["nprobe"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/adbpg/config.py b/vectordb_bench/backend/clients/adbpg/config.py new file mode 100644 index 000000000..cc69085bc --- /dev/null +++ b/vectordb_bench/backend/clients/adbpg/config.py @@ -0,0 +1,137 @@ +from collections.abc import Mapping, Sequence +from typing import Any, TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType + + +class AdbpgSessionCommands(TypedDict): + session_options: Sequence[dict[str, Any]] + + +class AdbpgConfigDict(TypedDict): + """These keys will be directly used as kwargs in psycopg connection string, + so the names must match exactly psycopg API.""" + + user: str + password: str + host: str + port: int + dbname: str + + +class AdbpgConfig(DBConfig): + user_name: SecretStr = SecretStr("tester") + password: SecretStr = SecretStr("") + host: str = "localhost" + port: int = 5432 + db_name: str = "postgres" + + def to_dict(self) -> dict: + user_str = self.user_name.get_secret_value() if isinstance(self.user_name, SecretStr) else self.user_name + pwd_str = self.password.get_secret_value() + return { + "table_name": "vector", + "connect_config": { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + "options": "-c gp_session_role=utility", + }, + } + + +class AdbpgIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + create_index_before_load: bool = False + create_index_after_load: bool = True + + # ADB PG specific parameters + hnsw_m: int = 48 + ef_search: int = 150 + ef_construction: int = 600 + nlist: int = 1024 + algorithm: str = "novamr" + build_parallel_processes: int | None = None + # rabitq quantization params + rabitq_bits: int = 7 + quantize_rescore_amp: float = 0.0 + nova_adaptive_gamma: float = 0.0 + max_scan_points: int = 20000 + index_scan_mode: str = "snapshot" + auto_reduction: bool = False + pca_dim: int | None = None + # novad-specific search param (no-op for novamr/HNSW algorithms) + nprobe: int = 5 + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2" + if self.metric_type == MetricType.COSINE: + return "cosine" + if self.metric_type == MetricType.IP: + return "ip" + msg = f"Metric type {self.metric_type} is not supported!" + raise ValueError(msg) + + @staticmethod + def _build_forced_set_options(set_mapping: Mapping[str, Any]) -> Sequence[dict[str, Any]]: + """Always emit SET commands regardless of value (including 0 / 0.0).""" + return [ + { + "parameter": { + "setting_name": name, + "val": str(value), + }, + } + for name, value in set_mapping.items() + ] + + def index_param(self) -> dict: + with_options = [ + {"option_name": "algorithm", "val": self.algorithm}, + {"option_name": "hnsw_m", "val": self.hnsw_m}, + {"option_name": "hnsw_ef_construction", "val": self.ef_construction}, + {"option_name": "nlist", "val": self.nlist}, + {"option_name": "rabitq_bits", "val": self.rabitq_bits}, + # Covering index key length. + {"option_name": "max_key_len", "val": 1}, + ] + # Optional: auto_reduction=on — only include when True. + # Uses raw=True so the value 'on' is emitted as a bare identifier + # instead of a quoted string literal. + if self.auto_reduction: + with_options.append({"option_name": "auto_reduction", "val": "on", "raw": True}) + if self.pca_dim is not None: + with_options.append({"option_name": "pca_dim", "val": self.pca_dim}) + + return { + "metric": self.parse_metric(), + "build_parallel_processes": self.build_parallel_processes, + "create_index_before_load": self.create_index_before_load, + "create_index_after_load": self.create_index_after_load, + "index_creation_with_options": with_options, + } + + def search_param(self) -> dict: + return { + "metric": self.parse_metric(), + } + + def session_param(self) -> AdbpgSessionCommands: + # All CLI-driven search GUCs are always sent, regardless of value, + # so that callers can explicitly tune any parameter — including to 0. + session_parameters = { + "fastann.quantize_rescore_amp": self.quantize_rescore_amp, + "fastann.nova_adaptive_gamma": self.nova_adaptive_gamma, + "fastann.hnsw_ef_search": self.ef_search, + "fastann.hnsw_max_scan_points": self.max_scan_points, + "fastann.index_scan_mode": self.index_scan_mode, + "fastann.nova_nprobe": self.nprobe, + "optimizer": "off", + "elog_process_parameters": "off", + } + return {"session_options": self._build_forced_set_options(session_parameters)} diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 5a89c018a..6c3e868ec 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,3 +1,4 @@ +from ..backend.clients.adbpg.cli import AdbpgNova from ..backend.clients.alisql.cli import AliSQLHNSW from ..backend.clients.alloydb.cli import AlloyDBScaNN from ..backend.clients.aws_opensearch.cli import AWSOpenSearch @@ -48,6 +49,7 @@ from .batch_cli import BatchCli from .cli import cli +cli.add_command(AdbpgNova) cli.add_command(PgVectorHNSW) cli.add_command(PgVectoRSHNSW) cli.add_command(PgVectoRSIVFFlat) diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index d3305f47d..ac1703dbe 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -3131,6 +3131,122 @@ class FilterType(Enum): CaseConfigParamInput_SQType_PolarDB, ] +# ADBPG (Aliyun AnalyticDB for PostgreSQL) configs +CaseConfigParamInput_Algorithm_Adbpg = CaseConfigInput( + label=CaseConfigParamType.algorithm, + inputHelp="Select Nova algorithm variant", + inputType=InputType.Option, + inputConfig={ + "options": ["novamr", "novad"], + }, +) + +CaseConfigParamInput_HnswM_Adbpg = CaseConfigInput( + label=CaseConfigParamType.hnsw_m, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 256, + "value": 16, + }, + inputHelp="HNSW M parameter", +) + +CaseConfigParamInput_EFConstruction_Adbpg = CaseConfigInput( + label=CaseConfigParamType.ef_construction, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 2000, + "value": 200, + }, + inputHelp="HNSW ef_construction", +) + +CaseConfigParamInput_RabitqBits_Adbpg = CaseConfigInput( + label=CaseConfigParamType.rabitq_bits, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 8, + "value": 7, + }, + inputHelp="RaBitQ quantization bits", +) + +CaseConfigParamInput_AutoReduction_Adbpg = CaseConfigInput( + label=CaseConfigParamType.auto_reduction, + inputType=InputType.Bool, + inputConfig={"value": False}, + inputHelp="Enable auto_reduction=on for index build", +) + +CaseConfigParamInput_EFSearch_Adbpg = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 2000, + "value": 100, + }, + inputHelp="fastann.hnsw_ef_search", +) + +CaseConfigParamInput_MaxScanPoints_Adbpg = CaseConfigInput( + label=CaseConfigParamType.max_scan_points, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 100000, + "value": 2000, + }, + inputHelp="fastann.hnsw_max_scan_points", +) + +CaseConfigParamInput_QuantizeRescoreAmp_Adbpg = CaseConfigInput( + label=CaseConfigParamType.quantize_rescore_amp, + inputType=InputType.Float, + inputConfig={ + "min": 0.0, + "max": 100.0, + "value": 1.0, + "step": 0.1, + }, + inputHelp="fastann.quantize_rescore_amp", +) + +CaseConfigParamInput_NovaAdaptiveGamma_Adbpg = CaseConfigInput( + label=CaseConfigParamType.nova_adaptive_gamma, + inputType=InputType.Float, + inputConfig={ + "min": 0.0, + "max": 1.0, + "value": 0.0, + "step": 0.05, + }, + inputHelp="fastann.nova_adaptive_gamma", +) + +AdbpgLoadConfig = [ + CaseConfigParamInput_Algorithm_Adbpg, + CaseConfigParamInput_HnswM_Adbpg, + CaseConfigParamInput_EFConstruction_Adbpg, + CaseConfigParamInput_RabitqBits_Adbpg, + CaseConfigParamInput_AutoReduction_Adbpg, +] + +AdbpgPerformanceConfig = [ + CaseConfigParamInput_Algorithm_Adbpg, + CaseConfigParamInput_HnswM_Adbpg, + CaseConfigParamInput_EFConstruction_Adbpg, + CaseConfigParamInput_RabitqBits_Adbpg, + CaseConfigParamInput_AutoReduction_Adbpg, + CaseConfigParamInput_EFSearch_Adbpg, + CaseConfigParamInput_MaxScanPoints_Adbpg, + CaseConfigParamInput_QuantizeRescoreAmp_Adbpg, + CaseConfigParamInput_NovaAdaptiveGamma_Adbpg, +] + # Map DB to config CASE_CONFIG_MAP = { DB.Milvus: { @@ -3232,6 +3348,10 @@ class FilterType(Enum): DB.TurboPuffer: { CaseLabel.FullTextSearchPerformance: TurboPufferFtsConfig, }, + DB.Adbpg: { + CaseLabel.Load: AdbpgLoadConfig, + CaseLabel.Performance: AdbpgPerformanceConfig, + }, } diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index 268a2cd7d..4b162e9ed 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -75,6 +75,7 @@ def getPatternShape(i): DB.Endee: "data:image/svg+xml,%3c?xml%20version=%271.0%27%20encoding=%27UTF-8%27?%3e%3csvg%20id=%27Layer_1%27%20xmlns=%27http://www.w3.org/2000/svg%27%20version=%271.1%27%20viewBox=%270%200%20600%20600%27%3e%3c!--%20Generator:%20Adobe%20Illustrator%2030.0.0,%20SVG%20Export%20Plug-In%20.%20SVG%20Version:%202.1.1%20Build%20123)%20--%3e%3cdefs%3e%3cstyle%3e%20.st0%20{%20fill:%20%233266a4;%20}%20%3c/style%3e%3c/defs%3e%3cpath%20class=%27st0%27%20d=%27M106.22,490.02H10.36l-.04-184.85c11.19-163.31,232.1-211.74,306.42-61.94,23.96,48.3,15.59,99.83,17,152.01h61.62c15.25,0,42.7-13.75,54.75-23.2,66.11-51.86,57.28-158.2-16.28-198.49-9.65-5.28-33.16-14.19-43.74-14.19h-154.31v-91.09c0-.87,2.55-1.86,3.63-1.63,104.05,4.23,201.15-21.64,284.48,55.84,119.18,110.8,69.12,325.47-91.33,362.6-28.65,6.63-85.47,7.76-115.02,4.8-19.71-1.97-43.29-16.57-55.97-31.45-37.98-44.56-20.77-98.07-24.7-151.16-5.18-69.99-100-85.31-125.9-20.47-1.16,2.92-4.76,13.88-4.76,16.3v186.91Z%27/%3e%3c/svg%3e", DB.Lindorm: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAJgAAACKCAYAAABW3IOxAAAMT2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSIQQIREBK6E0QqQGkhNACSC+CqIQkQCgxJgQVO7K4gmsXESwrugqi2FZAFhvqqiuLgr0uFlSUdXFd7MqbEECXfeV7831z57//nPnnnHPn3rkDAL2LL5XmopoA5EnyZbEhAazJySksUg8gAmNABjZAiy+QSznR0REAluH27+X1NYAo28sOSq1/9v/XoiUUyQUAINEQpwvlgjyIfwQAbxFIZfkAEKWQN5+VL1XidRDryKCDENcocaYKtyhxugpfGrSJj+VC/AgAsjqfL8sEQKMP8qwCQSbUocNogZNEKJZA7A+xb17eDCHEiyC2gTZwTrpSn53+lU7m3zTTRzT5/MwRrIplsJADxXJpLn/O/5mO/13ychXDc1jDqp4lC41Vxgzz9ihnRrgSq0P8VpIeGQWxNgAoLhYO2isxM0sRmqCyR20Eci7MGWBCPFGeG8cb4mOF/MBwiA0hzpDkRkYM2RRliIOVNjB/aIU4nxcPsR7ENSJ5UNyQzQnZjNjhea9lyLicIf4pXzbog1L/syIngaPSx7SzRLwhfcyxMCs+CWIqxIEF4sRIiDUgjpTnxIUP2aQWZnEjh21kilhlLBYQy0SSkACVPlaeIQuOHbLfnScfjh07kSXmRQ7hzvys+FBVrrBHAv6g/zAWrE8k4SQM64jkkyOGYxGKAoNUseNkkSQhTsXjetL8gFjVWNxOmhs9ZI8HiHJDlLwZxPHygrjhsQX5cHGq9PESaX50vMpPvDKbHxat8gffDyIAFwQCFlDAmg5mgGwgbu9t7IV3qp5gwAcykAlEwGGIGR6RNNgjgdc4UAh+h0gE5CPjAgZ7RaAA8p9GsUpOPMKprg4gY6hPqZIDHkOcB8JBLrxXDCpJRjxIBI8gI/6HR3xYBTCGXFiV/f+eH2a/MBzIRAwxiuEZWfRhS2IQMZAYSgwm2uIGuC/ujUfAqz+szjgb9xyO44s94TGhg/CAcJXQRbg5XVwkG+XlJNAF9YOH8pP+dX5wK6jphgfgPlAdKuNM3AA44K5wHg7uB2d2gyx3yG9lVlijtP8WwVdPaMiO4kRBKWMo/hSb0SM17DTcRlSUuf46Pypf00fyzR3pGT0/96vsC2EbPtoS+xY7hJ3FTmLnsRasEbCw41gT1oYdVeKRFfdocMUNzxY76E8O1Bm9Zr48WWUm5U51Tj1OH1V9+aLZ+cqXkTtDOkcmzszKZ3HgjiFi8SQCx3EsZydnNwCU+4/q8/YqZnBfQZhtX7glvwHgc3xgYOCnL1zYcQAOeMBPwpEvnA0bbi1qAJw7IlDIClQcrrwQ4JeDDt8+fbi/mcP9zQE4A3fgDfxBEAgDUSAeJINp0PssuM5lYBaYBxaDElAGVoH1oBJsBdtBDdgLDoJG0AJOgp/BBXAJXAW34erpBs9BH3gNPiAIQkJoCAPRR0wQS8QecUbYiC8ShEQgsUgykoZkIhJEgcxDliBlyBqkEtmG1CIHkCPISeQ80oHcRO4jPcifyHsUQ9VRHdQItULHo2yUg4aj8ehUNBOdiRaixegKtAKtRvegDehJ9AJ6Fe1Cn6P9GMDUMCZmijlgbIyLRWEpWAYmwxZgpVg5Vo3VY83wOV/GurBe7B1OxBk4C3eAKzgUT8AF+Ex8Ab4cr8Rr8Ab8NH4Zv4/34Z8JNIIhwZ7gReARJhMyCbMIJYRywk7CYcIZ+C51E14TiUQm0ZroAd/FZGI2cS5xOXEzcR/xBLGD+JDYTyKR9En2JB9SFIlPyieVkDaS9pCOkzpJ3aS3ZDWyCdmZHExOIUvIReRy8m7yMXIn+Qn5A0WTYknxokRRhJQ5lJWUHZRmykVKN+UDVYtqTfWhxlOzqYupFdR66hnqHeorNTU1MzVPtRg1sdoitQq1/Wrn1O6rvVPXVrdT56qnqivUV6jvUj+hflP9FY1Gs6L501Jo+bQVtFraKdo92lsNhoajBk9DqLFQo0qjQaNT4wWdQrekc+jT6IX0cvoh+kV6ryZF00qTq8nXXKBZpXlE87pmvxZDa4JWlFae1nKt3VrntZ5qk7SttIO0hdrF2tu1T2k/ZGAMcwaXIWAsYexgnGF06xB1rHV4Otk6ZTp7ddp1+nS1dV11E3Vn61bpHtXtYmJMKyaPmctcyTzIvMZ8P8ZoDGeMaMyyMfVjOse80Rur568n0ivV26d3Ve+9Pks/SD9Hf7V+o/5dA9zAziDGYJbBFoMzBr1jdcZ6jxWMLR17cOwtQ9TQzjDWcK7hdsM2w34jY6MQI6nRRqNTRr3GTGN/42zjdcbHjHtMGCa+JmKTdSbHTZ6xdFkcVi6rgnWa1WdqaBpqqjDdZtpu+sHM2izBrMhsn9ldc6o52zzDfJ15q3mfhYnFJIt5FnUWtywplmzLLMsNlmct31hZWyVZLbVqtHpqrWfNsy60rrO+Y0Oz8bOZaVNtc8WWaMu2zbHdbHvJDrVzs8uyq7K7aI/au9uL7Tfbd4wjjPMcJxlXPe66g7oDx6HAoc7hviPTMcKxyLHR8cV4i/Ep41ePPzv+s5ObU67TDqfbE7QnhE0omtA84U9nO2eBc5XzFReaS7DLQpcml5eu9q4i1y2uN9wYbpPclrq1un1y93CXude793hYeKR5bPK4ztZhR7OXs895EjwDPBd6tni+83L3yvc66PWHt4N3jvdu76cTrSeKJu6Y+NDHzIfvs82ny5flm+b7vW+Xn6kf36/a74G/ub/Qf6f/E44tJ5uzh/MiwClAFnA44A3XizufeyIQCwwJLA1sD9IOSgiqDLoXbBacGVwX3BfiFjI35EQoITQ8dHXodZ4RT8Cr5fWFeYTNDzsdrh4eF14Z/iDCLkIW0TwJnRQ2ae2kO5GWkZLIxigQxYtaG3U32jp6ZvRPMcSY6JiqmMexE2LnxZ6NY8RNj9sd9zo+IH5l/O0EmwRFQmsiPTE1sTbxTVJg0pqkrsnjJ8+ffCHZIFmc3JRCSklM2ZnSPyVoyvop3aluqSWp16ZaT5099fw0g2m5045Op0/nTz+URkhLStud9pEfxa/m96fz0jel9wm4gg2C50J/4Tphj8hHtEb0JMMnY03G00yfzLWZPVl+WeVZvWKuuFL8Mjs0e2v2m5yonF05A7lJufvyyHlpeUck2pIcyekZxjNmz+iQ2ktLpF0zvWaun9knC5ftlCPyqfKmfB34o9+msFF8o7hf4FtQVfB2VuKsQ7O1Zktmt82xm7NszpPC4MIf5uJzBXNb55nOWzzv/nzO/G0LkAXpC1oXmi8sXti9KGRRzWLq4pzFvxY5Fa0p+mtJ0pLmYqPiRcUPvwn5pq5Eo0RWcn2p99Kt3+Lfir9tX+aybOOyz6XC0l/KnMrKyz4uFyz/5bsJ31V8N7AiY0X7SveVW1YRV0lWXVvtt7pmjdaawjUP105a27COta503V/rp68/X+5avnUDdYNiQ1dFREXTRouNqzZ+rMyqvFoVULVvk+GmZZvebBZu7tziv6V+q9HWsq3vvxd/f2NbyLaGaqvq8u3E7QXbH+9I3HH2B/YPtTsNdpbt/LRLsqurJrbmdK1Hbe1uw90r69A6RV3PntQ9l/YG7m2qd6jfto+5r2w/2K/Y/+xA2oFrB8MPth5iH6r/0fLHTYcZh0sbkIY5DX2NWY1dTclNHUfCjrQ2ezcf/snxp10tpi1VR3WPrjxGPVZ8bOB44fH+E9ITvSczTz5snd56+9TkU1dOx5xuPxN+5tzPwT+fOss5e/ycz7mW817nj/zC/qXxgvuFhja3tsO/uv16uN29veGix8WmS56Xmjsmdhzr9Os8eTnw8s9XeFcuXI282nEt4dqN66nXu24Ibzy9mXvz5a2CWx9uL7pDuFN6V/Nu+T3De9W/2f62r8u96+j9wPttD+Ie3H4oePj8kfzRx+7ix7TH5U9MntQ+dX7a0hPcc+nZlGfdz6XPP/SW/K71+6YXNi9+/MP/j7a+yX3dL2UvB/5c/kr/1a6/XP9q7Y/uv/c67/WHN6Vv9d/WvGO/O/s+6f2TD7M+kj5WfLL91Pw5/POdgbyBASlfxh/8FcCA8miTAcCfuwCgJQPAgOdG6hTV+XCwIKoz7SAC/wmrzpCDxR2AevhPH9ML/26uA7B/BwBWUJ+eCkA0DYB4T4C6uIzU4bPc4LlTWYjwbPD9tE/peeng3xTVmfQrv0e3QKnqCka3/wLmpoMnuLWGFQAAAIplWElmTU0AKgAAAAgABAEaAAUAAAABAAAAPgEbAAUAAAABAAAARgEoAAMAAAABAAIAAIdpAAQAAAABAAAATgAAAAAAAACQAAAAAQAAAJAAAAABAAOShgAHAAAAEgAAAHigAgAEAAAAAQAAAJigAwAEAAAAAQAAAIoAAAAAQVNDSUkAAABTY3JlZW5zaG90rCu3yAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAdZpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDYuMC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iPgogICAgICAgICA8ZXhpZjpQaXhlbFlEaW1lbnNpb24+MTM4PC9leGlmOlBpeGVsWURpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6UGl4ZWxYRGltZW5zaW9uPjE1MjwvZXhpZjpQaXhlbFhEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PlNjcmVlbnNob3Q8L2V4aWY6VXNlckNvbW1lbnQ+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpf6GgjAAAAHGlET1QAAAACAAAAAAAAAEUAAAAoAAAARQAAAEUAACFD9bmq1QAAIQ9JREFUeAGcncu2JFdxhk+LBQJzsw1mgO0ZSIwl8RDm5ocw6pY9smH5LUD2DHXDU3DxQ4AaewYC7JFnXJbBmJuE2vFF5rfPX3Eyz2nYUtXeEfHHH7FzR+3MyqpTfe/tt99+clXt3r17V0+ePLno0duw0cCVT/fasseeTT90jJ955pk03znWP3kzfuqNISm29Gc88WKzT5/UOyb+XfM4iiWvPGe5pK8+ic35n3HJkf6Oz3zUZ6/PWXz1E4eM7V4lu1VOsaLAwMMDKJB+NvH22pFp00e9uLPemNjnYibnEV8e/OQ5wh7Fx4cH8zdW+h7pkgc7eHoeHseJUU5udNPvrvkbTz5leeS3N4b41KuTA/mu+OnjGM4VvwZdObkwAukzgR26dMriJT7zm3r95El/bfZpMyf9znj1fRrc2fzlmPHh5GEu5pCxjnTixck7uYxrL05O+vTJsT639TM+2Iwxfc1bPf48jl5AYujvFagLbO/axlhCe+30Sao+SZv4KXcx/TMOOuXJexb/NnxyPA3uaeObe/LfNTb+bb5PGz9jySd/2o7G4rXph/4oftqPxvLMfp0icZLYfoLV2087sjsBfI7RTxnd5EH21OQkwM02/bCjs+lL/NteDMbTXznznpxgxGnLXt85X2Xzkcdc6dGJS7382ulnEy8G+5z/lBPLmAf5OQc45GVsA2fTfqRrnnpaaME6Z58JpH6OM9C03SbjR/y7cjiz628M85h49eK0e1CR5Uqs+umnbJ8+6rI33hEubcZLHT5ZoPBOnLHkT3/xYrI/m7/+YOFUPuLXJpb+3u9///vCXieOkgYYvUQ5MW3g0u5YPTgSp+mfSbRhPMlhjMRjU04cFMg8jCPW+PoZbsrypV1O+ne84x1Xr//gN1eP3/jd1YPPfmDNO3mSgzE2F8685NeesmN6fM8w2sTpl/GnTYz95FZvj51H5i3/UfyJd95dYDjYACKrqwLsIMrgkswFNBGTkEc5OWcsZfqV2J7TEY84YyJnHGViwocNLHLi5BZvn7ky/+/+6M2rr37zf6vAfntVR+bq/mfeT8CrB5/7IC4XTX65nU/migN2HuqnHzmIO8o7fRu4P8mDKEfaHYM74kBnvNnLeeSLTju9814X+Sgld0x/V3MS6Tt9ZnDt0yfl6XOXvCa0L0zGkNcDhi3zVoZjLvhXv/XLq4ff/EUfGznFP/jMB7ZiK8WMb75g/5j4+NGSZ9McPxtDa85P2x8yf3nsZx7JD8YY4rXfWmBHThlIEklv6/VzktnrJyblXPCMN3ObcnI4tgebvOjhzgLjdHj/Sz++urffGG7+2r3qdd80nQvD2mgotJc//b5VtADAn7Xb4jtH+skxdcqzJy462uRQd9f8xSWPcbAd6Y/s/S5SQzpBQlO3SduzSU8/rL6SwWA/wugv520YbROLjM14jGnKM34b60lc5mAMek6Dng7bZy8i/amvLjIKiJh7jy+F9vlPvXfFyFiO4TG3HKdOPT1+t9mcr/zK+qiHKxt2mnb8LLrEMRbL2HwcGw/ZJob+4jYFgCRLh7SZlHZtEiPnGE6TP0pIPL3NPPSl59FJs7B7E4eYMXOcfo7TD9/v/uh3V4++sV1nIa/WC1HxCEkO1XEdxsgebGdUed3/9PuvXnz+XVcvPfdu1J0zseb8yU87vXIr68n80lddYtXJoez8sweDXZ0+6lNmTLPw8NP3KP6RDf8+RWqchA2IxUSmgbfPYOhMXk7ldgj79MMub/qc4RIjN736s/hgtIl/+I1fXD2qi/iukp7bXlBiMWz1UM6lXOMS5vHZ7fc/9b6r+/Vu03yIZSO+BafO3gVVtj86NnMeR9ipQ57xk2fG10ZPy/VQd6TXr99FAnDC6ZSOOqATc9vBEwPehs44qXNM7wT0T/lsnP7G0P/MBpfXWR13v9ZKfI/ruG671dXVS8+/++rF557ti35tq9h03At02+u267N+11n2zC3nguuZLK07PzjnNsdgZ4Hoj89RfPA01yW5py/xzDNtjNUbf/G89dZbTzAaQMeckDp69YwlZSwhY1rakLWjZzztYGzYxaSf9uTgNgKy+U+8XNrh+M4bv67rrF9utx3K96LtO9ATDnzZXICv/fNHrl78+LMN5Z3la9/4edXW9c4247rTPan72G8/efvq7z/3p/2OM3PPuOmfY+ObP7bkYP407diygdcHvb5iJn/aGWPHX365jDP95bW/2MF01jhl9OiyGUgsAdGpF6vf1GunnxjkqdNfvb0HYMbPA/D4h7+9esh11hu/WWHlawVzY4H2KVb0q5c51dV9L/nFE5fT6qNv1am1GnIXHM6xyMTHhj9vAl58/tm6PtsKtR33JzDNMY5d5j/xyPjQyIuHHK2sp4wvjl6/5NfXOYKb/uhoYtIfvRyMaes2hQYDb+ZLIieBbeLE2x9NVpuxlOmnTn4nknZt+KU9ZcbpQzH0rsPisyjd19Bqgqv+U/5knQ7vf7Yu2GvXgkeuo3hyE9PWxdahLLwtFv4P6o3Ay1W0cuEjPz0tbcgsJDoeYtHb9FMWhyyXGHpfMOrEKeuDHh0PdKnHZtNPWdx6F4kBpcBJqF5HZQnF26tPvGNt9MmT9jygZ3j0+ohXJ6/vDr/z/V8D3vCs4X4m6VPh5tQF9uJz71qFBbc8QNgBv/uDN3tHU298TrvYHtapc8UpH3E1uNbXkELOG7VbCpfx5Ma2eBCqpQ3Z+YObNuypTzt+NHQ8Ms6UxbXDeJrxjXFxHwxyK1t/A85gEpzhEp9j8fo7cXrHYNKunHZ50GlP3eMfctvhF+vjHRY0W/LXoa2AdUFeOwvv/Gjy6rOuu2oRLIzMR/zC7ZyNqfGMLy/9K5/9YN+oTb7MDwxy2tHRjLtJlzht+oqhl18dWB5z/bUf9fJjSz70xryxg02iJJFIgsQm7sgudgWuA0ZLP23oTTjtqWdM0y6ed4bcdnj8g99tgPFMMbHY4vGnYPgAOxt6MP1O88s/WSaLcfqZB0DGX/u3/zv+iOmk2Lw+e/Hj7+pYmd8KXgP16jJu6iZOG3p8pv1Ip89RP+PKhz65btwHk0wCE0KPbla4OP0MpHzWZxJzjI882HgQl96WdnXzohu9uN7A9tNi44uL2w5cC7GoC7eTcTpcN17JAX31+dERKvwoDj4qUqY3V3Lis8xZ2HCVcwEbXU81rv+5UettDSw0uIjjcVDXxv3J/I2rnBh50HlKA8f4bF2Ni89R/BnH+OBp6yIfIRMQKMGUwdNMVF/xm/X6+cz/GrHFV5Zn+hlHHPG5znLXyh0GjDw1uc2lDij3sh7sF/Dy0MPtNRsfGWW7yKOoehekIqiQ4swbq/iJZ0yREb8LjfttpLK7XvZ7IZfZ0zD+2ZyPx8HjD2ba0g8czULS3x7bHKNLzilj08ceDGPbKrALZTnO+0s6SopscMnEpE3eI+wRXq55QNSnTxfW17frrLQzvhFvLwqud+YOIWfuNr3wbYCsHrv/2omw1XGyOU8Lw/jqkeXXJ/vEEYt7Z+j+4W//bH2QjizvXB/95VS2oNCnP8cXOe36GkM5/dTN+2/q9XX91ikSAEYM2esggf2RnkRMBjtjW+LFYNPHiWpbCe6LqL+cFNd9ro32he84xiufxiPv/p56zEs+/LjO4nTIdduTWlh8ln3nvJ5JRwp7yVVnnVfE98J9Q2/H1jFvPh6yq1mf5Qd/776MKv4qgBo//OKH12eb5uXxyePmsQGjnS9K0tLmmB6svX70PGjyGKeVQz/55GneIug7+YIkmL0BTeYIL2b66pP69Mcv5cQdjcHfuPjmgGyrdFF03M96+TPvW3fhzZF4XGf1Hf03fluulyUkbsb3Dr96cT1HKoZ1KSr4uKHa13nxVZ7G7fNlR+sbtTsezuRTdldEZsHFaKe3JT86sOrE3NaDpWWMxBt/2o2RffPUVtcFJomOEFG1ZwHV2+tHbxA57RPreGLlmfHl0L4KzAPCtY2tdFxnccsh75rnweGabbv4Lqc9Z9y3XYQRB3ovGMS9mbcy9UTblmVfmIq/eW/+Fgi+PPK48tUg4nMTmGZ8dtJ792pORaE/9hkfHc3jYgxkdRvi2jf14FNO7Jl+xtPHXk781ykS4Sj5qTeoJJDmOIMw1l8/dBnnTA/OBt4tWt8+RX5pu32Quw98uSBy2Hdh8TmiBbnXUdsrTiUs9LovvQWksvNWWS7mpd2egsHbvHi3mfMBh9zfnN1v0uprn/fn1M1+xs/jeoQ9ssOhXj5kx/Ck3TF6MalrvDuYBoFtjIOtHhxjHvpMLLJ4x2ATr789OFr6GSv9wLATUWAPXv3pwudCPn7tL4GtBqenw+/U55Bg+X+rmpOiwpsC2nGWWMfp3QUOjHsrrBhVq/dY7ccu32jkrvrg1Z/0jeFrvxpViPmC8RjN46Ie/7SpT50xsD2NPjkY65e+6uTuPGqCpb8Okg4AsNmmTb198ug3faYemUfuUOkjnj713Km//+UfG3rrybUWMQuMQlz3sy7RXUBdFHDvO1rHsQALb/wupb1AoFlFunM2Lnh29Wm3fTFx++DbGBZYcmOjwPxKdh4DbMqMeeRxPA1eBv3AJI8+6GZLn2lDPuK5uJMPaJIYaOrB3tZu8zMRMfDIr03uxKijZ0d68OXawbZtqE29u5ScBQbvxe0BjpsbD4vSniP+UYEVD818Ot/g6rzDb6fdusB1wKLi5i5vPj75/Hsag/9tBeatlduOkzYIzZMxemXH2YOZTbz65Fb3NP26BgMsKWSTMG1s675SDKKdHl8wNHDKcopF5qFP6vVPDHzi120KlLWAq1Qq/uOHf9Xx4TM+MD8nrID7Ou/zjIIDV0Eu8rco2kYs7lG1sOVDYVu0vsvsuezx4SMmvYX1wse2Tw/Iz3nff/XH9YF5fcQ14h+dIjkONHwdT1lej5k9+lw/ORLf5Af86F0bxnDKi0xDpjWv12CtiSdBAsO0CFJncupIAg4noh2dttRlPDjAYHcBtNuzg/V9sD2gXIjsYB4E/PXBxm722tf/p7hLqOPwDAdjPyDYbenf4B1DHAqKG6E0ue3Nwzn2/PdYj+p+Fl//0SZWX3YwTv20jH/bRf7kaOc7nuB2XczFHKYr/DzEMwZrftPvhlwOvhgXtyQophmCqQMncdrUyZO8acOeTY7EpI7xLLD09xSZ/tjlYNz3oKrY3PkoGsb2YHKMTENXTEy4ZTmPYunvDgRWnH369ymy7sl1CNgrDDnpjwq/LJD0x24zVtrViaFPu/rE/TF2eejXR0UqIeRhxTqZGQiZydrr7ysCOXkSl2P97LHpR08zjmN67oN5DeZCoi/nPkVuwydX//6fb/af+3sNo56eltdni6fnVqeuXmHmuY8Lb2G0814AXXRbvXX8AvUc+G4Z39Lwq9b6Mi9O8fwMQX5Ifp8djM9Ae97srDWs3eaV+rq1uI4becAlLzZlesfoE4NsA0PTrt9cf/GzzxjYUu5xPXUECyMDJRn6HdrJMOZhIomdY/3QT355wTA+sssnD5jeweoPY7fDsyEsEHcwtF535a0B9JPr9e/V13z2rz9jt5lP11o9tbwXVpGsd5+NJ5kqCm7u+unB8peweosa3MMvfLhzIZ9X/uWnl7cpdh92MH8PQz7wPDz+jLHRZxOv7sx+tv7pb4zkSrv67NcOZuCjJNMB3JxU2udYXvUmNPVpT9vEG59vO3BKSawcWWAupjyebvRTjyy2SHklNF3aUezatrHDaW9FPXmdNfmxL/4dTIG99k8f2qW63+V9sIiP0ZwFGpMYOdaevXZ15qWsXT2yYzDTnjo5busvblNATPHQT2IDZw+xyST+aCw2bcYzQeN6WtaHfvpx6uMi3/hyUAC8i6TB03fI/T4WF+Z74bBonHZ8sYA3fn8YHX/Mgc3TYMfj+NRxohnfHTJ3AvnYbfuPTepr2/rhyyn00Rf+gmE33kVy2ixWNsxuzNsCMxa9eavb4cuHQR5HZPNxnDJxlKcf+Gy3xZQDfI+LrOeiwYWUMMkYOzHti2hfuNTfNpbXeMaffIlzDGbdpqjsXQ6LgB1M7Nw1MicW7qVPPNvXSBlfDL7rM0LmV/PPhQfXn3nW/az8S270zIvC8iav+ThfMJ4iGWPvU6QX+fu8wPtiyByT54j7SEccm3ZlejkzTtpTrz8+qU98c5bRY9Y2RR11eJrgM5A+cMgrn6+SxCQOPA8LeuL4Iwt2sAt94dkhLDBsFthRfAry3jPbAvqVaXHG/4//eqt3FT7D3M+PVPTVCx9/Z/9xiDdKM3fGD7/+876mMz/75OebFo/qGoyGnVMkH2XROD51AK6eqa/bUGDzGqzthfP44AN3xmFsPPXgsk17ysmXPjmeeG3kR8x1DXbDUEaahaCdPknVq1Omz0mZrDh6Hn5fST8Ts0efB1Hct7//q6tXXv3Z9aLvBmKeFRgF5R7UebAg3Cejcup/FtIfL8Fu/vQWKsXFYv/d3/xJ5yXGvPxu2be/9ysOwMq9cfuuBBZ+TpFf/eJHVpwusP2vnzo+SVXAfBdpPL7wx3FRNr69+dPTjnBHNo+7eHnktVc/OZTJD451DYagkyTZ65iBsacsx9TLK/aMVxz2u+J5H2zhtqDtR4EZy3ebpWCtupCAXox3mTv0L33iPX3qytsajb/jicLyr8WbLhe2xlvorZCxH31t+/oarADUVjXmR+Gbj/ParNfHCflpjv86Xvt6y4NvbiaJA3Mm648dDvNb+CLtEs/kcFJOB8fYJWScbRGzoNWUxRzxYjvS4+vuNeNthVMX+fWfOxCriJwFZty1A6Eo3l7BfREbg28VmBfhXvfknBt38HR9/2qjBeK89U/57M58F1j+NVTlSaYWmFzw2+ZxyThglMWr47imDW53L7HYz44/Nnxmn76MewdLIEqdGJ81MSbl5JXxU8cYPC11yPLYo6Mlftqwcw3G13W6RvZibr869/MuUp85Ny/czaNxxbK+4AdJFJ6Li9pmboff4WKe5LP68mpx+4br/BKknBy3V/71Z3W9d/ntWl487F5eI4p3fh7vi/nEwou/q4fnrJiOfI1vL0Z55VWDXvlMEHDKR+MkdAyOADR9GKfOBNDT0rZptmf19EcT907+KoZ9QcG7gx1xMHHegfZfYZ/8NGbm0YVSQVhg76bzeWH/SJ3v+HDY4y9fDsNeqP3X4lUkvNMkpzw24tGv+2Aq6UvPNRhFduTrHIEe8W4U1zGPOMTQ2+RKfI7F2R/ZWldPtE6Onia5zvRHNv2m/QwrX/oRSzn9HOMjxjE9BZY/cYmuW03h9dc+ujgpBm4HyCcXfZ82fXdYftvpFpaqjL04OrcSsLGbwcePp+BfpCS3xfWZQ4hqt7kDHsXnNO9vX8DH99s27uuPpqD1Hhtjmlyd2x6/89nMCzN1+k6/lHMnC7obQ3zSL2Olbf26jgD7ybi2vHlAdyCkNHp3nN209NroiUPPg2Zc+9SLw6b9xg7WLBWfHerRXy9eTmN8vvcC378aN1ZxgZvd6LXazWjyt8BTpUdxFbCLiVMWcuNIfdRX40r5Uv2KDh8D2ZwDx4bC4sYrrl/5xz9vCHxHO5inSHYwc5vHBoK0OW7ieMIPW/ZyuWbKk0Mf6ZQn3gJd+lJwmFYzARQ5XoBdPwtuEdYEaMg8SJw+E9amHXza9c/42uXyIt8FRm/zFInsxT12dhNOdeRuk5f+ckfbCkqc/D27fY5ddzxFY7fkV3m88YpJX8YP6vPGx9zrKo6jj4r6Jw8Iwnz2OOTt6RkOc2Yst7opg6GlnvkrW1gbauPGNtdXOz2xpl0+ccrrNgUGlDibrDqd6LW5SCaor72T0K6fXPqrp9dXDD06mrgW6qkLLH4zon2r2rhY5yJffBZY8xSG3cAFE2f87cJ9280sXvzO8lC/FdYHrl742Ds7trzazYMdSd5ZYP2Fwx++WcGIWG17rfYLY+a7AW7mNRf+CEdO5mWe4NAheyyU5aAX7/q5vmKNb39RYBInEeOZjGRpY0wzAcbTTx16E0NHS85Nc/k87X0NFnfyMxY7mG3tSihqwRrHgaydNXeGzBuoBdGLjR+7Hgcfkn3hwdG8ztqky2dPh7w7XH4scMHyTj559e42frIAoG8wZo5Ect5GnRjsqUPm4fFPf3DKRz7oxKQ980g9XBfXYDpnsjPgmdxklQBt8ky+mUTaHSefuuxvFNgqgOsPu8FfF0qt1H4As0j4TQna/Fc7jK//nDc+ng7zQl0/CsvPIcHWynX8bVxi/Yd/XqddFFjgbytg8zIu/Lcd38SBdSfCZ67blMHbMgY6dyzG6bfug2HI4CaOnpZOKRtIvHI71ZOc2tVPHHqxjsXSG1+/s6/rUDyPH97cwbY945pRno5ZPp421V8jt9OxtzXQUxjzV3mcH/58bklhErP5qraMLz94eB59cXybYr/RmnxPU2DmKz9ycjgWZy8eOw/laVeWB1ziU5/YdYqcxDrrqJPEKTMWp10+9YmXW91RL+bIH7zXYEf217/y0XWgJo4CXAtfPOmPjULjQXMujikaGnb9cp7udmXEubFpR0EM/sefwskbqOsTgcJhF/uHFpi5dQLjKeeEKfM7sw2KNffpr5zx+8NutjfOyReG/QDh1JPdKxb5qJmotuRCN+1uqeiNr+/sj+Kvr+tMcMl5DYYvC++OIryXnzmyu/AR0f5n+uadi4puHp/FUxw3TodllGfhqlzcxbYdcPuKT84/v02x/Iqfj5bYMc1D2x/Tmxc9sTO+usmr3h4742zw0NQrr1MkhiQQkCQSYBNroWBLHwNNfzH62yf39ElZfO9M/dMB+27BhJlkdXmKNA967nfxtWh3hi6uerqWLYEtIkVGY8c6ypubro/q/lnfWoj4RbgO9MZUz9irPaifj2LXch722F588N+kdNGISx63vYsEkzwXBLuA3ZZ4/eh5ePEv9qzX72j9teF7eorEkZYB0zHH4JCzGfjM/4h/+iO7c7jA6IhFgfFZ5GzmMe9+6wfPOpWFs34dp6bin6U9UzsbBeFpUx6LVQr9lTNfdPOLieLwm/msggdUuRA7i7LV5UeMGVfeGV/97Ke/fkfrA9aYfh0n1ze55e0CQxAoiQEMqPOU1eO3SGvi+vt9r7Q5pp8FhI4Y+DsmRuaHvE6R1PW2O/cu0dzl33kWFzvGvGaSdy0suTcNRDWK/PvvJkuLjp3khefqi4Zxe2R33OZePN0iPrciuPHKH9rSyMv4vEge8gN6+41XCovTNbE8ztwe6dN1Fbn+PYinnnPJ9vhmnID20PgIjJXTx/UzjyMOdK4f44y//EvZR0UjQNptxCaSOPzTRz6x2iaug9VT4tFNWRyJw9WnyLjRir0XqFacnv9p8HANw30vC82csHtjdSux63nP+MpFCAjXi9bzKxM85pF/AOL8cepTa73T9Gc6xSeh+M4/3gw4/7Trhw68NvWpy7F2evS06dvKeDrzD0gPF18lfPEDdAa4i0iCSewk0cslBh8e7kbojSOfPhzIxCWH496B9t/Vulh0DnIVFLvLKpxa9gd1qvn8/nmk8eAi5vXvhW3Fkxf+YBKPvJoLQ7xq4Py+1zwWzIlT66P6fNS8yuGiYPvFsTHVM78nu/1uvx89mUceJ+DKjMGcHbtpQ555ypGc6I7azAdM6tZXplHyOEtsks/gSTqx2tQfxUGX+vQ505vDOtWNyXU8Xpix4cBFAeSNVWPRz58f7xjktidvTLCrGHZ+3x1y41UcbmDN8QZfscA+9f5LI5xa8Z/r0vH3nZTxbBnfHMSc8aU+ORnLZ5+6xGo31sW7SJQTIFAS7HObTsz0xy99Zwxs+ExM6icn8eXRxgLyrQn/aEJ9A12APU4XRhUF1zZ+B79x9YQfp9/Xv1934ouzi7PWzxLbimqfU2HBe+OVfs6DTxzYHbePijY81dpFRXGNay5+HCU/LPc4mB89uqknD2ODyflP/LSlrzb5s5d38qVef3S03sEk2VTbM7rZZiLpN4MaaPKol/vML+2MMxayucgnT19T7V+9EUdvHomnWLgu48I9/zpIvLsOchEQtIc8daGV+PALH1o/DZA58iYkPyqa8ZvIwi3u/GzUHMHol9zoEyMOjDudfuKO5LQdjeWlp4nZpMs1gT/ji1mnSBX2JqQseerVgUGPrF3bmV4f+e31U5ZPOfvEzjgWR2NcyHK2MJq3crZo5m2NjCtXxhafOGIhg/dfYWufg/ibnn1s++jpwee237DAf84r4zpODLozP3HmmTI6C1Leu3huw8kthn7tYAiZhKfBmQA4bGKxQ6xsguqU7adeecaXD3tiGBufsfmJhwe9fP6Vdmt2fSW7FRarW21bZgZ1fVanTf+l2rbFglM4vPN7+dPvXbsWsYzXhZi/s1q+nRdx95iN3cfbvbHtXxoxFn1yItOcHzbGxtys13Zt4pTt1ae/NnrHaZ/xPf7o5/qjw1dMf5tikgEiEA0CgyI75kYbzQVuIZ7kFC8fesf2YsO9E0Se8dGBZwI0459xob84bfa0eKpFZ93hKb7ZcoeCI+NkfMZcs+WfrCUXvuyaFjFj/k3v+1XIXLMxj6P5p865wZv6GSdlcR0/jjl6deCNn3j0yowzvjaPf+K00WPvOPW0v46vi6cNBwcdR5pJMTaANMrYZjviTT/tqZscyOIcH2HMI7nYzfxqND5z4SkCdfizmx19RGN8CiuvszKfjl9H1sLqHOtQe+P3KL/G1JP8N/h2QB5/fY56YuT8HYs1B2Xj2quf/ZFdbjnFrGswFfQ85ivWINhskiEf6dElRr/spx8yD3cusLdxGEMeZXzUTQ7veWUejXGn6byf6e/Vf75Oh/MNQPptp8XrnxVwR+ycmcsOZrfyT9bMEZM5OkdkHh5/islx4hk7R3vtyeXYXgw9Tb05zfjqN/T5MzgbnPKsa7C7iHSCJJOSNPWpY5zBtcmh7KtS/fSZ8Z1AHny56PXXT97EWByUQZ8k68DQtmLY/rXbxE9OZHjhodFfl9Tm2R8VxY+jbNqbz85/Wo7yFmN8ZXpzVKe/emXt+sg17eknJn08/mnTTr8+7EawTbB6emySKqf9LMHEMD7DoSfGWcv4d+WZHDOeNncz7Z4WlcWZ04yvnb4LNm6R+FERtsmHLpv86HJe0y9t6e84edDpr175NnzG0E/85JRP3A25Xjm9mhgktk/SHEuWOol9JSIzzmLUZpKTBxl8xhdjfviKc0yfLXOZ8ZXhSB4Kjest7JmnvHIim5O27LFxD8w/stXm8ZCHHqwyYx63zR+uxCkbQxnOyZ82x8bTH1k/deanj30eR7H2+nAc/x8AAP//4/JLOgAAIrJJREFUnZwJ1GVVdedPzSNUUQwWQ0xWEzEYjVFBMYDGqEA3Dq0SVBBMbBEihIA4QZYDmABL44AE0RZjWu2lcaVNFCFZ6IqmUUHRdEgMajRGoxQWWEzFUHPl/M59v/vtt+ve7/voE9/bZ+/9///3Pvedd+999yuyYNeuXbvLZCxYsMDpoN29e3cRw5yBPxRXIOPH4mqIF2cdfDHMI07MbLGMyX49Dk0TDeuIsZ5xfHPiI8Z5xDGPI/ZKPNaPuKE65CN/CBPzUc955BDDz5wYYx79iCXOGIzVZL/BAOlieS1cuJBwI5tDaOfOnS1mHox55kN54g6biRxyxjnguT55YmDIM8Rjoxb16c3+zIlv5PpmXD/mYy7qj2HAm8sbxrh1tLEGsYiLOeL6ERN1yMf1Rpxc8TFHzHyME4u+uBwzHvF+/gtqcGqDeWAEY2laGOLMEVi0aNFgAxRUJzYjl/zY3JwbCN+DxpxhfeLqa+1zaIOJ6VSmD6o8chFHH0PrFxd56mLz+qOmOLnknJOL2LnqD2kRQy/q6FvHmmKMwzU2pGNsCEfM0a+/TtoGywUFxsLELJ7jMScXC46Xm0Se9cTqz6YPdiyPrjk1xY/lIo55xNmnmCFtc1gPqLGIVyvGxEXrZiKWOfPxx/Qz15rEI2eovlhtxBNTmzk5fXELZttggiEzJDG3OS2xoaGGXHxe8awALzanTtZWg7y5ISsfGznG5ehrY5x59Id05GHFYhkZH+NiGzC85Q8YXPxiRs0xjSA3NR2qnzWG6seaca541oh1wPSXyCguOdpMBG9BrPkmOvHNEzMvNubIz1ZfLjj5cR5jOY4PP9bLehkzlAfjMI9mPA7m7WeoprGoAQ8/5pybi9rMrStPvHEwxsQQY1i78zrcbMc/crKm2mpGH2y/wQQoYPFIYD72jXJhmZ+bG8ON1Xfh5O1lqMZsurFvca4va9mHeawYc/rkjGGJxxx5hhhy4mKsQ3XvOZ7Xn4+/eNjqO8eS5yUvrx9MHLl/uOpGnchhnnXl9BsMUG6GmIMcwwYydigfsepgwbpgfbHxgJIzzjyOXD/mmLtg+GDVmQ9vrDdrqIfNw3rErZkxkZcx/7/rV1O9uP5YHxwvcVoxMWcsWzHYPNTr69fJnqjKEqhAT6gfGMMi5mOxyI1x5nx4kYsuww814omrFTnEs0+MYXxIJ+aiLrwxX82MyXjyDGq4li4y8259I7HHMb0Ydw5/Li21hzhyxaAXcfh5yCGe5xmLltpTN/kC+2QAxhwCczUkHqsec4vPxfc5Sv6w8kZH85GOoQMU+yFvHeu7BrlaeeajT0x+7FFujuHDJ58fs4i1Lyza1jOvRQMMj5JyPeLEGPaXdTz+xrXWV9O4etFvmEpolUhkEA0YYx6HQjHGvIlWLUfmj/EyFx4vD0DWG9PJ9eWNWfvL68/1xanjgbY/62rJM8zLG7PWow97IZb5xBi5vrqxfuTH40VcHePW1CdPDevHOLXkm8ePGq6/vweLAJvFSnIec8wtZAMxP5az1hBfjjpD9TNGLDb2Ic6YPpaY8cgfm8sxr5b+fLTkYOMHMx8NN5R11Brz0QRj3hpj8YzN+rm+euL0c73+EgmApIRcMBMVFB/5UUec+egzd6fnAz5WP9aLWhE/VH+ufF5fxMc6zof6sG7WguMHxDzmXT+xGM/1rScm+1k/57MefTDEMVebuXrGxOmDYRiP+hEzdQYD5Afd0WcEIClGTpEYM24xc5krDhuxzB9pfTQYsVYX6d6jfozbf4yJzTH8ITzxsbrk4hBnTD1rYp2LiVZ8jMW5+uDU0UbcbPOIj3pyjOHbT4wZjzpTG0ySgti5BHJezpAWuXgAxGIZQ5ysn/kdc6bPnJ8vX51s5Q/1BnYoTyzjIy7mjaNl78wZ0c96HWLmPepE/bl4MwrTs6hBRn17Utd4ZBPzRNE2GKdDA7MRyCmsoHjj8/XVyg0P6aqJtc+M07cPfbnWMY4Vay9acvKcy5dDnKEf8V2mezdvTJy1yHv8jYnFEjOOnWv9cq0Lh5H9GHsk9dVpouEtx/u6VXy3SYOB10/F9IHJhObMadGJ88whFzHmY8xeso6+HOvLzXl1tOQzBi3yvPwAxVsnWvjko7UPY+KtFfXEZBvry488YuqZt679RE0wmU8sasjD8nL94BhjfHnWB5t1W6wC2xZXKIIAMBTrvJmiLsZ4tOrlhiOGuTjjQ/XN8UDl4a27y6rlMx+wufnoxHVwYBhz9QdG3oNbdpcHt3TPkJYs2lXWrl68B98+PPBRXx0085Dn+mfrL2PRMqau/lh99a2nhedcLW3M+ZxMfeuJVaNdIk0SjCKAbSQLKQBGvpacPB70MV79J3eV7TvatLz0WSvLf3vaqubAseZQfTVv+NaWcun/vqc88PDu8rRfWVauPHe/sqj7t5BTOjjoZC11jIMjxrqsbwyMw3WAO/nijeUHG7a31NG/urRccc5+UxsMHbWcu358hvVjjaGcdcVpI7YJTt6sF2sQG6sPjrz9+PlSl1iuNynTx+1PXq6vbn+TrwBARi4QfedysLFZxYkz51nuEb93O24bZz1/7/Ka5+2t26x1p4LV6bRKeeb5t7fN1T76qnnpq9eV449Y0cOt3wfCJObiPEBGp7Gvky/ZWP7t9rrBav1jn7C8vO/sfaeOk+vGOuDnmuLARKwcbKyLL84NQCwO87mW9eNGEAtffK4XtZnLsb6+vOg7b7xK6O/BomguHEkZF33mcuWwZ4846/Ya7y5LZ71gTb/BxOZGWcjMQSnl6b9/e9m2feb0fcFvrymnPHv14Aeh5lBf5NSNvQ5h6d++yLcNtqE7DbvBMg8/1pCPFmtiUN84vseJucP1ixOjL07fNeHzij5Y1xJzxKmjtjjjsU8x1tOPa4KX60+dwUhKBMzIBAuQExt5MS+GM9iRr92A2/TOPHF1OfMFa5tvDK3MFUD8musfKB/6/OaGWb1iQbn+sgPL6hXdNTLWjxzm9hjj4P0AiMvHOiLPeNxgz/i1Fe0MJn+IZyxi8jr1tXJyfX170Y/azI1HHHPj4rM/FFdPLX1sHFmfnJz2JD8Wi0SBOe+ujXHnCkcdNthT2WCTTRQvkX6DhvjEYvM/u3tn+cldO8qTfnlJWVivQnKoFef2bdyesh75sfpDemywH27YWXvaVY594opyxdn7zRzI1CvaDuq7DnX151tfLWzkjK3NOHhrWZuYQy39aCNePWJj88jtMXXCmPUDgjiEiYJijE03Vy+R4R7szOft1V8i5UW8MbWwP7pjR7n1h9taH0uXLCgnHrWqX+h3frytfO8n25u/csWidm9224+2lxu/vaXc+oNtTebQgxaXo+t909MftzzKtjlrc8C75V+3ln/6t23l/od2lafWHxTPeuLycujBi8tL33Fn+eEdOxv0GO7BXruuP25otFdZWG7854fLv/xoW/nuj7eXLdt2N+7hj15ajn788rLfmu5Hj8cTu/GeHeXm27o+ET/+yJX1Xm9b+cQXHygbNu0qh+y/qJz6nJXlcb+4rPBj5+H6S7bUlp902LJ2Fv/8zQ+Wb353a9lZL3dPPHRZeXKNH/nYmXXe/J2t5evf2VK+XY/fsnrsDv+lpeVFx6wqB+073YvHYMjmjTj0ebEWrwzMGVOXyCjcA+qOfSRjiEetfoPV+ZnP7zYY2NyotdTRv/pzm8tH/qa7RPLr8RsfOLhxwb3rL+4rn/rSA6X+9inLl5Xy8t9aXT76tw9IbR9G/fibf2y9tL3rzHVl6eLps+POepZ996fvK3/x5Qcbvkq1YR8XnrK2fPrLD8xssMcva78iLQLuxxt3lvOu2tTOsvLI0xf1+WJc8sp15bgjV3SbcbL+r9Yvwh9cdbdS5U0vW1su/+Q9zff4nP3CvcornrO63ove0ePY5N+oG4t7U+pZB8BLnrG6vPGla8r5tZ+bbtva6hMHw/84hhe+fE150bGrCc854noA25fEVj/sFfFTjyn6YAVmAYWw4DJWHwtXTKezoBzJGWwSz/dgURM8L74xaPCNwP/AZ+9vG4x9smjh7nLzVQe1lsizwT75d5ub3/bFhF8Fum9U9Zlb/7Uv2Ku86r/u1XS7/ko5+3131Q9iS4uBmxrsTUNtn+6uZ6Nl5b31DGZ///D9re1RjHWaLjUZVW83N/gT3dOfs6qc99v7dLn6zgb7/St/3npsvIpj/Yx2RqgyZ//3yQY7hw1WA7HH6voFsj7HbsXSUrZur42DDetnvqAeN2Q++ZYDymGHLGm14DA8JsyJ4cecvZFv/TGZDPG4bV4fmA3+ihwqogjWIhEX87Gh2vLUGew1J67qb/JtaBo/rU/ug9c+0G2wWmRRPV43X3VgK7fHBpsc+KcfvrScUr/x1P7c1x4qX/jWw1179RguXLCr8g+pB647eF//7rby2rrBGKyHb/cFJ68tj//FJWXrjt3lw9fdX88UM5cwPpijH7+0f0yxY+fucuJFG8vP7+sun+gcf8TK8vzfWFmWL11Q/l+9TF/11/cRrt3wYe0qn3rr+vbBsrav/cvWboORn/TP8eWMd8A+i8vGTTvK+fVX80uOXdWdwdgHkw2/ZtXC8saXrSn77b2wfPEfHq5n2XoGrgM+WhyfY+ql+bTjVrczHWfoG299qObqIqvGr9f72Y+8/oDGGXqjP172BYZ5ftA6xG3YSu627QgipxHPBaEO4YxTof2KZFL53IOdcWJ3BpE3l65nMD4gzmBfr5dIh5dI/aN/dXl7EGt97IUfvrvc8M2HWn38a/94fTl4v8Wt7xe/bWP5j3p544CzeT9zyaPqfc9iYC1P23/wp3eXr9UzHJuLDfKMJ67sf0V+9G83lyv/arKB6vpOf+7qcu6L9576UG79wdbyu++8sztzVN3DDl7Szh6sv10iqz5nIdbH/06rZ7lzX7y2bpDWRnvbXjf7UWdvaPU5jnwRrr9sfbuvcwO85c/uKdd/46HWN1qH1y/JJy7qNpCf2yl/dGf53k/r87y6lqX1C3DTld3VYKbSnrP8OYmwrn62g/dgNgKYbwLfglggi+Cbl+uG6XJusI7JPRgbLA+bVYO89TkDcA8GhgP79XqJtMaffPr+/hJZGyl/efH68l8OXNL3hA6XsNe8exPT9kF+4Lz9y1GHL6t/etpVjj53Q7cZ6gF/4TEry1tesbbzK9b6P62/Xl/4lo29ZnxMcfrld5Vv//vWps1N9P9930Flcb1/Zh1eQphf9JF7yg231DNp3UCMr77/oHrT3V0iz/3TTTXcba4jH7u0fPD8/ftaHhc22NPOnnlg/cKjV5W3njbTK5pf/seHywUfrJu11qtv5ZJX7Vued9RKUv245vrN5erP3d/7N15xUFm5bNJUjcbj77zp1Zy9GO9FJpMYb/P6xpgSBRuF9MHFOH48gOSIiW+T+kboKWf9VLedwc58/preZ+IpfSoYHG7yr7m+OyiL6zOKb1x9cF/LMxgf0MJ6dvMHgHR64hHHiRf9rO/fvwR8vz6Zf1n9deg90mX1LwTHhb8QqMHanvP6DeXuzfXSU//v2F9b3u7BiD/z/A3trwycEZ77lOXlsjPWSWv1PG6cqdhIjk9cuH89wyxtZ7BzuQerWoz3/N6+5Zn1lysjHs9t23eVo86pX4b6f5ztzj9p73Lac7u/iFjjth9tLa+4rLvcw/+fr9u/POWwpf26iXHLcPHHuh8R9Px371lf9qrPFlkLL4Z1m1PfYnxoLm4PXv1gux0holpAceOEVJsOFdhDeNIohPlssMhnbg1re4lEbHE9hbHBOu2ZX5H4K+o38Sv1GylfXe6Pjn/Tnhvs72/dUl539ab+gF7z+v3Lrx+6pOe3IpO3V07OVNy/HFM32BX1T0U8huAM6Di93uuc+6LpP4ORo59/r49aTqp/z3Rc/Dv7tLNL3nh/dfEB5dGPmr7xhtNdIm+vWt118031V+DJv9n9CnSd3/2P7eXUS++0RGE9T/rl6Q32+ZsfKm/7826DwfvSew4se6/sNOlTrV6kTjyentFjLs4zt/8n0wprJSmsj0VkKJ5z4iq8/amoMvnC9Gcw8/AY+tou2r33G6y6i+rNeb4Ha48Xam75kt3lK+/vHmHEs+Km+3eV497QPexF8bIz9i3H1bMNN+BnvPvn3UGtjV513gH10jn9gYBnvKw+aP0+fyqqOB538LfI+hup3l/OXLZe/qzV5Q31MQMjfxicXU67vJ6p6mCN73zNPuXZT17ZncE4s9UYm+faPz6gHLhvdw/YwPWN4902WD2DgeM4vvnla/fYYDwT5AzGWa4CyzVvOKBtMHQ8Hmywt/+ve5tOFW4bjDMYPXk7NPb5ojP0+RCLOfgNV4t2mZrNolko+01xHm/U9jkYGtzk50vkXNptg3GJrI17D0Zpen7np+7tn1/xHIwzWB6ewYyzwfhj+V337iwnvPln/SXyD0/dpz5D6v6lh1gs/R1z7u1lCz/764h/izzhzXeUO++p/++s6v8d8dhl5UOvm37CDx7+Z+ul6R0frx/sxP/r+mPiFw5YPHWJROPzlz6qbTA4DtbJJdLnYOTyBgPDBuMM5mfpGSxq9RtsIs4ZbHW9IsORZ91o0RjLm7OOuP4mPyfY7bz85x4QMiYWZ24R5v6M7XgzG4xc/BWJz7ChWCPqscG4BwPnYwryfOO4B+sekNZnP8sXtg1mfS/1d927Y+oS+UevWltOqE/M0XtqPQPt7B47lcc9ekn5WL03QpuX6+dh5dlX1DNDxXP64B6MMxjjnCt+Xm6qT8sroX0Brrt0fdl/cotpfXgvefvGepmsv97qwL+lXub5k1e8RFITPmcw5q1eY5SZDVbL7Kq/ZNlgL33WzI8l8N/7yY7uEjnpZWyDvfWjd7cvxILaQLxEomF/uf6kjSkjnmDEe7bcY4MBZFECIHnaNKdoXDy5OPiAGR2322B8Ozm5/4/6kPN3T+j+1MMHGHWsa2xpvVIw739FVg3+idlNV65vcXL9Bqv1vAcb2mAnvKne/3QnoMIG4xJJ/Us+fk/57FfrI4w6uNk/76S15dRnr2y9E9t4z65y+uV3tudcrIH/HfOE+iS//i2Sccv3tpaz3ttd+vB/oT7iuOaCdWWfvbr/B3h8Zpd/6r7yf/7+wbZ+MDxpv+iUNW0NX/lnnuRPfuHW+tfVP+R7ifRDw3KJfDoPWmv9XfX4Xnjquv4SiSajvwejaD02cYNxrNDhDMYG4wuxsK4/brB8/K3fqc+8G8cOjV6nThrC4tg4xgQihrkF5cvDp0T3HKwCla9V2Wzi6xSVGcAE960Pdjfz/QarevkS2X5F8iS/5txgqDHog9fdm3eX499YL4VdoXLZq+s92BHLW/3u/mzyhHzS4GPq3x6fcOjScm/91cjfNPlzDMN+j6lP8uO/B7vg6rvKl/5xa59fUr8Y/MPINasW1Q24pdx5b/f3Q+rT4xfe1T0aoDfOYDxnc/3X1Wdbj9qnu+luRSd1+3uwSZAz2EnP6B5BoENvbjD7/PAF+/U/Woy1m/yP1pv8yTF2g7EpPOOi90iH+pE3eJMvIBdBIMb0o83cLjd9iawiFdadzci30dZT3/QJVty3PnRIS7PB/uxv6t8XK9wNJtfHFOydFfWfU8d7MPtlE/ErsvlV97Iz9us3GAW+8M2Hyx9+ZFOpf9do9cTFfjx7AWCD8S9aHfc+sLM+jb+73FbvgRjWtUdxy+ufb95/zr710cHMY4i4wTgMXCLXr5v+T/7R2WOD1R8TJz2zu8xTD0y+B2OD8SvSAebamx4sb//zei84xwZTM3JdFzG0MkasduoSCWFoZBF8Y1q5+lr0+jMYTj2CnkVwG4+jSmKyYvNspFuu7jbYh6/bXP9cVG/yKyw+6+Jb9/7P3F8+/sXuTyT+irS+B8ENZn2fg1FZzA/rYwQuVXds4p/ktKZqR/WY1P+98vi9yuYHd5XP3Nhd5rgH4xLZXwrawS7lYzfcX66+dnPZzhmP4zlZFnr8q4y3/c66/pkTtYnfUv8MddZ7uTHvzlo3vOvAsm/9008c4OIlkkv5m0/Zp53B6L8dx0r4wU+3lpPfUZ+D0X+NX1P/DPTkx8xsMDTbBuNXZB3o8qub4+bZyzjWY6M+MQY8Yq6/i87EzfcbTEC2Q6fNWBQhR26COHleNp8bypyol3XNRY4xsPaV57PVzzXw+Y87vvPjreWO+nD20AMXl8ccsrQsmfzrC7XljdXnksg/IeI/EjmsXm5/af2Ser8384G4hsjP2vjkXVdcR4zbizh1IsZ6Hn99MHHk+FDNiJdvLXP20jaYScEmAVtQIlacPGIRZ96YPjhG5umLkxexMedcXhOdvMElbk4tOeblmLcWdjaMvIiXYy7WMqa1N79wxO2VefxC4882Ii/WjHP4Eaevrmv1OIhVA5yYPBdrHMuQC29B/bXVtrAFtBEkaSiXcfkAmVeDA0sMLax5tbXgHeLIySMX5xFrDTFYNfIHK09r/dhXnIvT5lxevzgsWHu2DniGfWW9lqxvcvWzlYdVC4zxXNec+cgxh5UHjrl+zDnH5vVP3eQDYFg0i5FDwEL4ecglzpzGYwxubiJryAXrUAOrRuxDrDj9zJfrAR3Cx5hzdbBR27wxfXDMXb954kPrlyfOxyz6WnDM/RzQGxriyanNXK4xj0PEyyEmjphDDfzME4NtvdZG+zOYYpGUY9mPgsxdOJbhAppT32w61jDXGgqbyjjWusb01dFaX38IH+tEnTjPPP3ZdMFYP68/8mJ9dYnx8njpy9OKRz/iiYOxvngwcRBXW46WuDw5Q3xyc9Xv+6iTvoMsHotYPGNswHj04xwtNXLcOtFmTN/wHBvQPmK9qJvjuY6+HPWIO1cDa0yevpghfwgLnjGUyzF98FEff7YhD+tGFk9sNq18/NWSny1aTTPeg8UCFsxCEYNozJOzkVxQPfmRB9a4PHWMi9cXpxWPDybjrS9eC4/hAZcnfqyeOPNz1TcvXr59DMWJiRvKw41x8cbUxlqfecyrPxQfwhmLPLj45LTEGP1/tkYiJhXqYN17zBPBz2OoSMSqKzf6Ud941JdDzHyMGVfHHFjn6kW+eXnqYOWJkaeOdgynpnnxQzoZIxY7hI/5obl69h4xWQ8srxyXMxa3xhCu6dWd3e+S2EgWVMi4vsLRkvOMYFx85kefecYN8cFEnhhtzBvTWmOIbw7sUD7G1dPaNz5z1x/j5KyhPrE4Il4s+TF85DqPGsa0amY9OOYyFj/jxeRae2hUAKMXYB6HBK05Cw7hxWSb60QN53LUta7WfMYb10a+sWhjL1w+8oZQHxwv8mqiY17NmDOWcWDkecnCt37Mq0feOTb3GWvNNrdu1CKmfrRZJ3PyujI++lNP8hWaAtQm4gBjs8Q9UMZjLvLAMTxAMedcDXx7yXoRI27o4GR+9merSc66mTdW3+NgL+qrk9evTrTMPT7M59OHddXJ9e0jWjH2Rg5+rB/x5o1l3pBej62ibSUT0+IQfA7jgiVowStMLBYVg1XXvL5WfX04YpkzyM0Vi3w4+QPNfDCMzBvC5fpyItZYpzqzBuNi9XN/maePhSPfOJ8Pw+NnXNyQ/pCO/chTJ/LHeHDky9Ma7/9URAICwtFaWKtAtDbQi07OevpqY4lFPAdI3FDduNBYM8/VJa4OMf/BYOzf+mDlkecVc+RzffKMqIevjjm18BnixZmPNs471p71Yxwt1mdPWDR40bd5OGJyDeK83KjMI585w3xz0pvaWms0rfo2ddOV3KkDg24jTYri24DxaM1HTfHkhoZYcfpi1c/56DOPPDlqiMU3F/HisGJj3hj5HFePnCPG4tx8tOrFGuSNMycXfWKMzOmi01xx8ufi5LwnILW14tB13mpVwuB/2Y1Q3LU2pOBs1iKxEHibi3GxWc/4WF010GTYK3i5YOQzz1h49sScEXXVJB518B1qyiMONnLFEs85/MiN2BgHxzCmrzWOHdO0V3qTFzVbgckb2LE1RJw6YvF56feXSBuTbMP4EKIvRhvzzBkZH+MRD9ZcjqtjXl9OrCHGWPSzbvbVI+6BIeYYw1sLHJg4xnLEI1YcMV65vnFx2lgrztWOOGL6s+llnLrEh/pSExyYOMy1f02BY0CgfiQ5j5g4J599Y8RjHbXyGcQ4NvYQdZkP6RmDK975UO2YEx9rkh8aYK0V8WrAMS7WWIwzV0eueesO5cewcHIOX41cL/rW08rRGsdmTXPEHa6jP4PlhGCBeSMYlxeLGos242MO7mx5sNaPuMgbqy+ePCP6cd6SKW8MG2vhw1UTn5ExxKzBfGxEXpyDjzVizSHdjI38Ibz5WHM2XK4feWph4+ifg9mcBcZ8yeL0LSbPODZjzcnRn82qq1bm6otTK+KJRR+sp355Ma/GmAUrT+3oGxvjg7WemBxTT1z25WlzPuuJw4qNMevEWJxHzhA25uFNncFMQuSMweADwI855tGPheKZJmKYi4vzVmTyFvHWh5Prk3NjRD7zXN9aWjDMebk2Ygzj1uyi3Ts5BrnZ6oMDw4h6xl2LmAZMWGuQG1rnbPVdv7Vdo3E0yTGoYz/4OU7MEXNo4cPN/VlH3f4eTCEtQAYCiuFDZJjX18ZGGnDgLerFeYS6iFwfjLUi3nmur36MexDMRb2Iy5r6EW9szFoj6hqD4zzmiXt88/rFgxka6mjdAPr0rrZaQ+uJMbnWixrqmxMrv79EArBgBpnDKp6FyTGGuF1mJodvA+aw1s9zfaz1h/jk4wAT+3EORr4xfXIxFufkGHxAY+vvEDNriXznYob6yzGxcsn7BTGHjf3HeOQxF+c85uWZ08dGXMwbj1jm1uk3mCQsLw6gZMDOFVJAX+vi5Qzh0MrxGHNuTbWsoTWuJS43z2O9jAFrnhwv1+8cDAOfIb454Y28/TjPeDUCrU0jz/pyhzjiYy+xZuREDJoxZw3j5PwCqSd+SMc+5OMb22ODAZrvsCh4CxvTn48WHF6RE+dqRIycsQMhRx3wQyPmmYuLcXiz5cxj5TNnRJ5+S6Q3v5gp3LtRxzm1xtYPxgEu+sTt03jGmFcj48jzsr64PXg1sMeRJ6SgRG3OSReffXljVnzME5ut8VgrzqOG8/nmxVM71s/9xRwc83PVUV+cvlYdfOaz4Tw2ERf5aMg3rk+OMRaPuQacvMmPtwexvlh19afOYCRpfogIwXgUsbAxMcRjM/Dzt1QsOfD6ahnHxjGUH4rNVj/WQxs+L9dvPdcnRp75bNEAwyuuN/vquVn0xcX4WA05WHj6Q/Oc02+k+gbH3rURM5aXr43ctv4aaGcwD4bNSdAC4+XCJ7Q9FmZcnno5bj5ascbgxFj01TMfc/CH8upi5RkbWn/UZM5raP2xlnN157L2AW9oHvkRM594xIzNY7/Wj9iYZ+76I2ZoLm/wv4uUIAif4vrMh/49Umwg520+alhnzIKVJ8ZY1icf64uTpyXOUFffvJYNx/Cf+xifTTfnrCF3yOb6kZP1om9/8QOfSyvWVyvWI68Glpf64uRFLXkxx+cDt/+PPiREEDEWApA4g0LMe4Hqx2Ej5Jnra7OOcTVy/eyDm0/9IZ5cLHXB5Lk8rfkGnLxFnnFj+ti8tphjLif2Qjzy7CNiwYwdf3Fz6bhB8xeIOPW16LjJmDOskfvsst27/KkzGMExMcmKWsQ41lyMgeNFLuejBjlxmR/92eZRX2114cX8kM5QfXFRj9iQH/XNiyU3Vy+PpH7uS649zKe+nKg1xDePHctnLXGzbrDYZBRnrqCW2NAY0pATczRkXJ2YJybGvL6LMY6Vay76uY68HI9+nKuvtvxowY8NcvmLDHa2GjmntnFtjDvPFqxXpbgG4nFkzYzFFxMtGmL/E7AKXaYE1pgUAAAAAElFTkSuQmCC", DB.PolarDB: "data:image/svg+xml;base64,PHN2ZyB0PSIxNzczNTU4MDAyMzQ5IiBjbGFzcz0iaWNvbiIgdmlld0JveD0iMCAwIDMwNTEgMTAyNCIgdmVyc2lvbj0iMS4xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHAtaWQ9IjE3MzgiIHdpZHRoPSIyMDAiIGhlaWdodD0iMjAwIj48cGF0aCBkPSJNMCAwaDQ3Ny40NzA3MmMyNjMuMjgwNjQgMCA0NzYuNzIzMiAyMTMuNDMyMzIgNDc2LjcyMzIgNDc2LjcyMzJ2NjguMDg1NzZjMCAyNjMuMjgwNjQtMjEzLjQ0MjU2IDQ3Ni43MjMyLTQ3Ni43MjMyIDQ3Ni43MjMySDBWMHoiIGZpbGw9IiNGRTY5MDIiIHAtaWQ9IjE3MzkiPjwvcGF0aD48cGF0aCBkPSJNMzY0LjkwMjQgMTAyNGMyNC43ODA4LTEzMC4yMzIzMiA2Ni44MzY0OC0yMzIuNzk2MTYgMTI2LjE3NzI4LTMwNy42NzEwNEM1OTkuODQ4OTYgNTc5LjA1MTUyIDcwNi41NiA1MTAuMDQ0MTYgNzE1LjAwOCA0OTkuMTU5MDRjMTguNjQ3MDQtMjQuMDQzNTIgOS4wNDE5Mi01MC4wODM4NCAxNC4yMDI4OC02MC45NDg0OHMzNS4zMjgtMzEuMTkxMDQgNDEuMDAwOTYtNDkuMDQ5NmM1LjY3Mjk2LTE3LjgzODA4IDUuNjcyOTYtOTEuMjA3NjgtMjIuODk2NjQtMTA1LjQwMDMyLTE5LjA0NjQtOS40NzItNjcuNzY4MzItMi4zNTUyLTE0Ni4xODYyNCAyMS4zNTA0bC0xMjkuMDU0NzItMzEuNDA2MDhjLTk1Ljg3NzEyLTEwLjQ4NTc2LTE1Ni44MTUzNi0xMC40ODU3Ni0xODIuNzg0IDAtMjUuOTg5MTIgMTAuNDc1NTItMTIyLjQxOTIgNzMuOTYzNTItMjg5LjI5MDI0IDE5MC40NjRWMTAyNGgzNjQuOTAyNHoiIGZpbGw9IiNGRUZGRkEiIHAtaWQ9IjE3NDAiPjwvcGF0aD48cGF0aCBkPSJNMzM5LjY3MTA0IDM2Mi4xODg4YzM1LjE3NDQtMjkuNzQ3MiA1Ni4wMTI4LTQ0LjYxNTY4IDYyLjUyNTQ0LTQ0LjYxNTY4IDExLjQ3OTA0IDAgMTkuMjIwNDggNS43MjQxNiAyNS40NjY4OCAxMy41ODg0OGE0MjkuMDU2IDQyOS4wNTYgMCAwIDEgMTQuMjMzNiAxOS41ODkxMiA1LjE4MTQ0IDUuMTgxNDQgMCAwIDEtNC4wNDQ4IDguMTUxMDRjLTI2LjM1Nzc2IDAuOTYyNTYtNDUuOTY3MzYgMi4zMjQ0OC01OC44MTg1NiA0LjA3NTUyLTkuMjg3NjggMS4yNjk3Ni0yMC44MDc2OCAzLjk3MzEyLTM0LjU3MDI0IDguMTEwMDh2MC4wMTAyNGE1LjE4MTQ0IDUuMTgxNDQgMCAwIDEtNC44MTI4LTguOTI5Mjh6TTYyMS41MTY4IDMzMS4yMjMwNGMzLjUwMjA4LTE4LjczOTIgOTUuOTQ4OC01NC4zNDM2OCAxMTUuNTg5MTItNDAuNDI3NTIgMTkuNjQwMzIgMTMuOTI2NCAxMC40MTQwOCA0MC40Mjc1Mi0xMi45NjM4NCA2OS40Mzc0NC0zMC41NjY0IDMzLjI4LTEwNi4xMTcxMi0xMC4yODA5Ni0xMDIuNjI1MjgtMjkuMDA5OTJ6IiBmaWxsPSIjRkU2OTAyIiBwLWlkPSIxNzQxIj48L3BhdGg+PHBhdGggZD0iTTExOS42MzM5MiA0NjMuOTg0NjRsLTQ4LjQ4NjQtNDYuNTkyYy0yNi43ODc4NC0yNS43MjI4OC0yNS45MTc0NC02OS44NDcwNCAxLjkzNTM2LTk4LjUzOTUyIDI3Ljg1MjgtMjguNzAyNzIgNzIuMTQwOC0zMS4wOTg4OCA5OC45MTg0LTUuMzc2bDQ4LjUwNjg4IDQ2LjU5MiIgZmlsbD0iI0ZFRkZGQSIgcC1pZD0iMTc0MiI+PC9wYXRoPjxwYXRoIGQ9Ik02OS4zOTY0OCAzMTUuMjM4NGMyOS43ODgxNi0zMC43MDk3NiA3Ny4zMjIyNC0zMy4zMTA3MiAxMDYuMjA5MjgtNS41Mjk2bDQ4LjQ1NTY4IDQ2LjU5Mi03LjE4ODQ4IDcuNDc1Mi00OC40NTU2OC00Ni41OTJjLTI0LjY0NzY4LTIzLjY5NTM2LTY1LjY1ODg4LTIxLjQ1MjgtOTEuNTc2MzIgNS4yNzM2LTI1LjkwNzIgMjYuNzE2MTYtMjYuNzI2NCA2Ny41NjM1Mi0yLjA5OTIgOTEuMjM4NGw0OC40NTU2OCA0Ni41OTItNy4xODg0OCA3LjQ3NTItNDguNDU1NjgtNDYuNTkyYy0yOC44OTcyOC0yNy43ODExMi0yNy45NTUyLTc1LjIxMjggMS44NDMyLTEwNS45MzI4eiIgZmlsbD0iI0ZCNkQwMSIgcC1pZD0iMTc0MyI+PC9wYXRoPjxwYXRoIGQ9Ik00NDQuNzMzNDQgMjk0LjA3MjMyYTYyLjIyODQ4IDU2Ljc1MDA4IDAgMSAwIDEyNC40NTY5NiAwIDYyLjIyODQ4IDU2Ljc1MDA4IDAgMSAwLTEyNC40NTY5NiAwWiIgZmlsbD0iI0ZFRkZGQSIgcC1pZD0iMTc0NCI+PC9wYXRoPjxwYXRoIGQ9Ik0xOTEuMzY1MTIgNjEwLjg1Njk2YzAgOS44OTE4NCA2MC44MjU2IDU1LjI5NiAxMDAuNzIwNjQgNjIuOTU1NTIgMzkuODg0OCA3LjY2OTc2IDg2LjI3Mi0yNi4yNzU4NCA4Ni4yNzItMzIuODI5NDQgMC02LjU1MzYtNDcuODIwOCAxMC4zNDI0LTg2LjI3MiA0LjE0NzItMzguNDUxMi02LjE5NTItMTAwLjcyMDY0LTQ0LjE3NTM2LTEwMC43MjA2NC0zNC4yNzMyOHpNNzI3LjM3NzkyIDQ1NC4yODczNmMtMC4wNjE0NCAxMi42MjU5Mi0wLjE5NDU2IDI5LjE3Mzc2LTEyLjM2OTkyIDQ0Ljg3MTY4LTguNDM3NzYgMTAuODg1MTItMTE1LjE1OTA0IDc5Ljg5MjQ4LTIyMy45MjgzMiAyMTcuMTY5OTItNTkuMzQwOCA3NC44NzQ4OC0xMDEuMzk2NDggMTc3LjQzODcyLTEyNi4xNzcyOCAzMDcuNjcxMDRINzIuMzA0NjRjMTExLjYxNi0xNzQuMjc0NTYgMjIxLjMzNzYtMzA3LjA5NzYgMzI5LjE0NDMyLTM5OC40OTk4NEM1MjUuMjA5NiA1MjAuNjAxNiA2MzUuMjU4ODggNDYzLjM3MDI0IDczMS42Mjc1MiA0NTMuODQ3MDR6IiBmaWxsPSIjRkVEQkJCIiBwLWlkPSIxNzQ1Ij48L3BhdGg+PHBhdGggZD0iTTExNTMuODYzNjggMzMwLjM0MjR2MzYwLjk5MDcyaDU1LjM5ODRWNTUwLjc3ODg4aDk0LjAxMzQ0Yzg2LjkwNjg4IDAgMTMwLjYxMTItMzYuOTA0OTYgMTMwLjYxMTItMTEwLjcyNTEyIDAtNzMuMzE4NC00My4xOTIzMi0xMDkuNzIxNi0xMjkuNTg3Mi0xMDkuNzIxNmgtMTUwLjQyNTZ6IG01NS4zOTg0IDQ3LjAxMTg0aDkwLjQ2MDE2YzI2LjkzMTIgMCA0Ni43NTU4NCA1LjA1ODU2IDU5LjQ2MzY4IDE1LjE3NTY4IDEyLjY5NzYgOS4wOTMxMiAxOS4zMDI0IDI1LjI3MjMyIDE5LjMwMjQgNDcuNTEzNiAwIDIyLjI1MTUyLTYuNjA0OCAzOC40MzA3Mi0xOC44MDA2NCA0OC41Mzc2LTEyLjY5NzYgMTAuMTE3MTItMzIuNTIyMjQgMTUuMTc1NjgtNTkuOTY1NDQgMTUuMTc1NjhoLTkwLjQ2MDE2VjM3Ny4zNTQyNHpNMTYwMS4wODU0NCA0MjIuODYwOGMtMzkuNjI4OCAwLTcxLjY0OTI4IDEzLjE0ODE2LTk1LjUzOTIgMzkuNDM0MjQtMjMuODg5OTIgMjUuNzg0MzItMzUuNTczNzYgNTguNjU0NzItMzUuNTczNzYgOTguNjAwOTYgMCAzOS40MjQgMTEuNjgzODQgNzIuMjk0NCAzNS4wNjE3NiA5Ny41NzY5NiAyNC40MDE5MiAyNi4yOTYzMiA1Ni40MjI0IDM5LjkzNiA5Ni4wNTEyIDM5LjkzNiAzOS42MzkwNCAwIDcxLjY1OTUyLTEzLjYzOTY4IDk2LjA1MTItMzkuOTM2IDIzLjM3NzkyLTI1LjI4MjU2IDM1LjA3Mi01OC4xNDI3MiAzNS4wNzItOTcuNTg3MiAwLTM5LjkzNi0xMi4xOTU4NC03Mi44MDY0LTM1LjU3Mzc2LTk4LjU5MDcyLTIzLjg4OTkyLTI2LjI4NjA4LTU1LjkxMDQtMzkuNDM0MjQtOTUuNTM5Mi0zOS40MzQyNHogbTAgNDMuOTkxMDRjMjQuOTAzNjggMCA0NC4yMTYzMiA5LjYwNTEyIDU4LjQ0OTkyIDI5LjMxNzEyIDEyLjE4NTYgMTYuNjkxMiAxOC4yODg2NCAzOC40MzA3MiAxOC4yODg2NCA2NC43MTY4IDAgMjUuNzk0NTYtNi4wOTI4IDQ3LjAyMjA4LTE4LjI4ODY0IDY0LjIxNTA0LTE0LjIzMzYgMTkuMjIwNDgtMzMuNTQ2MjQgMjkuMzI3MzYtNTguNDQ5OTIgMjkuMzI3MzYtMjQuOTAzNjggMC00NC4yMDYwOC0xMC4xMDY4OC01Ny45Mjc2OC0yOS4zMjczNi0xMi4yMDYwOC0xNi42OTEyLTE3Ljc4Njg4LTM3LjkxODcyLTE3Ljc4Njg4LTY0LjIwNDggMC0yNi4yOTYzMiA1LjU4MDgtNDguMDM1ODQgMTcuNzg2ODgtNjQuNzE2OCAxMy43MjE2LTE5LjcyMjI0IDMzLjAyNC0yOS4zMjczNiA1Ny45Mjc2OC0yOS4zMjczNnpNMTc4OS42MzQ1NiAzMjMuMjU2MzJ2MzY4LjA3NjhoNTMuODYyNFYzMjMuMjU2MzJ6TTIwMjkuMDA0OCA0MjIuODYwOGMtMzIuNTMyNDggMC01OC45NTE2OCA1LjU2MDMyLTc4LjI2NDMyIDE3LjY5NDcyLTIyLjM2NDE2IDEzLjE0ODE2LTM2LjU5Nzc2IDM0LjM4NTkyLTQyLjE4ODggNjIuNjk5NTJsNTMuMzcwODggNC41NDY1NmMzLjA0MTI4LTE0LjY2MzY4IDEwLjY3MDA4LTI1LjI4MjU2IDIyLjg2NTkyLTMyLjM1ODQgMTAuMTY4MzItNi4wNjIwOCAyMy44ODk5Mi05LjEwMzM2IDQwLjY1MjgtOS4xMDMzNiAzOS42MzkwNCAwIDU5LjQ2MzY4IDE4LjIwNjcyIDU5LjQ2MzY4IDU0LjYwOTkydjEwLjYxODg4bC01OC45NTE2OCAxLjUxNTUyYy0zOC42MjUyOCAxLjAxMzc2LTY5LjEyIDguNjAxNi05MC40NjAxNiAyMy43NTY4LTIzLjM3NzkyIDE1LjY3NzQ0LTM1LjA3MiAzOC40MzA3Mi0zNS4wNzIgNjcuNzU4MDggMCAyMS43Mzk1MiA4LjEzMDU2IDM5LjQzNDI0IDI0LjkwMzY4IDUzLjA4NDE2IDE1LjI1NzYgMTMuNjQ5OTIgMzYuNTk3NzYgMjAuNzM2IDY0LjA0MDk2IDIwLjczNiAyMy4zNzc5MiAwIDQzLjcwNDMyLTQuNTU2OCA2MC45NzkyLTEyLjY0NjRhMTA5LjMxMiAxMDkuMzEyIDAgMCAwIDM4LjExMzI4LTMxLjM0NDY0djM2LjkwNDk2aDQ5LjgwNzM2VjUyNC40OTI4YzAtMzEuODU2NjQtOC4xMzA1Ni01Ni4xMjU0NC0yMy44Nzk2OC03Mi44MDY0LTE4LjI5ODg4LTE5LjIyMDQ4LTQ2Ljc1NTg0LTI4LjgyNTYtODUuMzgxMTItMjguODI1NnogbTU1LjkwMDE2IDE0Ny42NDAzMnYxNS4xNjU0NGMwIDIwLjIyNC04LjY0MjU2IDM3LjQxNjk2LTI0LjkwMzY4IDUxLjA2Njg4LTE2LjI2MTEyIDEzLjY0OTkyLTM1LjU3Mzc2IDIwLjcyNTc2LTU4LjQzOTY4IDIwLjcyNTc2LTEzLjcyMTYgMC0yNC45MDM2OC0zLjUzMjgtMzMuMDM0MjQtMTAuMTA2ODgtOC42NDI1Ni02LjU3NDA4LTEyLjcwNzg0LTE0LjY2MzY4LTEyLjcwNzg0LTI0Ljc4MDggMC0zMi4zNTg0IDI0LjM5MTY4LTQ5LjU0MTEyIDczLjY4NzA0LTUwLjU1NDg4bDU1LjM5ODQtMS41MTU1MnpNMjMyMS43MzU2OCA0MjIuODYwOGMtMTYuMjcxMzYgMC0zMC40OTQ3MiA0LjU0NjU2LTQyLjcwMDggMTQuNjYzNjgtMTAuMTU4MDggNy4wNzU4NC0xOC44MDA2NCAxNy42OTQ3Mi0yNS4zOTUyIDMxLjg0NjR2LTM5LjQyNGgtNTMuODgyODh2MjYxLjM4NjI0aDUzLjg3MjY0VjU1Mi44MDY0YzAtMjIuNzUzMjggNi42MDQ4LTQxLjQ3MiAyMC4zMjY0LTU1LjYyMzY4IDEyLjcwNzg0LTEzLjE0ODE2IDI3LjQ0MzItMTkuNzEyIDQzLjcwNDMyLTE5LjcxMiAxMi4yMDYwOCAwIDI0LjkwMzY4IDEuNTE1NTIgMzguMTIzNTIgNS41NjAzMnYtNTMuNTk2MTZjLTkuMTU0NTYtNC41NDY1Ni0yMC44Mzg0LTYuNTc0MDgtMzQuMDQ4LTYuNTc0MDh6TTIzOTMuODk2OTYgMzMwLjM0MjR2MzYwLjk5MDcyaDEzMS4xMTI5NmM1OC40NDk5MiAwIDEwMi42NjYyNC0xNi4xNzkyIDEzMy4xNTA3Mi00OC41Mzc2IDI4Ljk3OTItMzEuMzM0NCA0My43MTQ1Ni03NS4zMzU2OCA0My43MTQ1Ni0xMzEuOTYyODggMC01Ny4xMjg5Ni0xNC4yMzM2LTEwMS4xMi00Mi43MDA4LTEzMS40NTA4OC0zMC40ODQ0OC0zMi44NzA0LTc0LjcwMDgtNDkuMDQ5Ni0xMzMuMTQwNDgtNDkuMDQ5NmgtMTMyLjEzNjk2eiBtNTUuMzk4NCA0Ny4wMTE4NGg2Ni41NzAyNGM0NS43NDIwOCAwIDc5LjI3ODA4IDEwLjYxODg4IDEwMC42Mjg0OCAzMi4zNTg0IDIwLjMyNjQgMjEuMjM3NzYgMzAuOTk2NDggNTQuNjA5OTIgMzAuOTk2NDggMTAxLjEyIDAgNDUuNTA2NTYtMTAuNjcwMDggNzguODc4NzItMzEuNTA4NDggMTAwLjYxODI0LTIxLjM0MDE2IDIxLjczOTUyLTU1LjM5ODQgMzIuODcwNC0xMDEuMTMwMjQgMzIuODcwNGgtNjUuNTU2NDh2LTI2Ni45NTY4ek0yNzU4LjI4NzM2IDMzMC4zNDI0djM2MC45OTA3MmgxNjUuNjcyOTZjMzguNjI1MjggMCA2OC42MDgtNy4wNjU2IDg5Ljk0ODE2LTIxLjIyNzUyIDI0LjkwMzY4LTE3LjIwMzIgMzcuNjExNTItNDMuNDg5MjggMzcuNjExNTItNzkuODkyNDggMC0yNC4yNjg4LTYuMTAzMDQtNDMuOTgwOC0xOC4yOTg4OC01OC42NDQ0OC0xMi4xODU2LTE0LjY2MzY4LTMwLjQ4NDQ4LTI0LjI2ODgtNTQuMzc0NC0yOC44MjU2IDE4LjI5ODg4LTYuNTc0MDggMzIuMDIwNDgtMTYuNjgwOTYgNDIuMTg4OC0yOS44MjkxMiAxMC4xNTgwOC0xNC4xNTE2OCAxNS4yMzcxMi0zMS4zNDQ2NCAxNS4yMzcxMi01MS41Njg2NCAwLTI3LjgxMTg0LTkuNjU2MzItNDkuNTUxMzYtMjguOTY4OTYtNjUuNzMwNTYtMjAuMzI2NC0xNy4xOTI5Ni00Ny43Njk2LTI1LjI4MjU2LTgzLjM1MzYtMjUuMjgyNTZoLTE2NS42NjI3MnogbTU1LjM5ODQgNDUuNDk2MzJoOTYuNTUyOTZjMjQuMzkxNjggMCA0Mi42OTA1NiA0LjA0NDggNTMuODYyNCAxMi42NDY0IDExLjE5MjMyIDguMDg5NiAxNi43NzMxMiAyMS4yMjc1MiAxNi43NzMxMiAzOS40MjQgMCAxOS4yMjA0OC01LjU4MDggMzMuMzgyNC0xNi43NjI4OCA0Mi40NzU1Mi0xMS4xODIwOCA4LjYwMTYtMjkuNDgwOTYgMTMuMTQ4MTYtNTQuODg2NCAxMy4xNDgxNmgtOTUuNTM5MlYzNzUuODM4NzJ6IG0wIDE1Mi42OTg4OGgxMDQuMTcxNTJjMjYuNDI5NDQgMCA0Ni4yNTQwOCA0LjU0NjU2IDU4Ljk1MTY4IDE0LjE1MTY4IDEyLjcwNzg0IDkuNjA1MTIgMTkuMzEyNjQgMjUuMjgyNTYgMTkuMzEyNjQgNDYuNTIwMzIgMCAyMC43MjU3Ni04LjYzMjMyIDM1Ljg5MTItMjQuOTAzNjggNDUuNDk2MzItMTMuMjA5NiA3LjA4NjA4LTMxLjUwODQ4IDExLjEzMDg4LTU0Ljg4NjQgMTEuMTMwODhoLTEwMi42NTZWNTI4LjUzNzZ6IiBmaWxsPSIjMTExMTExIiBwLWlkPSIxNzQ2Ij48L3BhdGg+PC9zdmc+", + DB.Adbpg: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAycAAAMkCAYAAACiPV8HAAAACXBIWXMAACxKAAAsSgF3enRNAAAgAElEQVR4nO3d+1XcWLrw4a1Z/b89EUBHYCYC6AjsCWBkOgK7I2g6gqEjaFwJHBzBQAQDEQxE8JkI9C2VVKbMtS6Sal+eZy0f97mMjaXThh97v3tXzb/CfgjhOPCSI0+HxL0NIbzzEhnRdQjh2zO//NUT/7uH/7ObMGtuvCCAsrVx0n7h/Z/SHwQAUVmOnZv+x8N/vgqz5rkgAiBBP3lpAERoeaXv8NkPr64W/3TZ/7yIl2/96kwIs+bCCwZIgzgBIAeLgHkcMvcBc70ULT/+bAUGIAriBIBSLFZjnguYy6VoWazACBeACZk5AYDXXf4QLN0A/5XnBjAsKycA8LrDRysu3WrL9VKwXPUrLU4dA9iQlRMAGN7l0vawK0P5AKuxcgIAw/txpeV+lWV5hUWwADxg5QQAdmc5WC7MsQClEycAEJfL77FihgUojDgBgLjdfg8VqytA5sQJAKTlro+VC7MrQG7ECQCkr90Kdm5lBUid07oAIH33p4PV1fLKilgBkmLlBADydrsUK+dh1nzzvoFYiRMAKMv1UqiYVwGiIk4AoFx330PFqgoQAXECACy0qypnZlWAXREnAMBTbpdWVGz/AiYhTgCA19wthcq5pwWMRZwAAOswpwKMRpwAANv4KlSAofzNkwQAtvA+hPBXCOH/hbo6C3X1wcMENmXlBAAYmhkVYCPiBAAY011/PPGZ44mB14gTAGAq7fHEp/2Kyo2nDjwkTgCAXbjsV1QM0gPfGYgHAHbhsB+kv+kH6Q+8BcDKCQAQi8W2rzOrKVAmKycAQCz2Qgj/XjqW+MibgbJYOQEAYmY1BQpi5QQAiJnVFCiIlRMAIDXXS0cSW02BjFg5AQBS827ppK/TUFf73iDkQZwAAKl6E0L4FEL4X6irc1u+IH3iBADIwfv5NvW6ugp1deyNQprMnAAAObrtb6A/NZcC6bByAgDkqD3l6/elG+jNpUACxAkAkLN2LuVjP5fSRsqBtw3xEicAQCnaSPlvqKsLw/MQJ3ECAJTmsB+ebyPlg7cP8RAnAECp2kj5v1BXN074gjiIEwCgdHvzSx1FCuycOAEA6IgU2DFxAgDwI5ECOyJOAACeJlJgYuIEAOBlIgUmIk4AAFaziJQr96TAOMQJAMB63i3dkyJSYEDiBABgM8uXOe57hrA9cQIAsJ02Uv4X6upMpMB2xAkAwDA+hhDaeZSTUFdvPVNYnzgBABjOmxDC7yEEJ3vBBsQJAMDw3iwdP2xoHlYkTgAAxrNnaB5WJ04AAMa3GJo/NY8CzxMnAADT+dTPo3z2zOExcQIAMK12HuXfbpqHx8QJAMBuLG6aP7PVCzriBABgtz7a6gUdcQIAsHu2elG8IE4AAKJiqxdFEycAAPH56JZ5SiROAADitLhl3gWOFEOcAADEbXGB44n3RO7ECQBAGn43ME/uxAkAQDoWA/OnBubJkTgBAEjPpxCCVRSyI04AANK0ZxWF3IgTAIC0WUUhG+IEACB9VlHIgjgBAMiHVRSSJk4AAPJiFYVkiRMAgDy1qyjt7fIH3i+pECcAAPlq70X5r9vlSYU4AQDIX3u7fLuKsu9dEzNxAgBQhsN+WP7Y+yZW4gQAoBxvQgh/hbo6NyxPjMQJAEB53verKIbliYo4AQAo055heWIjTgAAyrYYlrfNi50TJwAAtMPyN26WZ9fECQAAoR+W/49tXuySOAEAYJltXuyMOAEA4CHbvNgJcQIAwFMW27w+ezpMRZwAAPCSf7u0kamIEwAAXtNe2njh0kbGJk4AAFjFuz5Qjj0txiJOAABYVTuH8leoq1NPjDGIEwAA1vXJccOMQZwAALCJ9rjhK3MoDEmcAACwqb1+DuWDJ8gQxAkAANto51D+L9TViafItsQJAABD+D3U1Zk5FLYhTgAAGMrHfpuXQGEj4gQAgCG196HcGJRnE+IEAIChvXFhI5sQJwAAjGFxYeNnT5dViRMAAMb07/mgPKxAnAAAMLaPoa7ODcrzGnECAMAU3jvJi9eIEwAAptKe5HXlJC+eI04AAJjSXr+CIlB4RJwAADC19iSv/zpqmIfECQAAu/KXQGGZOAEAYJfaQDnxBgjiBACACPzuLhSCOAEAIBIfBQriBACAWLSB4i6UgokTAABicuiyxnKJEwAAYvNOoJRJnAAAECOBUiBxAgBArNpAuXKbfDnECQAAMdvrV1AESgHECQAAsXsjUMogTgAASIFAKYA4AQAgFQIlc+IEAICUCJSMiRMAAFIjUDIlTgAASJFAyZA4AQAgVQIlM+IEAICUCZSMiBMAAFInUDIhTgAAyIFAyYA4AQAgF4tA2fdG0yROAADISRso56Gu3nqr6REnAADk5l2/giJQEiNOAADIkUBJkDgBACBXAiUx4gQAgJy1gXLqDadBnAAAkLuPoa7OvOX4iRMAAEogUBIgTgAAKEUbKJ+97XiJEwAASvLvUFfH3nicxAkAAKX5K9TVB289PuIEAIASnYW6OvDm4yJOAAAo0Zv+DhSBEhFxAgBAqd70KyguaYyEOAEAoGRukY+IOAEAoHRukY+EOAEAgO4OFIGyY+IEAAA6n9yBslviBAAA7rV3oBx5HrshTgAA4EfnjhjeDXECAAA/csTwjogTAAB47N18BYVJiRMAAHjaYairM89mOuIEAACe99EJXtMRJwAA8DIneE1EnAAAwOvaE7z2PadxiRMAAHjdmz5QnOA1InECAACraU/wOvWsxiNOAADidO29RKkdkP9c+kMYizgBAIjDXQjhSwjhnyGEv/sOfdT+bUB+HD/l+IcCAEjEbX/R31mYNVc/fMh19cFLjFo3ID9rvpX+IIYkTgAApnXdB8n5oyD50XvvJWrtgPxFCOGg9AcxJHECADC+6/nqSBckN6/+blZNUvEu1NVpmDVmUAYiTgAAxrFekPxInKTjU6irqzBrzkp/EEMQJwAAw7ntg+RsgyBZJk7SctoHykvb9FiBOAEA2M7zQ+2bqKuDfp6BdLyZv//2BC8D8lsRJwAA67tbGmo/H/j5HXsfSVpc0Oj9bUGcAACs7nJpjmSs75Db0pWu9oLGC/MnmxMnAAAvG2qO5HXdlq497yNp5k+2IE4AAJ72pQ+Siwmfjy1B6TN/sgVxAgBw77afGzjb0ReWtnTloZ0/OQkhuP9kTeIEAGA3qyQ/sqUrN5/6+ZOhD0zImjgBAEq161WSh2zpys/ZPDrHnlXKiDgBAErzdR4lu1wleZotXfl50x+mcFT6g1jV39L4MAEAttLeS/JnCOHnMGs+RBcmtnTl7DDU1UnpD2FV4gQAyFm7devXEMJ+mDWfI95eY0tX3n7vA5RX2NYFAOTocn5aUnxbt55jS1f+zvv5E8cLv8DKCQCQky/91q2jZMLElq5S7PXHC/MCcQIApK6dJ/kjhPD3MGuOEzwZyZaucrTHC1sle4FtXQBAqmI7CnhT4qQs7fHC+7Z3PU2cAACpue3nSc6Sf3Pdd9HfRPCRMJ3F8cJWUJ5gWxcAkIrL+clbs2Y/izDp+AK1TO9DXVkxe4I4AQBi10bJL/2Qey5R0q6avBUnRTudb+/iB7Z1AQCxSu044HXZ0lU2t8c/wcoJABCb5ZWSXMMkGISnvz3+swdxT5wAALEoJUpCv53nMIKPhN07sb3rnm1dAMCu5b596ylWTViwvWuJlRMAYFdui1kpeUycsMz2rp44AQCmdrt0JHBpUdJu6Wq/Q74XwUdCXNrtXW9Lfye2dQEAU7kLIXzO6jjgzVg14SnFX84YrJwAABNoo+SPEEJOlydupvvO+McUP3Qm0V7OKE4AAEbyZx8l7cD7Nw/ZpYu86qzk7V3iBAAYw9cQws9h1nwWJT8w9Mxr3sznTwolTgCAIS3uKvkQZs2NJ7ukrg5CCO+i+XiI2af+4ITiGIgHAIZw299VUvqw+0usmrCOs/mWyMJYOQEAtrEYdj8QJi/oZgjMm7COvVBXxW3vEicAwKa+9lFi2P11H/pZAljH76Guilo9sa0LAFjXdX9fSXkXKG6u2AFnttauSBYzf2LlBABYVbuF67cwaw6EyRrcCM92Dku6+0ScAACr+NLfV3Lqaa3NjfBs67SUu0/ECQDwkuv+aOBjcyUb6OYF3AjPtvZKOe1NnAAAT+lO4bKFa1tWTRhKEcPx4gQAeOj+FC62JU4YUvbHdYsTAGChvUjxn253H0hdHRuEZ2DZD8eLEwCg9We/WnLuaQzGqgljyHo4XpwAQNkWA++fDbwPqDs++DCbPw8xyXo4XpwAQLkMvI/Hqglj+pzrcLw4AYDytKsl/zDwPhLHBzO+NyGELP/9FScAUJbFasmV9z6aIu6jYOc+9tsHsyJOAKAMVkum0A0q29LFVLL791mcAED+rJZM57jfcgNTOOyPrM6GOAGAfFktmZ4tXUwtq3+/xQkA5MlqydRcushu7IW6yiZQxAkA5OXWasnOeObsyudcLmYUJwCQj8Ut71ZLptadmmTVhF3J5mhhcQIA6bsLIfzTLe87ZdWEXfuUw8WM4gQA0nYZQtgPs+bce9yRbtXksMg/O7FJPpLFCQCk67cwa46sluycVRNikfzFjOIEANKzOCL41LvbsW4bjVUTYpJ0LIsTAEjLlxDCkaH3aFg1ITaHKa+e/BTBxwAAvO5uflzorDnzrCLRrZp8LP0xEKWT+TcxEmTlBADid92vlgiTuFg1IVbJrp5YOQGAuH0Js+bYO4qMVRPidzY/yS8xVk4AIE7tNq5fhUm0rJoQu71QV8n9/SFOACA+tnHFzKoJ6UguosUJAMTFaVzxs2pCKpJbPREnABCP3+bbuFyqGC+rJqQnqZgWJwCwe3cuVUyGVRNSk9TqiTgBgN26np+oYxtX/KyakK5kolqcAMDutMcEH9jGlQwrW6QqmdUTcQIAu+GY4JR0F9q9L/0xkLQkVk/ECQBMazFf4pjgtJg1IXVJrJ6IEwCYjvmSFHWrJoelPwayEH1kixMAmIb5knRZNSEXe31sR0ucAMD4fjNfkqi6+mDVhMxEHdviBADG086X/NP9JUnz7sjNYcyrJ+IEAMZxG0I4CrPm3PNNVDc8vFf6YyBL0a6eiBMAGF47+H5g8D1hdfXWqgkZa1dPDmL844kTABjWl37FxOB72j6HEN6U/hDI2ucY/3DiBACG8+d88F2YpK2u9mP9wg0G9LH///WoiBMAGEZ747svaPNwYtWEQkT3d5Y4AYDtLE7kcuN7Drp9+B9LfwwU47ifr4qGOAGAzd05kSs7huApyZvYVk/ECQBs5roPEydy5cKFi5QpqgtixQkArE+Y5MmqCSXa6+/0iYI4AYD1XDoqOEN1deLCRQoWzdYucQIAq/sSZo0wyU03EOykNUr2LtTVUQx/fnECAKv5Mr/DhBydOjoY4pg9EScA8LrfhEmmuu8WOzoYIrmUUZwAwMvayxUNSufLu4V7O/8mjDgBgOf96nLFjNXV5/lee2Bh57NX4gQAHrsTJpnrhuBPSn8M8MCbXR8rLE4A4EeLW9+FSd4MwcPTxAkAROLO5YoFMAQPLzkMdXWwqyckTgCgI0zKYQgeXraz2RNxAgDCpBzdTfCG4OFlH/q5rMmJEwBKJ0xK0d3h4CZ4eN2beaDsgDgBoGTCpCyG4GF1Owl5cQJAqYRJSeqq/S7w+9IfA6zh3S4G48UJACUSJiXp9s47GhrWN/nqiTgBoDTCpDwntnPBRiYfjBcnAJREmJSmu9PkU+mPATY0+WC8OAGgFMKkNLZzwRAmvTFenABQAmFSpna//F7pDwG2dNgfwz0JcQJA7oRJibpThn4v/THAQCYbjBcnAORMmJTLdi4YzmRzJ+IEgFwJk1LV1cn8jgZgKHv9XUGjEycA5OqDMCmQ7VwwFnECABv6NcyaCw+vSLZzwTgmufNEnACQmzZMfIFaItu5YEyT3HkiTgDIiTApVXfZou1cMC5xAgAr+lOYFMplizCV92PfeSJOAMjBlzBrJjuHn+icuGwRJjPq6ok4ASB1bZgce4uF6rZzfSr9McCERv37VpwAkLLrKW8uJjLddq5zrwUm9W7MrV3iBIBUXfeXLH7zBot11p8gBExrtK1dP3mRAKyhvXV9lYsND0b+ovFOmBSurj7Ph3OBXWi3dp2O8fuKEwCesoiQi/7nm41uW+9u697vY+VooGgRJqXrtpSclP4YYIfezf9+3+TzwivECQAL1/3+/fPBPuF0v87VD3MBXbB86H9scmHehzE+IZKUc9u5YOeOx5j5q5p/zb+T9R/vF6BIt/3SfBskN5M/gO474B/6T3CrHAXrksXS1dWp07kgCrdh1gw+GC9OAMr0dR4ls+Yimj99dyRs+524j8/8X/wRZo2tPCXr/n/E1ywQj38MvZLttC6AsnwJIfwcZs2HqMIkzLeAXfT3lfzcf5zLvgiTwjk2GGI0+Kld4gSgDJd9lBzvZPvWOtqP7z5Svs5nYVyyiDkTiNHgcWJbF0DebufzHLPGd5xJV121q2a/e4MQpZ+H/KaXlROAfP05P7pXmJCybs5EmEC8Bl09cZQwQH7u+uN245opgXWZM4EUDHoho5UTgLxczy89FCbkwZwJxO9dfyz8IMQJQD7aE60O3JxOFro5k0MvE5JwNNQHKU4A8vCbE63IhjkTSM1gcydmTgDS59Z08tFtDzFnAml5P9RHa+UEIG3ChNyYM4EU1dUgqyfiBCBdwoS81NXZfLgWSJE4ASiYMCEvddXOTH30ViFZgwzFixOA9AgT8lJXB0PekwDsxF7/7/JWxAlAWoQJebm/aNGcCaRv661d4gQgHcKEHJ3Pv+MK5ECcABRCmJCfujp10SJk5V2/GroxcQIQP2FCfroB+E/eLGRnq9UTcQIQN2FCfrqh2b+8WcjSVqd2iROAeAkT8tNt+bjwZiFb4gQgQ8KE/NyHiZO5IF9bHSksTgDiI0zI1akb4KEIG6+eiBOAuAgT8tSdzOUGeCiDOAHIgDAhT07mgtKIE4DECRPy5GQuKNGbUFcbBYo4Adg9YUKeujBxMheUSZwAJEiYkKfuZK4zJ3NBscQJQGKECXm6PzLYyVxQrsNN/uTiBGA3hAk5c2QwEDaZOxEnANMTJuSrrs4cGQz0xAlA5IQJ+eqODBYmwII4AYiYMCFfXZg4MhhYtvbciTgBmIYwIV/uMgGes+bciTgBGJ8wIV/uMgFedrDO8xEnAOMSJuTrPkzcZQI8x8oJQCSECflyySKwGnECEAFhQr5csgis7k2/yroScQIwPGFCvoQJsD5xArADd8KEApwJE2BNK2/t+smTBRjE3fwv31lztdYvVlf7IYT2x9ul7yy9Xfd0kxBC+/t+e/DPN2HW3Hi9DKa7/f29BwqsaeXPaeIEYHuvh0m33/agD5Gj/ue9AZ/90xdd1VX7X6/7WLmYB0sbL+tGFHRh4vZ3YBMrr7ZWzb/mnyT/4zEDbORxmHR78o/6Hweb3JA7kcs+WK7mP8+ab1F8VMRHmADb+yXMmlfvRLJyArC5+zCpqw9LQZLKfvzDH8Kprq77WDlf5RMIhRAmwDCOVrmw1coJwOb+XNqmldtdD3fzSLmPFasqJaqrkxDC76U/BmAQX8Os+fDaLyROAFjFP8ypFKaujkMIf5X+GIDB3IZZs//aL+YoYQBecylMCiNMgOHt9TOZLxInALyk3d517AkVRJgA43n1SGFxAsBLTtyVUhBhAozr1csYxQkAz7kOs+bU0ymEMAHGZ+UEgI3ZzlUKYQJMQ5wAsJE/DMEXQpgA03l1KF6cAPDQbQjBdq4SCBNgei+unogTAB46duliAboLFoUJMLUXh+J/8joAWNLe4HvhgWSurs5CCB9LfwzATrx4EaOVEwAW2jtNPnsamRMmwG7Z1gXASk7daZI5YQLs3ruXPgJxAkDo7zQ58SQyJkyAWNTVs6snZk4ACLZzZaw7tvM8hHBY+qMAotHOnTx5XL2VEwC+GILPVBcmF8IEiMyzKyfiBKBshuBzdR8mL+7vBtgBcQLAk07daZKhbj+3MAFi9exxwmZOAMp1awg+Q/dh8qb0RwFE69lvnFg5ASjXsXefmbo6EiZAEp45sUucAJTp0hB8Zuqqjc3/CBMgEU9u7RInAGWyapKTumoPNfir9McAJOXJlRMzJwDl+dNN8BlxuSKQpidXTsQJQFnao4MNweegOyr4VJgAiRInAIQTRwdnwB0mQPoMxAMUrj06+LT0h5C87oSbK2ECJO5N/42WH4gTgHK4CT5190cF75X+KIAsPFo9EScAZWiPDj73rpN34qhgICNWTgAKZQg+D94jkBMrJwAF+uLCxUx07/Gy9McAZOPRiV3iBCB/vtueF+8TyIU4ASjMFxcuZsbqCZAPcQJQEBcu5suR0EAOHp08KE4A8vboJBQy0J28dutVAsmrqx9WT8QJQL7ezO/E6C7tIz+OhgZyIE4ACiJQ8mVrF5ADcQJQGIGSo+6gg+vSHwOQPHECUCCBkif31wCp+2E2UpwAlEOg5MfcCZC6Hz4niROAsgiUnLj5H8iMOAEoj0DJiwsZgZQdLn/s4gSgTAIlH1ZPgGyIE4ByCZQ8XJX+AIDELX0eEicAZRMo6bsp/QEAyft+Ypc4AUCgpGzWWDkBsiFOAAgCJXl3pT8AIGlHiw9enABM52sI4Z8R3+otUNJl9QTIgjgBGFf7He0/Qgg/h1nzIcya8/47RAIFADpmTgBG1t498WuYNW/DrDkJs+Z+aHnWfBMoAPCd07oARnC7tEpyFGbN2bO/hUBhWN88TyAH4gRge90syazZf7RK8hKBwnDMnAAps60LYEvtKslvIYS/L82SrE+gMIy3niOQsHeLD12cAKyuHW7/EkL4R79KctrHxXYECtvzboAsiBOA131dGm4/HuXSO4ECAOIE4BnX/batxRHAzw+3D0WgAFCquppfxChOAO61cyR/9tu2DvptW6sNtw9FoLCZfc8NyIE4AUq3mCP5pZ8j+TzKtq11CBTWt+eZATkQJ0CJFkHyz6U5kouonoNAYVV1ZdUEyMH81EFxApSkG2xvt8B0QbLZ8b9TESjTqKsPoa6m3b43LHEC5GD+uUScALlbBMnfvw+2D3H871QEynjaFYe6alfM/m++LaqNlDQdJfpxAzwiToAcpR0kDwmU4dXV5/5W9cOlXzvVL/JtrQOy8ZNXCWTgbv7FcQjn8x8ph8hz2j9Td8zixfJNuhFZBMrRzg8UeEkXUGfPPMN25eRzPB/sysQJkI2q+df8O0X/8UqBxNwtxUjcsyNDqqu3EQdK6N9LfIHSPbeTEMKnV/4v/xF1XD3UDcP/L64PCmAjl2HWHNnWBaRkcQ/JL0unbJUTJsEWr410syRXK4RJSHBrV6pzMgBPsq0LiN11vw3nIqnvaI/JFq/VdKsKZw/mSl7TPtfTnX3M6zMMD2RFnACxyX9+ZAgC5WV1ddLPj7xZ8z/5fuoPdWPdVrV0Pl6AFYgTIAa3fYxcFLdNaxsC5bHueZxtdWN69/HGdSnn02zpArIjToBd+fp9hWTWpHwB3m4JlM5mW7iec9A/z9gdJ/AxAqxqfqGsOAGmcr0UIyl84ZeOkgOl29rUbt/6fcBfNf65ky7GhggxgFjMV7zFCTCW2/6L5QuzIxMoMVDq6riPiHXnSl6Twr0hJxF8DACDc88JMJS7BzFiq9YulHAPShdhJyOvHPw92qDu3vHNCFEGsFuzpmpXTr712y1i/UQGxGk5RhzzG4ucV1C6rUynE51QFfPcySankAEkoWqapvs4u+/EHC39ECvAMjGSkpxWUMaZK3nNb2HWxDd3YtUEyNmsqe7j5CGxAqW77W/VFiOpyiFQNr+vZFt/hlnzeeLf83V11Z5I9jG6jwtgCC/GyUP3sXLQ/+yUEMjL4jStqz5GzIzkINVA6YbdT7a6r2Q7l2HWxHX7el21n3//G8FHAjCOteLkKd2+5oOl1RXLzJCGux9WRdp/dppWvlIKlO7zymkEH+ttmDX7O/4YflRXF74xCGRt6zh5qBtWXATLgb9EIRqXfYxYFSlVGoFyFdXnjVlTRfBRdOqq3WL27xg+FIAR/TxsnDzlfnVl8cPsCozreilErlx4yHfxB0psfo4i5LvtXBd2JwAF+GX8OHmKYIGhXPYn9wgRViNQ1vFLFP9O1dWV9wUUYkdx8pT7YNm3JQweWZ4RuelDxOlZbEagrGr3ceJ0LqAsEcXJU7oZluUVln2fTCnAYjXkxrA6oxEoq/gjzJqTnf3u3Yllf+3s9weY3i8/Rf3Qu72+7Y/zH/7n3SrLfv+j/ee3PsGSoIcRcmNQncnEf5N82erqgzABShR3nDznuWX2bmhwf2mVZd/2MHasHU7/1n8B+G1pNsRKCLsnUOLUfS47K/0xAGWKe1vXULrtC8vBctCvthw4/YQBPBUgVkFIhy1ez5n+lngncwFli3zmZCrddw5Dv0Vs+WerLoSl+Lh69LMVEHIhUJ4y7S3xwgRAnKzkPl4WKy6LFZggYJK3CI/F7Efovzh4fvsg5EqgPDRdnAgTgCBOhtSdLLYIlsUns8XWsWBof1KL4AhLqx3he3S0/71jeOFpAmXZNHEiTAAWxMlO3M/ALCxWZBYefjIscXVmca/HsuWVjG8P/ve2WMFQBMrC+HEiTACWiZNk3W81e+i1T6QPw2hIy6sUz3l6q5QtVBAXgRJGjxNhAvCQOAHgGQJlvDgRJgBP+cffPBYAntRtlTzq57gYijABeNqsuRInADxPoAxLmAC8SJwA8LJyA2XYU/2ECcCrxAkAryszUIY7AVCYAKxEnACwGlu8NiNMAFYmTgBYnUBZjzABWIs4AWA9XaB86C9Lzdl29y8JE4C1iRMA1jdrbvoVlNwDZTPCBGAj4gSAzcyaq8y3eN1s9J8SJgCbmH+zS5wAsLmcA6VbHVqPMAHY1Pz4dnECwHbyHJK/Xfs/IUwAtiZOANhefhdVUFsAABTeSURBVIGy3qqJMAEYhDgBYBh5Bcrqt8MLE4DBiBMAhtMFyupf2MdrtZUTYQIwKHECwHDq6m1/B0rqXg8sYQIwJAPxAAzuQxZfrM+aly9gFCYAQ2tX3sUJAIP6nMHjfHlmRpgAjEacADCMumqH4d9l8DSf39IlTABGJU4AGMpxJk/y6TgRJgBjmh9EIk4A2F43CP8xkyf5eN5EmACMTZwAMJhcVk3uwqz5ceVEmABMRpwAMIRc4uTHVRNhAjApcQLAdrov4HMYhG+df/8nYQIwJfecADCIHI4PXuhWToQJwLRmjXtOANhSPjfCh/n9JrPmRpgA7I44AWAbedwI37no72oRJgDTul38bj958ABsIZdB+IX/xPFhABTlZvGHtXICwGbqaj+EcJjR0/sUwccAUDRxAsCmchqEB2B3vt8vJU4A2FQug/AA7Na3xe8uTgBYX121YbLnyQEwJHECwCasmgAwlIvFryNOAFhPd7fJR08NgKGJEwDWZdUEgCE5ShiAjeV2twkAuzRrxAkAG8jvbhMAIiJOAFiHLV0ADOly+dcSJwCsw8WLAIxGnACwmro6cLcJAAO7Wv7lxAkAqzIID8DQvi3/euIEgFWZNwFgaDfLv544AeB1dfXBli4ARiBOAFibVRMAxiBOAFibOAFgeEsXMAZxAsCrui1dbzwoAAZ2+/CXEyfEqbuFGoiDVRMAxnDz8NcUJ8TqONTVkbcDURAnAIzh6uGv+ZPHTJRmzYkXAxGwpQuA8Xx7+CtbOQHgJVZNABjLo5UTcQLA0+rqbQjho6cDwEisnACwMnNfAIxn1lw8/LXFCQDPsaULgLE8OkY4iBMAXiBOABjLo2OEgzgB4ElO6QJgXI+G4YM4AeAZVk0AGNOjYfggTgB4hjgBYEyPhuGDOAHgEVu6ABifmRMAVmLVBIBxzRpxAsBKxAkAY7p87tcWJwDcq6sDW7oAGNmTqyZBnADwwLEHAsDIxAkAK7GlC4CxPXlSVxAnAHzXbena80AAGNmTFzAGcQLAkiMPA4CR3YVZ8+QFjEGcALDEvAkAY3t21SSIEwDm6uptCOGdhwHAyMQJAK8yCA/AFMQJAK8SJwBMQZwA8CrD8ACMb9aIEwBeUFdHboUHYAKXr/0W4gQAW7oAmMKLqyZBnAAgTgCYiDgB4AV1te9WeAAmIk4AeJFVEwCm8cowfBAnAMVzShcAU3h1GD6IE4DivS/9AQAwiVdXTYI4AShYd4QwAExBnADwIvMmAEzlYpXfR5wAlMvKCQBTuAuz5maV30ecAJSort6GEN559wBMYKVVkyBOAIplSxcAU1lp3iSIE4Bi2dIFwFSsnADwInECwDRmjTgB4Bl1dRBC2PN4AJjA9Tq/hTgBKI9VEwCmsvKqSRAnAEUSJwBMRZwA8CJxAsBUxAkAz+jmTd54PABM4DrMmm/r/DbiBKAsVk0AmMpaqyZBnAAUR5wAMBVxAsCL3ns8AExEnADwjLqyagLAVNaeNwniBKAoB143ABNZe9UkiBOAolg5AWAq4gSAF4kTAKYiTgB4hvtNAJjORvMmQZwAFMOqCQBTOd/09xEnAGUQJwBMZaMtXUGcABRDnAAwhbswa8QJAM+oq33zJgBMZOMwCeIEoAhWTQCYysbzJkGcABRBnAAwFSsnALzIzfAATKE9Qvhmm99HnADkrK7ehhDeeccATGCrVZMgTgCyZ0sXAFPZat4kiBOA7NnSBcAUtjpCeEGcAOTNygkAU9h61SSIE4DsHXrFAExg61WTIE4AMlZXtnQBMBUrJwC8SJwAMIXLMGu+DfH7iBOAfJk3AWAKg6yaBHECkDUrJwBMQZwA8CqXLwIwtq1vhV8mTgByVFe2dAEwhbMhfw9xApAnW7oAmMIgRwgviBOAPIkTAMZ2G2bN1ZC/hzgByJM4AWBsgw3CL4gTgDwZhgdgbIPOmwRxApAhw/AAjG/wLV1BnABkyZYuAMY2+JauIE4AsiROABjb4Fu6gjgByJI4AWBMo2zpCuIEIEuG4QEY0+lYv7Y4AciJYXgAxjfKvEkQJwDZsaULgDFdh1lzM9avL04A8rLvfQIwolEG4RfECUBerJwAMKbRtnQFcQKQnUOvFICRfB1zS1cQJwAZqStbugAY06irJkGcAGTFli4AxnInTgBYhzgBYCznYdZ8G/vpihOAfIgTAMYy6ildC+IEIB9mTgAYw22YNRdTPFlxApCPd94lACOYZNUkiBOATNSVLV0AjEWcALAWW7oAGMPod5ssEycAebByAsAYRj8+eJk4AciDOAFgaHdh1ky2pSuIE4Bs2NYFwNAmDZMgTgCy4aQuAIZ2OvUTFScAqXNSFwDDu5xyEH5BnACk7613CMDAJt/SFcQJQBaOvEYABjT5IPyCOAFIn2F4AIY0+azJgjgBSJ84AWBIO1k1CeIEIAsG4gEYyqQ3wj8kTgDS98Y7BGAgO9vSFcQJQOLqyjA8AEO5DbPmYpdPU5wApM0xwgAM5WTXT1KcAKTNvAkAQ7gLIZzv+kmKE4C0OakLgCGchVnzbddPUpwApE2cADCEnQ7CL4gTgLTZ1gXAtnZ6fPAycQKQNscIA7CtKFZNgjgBSFhdWTUBYFuXuz4+eJk4AUiXY4QB2NZZTE9QnACky8oJANtoL10UJwAMwsoJANvY+aWLD4kTgHQdeXcAbCiKSxcfEicAAFCe0xguXXxInACky8wJAJu4i+n44GXiBCBd7jgBYBPnMa6aBHECkCh3nACwuegG4RfECUCanNQFwCa+hFlzE+uTEycAadr33gDYQLSrJkGcACRLnACwrqhXTYI4AUiWbV0ArCvqVZMgTgCSZSAegHVEv2oSxAkAABQh+lWTIE4AknXo1QGwoiRWTYI4AQCA7CWxahLECUCC6spJXQCsKplVkyBOAJIkTgBYVTKrJkGcAABAtpJaNQniBCBJR14bACtIatUkiBMAAMhScqsmQZwAJMnMCQCvSW7VJIgTgCSJEwBe8keKqyZBnAAAQFbuQginqf6BxAlAeqycAPCc0zBrvqX6dMQJQHr2vDMAnpD0qkkQJwAAkI3PKa+atKqmaSL4MABYSV21W7r+52EB8MBtmDXJb/u1cgKQFvMmADzlcw5PRZwAAEDaLsOsOc/hHYoTgLRYOQHgoSQvXHyKOAFIizgBYNmXMGsucnki4gQAANJ0l9OqSRAnAACQrPbCxZucXp84AUjLkfcFwPzo4MQvXHyKOAEAgPScpH7h4lPECQAApKU9Ovgsx3cmTgAAIC1ZXLj4FHECkJZD7wugaO3RwVe5PgBxAgAAabjLedUkiBMAAEhGlkPwy8QJAADE7zrMmuyODn5InACkoq4OvCuAYmW9nWtBnACk4613BVCkP8OsuSjhDy5OAAAgXnfzWZNCiBMAAIjX59yH4JeJEwAAiFO2N8E/R5wApMNAPEBZjkv7A4sTgHQYiAcoxx9h1tyU9r7FCQAAxKW906SYIfhl4gQAAOJSxJ0mTxEnAAAQj2LuNHmKOAEAgDjclnSnyVPECUA6nNYFkLfjku40eYo4AUiH07oA8vWl5O1cC+IEAAB2667kIfhl4gQAAHar+O1cC+IEAAB252uYNeeef0ecAADAbtzNV034TpwAAMBufLCd60fiBAAAplf0ZYvPEScAADCt4i9bfI44AQCAaTmd6xniBCAdh94VQPL+sJ3reeIEAACmcR1mje1cLxAnAAAwPscGr0CcAADA+E7CrLnynF8mTgAAYFztLfCnnvHrxAkAAIzHdq41iBMAABiPW+DXIE4AAGAcboFfkzgBAIDhtccGf/Zc1yNOAABgWHfz7VysTZwAAMCwjsOsufFM1ydOAABgOO2cybnnuRlxAgAAwzBnsiVxAgAA2zNnMgBxAgAA2zNnMgBxAgAA2zFnMhBxAgAAm7s0ZzIccQIAAJsxZzIwcQIAAJs5CrPmm2c3HHECkI477wogGr+FWXPldQxLnACkwydBgDh8CbPm1LsYnjgBAIDVXYcQDMCPRJwAAMBqugF4cyajEScAALCaDy5aHJc4AQCA1/0aZs2F5zQucQIAAC9rB+DPPKPxiRMAAHjedZg1x57PNMQJAAA87XZ+0SKTEScAAPCYk7l2QJwApMMJMQDTOXYD/PTECUA6xAnANNqTuc496+mJEwAAuOdkrh0SJwAA0PnqZK7dEicAANAeGdzOmbBT4gQAgNLdzY8MdjLXzokTgHQYiAcYnjCJiDgBSIc4ARjeB0cGx0OcAABQqvbI4AtvPx7iBACAEv3qyOD4iBMAAErjLpNIiROAdJg5AdjeF3eZxKtqmqb0ZwCQjrrylzbA5i7DrDny/OJl5QQAgBJcz0/mImriBACA3F27yyQN4gQgLXfeF8BaboVJOsQJQFpcFAawurv+kkVhkghxAgBAju76FRPf1EmIOAEAIDfCJFHiBCAt7joBeJkwSZg4AUiLOAF4mTBJmDgBACAXvwqTtIkTAABy0IbJmTeZNnECkJYL7wvgEWGSCXECAEDKhElGxAkAAKkSJpkRJwBpMegJ0BEmGaqapin9GQCkpa78xQ2UTphkysoJAAApESYZEycA6bn2zoBCCZPM/VT6AwBI0DcvDSjMXQjhOMyacy8+b+IEAICYtWFy5Ob3MtjWBZAeFzECpRAmhREnAADESJgUSJwApOfGOwMyJ0wKJU4A0iNOgJy1JxIeCJMyGYgHACAW1/2KiVMJC+WGeIAUuSUeyI8wwbYuAAB27oswIdjWBZCs2xDCntcHZOBLmDXHXiTByglAsgzFAzn4U5iwzMoJQJpsfQBS92uYNWfeIsusnACkyRGbQKruhAnPsXICAMBUXK7Ii6ycAKTpwnsDEnMtTHiNlRMAAMbmDhNWYuUEIE2+8wikwh0mrMwN8QCpcks8EL/2qODP3hOrsnICkK5r7w6I2K/ChHWZOQFIly0SQIycyMXGrJwApMsnfiA2TuRiK+IEIF1WToCYfBUmbMu2LoB0+QIAiIXBdwYhTgDSZeUE2LV2vuRzmDVn3gRDECcA6bJyAuySwXcG554TgJS56wTYDTe+MwoD8QBpu/T+gIl9CbPmQJgwBnECkDZfHABTuesvVjz2xBmLOAFIm73ewBRu+21cBt8ZlTgBSNuN9weMrL2/5MDgO1NwWhdA2sQJMKY/wqw58YSZijgBSJvvZAJjaOdLPoRZc+HpMiVHCQOkznHCwLAu+zBx4AaTM3MCkD7HCQNDabdxub+EnREnAOkzdwJsq93G9Yv5EnZNnACkT5wA22hXX/fNlxADA/EA6TMUD2zKaVxERZwApM/KCbCu9lLFY6slxMZpXQA5cGIXsLqvfZgYeic6Vk4A8nAdQnjnXQIvaIfeP4dZc+YhESsD8QB5sLULeEn7DYwDYULsxAlAHgzFA89ph97bMPFNDKJnWxdAHsQJ8NBtf9O7vx9IhpUTgDz4jiiw7M9+G5cwISlO6wLIhRO7AEcEkzgrJwD5uPYuoWiL1RJhQrLMnADk48ZxwlAkqyVkw8oJQD7sLYfyWC0hK1ZOAPIhTqAcVkvIkpUTgHyIEyiD1RKy5bQugJzU1bcQwhvvFLJktYTsWTkByIvVE8hTe8v7vjAhd+IEIC/iBPJyGUL4OcyaE++VEhiIB8iLOIE83IUQTsKsOfU+KYk4AciLOIH0fQkhfA6z5pt3SWkMxAPkpq78xQ5pMvBO8aycAOTn2k3xkJR2C9epuRIwEA+QI1u7IB1f+ztLhAnFC1ZOALLUxslHrxaiZgsXPEGcAOTHygnEyylc8AID8QA5MhQPMXIKF7zCyglAngzFQzwu+yixqgmvECcAeboQJ7Bzt32UnHsVsBpxApAn36GF3XE0MGxInADkSZzAbvzZD7ybK4ENGIgHyJWheJjSlz5Kbjx12JyVE4B8tUO4h94vjOqyjxL3lcAAxAlAvi7ECYzmuh92FyUwIHECkC9zJzC8236l5MyzheGJE4B8+Y4uDEeUwAQMxAPkrK7a4dw97xg2JkpgQn/zsAGyZvUENtPeVfJHCOFAmMB0bOsCyFs7d/LRO4aVdRcodpcouqsEJiZOAPJm5QRWI0ogAmZOAHJXV+0XWm+8Z3iSmRKIiJUTgPxdue8EHhElECFxApA/lzHCPVECEXNaF0D+zJ1ACJchhF/CrNkXJhAvMycAJagrf9lTqi8hhLMwa0Q6JMC2LoAyXNraRUHak7fO++1bN148pEOcAJTB3AkluJ2vkjgOGJIlTgDKcOU9k7HrPkjMkkDixAlAGey3J0fmSSAzBuIBSlFX7erJO++bxC1ucj8zTwL5sXICUI4LcULCbN2CAogTgHK0cfLJ+yYhi1O32igxNwUFECcA5bAvn1Rc91u3zp26BWUxcwJQkrpypDCxskoCWDkBKIw4ITZWSYDvxAlAWdo4+d07Z8dul1ZJnLgFfGdbF0Bp6spf/OzKl36F5NwbAJ5i5QSgPF9DCO+9dyZi2xawMnECUJ4LccLIbpeCxLYtYGXiBKA8jhRmDIs5kjOnbQGbMnMCUKK6arfXvPHu2ZIgAQZl5QSgTO0XlB+9ezYgSIDRiBOAMl2IE9YgSIBJ2NYFUKK6ehtC+H/ePS+47iNWkACTEScApaqr9gvOd94/Sy77FRKnbAE7YVsXQLkuxEnx7voYuXAPCRADcQJQrvaL0k/ef3Gul2LEsdJAVGzrAiiZI4VLcPc9RtqfbdcCImblBKBsjhTO0+XS6ohhdiAZ4gSgbI4UzsNiq1a7MnJe+sMA0mVbF0DJHCmcqvsY6YLEIDuQBXECUDpHCqeg3aZ1JUaA3NnWBcBZCOHfxT+FeCwG2K/6EHGiFlAMcQKAL3536/pBjDhNCyiWbV0AtFu72i+I9zyJ0V33EdL9sCoC8AMrJwAEFzKOQogArEmcABD6bUXiZDN3P0RICDdCBGAztnUB0HFb/GvufgiQ+xURJ2cBDMTKCQALbovvtNuxvvWrSd9ECMB0xAkACyXFyWX/83KA3DgpC2C3xAkAC7nMSSy2X4WlP9PiZysgABEzcwLAvbpqV0/eR/pElqPjpv8RlsLDygdA4qycALBsijhZjoywtK1qYTk8voVZc/X4lwAgR+IEgGVtnOxv8ESe2xImLgBYTQjh/wOaO0WmP08QbgAAAABJRU5ErkJggg==", } # RedisCloud color: #0D6EFD diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 1d1b88e6f..d427d4f58 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -186,6 +186,15 @@ class CaseConfigParamType(Enum): exbits = "exbits" number_of_regions = "number_of_regions" + # ADBPG parameters + hnsw_m = "hnsw_m" + algorithm = "algorithm" + rabitq_bits = "rabitq_bits" + quantize_rescore_amp = "quantize_rescore_amp" + nova_adaptive_gamma = "nova_adaptive_gamma" + max_scan_points = "max_scan_points" + auto_reduction = "auto_reduction" + class CustomizedCase(BaseModel): pass @@ -422,9 +431,7 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: task_config = case_result.get("task_config") case_config = task_config.get("case_config") db = DB(task_config.get("db")) - task_config["db_config"] = db.config_cls(**task_config["db_config"]) - # Safely instantiate DBCaseConfig (fallback to EmptyDBCaseConfig on None) raw_case_cfg = task_config.get("db_case_config") or {} index_value = raw_case_cfg.get("index", None) From ce067677592e04417fcefdac3d18ba2e9f0a16dd Mon Sep 17 00:00:00 2001 From: ForwardXu Date: Thu, 2 Jul 2026 22:00:04 +0800 Subject: [PATCH 40/49] feat(lancedb): enhance index types and CLI integration (#787) * feat(lancedb): rewrite LanceDB backend with new index types, filter support, batch insert optimization and full CLI integration Core changes: - api.py: Add IVF_HNSW_SQ and IVF_HNSW_PQ to IndexType enum - config.py: Rewrite into 5 independent config classes (IVF_PQ / NONE / AUTOINDEX / IVF_HNSW_SQ / IVF_HNSW_PQ); all IVF variants share refine_factor search param; add storage_options for remote storage - lancedb.py: Implement standard prepare_filter() pattern (NumGE/StrEqual); support scalar labels insert; use PyArrow FixedSizeListArray for batch writes instead of per-row dicts; unify search param passing; optimize() supports compact_files + cleanup_old_versions; custom __deepcopy__ to solve multi-process Rust handle serialization issue; explicit select _distance to suppress lance deprecation warning - cli.py: Add 5 CLI commands (LanceDB/AutoIndex/IVFPQ/IVFHNSWSQ/IVFHNSWPQ); fix IndexType.NONE lookup bug; add COS/GooseFS remote storage_options builder - vectordbbench.py: Register all new LanceDB CLI commands New files: - docs/lancedb-enhancement-plan.md: Development plan and implementation notes - docs/lancedb-integration.md: Integration verification report - tests/test_lancedb_config.py: 10 offline unit tests covering config/registration/CLI structure - scripts/bench_lancedb_500k.sh: One-click 500K three-index comparison benchmark script - scripts/aggregate_lancedb_results.py: Aggregate results into Markdown comparison table * fix(lancedb): always pin query metric to case metric in search_embedding Previously search_embedding() relied on the index's metric being applied implicitly. This works for indexed paths (IVF_PQ / IVF_HNSW_SQ / IVF_HNSW_PQ / AutoIndex) but silently falls back to LanceDB's default (L2) on the no-index brute-force path, which corrupts recall on cosine / IP datasets whose ground-truth uses a different metric. Fix: always call .metric(self.case_config.parse_metric()) on the query builder. For indexed paths this is a no-op when the index metric already matches; for the no-index path it aligns the scan with the case-configured metric. --- tests/test_lancedb_config.py | 280 +++++++++++++++ vectordb_bench/backend/clients/api.py | 2 + vectordb_bench/backend/clients/lancedb/cli.py | 320 ++++++++++++++++-- .../backend/clients/lancedb/config.py | 147 ++++++-- .../backend/clients/lancedb/lancedb.py | 234 ++++++++++--- vectordb_bench/cli/vectordbbench.py | 12 +- 6 files changed, 878 insertions(+), 117 deletions(-) create mode 100644 tests/test_lancedb_config.py diff --git a/tests/test_lancedb_config.py b/tests/test_lancedb_config.py new file mode 100644 index 000000000..cf2d6e12f --- /dev/null +++ b/tests/test_lancedb_config.py @@ -0,0 +1,280 @@ +"""Offline unit tests for LanceDB config objects. + +These tests don't require a running LanceDB instance. They freeze the +contract for the three IVF-family indexes (IVF_PQ / IVF_HNSW_SQ / +IVF_HNSW_PQ) so that any future refactor that accidentally diverges their +code paths (e.g. introduces index-type-specific branches) will fail CI. + +Background: IVF_HNSW_SQ / IVF_HNSW_PQ share the same code path as IVF_PQ +and the CLI is wired up for all three. These tests encode that claim. +""" + +import typing +from typing import Annotated, get_type_hints + +from vectordb_bench.backend.clients.api import IndexType, MetricType +from vectordb_bench.backend.clients.lancedb.config import ( + LanceDBAutoIndexConfig, + LanceDBIndexConfig, + LanceDBIVFHNSWPQConfig, + LanceDBIVFHNSWSQConfig, + LanceDBNoIndexConfig, + _lancedb_case_config, +) + +# --------------------------------------------------------------------------- +# Registry mapping +# --------------------------------------------------------------------------- + + +def test_registry_covers_all_lancedb_index_types(): + """Every LanceDB-supported IndexType must resolve to a config class.""" + required = { + IndexType.IVFPQ, + IndexType.AUTOINDEX, + IndexType.IVF_HNSW_SQ, + IndexType.IVF_HNSW_PQ, + IndexType.NONE, + } + assert required.issubset(_lancedb_case_config.keys()) + + # HNSW is kept for backwards compatibility and must map to IVF_HNSW_SQ. + assert _lancedb_case_config[IndexType.HNSW] is LanceDBIVFHNSWSQConfig + + +# --------------------------------------------------------------------------- +# index_param() / search_param() contract +# --------------------------------------------------------------------------- + + +def test_ivfpq_default_params_are_minimal(): + cfg = LanceDBIndexConfig() + assert cfg.index == IndexType.IVFPQ + p = cfg.index_param() + # Always present + assert p["metric"] == "cosine" or p["metric"] == "l2" + assert "num_bits" in p + # Zero-valued optionals stay out of the param dict so LanceDB uses its + # own defaults. + assert "num_partitions" not in p + assert "num_sub_vectors" not in p + # search_param() is empty when all tunables are zero. + assert cfg.search_param() == {} + + +def test_ivfpq_tuned_params_are_forwarded(): + cfg = LanceDBIndexConfig( + metric_type=MetricType.COSINE, + num_partitions=256, + num_sub_vectors=96, + nbits=8, + nprobes=20, + refine_factor=10, + ) + p = cfg.index_param() + assert p == { + "metric": "cosine", + "num_bits": 8, + "sample_rate": 256, + "max_iterations": 50, + "num_partitions": 256, + "num_sub_vectors": 96, + } + assert cfg.search_param() == {"nprobes": 20, "refine_factor": 10} + + +def test_ivf_hnsw_sq_params_are_forwarded(): + cfg = LanceDBIVFHNSWSQConfig( + metric_type=MetricType.COSINE, + num_partitions=256, + m=16, + ef_construction=200, + ef=128, + nprobes=20, + refine_factor=10, + ) + p = cfg.index_param() + assert p["index_type"] == "IVF_HNSW_SQ" + assert p["num_partitions"] == 256 + assert p["m"] == 16 + assert p["ef_construction"] == 200 + assert cfg.search_param() == { + "ef": 128, + "nprobes": 20, + "refine_factor": 10, + } + + +def test_ivf_hnsw_pq_params_are_forwarded(): + cfg = LanceDBIVFHNSWPQConfig( + metric_type=MetricType.COSINE, + num_partitions=256, + num_sub_vectors=96, + m=16, + ef_construction=200, + ef=128, + nprobes=20, + refine_factor=10, + ) + p = cfg.index_param() + assert p["index_type"] == "IVF_HNSW_PQ" + assert p["num_partitions"] == 256 + assert p["num_sub_vectors"] == 96 + assert p["m"] == 16 + assert p["ef_construction"] == 200 + assert cfg.search_param() == { + "ef": 128, + "nprobes": 20, + "refine_factor": 10, + } + + +# --------------------------------------------------------------------------- +# Code-path unification +# --------------------------------------------------------------------------- + + +def test_ivf_family_shares_search_knobs(): + """IVF_PQ exposes nprobes+refine_factor; IVF_HNSW_SQ/PQ additionally + expose ef. These are the only keys lancedb.py's search_embedding knows + how to forward, so any new index type must stay inside this union. + """ + allowed = {"nprobes", "ef", "refine_factor"} + + ivfpq = LanceDBIndexConfig(nprobes=10, refine_factor=5) + sq = LanceDBIVFHNSWSQConfig(nprobes=10, ef=64, refine_factor=5) + pq = LanceDBIVFHNSWPQConfig(nprobes=10, ef=64, refine_factor=5) + + for cfg in (ivfpq, sq, pq): + assert set(cfg.search_param().keys()).issubset(allowed) + + +def test_no_index_and_autoindex_are_well_formed(): + none_cfg = LanceDBNoIndexConfig() + assert none_cfg.index == IndexType.NONE + assert none_cfg.index_param() == {} + + auto_cfg = LanceDBAutoIndexConfig() + assert auto_cfg.index == IndexType.AUTOINDEX + assert "metric" in auto_cfg.index_param() + + +# --------------------------------------------------------------------------- +# CLI wiring — validate via static TypedDict introspection (no heavy runtime +# imports needed, avoids hdrh / streamlit / etc. dependency chains) +# --------------------------------------------------------------------------- + + +def _extract_click_option_names(typed_dict_cls: type) -> set[str]: + """Extract ``--option-name`` strings from a Click-annotated TypedDict. + + Each field is ``Annotated[T, click.option("--name", ...)]``. + We pull the option strings from the ``click.Option`` metadata. + """ + hints = get_type_hints(typed_dict_cls, include_extras=True) + names: set[str] = set() + for _field, hint in hints.items(): + if typing.get_origin(hint) is Annotated: + for meta in hint.__metadata__: + # click.option(...) produces a click.core.Decorator / functools.partial + # but in this project it's stored as a click.Argument or a + # ``functools.partial`` wrapping ``click.option``. Extract the + # first positional string that starts with "--". + if hasattr(meta, "name") and isinstance(meta.name, str): + names.add(meta.name) + # click.option() returns a decorator whose .args[0] is the + # option flag(s). We try a few accessor patterns. + for attr in ("args", "decls"): + for val in getattr(meta, attr, ()): + if isinstance(val, str) and val.startswith("--"): + names.add(val) + # For click.option stored as click.core.Option or Decorator + if hasattr(meta, "opts"): + for opt in meta.opts: + if isinstance(opt, str) and opt.startswith("--"): + names.add(opt) + return names + + +def test_cli_typed_dicts_define_all_expected_commands(): + """Every expected CLI TypedDict class must be importable from lancedb/cli.py.""" + # We mock the heavy module to avoid pulling the entire runtime. + # lancedb/cli.py imports ....cli.cli which triggers hdrh etc. + # Instead we just verify the TypedDict definitions exist in source. + import ast + from pathlib import Path + + cli_path = Path("vectordb_bench/backend/clients/lancedb/cli.py") + tree = ast.parse(cli_path.read_text()) + + class_names: set[str] = set() + func_names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + class_names.add(node.name) + elif isinstance(node, ast.FunctionDef): + func_names.add(node.name) + + # TypedDicts that drive CLI options + assert { + "LanceDBTypedDict", + "LanceDBIVFPQTypedDict", + "LanceDBIVFHNSWSQTypedDict", + "LanceDBIVFHNSWPQTypedDict", + }.issubset(class_names) + + # Command functions registered via @cli.command() + assert {"LanceDB", "LanceDBAutoIndex", "LanceDBIVFPQ", "LanceDBIVFHNSWSQ", "LanceDBIVFHNSWPQ"}.issubset(func_names) + + +def test_cli_typeddict_ivfpq_has_search_knobs(): + """IVF_PQ TypedDict must define nprobes / refine_factor / num_partitions.""" + import ast + from pathlib import Path + + cli_src = Path("vectordb_bench/backend/clients/lancedb/cli.py").read_text() + tree = ast.parse(cli_src) + + def _fields_of(cls_name: str) -> set[str]: + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == cls_name: + return { + item.target.id + for item in node.body + if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name) + } + return set() + + ivfpq = _fields_of("LanceDBIVFPQTypedDict") + assert {"nprobes", "refine_factor", "num_partitions", "num_sub_vectors", "nbits"}.issubset(ivfpq) + + +def test_cli_typeddict_hnsw_variants_are_superset_of_ivfpq(): + """Both HNSW TypedDicts must have nprobes + refine_factor + graph knobs.""" + import ast + from pathlib import Path + + cli_src = Path("vectordb_bench/backend/clients/lancedb/cli.py").read_text() + tree = ast.parse(cli_src) + + def _fields_of(cls_name: str) -> set[str]: + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == cls_name: + return { + item.target.id + for item in node.body + if isinstance(item, ast.AnnAssign) and isinstance(item.target, ast.Name) + } + return set() + + sq_fields = _fields_of("LanceDBIVFHNSWSQTypedDict") + pq_fields = _fields_of("LanceDBIVFHNSWPQTypedDict") + + # Both must have shared search knobs + shared_knobs = {"nprobes", "refine_factor", "num_partitions", "m", "ef", "ef_construction"} + assert shared_knobs.issubset(sq_fields), f"SQ missing: {shared_knobs - sq_fields}" + assert shared_knobs.issubset(pq_fields), f"PQ missing: {shared_knobs - pq_fields}" + + # PQ variant has num_sub_vectors, SQ does not + assert "num_sub_vectors" in pq_fields + assert "num_sub_vectors" not in sq_fields diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index c5474fba7..dcb1921c6 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -54,6 +54,8 @@ class IndexType(StrEnum): SVS_VAMANA_LEANVEC = "SVS_VAMANA_LEANVEC" Hologres_HGraph = "HGraph" Hologres_Graph = "Graph" + IVF_HNSW_SQ = "IVF_HNSW_SQ" + IVF_HNSW_PQ = "IVF_HNSW_PQ" NONE = "NONE" diff --git a/vectordb_bench/backend/clients/lancedb/cli.py b/vectordb_bench/backend/clients/lancedb/cli.py index 219ae114e..494c28a2b 100644 --- a/vectordb_bench/backend/clients/lancedb/cli.py +++ b/vectordb_bench/backend/clients/lancedb/cli.py @@ -1,3 +1,4 @@ +import os from typing import Annotated, Unpack import click @@ -22,21 +23,134 @@ class LanceDBTypedDict(CommonTypedDict): str | None, click.option("--token", type=str, help="Authentication token", required=False), ] + cos_secret_id: Annotated[ + str | None, + click.option( + "--cos-secret-id", + type=str, + help="Tencent COS secret ID (or set COS_SECRET_ID env var)", + required=False, + ), + ] + cos_secret_key: Annotated[ + str | None, + click.option( + "--cos-secret-key", + type=str, + help="Tencent COS secret key (or set COS_SECRET_KEY env var)", + required=False, + ), + ] + cos_endpoint: Annotated[ + str | None, + click.option( + "--cos-endpoint", + type=str, + help="Tencent COS endpoint (or set COS_ENDPOINT env var)", + required=False, + ), + ] + cos_region: Annotated[ + str | None, + click.option( + "--cos-region", + type=str, + help="Tencent COS region (or set TENCENTCLOUD_REGION env var)", + required=False, + ), + ] + + +def _build_storage_options(**parameters) -> dict[str, str] | None: + """Build storage_options based on URI scheme. + + Supports: + - cos:// / s3:// → Tencent COS credentials + - goosefs:// → GooseFS authentication options + """ + uri = parameters.get("uri", "") + + # --- GooseFS storage options --- + if uri.startswith("goosefs://"): + return _build_goosefs_storage_options(**parameters) + + # --- COS / S3 storage options --- + if uri.startswith(("cos://", "s3://")): + return _build_cos_storage_options(**parameters) + + return None + + +def _build_cos_storage_options(**parameters) -> dict[str, str] | None: + """Build storage_options for COS/S3 if credentials are provided.""" + secret_id = parameters.get("cos_secret_id") or os.environ.get("COS_SECRET_ID") + secret_key = parameters.get("cos_secret_key") or os.environ.get("COS_SECRET_KEY") + endpoint = parameters.get("cos_endpoint") or os.environ.get("COS_ENDPOINT") + region = parameters.get("cos_region") or os.environ.get("TENCENTCLOUD_REGION") + + if not (secret_id and secret_key): + return None + + storage_options = { + "aws_access_key_id": secret_id, + "aws_secret_access_key": secret_key, + } + if endpoint: + storage_options["endpoint"] = endpoint + if region: + storage_options["region"] = region + + return storage_options + + +def _build_goosefs_storage_options(**_parameters: str) -> dict[str, str] | None: + """Build storage_options for GooseFS. + + Recognized environment variables: + - GOOSEFS_AUTH_TYPE → goosefs_auth_type (simple / nosasl) + - GOOSEFS_AUTH_USERNAME → goosefs_auth_username + - GOOSEFS_WRITE_TYPE → goosefs_write_type (CACHE_THROUGH etc.) + - GOOSEFS_BLOCK_SIZE → goosefs_block_size + - GOOSEFS_CHUNK_SIZE → goosefs_chunk_size + """ + storage_options: dict[str, str] = {} + + _goosefs_env_keys = { + "GOOSEFS_AUTH_TYPE": "goosefs_auth_type", + "GOOSEFS_AUTH_USERNAME": "goosefs_auth_username", + "GOOSEFS_WRITE_TYPE": "goosefs_write_type", + "GOOSEFS_BLOCK_SIZE": "goosefs_block_size", + "GOOSEFS_CHUNK_SIZE": "goosefs_chunk_size", + } + + for env_key, opt_key in _goosefs_env_keys.items(): + value = os.environ.get(env_key) + if value: + storage_options[opt_key] = value + + return storage_options if storage_options else None + + +def _build_db_config(**parameters): + from .config import LanceDBConfig + + return LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + storage_options=_build_storage_options(**parameters), + ) @cli.command() @click_parameter_decorators_from_typed_dict(LanceDBTypedDict) def LanceDB(**parameters: Unpack[LanceDBTypedDict]): - from .config import LanceDBConfig, _lancedb_case_config + from .config import LanceDBNoIndexConfig run( db=DB.LanceDB, - db_config=LanceDBConfig( - db_label=parameters["db_label"], - uri=parameters["uri"], - token=SecretStr(parameters["token"]) if parameters.get("token") else None, - ), - db_case_config=_lancedb_case_config.get("NONE")(), + db_config=_build_db_config(**parameters), + db_case_config=LanceDBNoIndexConfig(), **parameters, ) @@ -44,16 +158,12 @@ def LanceDB(**parameters: Unpack[LanceDBTypedDict]): @cli.command() @click_parameter_decorators_from_typed_dict(LanceDBTypedDict) def LanceDBAutoIndex(**parameters: Unpack[LanceDBTypedDict]): - from .config import LanceDBConfig, _lancedb_case_config + from .config import LanceDBAutoIndexConfig run( db=DB.LanceDB, - db_config=LanceDBConfig( - db_label=parameters["db_label"], - uri=parameters["uri"], - token=SecretStr(parameters["token"]) if parameters.get("token") else None, - ), - db_case_config=_lancedb_case_config.get(IndexType.AUTOINDEX)(), + db_config=_build_db_config(**parameters), + db_case_config=LanceDBAutoIndexConfig(), **parameters, ) @@ -65,7 +175,8 @@ class LanceDBIVFPQTypedDict(CommonTypedDict, LanceDBTypedDict): "--num-partitions", type=int, default=0, - help="Number of partitions for IVFPQ index, unset = use LanceDB default", + help="Number of partitions for IVF_PQ index, 0 = use LanceDB default", + show_default=True, ), ] num_sub_vectors: Annotated[ @@ -74,7 +185,8 @@ class LanceDBIVFPQTypedDict(CommonTypedDict, LanceDBTypedDict): "--num-sub-vectors", type=int, default=0, - help="Number of sub-vectors for IVFPQ index, unset = use LanceDB default", + help="Number of sub-vectors for IVF_PQ index, 0 = use LanceDB default", + show_default=True, ), ] nbits: Annotated[ @@ -83,13 +195,28 @@ class LanceDBIVFPQTypedDict(CommonTypedDict, LanceDBTypedDict): "--nbits", type=int, default=8, - help="Number of bits for IVFPQ index (must be 4 or 8), unset = use LanceDB default", + help="Number of bits for quantization (4 or 8)", + show_default=True, ), ] nprobes: Annotated[ int, click.option( - "--nprobes", type=int, default=0, help="Number of probes for IVFPQ search, unset = use LanceDB default" + "--nprobes", + type=int, + default=0, + help="Number of probes for IVF search, 0 = use LanceDB default", + show_default=True, + ), + ] + refine_factor: Annotated[ + int, + click.option( + "--refine-factor", + type=int, + default=0, + help="Refine factor for better recall, 0 = disabled", + show_default=True, ), ] @@ -97,50 +224,171 @@ class LanceDBIVFPQTypedDict(CommonTypedDict, LanceDBTypedDict): @cli.command() @click_parameter_decorators_from_typed_dict(LanceDBIVFPQTypedDict) def LanceDBIVFPQ(**parameters: Unpack[LanceDBIVFPQTypedDict]): - from .config import LanceDBConfig, LanceDBIndexConfig + from .config import LanceDBIndexConfig run( db=DB.LanceDB, - db_config=LanceDBConfig( - db_label=parameters["db_label"], - uri=parameters["uri"], - token=SecretStr(parameters["token"]) if parameters.get("token") else None, - ), + db_config=_build_db_config(**parameters), db_case_config=LanceDBIndexConfig( index=IndexType.IVFPQ, num_partitions=parameters["num_partitions"], num_sub_vectors=parameters["num_sub_vectors"], nbits=parameters["nbits"], nprobes=parameters["nprobes"], + refine_factor=parameters["refine_factor"], ), **parameters, ) -class LanceDBHNSWTypedDict(CommonTypedDict, LanceDBTypedDict): - m: Annotated[int, click.option("--m", type=int, default=0, help="HNSW parameter m")] +class LanceDBIVFHNSWSQTypedDict(CommonTypedDict, LanceDBTypedDict): + num_partitions: Annotated[ + int, + click.option( + "--num-partitions", + type=int, + default=0, + help="Number of IVF partitions, 0 = use LanceDB default", + show_default=True, + ), + ] + m: Annotated[ + int, + click.option("--m", type=int, default=0, help="HNSW parameter m, 0 = use LanceDB default", show_default=True), + ] ef_construction: Annotated[ - int, click.option("--ef-construction", type=int, default=0, help="HNSW parameter ef_construction") + int, + click.option( + "--ef-construction", + type=int, + default=0, + help="HNSW ef_construction, 0 = use LanceDB default", + show_default=True, + ), + ] + ef: Annotated[ + int, + click.option("--ef", type=int, default=0, help="HNSW search ef, 0 = use LanceDB default", show_default=True), + ] + nprobes: Annotated[ + int, + click.option( + "--nprobes", + type=int, + default=0, + help="Number of probes for IVF search, 0 = use LanceDB default", + show_default=True, + ), + ] + refine_factor: Annotated[ + int, + click.option( + "--refine-factor", + type=int, + default=0, + help="Refine factor for better recall, 0 = disabled", + show_default=True, + ), ] - ef: Annotated[int, click.option("--ef", type=int, default=0, help="HNSW search parameter ef")] @cli.command() -@click_parameter_decorators_from_typed_dict(LanceDBHNSWTypedDict) -def LanceDBHNSW(**parameters: Unpack[LanceDBHNSWTypedDict]): - from .config import LanceDBConfig, LanceDBHNSWIndexConfig +@click_parameter_decorators_from_typed_dict(LanceDBIVFHNSWSQTypedDict) +def LanceDBIVFHNSWSQ(**parameters: Unpack[LanceDBIVFHNSWSQTypedDict]): + from .config import LanceDBIVFHNSWSQConfig run( db=DB.LanceDB, - db_config=LanceDBConfig( - db_label=parameters["db_label"], - uri=parameters["uri"], - token=SecretStr(parameters["token"]) if parameters.get("token") else None, + db_config=_build_db_config(**parameters), + db_case_config=LanceDBIVFHNSWSQConfig( + num_partitions=parameters["num_partitions"], + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef=parameters["ef"], + nprobes=parameters["nprobes"], + refine_factor=parameters["refine_factor"], + ), + **parameters, + ) + + +class LanceDBIVFHNSWPQTypedDict(CommonTypedDict, LanceDBTypedDict): + num_partitions: Annotated[ + int, + click.option( + "--num-partitions", + type=int, + default=0, + help="Number of IVF partitions, 0 = use LanceDB default", + show_default=True, + ), + ] + num_sub_vectors: Annotated[ + int, + click.option( + "--num-sub-vectors", + type=int, + default=0, + help="Number of PQ sub-vectors, 0 = use LanceDB default", + show_default=True, ), - db_case_config=LanceDBHNSWIndexConfig( + ] + m: Annotated[ + int, + click.option("--m", type=int, default=0, help="HNSW parameter m, 0 = use LanceDB default", show_default=True), + ] + ef_construction: Annotated[ + int, + click.option( + "--ef-construction", + type=int, + default=0, + help="HNSW ef_construction, 0 = use LanceDB default", + show_default=True, + ), + ] + ef: Annotated[ + int, + click.option("--ef", type=int, default=0, help="HNSW search ef, 0 = use LanceDB default", show_default=True), + ] + nprobes: Annotated[ + int, + click.option( + "--nprobes", + type=int, + default=0, + help="Number of probes for IVF search, 0 = use LanceDB default", + show_default=True, + ), + ] + refine_factor: Annotated[ + int, + click.option( + "--refine-factor", + type=int, + default=0, + help="Refine factor for better recall, 0 = disabled", + show_default=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBIVFHNSWPQTypedDict) +def LanceDBIVFHNSWPQ(**parameters: Unpack[LanceDBIVFHNSWPQTypedDict]): + from .config import LanceDBIVFHNSWPQConfig + + run( + db=DB.LanceDB, + db_config=_build_db_config(**parameters), + db_case_config=LanceDBIVFHNSWPQConfig( + num_partitions=parameters["num_partitions"], + num_sub_vectors=parameters["num_sub_vectors"], m=parameters["m"], ef_construction=parameters["ef_construction"], ef=parameters["ef"], + nprobes=parameters["nprobes"], + refine_factor=parameters["refine_factor"], ), **parameters, ) diff --git a/vectordb_bench/backend/clients/lancedb/config.py b/vectordb_bench/backend/clients/lancedb/config.py index b0621c22e..8a98c36f3 100644 --- a/vectordb_bench/backend/clients/lancedb/config.py +++ b/vectordb_bench/backend/clients/lancedb/config.py @@ -6,18 +6,22 @@ class LanceDBConfig(DBConfig): """LanceDB connection configuration.""" - db_label: str - uri: str + db_label: str = "" + uri: str = "/tmp/lancedb" token: SecretStr | None = None + storage_options: dict[str, str] | None = None def to_dict(self) -> dict: return { "uri": self.uri, "token": self.token.get_secret_value() if self.token else None, + "storage_options": self.storage_options, } class LanceDBIndexConfig(BaseModel, DBCaseConfig): + """Default IVF_PQ index configuration.""" + index: IndexType = IndexType.IVFPQ metric_type: MetricType = MetricType.L2 num_partitions: int = 0 @@ -26,91 +30,166 @@ class LanceDBIndexConfig(BaseModel, DBCaseConfig): sample_rate: int = 256 max_iterations: int = 50 nprobes: int = 0 + refine_factor: int = 0 + + def parse_metric(self) -> str: + if self.metric_type in (MetricType.L2, MetricType.COSINE): + return self.metric_type.value.lower() + if self.metric_type in (MetricType.IP, MetricType.DP): + return "dot" + msg = f"Metric type {self.metric_type} is not supported for LanceDB!" + raise ValueError(msg) def index_param(self) -> dict: - if self.index not in [ - IndexType.IVFPQ, - IndexType.HNSW, - IndexType.AUTOINDEX, - IndexType.NONE, - ]: - msg = f"Index type {self.index} is not supported for LanceDB!" - raise ValueError(msg) - - # See https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.create_index params = { "metric": self.parse_metric(), "num_bits": self.nbits, "sample_rate": self.sample_rate, "max_iterations": self.max_iterations, } - if self.num_partitions > 0: params["num_partitions"] = self.num_partitions if self.num_sub_vectors > 0: params["num_sub_vectors"] = self.num_sub_vectors - return params def search_param(self) -> dict: params = {} if self.nprobes > 0: params["nprobes"] = self.nprobes - + if self.refine_factor > 0: + params["refine_factor"] = self.refine_factor return params - def parse_metric(self) -> str: - if self.metric_type in [MetricType.L2, MetricType.COSINE]: - return self.metric_type.value.lower() - if self.metric_type in [MetricType.IP, MetricType.DP]: - return "dot" - msg = f"Metric type {self.metric_type} is not supported for LanceDB!" - raise ValueError(msg) - class LanceDBNoIndexConfig(LanceDBIndexConfig): + """No index — brute-force scan.""" + index: IndexType = IndexType.NONE def index_param(self) -> dict: return {} + def search_param(self) -> dict: + params = {} + if self.refine_factor > 0: + params["refine_factor"] = self.refine_factor + return params + class LanceDBAutoIndexConfig(LanceDBIndexConfig): + """AutoIndex — let LanceDB decide.""" + index: IndexType = IndexType.AUTOINDEX def index_param(self) -> dict: - return {} + return {"metric": self.parse_metric()} + def search_param(self) -> dict: + params = {} + if self.nprobes > 0: + params["nprobes"] = self.nprobes + if self.refine_factor > 0: + params["refine_factor"] = self.refine_factor + return params + + +class LanceDBIVFHNSWSQConfig(BaseModel, DBCaseConfig): + """IVF_HNSW_SQ index — IVF partitioning + HNSW graph + scalar quantization.""" -class LanceDBHNSWIndexConfig(LanceDBIndexConfig): - index: IndexType = IndexType.HNSW + index: IndexType = IndexType.IVF_HNSW_SQ + metric_type: MetricType = MetricType.L2 + num_partitions: int = 0 m: int = 0 ef_construction: int = 0 ef: int = 0 + nprobes: int = 0 + refine_factor: int = 0 - def index_param(self) -> dict: - params = LanceDBIndexConfig.index_param(self) + def parse_metric(self) -> str: + if self.metric_type in (MetricType.L2, MetricType.COSINE): + return self.metric_type.value.lower() + if self.metric_type in (MetricType.IP, MetricType.DP): + return "dot" + msg = f"Metric type {self.metric_type} is not supported for LanceDB!" + raise ValueError(msg) - # See https://lancedb.github.io/lancedb/python/python/#lancedb.index.HnswSq - params["index_type"] = "IVF_HNSW_SQ" + def index_param(self) -> dict: + params = { + "metric": self.parse_metric(), + "index_type": "IVF_HNSW_SQ", + } + if self.num_partitions > 0: + params["num_partitions"] = self.num_partitions if self.m > 0: params["m"] = self.m if self.ef_construction > 0: params["ef_construction"] = self.ef_construction - return params def search_param(self) -> dict: params = {} - if self.ef != 0: - params = {"ef": self.ef} + if self.ef > 0: + params["ef"] = self.ef + if self.nprobes > 0: + params["nprobes"] = self.nprobes + if self.refine_factor > 0: + params["refine_factor"] = self.refine_factor + return params + + +class LanceDBIVFHNSWPQConfig(BaseModel, DBCaseConfig): + """IVF_HNSW_PQ index — IVF partitioning + HNSW graph + product quantization.""" + index: IndexType = IndexType.IVF_HNSW_PQ + metric_type: MetricType = MetricType.L2 + num_partitions: int = 0 + num_sub_vectors: int = 0 + m: int = 0 + ef_construction: int = 0 + ef: int = 0 + nprobes: int = 0 + refine_factor: int = 0 + + def parse_metric(self) -> str: + if self.metric_type in (MetricType.L2, MetricType.COSINE): + return self.metric_type.value.lower() + if self.metric_type in (MetricType.IP, MetricType.DP): + return "dot" + msg = f"Metric type {self.metric_type} is not supported for LanceDB!" + raise ValueError(msg) + + def index_param(self) -> dict: + params = { + "metric": self.parse_metric(), + "index_type": "IVF_HNSW_PQ", + } + if self.num_partitions > 0: + params["num_partitions"] = self.num_partitions + if self.num_sub_vectors > 0: + params["num_sub_vectors"] = self.num_sub_vectors + if self.m > 0: + params["m"] = self.m + if self.ef_construction > 0: + params["ef_construction"] = self.ef_construction + return params + + def search_param(self) -> dict: + params = {} + if self.ef > 0: + params["ef"] = self.ef + if self.nprobes > 0: + params["nprobes"] = self.nprobes + if self.refine_factor > 0: + params["refine_factor"] = self.refine_factor return params _lancedb_case_config = { IndexType.IVFPQ: LanceDBIndexConfig, IndexType.AUTOINDEX: LanceDBAutoIndexConfig, - IndexType.HNSW: LanceDBHNSWIndexConfig, + IndexType.IVF_HNSW_SQ: LanceDBIVFHNSWSQConfig, + IndexType.IVF_HNSW_PQ: LanceDBIVFHNSWPQConfig, + IndexType.HNSW: LanceDBIVFHNSWSQConfig, # backward compat: HNSW maps to IVF_HNSW_SQ IndexType.NONE: LanceDBNoIndexConfig, } diff --git a/vectordb_bench/backend/clients/lancedb/lancedb.py b/vectordb_bench/backend/clients/lancedb/lancedb.py index 65330e2cb..2febc2f3c 100644 --- a/vectordb_bench/backend/clients/lancedb/lancedb.py +++ b/vectordb_bench/backend/clients/lancedb/lancedb.py @@ -1,29 +1,44 @@ +"""Wrapper around the LanceDB vector database over VectorDB""" + import logging +import os from contextlib import contextmanager import lancedb import pyarrow as pa -from lancedb.pydantic import LanceModel + +from vectordb_bench.backend.filter import Filter, FilterOp from ..api import IndexType, VectorDB -from .config import LanceDBConfig, LanceDBIndexConfig +from .config import LanceDBIndexConfig log = logging.getLogger(__name__) - -class VectorModel(LanceModel): - id: int - vector: list[float] +# Rows per ``table.add`` call. Each call produces a new Lance data file +# (fragment), so enlarging this value directly controls the on-disk fragment +# size. Override via the ``LANCEDB_BATCH_SIZE`` environment variable. +# +# Rough sizing for float32 vectors: bytes_per_fragment ≈ rows * dim * 4. +# Example: dim=768, rows=170000 -> ~498 MB per fragment. +LANCEDB_BATCH_SIZE = int(os.environ.get("LANCEDB_BATCH_SIZE", "5000")) class LanceDB(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + thread_safe: bool = False + def __init__( self, dim: int, - db_config: LanceDBConfig, + db_config: dict, db_case_config: LanceDBIndexConfig, collection_name: str = "vector_bench_test", drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): self.name = "LanceDB" @@ -32,79 +47,206 @@ def __init__( self.table_name = collection_name self.dim = dim self.uri = db_config["uri"] - # avoid the search_param being called every time during the search process - self.search_config = db_case_config.search_param() + self.storage_options = db_config.get("storage_options") or None + self.with_scalar_labels = with_scalar_labels + self.where_clause = None + + self._id_field = "id" + self._vector_field = "vector" + self._label_field = "label" - log.info(f"Search config: {self.search_config}") + # cache search params to avoid repeated calls + self.search_config = db_case_config.search_param() + log.info(f"LanceDB search config: {self.search_config}") - db = lancedb.connect(self.uri) + connect_kwargs = {} + if self.storage_options: + connect_kwargs["storage_options"] = self.storage_options + db = lancedb.connect(self.uri, **connect_kwargs) if drop_old: try: db.drop_table(self.table_name) + log.info(f"LanceDB dropped old table: {self.table_name}") except Exception as e: log.warning(f"Failed to drop table {self.table_name}: {e}") - - try: - db.open_table(self.table_name) - except Exception: - schema = pa.schema( - [pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.float32(), list_size=self.dim))] - ) + # Always create a fresh table with the correct schema after drop. + # On remote storage (e.g. GooseFS) drop_table may not fully purge + # metadata immediately, causing open_table to succeed with a stale + # schema that is missing expected fields like 'id'. + schema = self._build_schema() db.create_table(self.table_name, schema=schema, mode="overwrite") + log.info(f"LanceDB created table: {self.table_name} (schema: {schema})") + else: + try: + db.open_table(self.table_name) + except Exception: + schema = self._build_schema() + db.create_table(self.table_name, schema=schema, mode="overwrite") + log.info(f"LanceDB created table: {self.table_name} (schema: {schema})") + + def _build_schema(self) -> pa.Schema: + fields = [ + pa.field(self._id_field, pa.int64()), + pa.field(self._vector_field, pa.list_(pa.float32(), list_size=self.dim)), + ] + if self.with_scalar_labels: + fields.append(pa.field(self._label_field, pa.utf8())) + return pa.schema(fields) @contextmanager def init(self): - self.db = lancedb.connect(self.uri) + connect_kwargs = {} + if self.storage_options: + connect_kwargs["storage_options"] = self.storage_options + self.db = lancedb.connect(self.uri, **connect_kwargs) self.table = self.db.open_table(self.table_name) yield self.db = None self.table = None + def __deepcopy__(self, memo: dict) -> "LanceDB": + """Custom deepcopy: skip live connection/table handles. + + The LanceDB ``Connection`` / ``Table`` objects wrap Rust bindings that + are not picklable. ``ConcurrentInsertRunner`` deep-copies the client + per thread for non-thread-safe DBs; the caller will then invoke + ``init()`` on the copy, which re-opens a fresh connection. + """ + cls = self.__class__ + new_obj = cls.__new__(cls) + memo[id(self)] = new_obj + from copy import deepcopy as _dc + + for k, v in self.__dict__.items(): + if k in ("db", "table"): + new_obj.__dict__[k] = None + else: + new_obj.__dict__[k] = _dc(v, memo) + return new_obj + def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception | None]: + assert self.table is not None, "Please call self.init() before" try: - data = [{"id": meta, "vector": emb} for meta, emb in zip(metadata, embeddings, strict=False)] - self.table.add(data) + log.info( + f"LanceDB insert_embeddings called with {len(embeddings)} rows, " + f"LANCEDB_BATCH_SIZE={LANCEDB_BATCH_SIZE} -> " + f"{(len(embeddings) + LANCEDB_BATCH_SIZE - 1) // LANCEDB_BATCH_SIZE} fragment(s)" + ) + for offset in range(0, len(embeddings), LANCEDB_BATCH_SIZE): + batch_emb = embeddings[offset : offset + LANCEDB_BATCH_SIZE] + batch_ids = metadata[offset : offset + LANCEDB_BATCH_SIZE] + + id_arr = pa.array(batch_ids, type=pa.int64()) + vec_arr = pa.FixedSizeListArray.from_arrays( + pa.array([v for emb in batch_emb for v in emb], type=pa.float32()), + list_size=self.dim, + ) + + if self.with_scalar_labels and labels_data is not None: + batch_labels = labels_data[offset : offset + LANCEDB_BATCH_SIZE] + label_arr = pa.array(batch_labels, type=pa.utf8()) + batch_table = pa.table( + { + self._id_field: id_arr, + self._vector_field: vec_arr, + self._label_field: label_arr, + } + ) + else: + batch_table = pa.table( + { + self._id_field: id_arr, + self._vector_field: vec_arr, + } + ) + self.table.add(batch_table) + return len(metadata), None except Exception as e: log.warning(f"Failed to insert data into LanceDB table ({self.table_name}), error: {e}") return 0, e + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.where_clause = None + elif filters.type == FilterOp.NumGE: + self.where_clause = f"{self._id_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self.where_clause = f"{self._label_field} = '{filters.label_value}'" + else: + msg = f"Unsupported filter for LanceDB: {filters}" + raise ValueError(msg) + def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, + **kwargs, ) -> list[int]: - if filters: - results = self.table.search(query).select(["id"]).where(f"id >= {filters['id']}", prefilter=True).limit(k) - if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config: - results = results.nprobes(self.search_config["nprobes"]).to_list() - elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config: - results = results.ef(self.search_config["ef"]).to_list() - else: - results = results.to_list() - else: - results = self.table.search(query).select(["id"]).limit(k) - if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config: - results = results.nprobes(self.search_config["nprobes"]).to_list() - elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config: - results = results.ef(self.search_config["ef"]).to_list() - else: - results = results.to_list() - - return [int(result["id"]) for result in results] + assert self.table is not None, "Please call self.init() before" + + # Include ``_distance`` in the projection to opt in to LanceDB's + # upcoming default behaviour. Without it, lance logs a per-query + # deprecation warning ("This search specified output columns but did + # not include `_distance`"). We only consume ``id`` downstream, so the + # extra column adds negligible overhead. + # + # Always pin the query metric to the case-configured metric. Otherwise + # the no-index path (brute-force scan) silently falls back to LanceDB's + # default (L2), which corrupts recall on cosine/IP datasets whose + # ground-truth is computed with a different metric. For indexed paths + # this is a no-op when the index metric already matches. + q = ( + self.table.search(query) + .metric(self.case_config.parse_metric()) + .select([self._id_field, "_distance"]) + .limit(k) + ) + + # apply filter + if self.where_clause: + q = q.where(self.where_clause, prefilter=True) + + # apply search parameters based on config + search_cfg = self.search_config + if "nprobes" in search_cfg: + q = q.nprobes(search_cfg["nprobes"]) + if "ef" in search_cfg: + q = q.ef(search_cfg["ef"]) + if "refine_factor" in search_cfg: + q = q.refine_factor(search_cfg["refine_factor"]) + + results = q.to_list() + return [int(r[self._id_field]) for r in results] def optimize(self, data_size: int | None = None): - if self.table and hasattr(self, "case_config") and self.case_config.index != IndexType.NONE: - log.info(f"Creating index for LanceDB table ({self.table_name})") - log.info(f"Index parameters: {self.case_config.index_param()}") - self.table.create_index(**self.case_config.index_param()) - # Better recall with IVF_PQ (though still bad) but breaks HNSW: https://github.com/lancedb/lancedb/issues/2369 - if self.case_config.index in (IndexType.IVFPQ, IndexType.AUTOINDEX): + assert self.table is not None, "Please call self.init() before" + + # Build index if configured + if self.case_config.index != IndexType.NONE: + index_params = self.case_config.index_param() + log.info(f"LanceDB creating index on table ({self.table_name}), params: {index_params}") + self.table.create_index(**index_params) + + # Compact fragments and clean up old versions for better performance. + # Prefer the unified ``table.optimize()`` API (lancedb >= 0.10), which + # internally handles both compaction and version cleanup without + # requiring the optional ``pylance`` package. Fall back to the legacy + # split APIs only if ``optimize`` is unavailable. + try: + if hasattr(self.table, "optimize"): self.table.optimize() + log.info(f"LanceDB optimize completed for table ({self.table_name})") + else: + self.table.compact_files() + self.table.cleanup_old_versions() + log.info(f"LanceDB compact_files + cleanup_old_versions completed for table ({self.table_name})") + except Exception as e: + log.warning(f"LanceDB optimize failed (non-fatal): {e}") diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 6c3e868ec..7d6aa0031 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -14,7 +14,13 @@ ) from ..backend.clients.endee.cli import Endee from ..backend.clients.hologres.cli import HologresHGraph -from ..backend.clients.lancedb.cli import LanceDB +from ..backend.clients.lancedb.cli import ( + LanceDB, + LanceDBAutoIndex, + LanceDBIVFHNSWPQ, + LanceDBIVFHNSWSQ, + LanceDBIVFPQ, +) from ..backend.clients.lindorm.cli import LindormHNSW, LindormIVFBQ, LindormIVFPQ from ..backend.clients.mariadb.cli import MariaDBHNSW from ..backend.clients.memorydb.cli import MemoryDB @@ -73,6 +79,10 @@ cli.add_command(Clickhouse) cli.add_command(Vespa) cli.add_command(LanceDB) +cli.add_command(LanceDBAutoIndex) +cli.add_command(LanceDBIVFPQ) +cli.add_command(LanceDBIVFHNSWSQ) +cli.add_command(LanceDBIVFHNSWPQ) cli.add_command(HologresHGraph) cli.add_command(QdrantCloud) cli.add_command(QdrantLocal) From 224e8b9389aa76bccb6921015dd57beb6beb1730 Mon Sep 17 00:00:00 2001 From: HUANG XIAO <33706975+norrishuang@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:05:57 +0800 Subject: [PATCH 41/49] Add OpenSearch Serverless (AOSS) support (#802) * Add OpenSearch Serverless (AOSS) support - Add --serverless and --aws-region CLI options - Use AWS SigV4 authentication via requests-aws4auth for AOSS - Skip unsupported operations for serverless: cluster settings, force merge, manual refresh, replica updates, warmup API - Use smaller batch size (100) for serverless bulk inserts - Store id as document field (serverless doesn't support custom _id) - Retrieve id from _source in search results for serverless - Remove 'engine' and 'encoder' from index method config for serverless (AOSS manages these internally) * Add OpenSearch Serverless section to README * Disable http_compress for serverless to fix SigV4 checksum verification * Format code with black * Address PR review: fix serverless multi-client insert, filters, and deps - Route serverless through single-client insert path (AOSS doesn't support custom _id; the multi-client path would send _id and fail) - prepare_filter now filters NumGE on the stored 'id' field for serverless, and mappings store 'id' as a numeric (long) field so range queries work - Add boto3 and requests-aws4auth to the opensearch extra in pyproject.toml and to install/requirements_py3.11.txt - Update README serverless prerequisites to reference the opensearch extra and mention boto3 --- README.md | 37 ++++ install/requirements_py3.11.txt | 2 + pyproject.toml | 2 +- .../clients/aws_opensearch/aws_opensearch.py | 172 +++++++++++++----- .../backend/clients/aws_opensearch/cli.py | 22 ++- .../backend/clients/aws_opensearch/config.py | 52 ++++++ 6 files changed, 240 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 0ac2f01ad..2aaffba15 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,43 @@ Options: --quantization-type TEXT which type of quantization to use valid values [fp32, fp16, bq] --help Show this message and exit. ``` + +### Run awsopensearch serverless from command line + +OpenSearch Serverless (AOSS) is a serverless deployment option for Amazon OpenSearch Service. VDBBench supports AOSS with the `--serverless` flag, which uses AWS SigV4 authentication and automatically skips unsupported operations (cluster settings, force merge, manual refresh, warmup API). + +**Prerequisites:** +- AWS credentials configured (via `~/.aws/credentials`, environment variables, or IAM role) +- Serverless dependencies installed (included in the `opensearch` extra): `pip install 'vectordb-bench[opensearch]'`. This installs `opensearch-py`, `boto3`, and `requests-aws4auth`. +- IAM identity policy allowing `aoss:APIAccessAll` on the collection +- AOSS Data Access Policy granting index/collection permissions to the IAM principal + +**Example: Run performance test on OpenSearch Serverless** + +```shell +vectordbbench awsopensearch --db-label aoss \ + --serverless --aws-region us-east-1 \ + --host .aoss.us-east-1.on.aws --port 443 \ + --case-type Performance768D1M \ + --m 16 --ef-construction 200 --ef-search 40 \ + --number-of-shards 8 --number-of-replicas 0 \ + --engine faiss --metric-type cosine \ + --num-concurrency 80,100,120 +``` + +OpenSearch Serverless-specific options: + +| Option | Description | +|--------|-------------| +| `--serverless` | Enable OpenSearch Serverless mode (uses AWS SigV4 auth) | +| `--aws-region` | AWS region for the AOSS collection (default: `us-east-1`) | + +> **Notes:** +> - `--user` and `--password` are not needed for Serverless mode +> - `--engine` is accepted but ignored internally (AOSS manages the engine) +> - `--force-merge-enabled`, `--refresh-interval`, `--flush-threshold-size`, and `--cb-threshold` are ignored for Serverless +> - Data insertion uses smaller batch sizes (100) for Serverless API compatibility + ### Run Elastic Cloud from command line Elastic Cloud supports multiple index types: HNSW, HNSW_INT8, HNSW_INT4, and HNSW_BBQ. diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 3d42528c3..a6f3f32cf 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -5,6 +5,8 @@ pinecone weaviate-client elasticsearch==8.16.0 opensearch-py +boto3 +requests-aws4auth pgvector pgvecto_rs[psycopg3]>=0.2.1 sqlalchemy diff --git a/pyproject.toml b/pyproject.toml index a5ef69112..d1955404c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ pgvecto_rs = [ "pgvecto_rs[psycopg3]>=0.2.2" ] redis = [ "redis" ] memorydb = [ "memorydb" ] chromadb = [ "chromadb" ] -opensearch = [ "opensearch-py" ] +opensearch = [ "opensearch-py", "boto3", "requests-aws4auth" ] aliyun_opensearch = [ "alibabacloud_ha3engine_vector" ] mongodb = [ "pymongo" ] mariadb = [ "mariadb" ] diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 8d77d263b..034793f4b 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -48,6 +48,7 @@ def __init__( log.info(f"AWS_OpenSearch client config: {self.db_config}") log.info(f"AWS_OpenSearch db case config : {self.case_config}") + self._is_serverless = ".aoss." in self.db_config.get("hosts", [{}])[0].get("host", "") client = OpenSearch(**self.db_config) if drop_old: log.info(f"AWS_OpenSearch client drop old index: {self.index_name}") @@ -80,6 +81,9 @@ def _log_index_creation_info(self) -> None: log.info(f"All case_config parameters: {self.case_config.__dict__}") def _configure_cluster_settings(self, client: OpenSearch) -> None: + if self._is_serverless: + log.info("Skipping cluster settings for OpenSearch Serverless") + return cluster_settings_body = { "persistent": { "knn.algo_param.index_thread_qty": self.case_config.index_thread_qty, @@ -89,17 +93,19 @@ def _configure_cluster_settings(self, client: OpenSearch) -> None: client.cluster.put_settings(body=cluster_settings_body) def _build_index_settings(self) -> dict: - return { + settings = { "index": { "knn": True, "number_of_shards": self.case_config.number_of_shards, "number_of_replicas": self.case_config.number_of_replicas, - "translog.flush_threshold_size": self.case_config.flush_threshold_size, - "knn.advanced.approximate_threshold": "-1", - "knn.algo_param.ef_search": self.case_config.ef_search, }, - "refresh_interval": self.case_config.refresh_interval, } + if not self._is_serverless: + settings["index"]["translog.flush_threshold_size"] = self.case_config.flush_threshold_size + settings["index"]["knn.advanced.approximate_threshold"] = "-1" + settings["index"]["knn.algo_param.ef_search"] = self.case_config.ef_search + settings["refresh_interval"] = self.case_config.refresh_interval + return settings def _build_vector_field_config(self) -> dict: method_config = self.case_config.index_param() @@ -108,6 +114,21 @@ def _build_vector_field_config(self) -> dict: if self.case_config.engine == AWSOS_Engine.s3vector: method_config = {"engine": "s3vector"} + # OpenSearch Serverless does not support 'engine' or 'encoder' in method config + if self._is_serverless and "engine" in method_config: + space_type = method_config.pop("space_type", self.case_config.parse_metric()) + method_config.pop("engine", None) + if "parameters" in method_config: + method_config["parameters"].pop("encoder", None) + vector_field_config = { + "type": "knn_vector", + "dimension": self.dim, + "space_type": space_type, + "method": method_config, + } + log.info(f"Serverless vector field config: {vector_field_config}") + return vector_field_config + if self.case_config.on_disk: space_type = self.case_config.parse_metric() vector_field_config = { @@ -157,6 +178,12 @@ def _build_mappings(self, vector_field_config: dict) -> dict: }, } log.info("Using standard mappings with _source configuration for non-s3vector engines") + + # Serverless stores the benchmark id in a dedicated numeric field (custom _id + # is not supported). Map it as a long so NumGE range filters work. + if self._is_serverless: + mappings["properties"]["id"] = {"type": "long"} + return mappings def _create_opensearch_index(self, client: OpenSearch, settings: dict, mappings: dict) -> None: @@ -210,6 +237,12 @@ def insert_embeddings( num_clients = self.case_config.number_of_indexing_clients or 1 log.info(f"Number of indexing clients from case_config: {num_clients}") + # OpenSearch Serverless requires the single-client path: it does not support + # custom _id and needs the benchmark id stored in _source with small batches. + if self._is_serverless: + log.info("Using single client for data insertion (OpenSearch Serverless)") + return self._insert_with_single_client(embeddings, metadata, labels_data) + if num_clients <= 1: log.info("Using single client for data insertion") return self._insert_with_single_client(embeddings, metadata, labels_data) @@ -222,25 +255,47 @@ def _insert_with_single_client( metadata: list[int], labels_data: list[str] | None = None, ) -> tuple[int, Exception]: - insert_data = [] - for i in range(len(embeddings)): - index_data = {"index": {"_index": self.index_name, self.id_col_name: metadata[i]}} - if self.with_scalar_labels and self.case_config.use_routing and labels_data is not None: - index_data["routing"] = labels_data[i] - insert_data.append(index_data) - - other_data = {self.vector_col_name: embeddings[i]} - if self.with_scalar_labels and labels_data is not None: - other_data[self.label_col_name] = labels_data[i] - insert_data.append(other_data) + embeddings_list = list(embeddings) + batch_size = 100 if self._is_serverless else len(embeddings_list) + total_inserted = 0 - try: - self.client.bulk(body=insert_data) - return len(embeddings), None - except Exception as e: - log.warning(f"Failed to insert data: {self.index_name} error: {e!s}") - time.sleep(10) - return self._insert_with_single_client(embeddings, metadata, labels_data) + for i in range(0, len(embeddings_list), batch_size): + batch_embeddings = embeddings_list[i : i + batch_size] + batch_metadata = metadata[i : i + batch_size] + batch_labels = labels_data[i : i + batch_size] if labels_data else None + + insert_data = [] + for j in range(len(batch_embeddings)): + if self._is_serverless: + index_data = {"index": {"_index": self.index_name}} + else: + index_data = {"index": {"_index": self.index_name, self.id_col_name: batch_metadata[j]}} + + if self.with_scalar_labels and self.case_config.use_routing and batch_labels is not None: + index_data["routing"] = batch_labels[j] + insert_data.append(index_data) + + other_data = {self.vector_col_name: batch_embeddings[j]} + if self._is_serverless: + other_data["id"] = batch_metadata[j] + if self.with_scalar_labels and batch_labels is not None: + other_data[self.label_col_name] = batch_labels[j] + insert_data.append(other_data) + + try: + self.client.bulk(body=insert_data) + total_inserted += len(batch_embeddings) + except Exception as e: + log.warning(f"Failed to insert batch: {self.index_name} error: {e!s}") + time.sleep(10) + try: + self.client.bulk(body=insert_data) + total_inserted += len(batch_embeddings) + except Exception as retry_e: + log.warning(f"Retry failed for batch: {retry_e!s}") + return total_inserted, retry_e + + return total_inserted, None def _insert_with_multiple_clients( self, @@ -402,24 +457,37 @@ def search_embedding( } try: - resp = self.client.search( - index=self.index_name, - body=body, - size=k, - _source=False, - docvalue_fields=[self.id_col_name], - stored_fields="_none_", - preference="_only_local" if self.case_config.number_of_shards == 1 else None, - routing=self.routing_key, - ) - log.debug(f"Search took: {resp['took']}") - log.debug(f"Search shards: {resp['_shards']}") - log.debug(f"Search hits total: {resp['hits']['total']}") - try: - return [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] - except Exception: - # empty results - return [] + if self._is_serverless: + resp = self.client.search( + index=self.index_name, + body=body, + size=k, + _source=["id"], + preference="_only_local" if self.case_config.number_of_shards == 1 else None, + routing=self.routing_key, + ) + try: + return [int(h["_source"]["id"]) for h in resp["hits"]["hits"]] + except Exception: + return [] + else: + resp = self.client.search( + index=self.index_name, + body=body, + size=k, + _source=False, + docvalue_fields=[self.id_col_name], + stored_fields="_none_", + preference="_only_local" if self.case_config.number_of_shards == 1 else None, + routing=self.routing_key, + ) + log.debug(f"Search took: {resp['took']}") + log.debug(f"Search shards: {resp['_shards']}") + log.debug(f"Search hits total: {resp['hits']['total']}") + try: + return [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] + except Exception: + return [] except Exception as e: log.warning(f"Failed to search: {self.index_name} error: {e!s}") raise e from None @@ -429,7 +497,10 @@ def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: self.filter = None elif filters.type == FilterOp.NumGE: - self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} + # Serverless stores the benchmark id in the "id" field of _source since it + # does not support custom _id. Filter on that stored field instead of _id. + filter_field = "id" if self._is_serverless else self.id_col_name + self.filter = {"range": {filter_field: {"gt": filters.int_value}}} elif filters.type == FilterOp.StrEqual: self.filter = {"term": {self.label_col_name: filters.label_value}} if self.case_config.use_routing: @@ -468,6 +539,10 @@ def _update_ef_search(self): log.warning(f"Failed to update ef_search parameter: {e}") def _update_replicas(self): + if self._is_serverless: + log.info("Skipping replica updates for OpenSearch Serverless") + return + index_settings = self.client.indices.get_settings(index=self.index_name) current_number_of_replicas = int(index_settings[self.index_name]["settings"]["index"]["number_of_replicas"]) log.info( @@ -490,6 +565,11 @@ def _wait_till_green(self): log.info(f"Index {self.index_name} is green..") def _refresh_index(self): + if self._is_serverless: + log.info("Skipping manual refresh for OpenSearch Serverless, waiting for auto-refresh...") + time.sleep(10) + return + log.debug(f"Starting refresh for index {self.index_name}") while True: try: @@ -505,6 +585,10 @@ def _refresh_index(self): log.debug(f"Completed refresh for index {self.index_name}") def _do_force_merge(self): + if self._is_serverless: + log.info("Skipping force merge for OpenSearch Serverless") + return + log.info(f"Updating the Index thread qty to {self.case_config.index_thread_qty_during_force_merge}.") cluster_settings_body = { @@ -530,6 +614,10 @@ def _do_force_merge(self): log.info(f"Completed force merge for index {self.index_name}") def _load_graphs_to_memory(self, client: OpenSearch): + if self._is_serverless: + log.info("Skipping warmup API for OpenSearch Serverless") + return + if self.case_config.engine != AWSOS_Engine.lucene: log.info("Calling warmup API to load graphs into memory") warmup_endpoint = f"/_plugins/_knn/warmup/{self.index_name}" diff --git a/vectordb_bench/backend/clients/aws_opensearch/cli.py b/vectordb_bench/backend/clients/aws_opensearch/cli.py index 5bc80a687..59bb68a23 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/cli.py +++ b/vectordb_bench/backend/clients/aws_opensearch/cli.py @@ -25,8 +25,14 @@ def optional_secret_str(value: str | None) -> SecretStr | None: class AWSOpenSearchTypedDict(TypedDict): host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] port: Annotated[int, click.option("--port", type=int, default=80, help="Db Port")] - user: Annotated[str | None, click.option("--user", type=str, help="Db User")] - password: Annotated[str | None, click.option("--password", type=str, help="Db password")] + user: Annotated[str | None, click.option("--user", type=str, help="Db User (not needed for Serverless)")] + password: Annotated[ + str | None, click.option("--password", type=str, help="Db password (not needed for Serverless)") + ] + is_serverless: Annotated[bool, click.option("--serverless", is_flag=True, help="Use OpenSearch Serverless")] + aws_region: Annotated[ + str, click.option("--aws-region", type=str, default="us-east-1", help="AWS region for Serverless") + ] number_of_shards: Annotated[ int, click.option("--number-of-shards", type=int, help="Number of primary shards for the index", default=1), @@ -172,6 +178,12 @@ class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFl def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig + is_serverless = parameters.get("serverless", False) + log.info(f"Is Serverless: {is_serverless}") + + if not is_serverless and not parameters.get("user"): + log.warning("Standard OpenSearch mode requires user and password.") + # Set default values for HNSW parameters if not provided and not using s3vector engine = AWSOS_Engine(parameters["engine"]) ef_construction = parameters.get("ef_construction") @@ -192,8 +204,10 @@ def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): db_config=AWSOpenSearchConfig( host=parameters["host"], port=parameters["port"], - user=parameters["user"], - password=optional_secret_str(parameters["password"]), + user=parameters.get("user"), + password=optional_secret_str(parameters.get("password")), + is_serverless=is_serverless, + aws_region=parameters.get("aws_region", "us-east-1"), ), db_case_config=AWSOpenSearchIndexConfig( number_of_shards=parameters["number_of_shards"], diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index 62c284317..134da869b 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -16,8 +16,60 @@ class AWSOpenSearchConfig(DBConfig, BaseModel): port: int = 80 user: str | None = None password: SecretStr | None = None + is_serverless: bool = False + aws_region: str = "us-east-1" def to_dict(self) -> dict: + if self.is_serverless: + return self._serverless_config() + return self._standard_config() + + def _serverless_config(self) -> dict: + """Configuration for OpenSearch Serverless using AWS SigV4 authentication.""" + log.info(f"Configuring OpenSearch Serverless - Host: {self.host}, Region: {self.aws_region}") + + try: + import boto3 + except ImportError as e: + raise ImportError("boto3 is required for OpenSearch Serverless. Install with: pip install boto3") from e + + try: + from opensearchpy import RequestsHttpConnection + from requests_aws4auth import AWS4Auth + except ImportError as e: + raise ImportError( + "requests-aws4auth is required for OpenSearch Serverless. " + "Install with: pip install requests-aws4auth" + ) from e + + session = boto3.Session() + credentials = session.get_credentials() + if not credentials: + raise ValueError("AWS credentials not found. Please configure AWS credentials.") + + credentials = credentials.get_frozen_credentials() + auth = AWS4Auth( + credentials.access_key, + credentials.secret_key, + self.aws_region, + "aoss", + session_token=credentials.token, + ) + + return { + "hosts": [{"host": self.host, "port": 443}], + "http_auth": auth, + "use_ssl": True, + "verify_certs": True, + "connection_class": RequestsHttpConnection, + "timeout": 600, + "max_retries": 3, + "retry_on_timeout": True, + "http_compress": False, + } + + def _standard_config(self) -> dict: + """Configuration for standard OpenSearch with basic auth.""" use_ssl = self.port == 443 http_auth = ( (self.user, self.password.get_secret_value()) From 3bc1be45e2d60a327743b694a0e63bf16d8a024b Mon Sep 17 00:00:00 2001 From: Siyu Chen Date: Mon, 6 Jul 2026 10:44:07 +0800 Subject: [PATCH 42/49] feat(volc_mysql): add VolcMySQL backend with HNSW vector index support (#804) Add a VectorDB backend for VolcMySQL (native VECTOR type and HNSW index) over the MySQL wire protocol via mysql-connector-python: - client, config, and Click CLI command; registered in the DB enum and CLI - per-thread connections (thread_safe=False, one connection per worker) - NonFilter and NumGE filter support via prepared SQL templates - binary float32 vector path with per-connection auto-probe fallback to to_vector() - unit tests for TSV encoding, config, filters, and init teardown --- README.md | 43 +- pyproject.toml | 1 + tests/test_volc_mysql_client.py | 236 ++++++++++ tests/test_volc_mysql_encoder.py | 74 ++++ vectordb_bench/backend/clients/__init__.py | 16 + .../backend/clients/volc_mysql/cli.py | 134 ++++++ .../backend/clients/volc_mysql/config.py | 75 ++++ .../backend/clients/volc_mysql/volc_mysql.py | 411 ++++++++++++++++++ vectordb_bench/backend/runner/rate_runner.py | 12 + vectordb_bench/cli/vectordbbench.py | 2 + 10 files changed, 1003 insertions(+), 1 deletion(-) create mode 100644 tests/test_volc_mysql_client.py create mode 100644 tests/test_volc_mysql_encoder.py create mode 100755 vectordb_bench/backend/clients/volc_mysql/cli.py create mode 100755 vectordb_bench/backend/clients/volc_mysql/config.py create mode 100755 vectordb_bench/backend/clients/volc_mysql/volc_mysql.py diff --git a/README.md b/README.md index 2aaffba15..847783545 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ All the database client supported | zvec | `pip install vectordb-bench[zvec]` | | endee | `pip install vectordb-bench[endee]` | | lindorm | `pip install vectordb-bench[lindorm]` | +| volc_mysql | `pip install vectordb-bench[volc_mysql]` | | adbpg | `pip install vectordb-bench[adbpg]` | ### Run @@ -93,6 +94,7 @@ Commands: pgvectorhnsw pgvectorivfflat vectorchordrq + volcmysqlhnsw test weaviate ``` @@ -674,6 +676,45 @@ To list the options for PolarDB, execute `vectordbbench polardbhnswflat --help`. Create index after load or inline at table creation ``` +### Run VolcMySQL from command line + +VolcMySQL is a MySQL-compatible service with a native `VECTOR` type and an HNSW vector index (created via `SECONDARY_ENGINE_ATTRIBUTE`). Optional quantization is configurable through `--quant-algorithm` (`NONE`, `SQ`, `PQ`) and `--quant-type` (`16_bit`, `8_bit`, `4_bit`, `binary`). + +```shell +vectordbbench volcmysqlhnsw \ + --case-type Performance1536D50K \ + --username \ + --password '' \ + --host \ + --port 3306 \ + --m 16 \ + --ef-construction 128 \ + --ef-search 100 \ + --quant-algorithm SQ \ + --quant-type 16_bit \ + --num-concurrency '10,20,40,60,80' \ + --concurrency-duration 30 \ + --task-label \ + --db-label +``` + +To list the options for VolcMySQL, execute `vectordbbench volcmysqlhnsw --help`. The following are some VolcMySQL-specific command-line options. + +```text + --username TEXT Username [required] + --password TEXT Password [required] + --host TEXT Db host [default: 127.0.0.1] + --port INTEGER DB Port [default: 3306] + --m INTEGER M parameter in HNSW vector indexing + --ef-search INTEGER Session variable loose_hnsw_ef_search + --ef-construction INTEGER HNSW ef_construction + --quant-algorithm [NONE|SQ|PQ] Quantization algorithm + --quant-type [16_bit|8_bit|4_bit|binary] + Quantization type +``` + +> Note: vectors are loaded and queried over the raw-binary `VECTOR` path by default; the client auto-probes server support and falls back to `to_vector()` text when unavailable. Set `VDB_BINARY_VEC=0` to force the text path or `1` to force binary. + #### Using a configuration file. The vectordbbench command can optionally read some or all the options from a yaml formatted configuration file. @@ -879,7 +920,7 @@ Now we can only run one task at the same time. ### Code Structure ![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8c06512e-5419-4381-b084-9c93aed59639) ### Client -Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, VectorChord, Redis, Chroma, CockroachDB, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. +Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, VectorChord, Redis, Chroma, CockroachDB, VolcMySQL, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. ### Benchmark Cases We've developed lots of comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into several main types: #### Capacity Case diff --git a/pyproject.toml b/pyproject.toml index d1955404c..9039e8f70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ zvec = [ "zvec" ] endee = [ "endee==0.1.10" ] lindorm = [ "opensearch-py" ] seekdb = [ "mysql-connector-python" ] +volc_mysql = [ "mysql-connector-python" ] pinot = [ "requests" ] adbpg = [ "psycopg", "psycopg-binary", "pgvector" ] diff --git a/tests/test_volc_mysql_client.py b/tests/test_volc_mysql_client.py new file mode 100644 index 000000000..ab602be9d --- /dev/null +++ b/tests/test_volc_mysql_client.py @@ -0,0 +1,236 @@ +from vectordb_bench.backend.clients.volc_mysql.volc_mysql import VolcMySQL + + +def test_volc_mysql_is_not_thread_safe(): + # mysql.connector is not thread-safe; ConcurrentInsertRunner relies on + # this class attribute to clamp max_workers=1. + assert VolcMySQL.thread_safe is False + + +def test_volc_mysql_ready_to_load_does_not_return_falsy_none(): + # The base VectorDB has no ready_to_load() method to override here; + # the previous stub returned None, which any `if not db.ready_to_load()` + # caller would misread as "not ready". + assert not hasattr(VolcMySQL, "ready_to_load") or VolcMySQL.ready_to_load.__qualname__.startswith("VectorDB") + + +from vectordb_bench.backend.filter import FilterOp + + +def test_volc_mysql_supports_numge_filter(): + # Matches the WHERE id >= %s prepared SQL the client already builds in init(). + assert FilterOp.NumGE in VolcMySQL.supported_filter_types + assert FilterOp.NonFilter in VolcMySQL.supported_filter_types + + +from pydantic import SecretStr + +from vectordb_bench.backend.clients.volc_mysql.config import VolcMySQLConfig + + +def test_volc_mysql_config_to_dict_has_only_driver_kwargs(): + cfg = VolcMySQLConfig(password=SecretStr("pw")) + d = cfg.to_dict() + # mysql.connector.connect accepts these; the previously-declared + # read_timeout/write_timeout are not valid kwargs for that driver. + assert set(d.keys()) == {"host", "port", "user", "password"} + assert d["user"] == "root" + assert d["password"] == "pw" + + +from vectordb_bench.backend.clients.volc_mysql.config import VolcMySQLHNSWConfig + + +def test_volc_mysql_hnsw_config_optional_fields_default_to_none(): + cfg = VolcMySQLHNSWConfig() + assert cfg.M is None + assert cfg.ef_search is None + assert cfg.ef_construction is None + assert cfg.quant_algorithm is None + assert cfg.quant_type is None + + +from vectordb_bench.backend.clients.volc_mysql.volc_mysql import _build_index_attrs_json + + +def test_build_index_attrs_json_drops_none_values(): + j = _build_index_attrs_json({ + "metric_type": "l2", + "M": 16, + "ef_construction": 128, + "quant_algorithm": None, + "quant_type": None, + }) + # Compact JSON, deterministic key order (insertion order from helper). + assert j == '{"algorithm":"hnsw","distance":"l2","m":16,"ef_construction":128}' + + +def test_build_index_attrs_json_includes_quantization_when_set(): + j = _build_index_attrs_json({ + "metric_type": "cosine", + "M": 32, + "ef_construction": 200, + "quant_algorithm": "SQ", + "quant_type": "8_bit", + }) + assert j == ( + '{"algorithm":"hnsw","distance":"cosine","m":32,"ef_construction":200,' + '"quant_algorithm":"SQ","quant_type":"8_bit"}' + ) + + +from unittest.mock import MagicMock + +from pydantic import SecretStr as _SecretStr + +from vectordb_bench.backend.filter import IntFilter, NonFilter + + +def _make_client_with_mock_cursor(): + """Build a VolcMySQL without touching a DB (drop_old=False does not connect), + then wire up the prebuilt SQL templates and a mock cursor so search_embedding + can be exercised offline. + """ + cfg = VolcMySQLConfig(password=_SecretStr("pw")).to_dict() + client = VolcMySQL(dim=4, db_config=cfg, db_case_config=VolcMySQLHNSWConfig(), drop_old=False) + client.select_sql = "SELECT id FROM t ORDER BY d LIMIT %s" + client.select_sql_with_filter = "SELECT id FROM t WHERE id >= %s ORDER BY d LIMIT %s" + client._binary_vec = False + cursor = MagicMock() + cursor.fetchall.return_value = [(1,), (2,)] + client.conn = MagicMock() + client.cursor = cursor + return client, cursor + + +def test_search_embedding_uses_unfiltered_sql_by_default(): + client, cursor = _make_client_with_mock_cursor() + client.search_embedding([0.1, 0.2, 0.3, 0.4], k=10) + sql, params = cursor.execute.call_args[0] + assert sql == client.select_sql + assert params == ("[0.1, 0.2, 0.3, 0.4]", 10) + + +def test_search_embedding_applies_numge_filter_after_prepare_filter(): + # Regression guard: runners apply filters via prepare_filter(), never by + # passing filters to search_embedding. Without prepare_filter wiring the + # client silently ran an unfiltered search and reported wrong recall. + client, cursor = _make_client_with_mock_cursor() + client.prepare_filter(IntFilter(int_value=500, filter_rate=0.99)) + client.search_embedding([0.1, 0.2, 0.3, 0.4], k=10) + sql, params = cursor.execute.call_args[0] + assert sql == client.select_sql_with_filter + assert params == (500, "[0.1, 0.2, 0.3, 0.4]", 10) + + +def test_prepare_filter_nonfilter_resets_to_unfiltered(): + client, cursor = _make_client_with_mock_cursor() + client.prepare_filter(IntFilter(int_value=500, filter_rate=0.99)) + client.prepare_filter(NonFilter()) + client.search_embedding([0.1, 0.2, 0.3, 0.4], k=10) + sql, _ = cursor.execute.call_args[0] + assert sql == client.select_sql + + +import tempfile + +import pytest + +from vectordb_bench.backend.clients.volc_mysql import volc_mysql as volc_mysql_module + + +def _make_uninitialized_client(): + cfg = VolcMySQLConfig(password=_SecretStr("pw")).to_dict() + return VolcMySQL(dim=4, db_config=cfg, db_case_config=VolcMySQLHNSWConfig(), drop_old=False) + + +def test_init_closes_connection_when_create_database_fails(monkeypatch): + """If admin_cursor.execute(CREATE DATABASE ...) raises inside init(), + the open connection and both cursors must still be closed and the + instance attributes reset to None. Prior to the fix this leaked because + setup ran outside the try/finally. + """ + client = _make_uninitialized_client() + conn = MagicMock(name="conn") + cursor = MagicMock(name="cursor") + admin_cursor = MagicMock(name="admin_cursor") + admin_cursor.execute.side_effect = RuntimeError("CREATE DATABASE failed") + monkeypatch.setattr(client, "_create_connection", lambda: (conn, cursor, admin_cursor)) + + with pytest.raises(RuntimeError, match="CREATE DATABASE failed"): + with client.init(): + pass + + cursor.close.assert_called_once() + admin_cursor.close.assert_called_once() + conn.close.assert_called_once() + assert client.conn is None + assert client.cursor is None + assert client.admin_cursor is None + + +def test_select_sql_does_not_hardcode_force_index(monkeypatch): + """Streaming/read_write cases search before optimize() creates idx_v, + so FORCE INDEX(idx_v) in the prebuilt SQL would fail every pre-optimize + search with "Key 'idx_v' doesn't exist in table". + """ + from vectordb_bench.backend.clients.api import MetricType + + cfg = VolcMySQLConfig(password=_SecretStr("pw")).to_dict() + case_cfg = VolcMySQLHNSWConfig(metric_type=MetricType.COSINE) + client = VolcMySQL(dim=4, db_config=cfg, db_case_config=case_cfg, drop_old=False) + conn = MagicMock() + cursor = MagicMock() + admin_cursor = MagicMock() + monkeypatch.setattr(client, "_create_connection", lambda: (conn, cursor, admin_cursor)) + monkeypatch.setattr(client, "_probe_binary_support", lambda: False) + + with client.init(): + assert "FORCE INDEX" not in client.select_sql + assert "FORCE INDEX" not in client.select_sql_with_filter + assert "idx_v" not in client.select_sql + assert "idx_v" not in client.select_sql_with_filter + + +def test_insert_embeddings_returns_actual_rowcount_on_partial_load(tmp_path, monkeypatch): + """LOAD DATA is already committed when the rowcount mismatch is detected; + reporting (0, err) corrupts the serial runner's already_insert_count and + sends its retry into a PK-collision wall. Return (actual, err) so the + runner slices past the already-loaded prefix on retry. + """ + client = _make_uninitialized_client() + client._batch_counter = 0 + client._binary_vec = False + cursor = MagicMock() + cursor.rowcount = 7 + client.cursor = cursor + client.conn = MagicMock() + monkeypatch.setattr(volc_mysql_module.tempfile, "gettempdir", lambda: str(tmp_path)) + + embeddings = [[float(i)] * 4 for i in range(10)] + metadata = list(range(10)) + actual, err = client.insert_embeddings(embeddings, metadata) + + assert actual == 7 + assert isinstance(err, RuntimeError) + assert "wrote 7 rows, expected 10" in str(err) + + +def test_insert_embeddings_returns_n_on_full_load(tmp_path, monkeypatch): + """Sanity: when rowcount matches, return (n, None). Guards against the + partial-load fix regressing the happy path. + """ + client = _make_uninitialized_client() + client._batch_counter = 0 + client._binary_vec = False + cursor = MagicMock() + cursor.rowcount = 10 + client.cursor = cursor + client.conn = MagicMock() + monkeypatch.setattr(volc_mysql_module.tempfile, "gettempdir", lambda: str(tmp_path)) + + embeddings = [[float(i)] * 4 for i in range(10)] + metadata = list(range(10)) + actual, err = client.insert_embeddings(embeddings, metadata) + assert actual == 10 + assert err is None diff --git a/tests/test_volc_mysql_encoder.py b/tests/test_volc_mysql_encoder.py new file mode 100644 index 000000000..fe656bf29 --- /dev/null +++ b/tests/test_volc_mysql_encoder.py @@ -0,0 +1,74 @@ +import struct + +from vectordb_bench.backend.clients.volc_mysql.volc_mysql import _encode_batch_to_tsv + + +def test_encode_batch_to_tsv_sorts_by_id_and_hex_encodes(tmp_path): + tsv = tmp_path / "batch.tsv" + _encode_batch_to_tsv( + metadata=[42, 7, 99], + embeddings=[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], + dim=2, + tsv_path=str(tsv), + ) + + lines = tsv.read_text().splitlines() + assert len(lines) == 3 + + # Sorted by id ascending: 7, 42, 99 + assert lines[0].startswith("7\t") + assert lines[1].startswith("42\t") + assert lines[2].startswith("99\t") + + # The (3.0, 4.0) embedding was paired with id=7 + _, hex_str = lines[0].split("\t") + floats = struct.unpack("<2f", bytes.fromhex(hex_str)) + assert floats == (3.0, 4.0) + + +def test_encode_batch_to_tsv_writes_dim_floats_per_row(tmp_path): + tsv = tmp_path / "batch.tsv" + _encode_batch_to_tsv( + metadata=[1], + embeddings=[[0.5] * 1536], + dim=1536, + tsv_path=str(tsv), + ) + + line = tsv.read_text().rstrip("\n") + id_str, hex_str = line.split("\t") + assert id_str == "1" + # 1536 floats * 4 bytes * 2 hex chars = 12288 chars + assert len(hex_str) == 12288 + assert bytes.fromhex(hex_str) == struct.pack("<1536f", *([0.5] * 1536)) + + +def test_encode_batch_to_tsv_empty_batch(tmp_path): + tsv = tmp_path / "batch.tsv" + _encode_batch_to_tsv(metadata=[], embeddings=[], dim=4, tsv_path=str(tsv)) + assert tsv.read_text() == "" + + +def test_encode_batch_to_tsv_text_mode_sorts_and_formats(tmp_path): + # binary=False (to_vector fallback): rows sorted by id, vector written as a + # delimiter-safe "[f1,f2,...]" literal with no tab/newline inside the field. + tsv = tmp_path / "batch.tsv" + _encode_batch_to_tsv( + metadata=[42, 7, 99], + embeddings=[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], + dim=2, + tsv_path=str(tsv), + binary=False, + ) + + lines = tsv.read_text().splitlines() + assert len(lines) == 3 + # Sorted by id ascending: 7, 42, 99; id=7 was paired with (3.0, 4.0) + assert lines[0] == "7\t[3.0,4.0]" + assert lines[1] == "42\t[1.0,2.0]" + assert lines[2] == "99\t[5.0,6.0]" + # field is delimiter-safe: no tab/newline inside the bracketed literal + for line in lines: + _id, vec = line.split("\t") + assert vec.startswith("[") and vec.endswith("]") + assert "\t" not in vec diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index 66cebce65..bf3d36312 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -63,6 +63,7 @@ class DB(Enum): PolarDB = "PolarDB" Pinot = "Pinot" SeekDB = "SeekDB" + VolcMySQL = "VolcMySQL" Adbpg = "AnalyticDB for PostgreSQL" @property @@ -270,6 +271,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return SeekDB + if self == DB.VolcMySQL: + from .volc_mysql.volc_mysql import VolcMySQL + + return VolcMySQL + if self == DB.Adbpg: from .adbpg.adbpg import Adbpg @@ -483,6 +489,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return SeekDBConfig + if self == DB.VolcMySQL: + from .volc_mysql.config import VolcMySQLConfig + + return VolcMySQLConfig + if self == DB.Adbpg: from .adbpg.config import AdbpgConfig @@ -694,6 +705,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return _seekdb_case_config.get(index_type) + if self == DB.VolcMySQL: + from .volc_mysql.config import _volcmysql_case_config + + return _volcmysql_case_config.get(index_type) + if self == DB.Adbpg: from .adbpg.config import AdbpgIndexConfig diff --git a/vectordb_bench/backend/clients/volc_mysql/cli.py b/vectordb_bench/backend/clients/volc_mysql/cli.py new file mode 100755 index 000000000..9ce563473 --- /dev/null +++ b/vectordb_bench/backend/clients/volc_mysql/cli.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class VolcMySQLTypedDict(CommonTypedDict): + user_name: Annotated[ + str, + click.option( + "--username", + type=str, + help="Username", + required=True, + ), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="Password", + required=True, + ), + ] + + host: Annotated[ + str, + click.option( + "--host", + type=str, + help="Db host", + default="127.0.0.1", + ), + ] + + port: Annotated[ + int, + click.option( + "--port", + type=int, + default=3306, + help="DB Port", + ), + ] + + +class VolcMySQLHNSWTypedDict(VolcMySQLTypedDict): + m: Annotated[ + int | None, + click.option( + "--m", + type=int, + help="M parameter in HNSW vector indexing", + required=False, + ), + ] + + ef_search: Annotated[ + int | None, + click.option( + "--ef-search", + type=int, + help="Session variable loose_hnsw_ef_search", + required=False, + ), + ] + + ef_construction: Annotated[ + int | None, + click.option( + "--ef-construction", + type=int, + help="HNSW ef_construction", + required=False, + ), + ] + + quant_algorithm: Annotated[ + str | None, + click.option( + "--quant-algorithm", + type=click.Choice(["NONE", "SQ", "PQ"]), + help="Quantization algorithm", + required=False, + ), + ] + + quant_type: Annotated[ + str | None, + click.option( + "--quant-type", + type=click.Choice(["16_bit", "8_bit", "4_bit", "binary"]), + help="Quantization type", + required=False, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(VolcMySQLHNSWTypedDict) +def VolcMySQLHNSW( + **parameters: Unpack[VolcMySQLHNSWTypedDict], +): + from .config import VolcMySQLConfig, VolcMySQLHNSWConfig + + run( + db=DB.VolcMySQL, + db_config=VolcMySQLConfig( + db_label=parameters["db_label"], + user_name=parameters["username"], + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + ), + db_case_config=VolcMySQLHNSWConfig( + M=parameters["m"], + ef_search=parameters["ef_search"], + ef_construction=parameters["ef_construction"], + quant_algorithm=parameters["quant_algorithm"], + quant_type=parameters["quant_type"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/volc_mysql/config.py b/vectordb_bench/backend/clients/volc_mysql/config.py new file mode 100755 index 000000000..aa4224848 --- /dev/null +++ b/vectordb_bench/backend/clients/volc_mysql/config.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +from typing import TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class VolcMySQLConfigDict(TypedDict): + """Keys used directly as kwargs in mysql.connector.connect; + names must match mysql-connector-python's API.""" + + user: str + password: str + host: str + port: int + + +class VolcMySQLConfig(DBConfig): + user_name: str = "root" + password: SecretStr + host: str = "127.0.0.1" + port: int = 3306 + + def to_dict(self) -> VolcMySQLConfigDict: + return { + "host": self.host, + "port": self.port, + "user": self.user_name, + "password": self.password.get_secret_value(), + } + + +class VolcMySQLIndexConfig(BaseModel): + """Base index config for VolcMySQL""" + + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2" + if self.metric_type == MetricType.COSINE: + return "cosine" + msg = f"Metric type {self.metric_type} is not supported!" + raise ValueError(msg) + + +class VolcMySQLHNSWConfig(VolcMySQLIndexConfig, DBCaseConfig): + M: int | None = None + ef_search: int | None = None + ef_construction: int | None = None + quant_algorithm: str | None = None + quant_type: str | None = None + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "M": self.M, + "ef_construction": self.ef_construction, + "quant_algorithm": self.quant_algorithm, + "quant_type": self.quant_type, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "ef_search": self.ef_search, + } + + +_volcmysql_case_config = { + IndexType.HNSW: VolcMySQLHNSWConfig, +} diff --git a/vectordb_bench/backend/clients/volc_mysql/volc_mysql.py b/vectordb_bench/backend/clients/volc_mysql/volc_mysql.py new file mode 100755 index 000000000..455286019 --- /dev/null +++ b/vectordb_bench/backend/clients/volc_mysql/volc_mysql.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +import json +import logging +import os +import struct +import tempfile +from contextlib import contextmanager +from pathlib import Path + +import mysql.connector as mysql +import numpy as np + +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import VectorDB +from .config import VolcMySQLConfigDict, VolcMySQLIndexConfig + +log = logging.getLogger(__name__) + + +def _encode_batch_to_tsv( + metadata: list[int], + embeddings: list[list[float]], + dim: int, + tsv_path: str, + *, + binary: bool = True, +) -> None: + """Sort (id, vector) pairs by id ascending and write them to ``tsv_path`` + for bulk ``LOAD DATA`` ingestion. + + ``binary=True`` (default) encodes each vector as hex of little-endian + float32 bytes -> ``\\t\\n``, consumed by + ``... (id, @h) SET v = UNHEX(@h)``. + + ``binary=False`` (to_vector fallback) writes the vector as a JSON-style + ``[f1,f2,...]`` literal -> ``\\t[..]\\n``, consumed by + ``... (id, @v) SET v = to_vector(@v)``. The literal contains no tab or + newline, so it stays delimiter-safe in the TSV transport. + """ + order = np.argsort(metadata) + with Path(tsv_path).open("w", buffering=1 << 20) as f: + if binary: + pack_fmt = f"<{dim}f" + for i in order: + hex_str = struct.pack(pack_fmt, *embeddings[i]).hex() + f.write(f"{metadata[i]}\t{hex_str}\n") + else: + for i in order: + vec_str = "[" + ",".join(repr(float(x)) for x in embeddings[i]) + "]" + f.write(f"{metadata[i]}\t{vec_str}\n") + + +def _build_index_attrs_json(index_param: dict) -> str: + """Build the SECONDARY_ENGINE_ATTRIBUTE JSON payload for CREATE VECTOR INDEX + from a case-config index_param dict. None values are dropped so the server + sees only the attributes the user explicitly set. + """ + attrs = { + "algorithm": "hnsw", + "distance": index_param.get("metric_type"), + "m": index_param.get("M"), + "ef_construction": index_param.get("ef_construction"), + "quant_algorithm": index_param.get("quant_algorithm"), + "quant_type": index_param.get("quant_type"), + } + attrs = {k: v for k, v in attrs.items() if v is not None} + return json.dumps(attrs, ensure_ascii=False, separators=(",", ":")) + + +class VolcMySQL(VectorDB): + # mysql.connector is not thread-safe; ConcurrentInsertRunner uses + # max_workers=1 when False. rate_runner branches on db.name to give + # each worker thread its own connection. + thread_safe: bool = False + + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + + def __init__( + self, + dim: int, + db_config: VolcMySQLConfigDict, + db_case_config: VolcMySQLIndexConfig, + collection_name: str = "vec_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "VolcMySQL" + self.db_config = db_config + self.case_config = db_case_config + self.db_name = "vectordbbench" + self.table_name = collection_name + self.dim = dim + + self.conn = None + self.cursor = None + self.admin_cursor = None + + # Active filter predicate selected by prepare_filter(). Runners call + # prepare_filter() (never search_embedding(filters=...)), so the + # filtered WHERE clause is chosen here, not per query. + self._filtered = False + + if drop_old: + self.conn, self.cursor, self.admin_cursor = self._create_connection() + try: + self._drop_db() + self._create_db_table(dim) + finally: + self.cursor.close() + self.admin_cursor.close() + self.conn.close() + self.conn = None + self.cursor = None + self.admin_cursor = None + + def _create_connection(self): + conn = mysql.connect( + host=self.db_config["host"], + user=self.db_config["user"], + port=self.db_config["port"], + password=self.db_config["password"], + allow_local_infile=True, + ) + cursor = conn.cursor() + admin_cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + assert admin_cursor is not None, "Admin cursor is not initialized" + + return conn, cursor, admin_cursor + + def _drop_db(self): + assert self.conn is not None, "Connection is not initialized" + assert self.admin_cursor is not None, "Cursor is not initialized" + log.info(f"{self.name} client drop db : {self.db_name}") + + # flush tables before dropping database to avoid some locking issue + self.admin_cursor.execute("FLUSH TABLES") + self.admin_cursor.execute(f"DROP DATABASE IF EXISTS {self.db_name}") + self.admin_cursor.execute("COMMIT") + self.admin_cursor.execute("FLUSH TABLES") + + def _create_db_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.admin_cursor is not None, "Cursor is not initialized" + + try: + log.info(f"{self.name} client create database : {self.db_name}") + self.admin_cursor.execute(f"CREATE DATABASE {self.db_name}") + + log.info(f"{self.name} client create table : {self.table_name}") + self.admin_cursor.execute(f"USE {self.db_name}") + + create_table_sql = f""" + CREATE TABLE {self.table_name} ( + id INT PRIMARY KEY, + v VECTOR({self.dim}) NOT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """ + self.admin_cursor.execute(create_table_sql) + self.admin_cursor.execute("COMMIT") + + except Exception as e: + log.warning(f"Failed to create table: {self.table_name} error: {e}") + raise e from None + + def _probe_binary_support(self) -> bool: + """Probe once whether the server accepts the raw little-endian float32 + binary VECTOR path: ``UNHEX()`` insert and the ``_binary`` query + introducer. Returns ``True`` only if both succeed; otherwise ``False`` + so :meth:`init` selects the ``to_vector()`` text path for insert and + search. + + Runs inside a session-local ``TEMPORARY TABLE`` with a throwaway + 4-dim vector, so it never touches the benchmark table and is + independent of the real vector dimension. A probe failure is expected + (not an error) on builds without binary VECTOR support and only flips + the path; it never raises. + """ + probe = [1.0, 2.0, 3.0, 4.0] + blob = struct.pack(f"<{len(probe)}f", *probe) + # Qualify with the (already-existing) benchmark schema: the client never + # issues USE, so the connection has no default database to host the temp table. + tmp = f"`{self.db_name}`._vdbb_binprobe" + cur = self.conn.cursor(buffered=True) + try: + cur.execute(f"CREATE TEMPORARY TABLE {tmp} (id INT PRIMARY KEY, v VECTOR(4))") + cur.execute(f"INSERT INTO {tmp} (id, v) VALUES (1, UNHEX(%s))", (blob.hex(),)) + cur.execute(f"SELECT id FROM {tmp} ORDER BY L2_DISTANCE(v, _binary %s) LIMIT 1", (blob,)) + cur.fetchall() + except mysql.Error as e: + log.warning(f"{self.name}: binary VECTOR path unsupported, falling back to to_vector() text path: {e}") + return False + else: + log.info(f"{self.name}: binary VECTOR path supported; using raw-binary insert + query") + return True + finally: + try: + cur.execute(f"DROP TEMPORARY TABLE IF EXISTS {tmp}") + except mysql.Error: + log.debug("Failed to drop binary-probe temp table", exc_info=True) + cur.close() + + @contextmanager + def init(self): + """create and destory connections to database. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + self.conn, self.cursor, self.admin_cursor = self._create_connection() + try: + # The binary probe creates a TEMPORARY TABLE qualified with self.db_name. + # When drop_old=False on a fresh server, the schema may not exist yet -- + # ensure it does so the probe lands on a real database and doesn't + # spuriously fall back to the text path. + self.admin_cursor.execute(f"CREATE DATABASE IF NOT EXISTS {self.db_name}") + + # Load-phase session tuning. SESSION-scoped; resets when the + # connection closes. No GLOBAL or instance-level changes. + try: + self.admin_cursor.execute("SET SESSION unique_checks = 0") + self.admin_cursor.execute("SET SESSION foreign_key_checks = 0") + except mysql.Error as e: + log.warning(f"Could not apply load-phase session tuning: {e}") + + # Per-batch TSV file numbering for LOAD DATA bulk load. + self._batch_counter = 0 + + index_param = self.case_config.index_param() + search_param = self.case_config.search_param() + + if search_param.get("ef_search") is not None: + try: + self.admin_cursor.execute(f"SET loose_hnsw_ef_search = {int(search_param['ef_search'])}") + self.conn.commit() + except mysql.Error: + log.warning( + f"Could not set loose_hnsw_ef_search = {int(search_param['ef_search'])}, " + "using server defaults" + ) + + # prebuild SQL strings + dist_func = "L2_DISTANCE" if index_param["metric_type"] == "l2" else "COSINE_DISTANCE" + # Raw-binary VECTOR path: send float32 vectors as little-endian bytes and let the + # server consume them directly -- UNHEX(@h) on insert, the `_binary` introducer on + # query -- with no to_vector() text parse and no Python str() formatting (+71% c80 + # QPS, recall identical). `_binary ` stays constant-foldable so the HNSW + # index scan is preserved (UNHEX() is NOT, hence hex is used only on the load path). + # + # Not every MySQL-compatible build accepts the binary path, so we AUTO-PROBE it once + # per connection (see _probe_binary_support) and fall back to the to_vector() text + # path -- for BOTH insert and query -- when it is unsupported. VDB_BINARY_VEC overrides + # the probe: "1" forces binary (skip probe), "0" forces the to_vector() text path. + force = os.environ.get("VDB_BINARY_VEC") + if force == "1": + self._binary_vec = True + elif force == "0": + self._binary_vec = False + else: + self._binary_vec = self._probe_binary_support() + vec_expr = "_binary %s" if self._binary_vec else "to_vector(%s)" + # No FORCE INDEX hint: optimize() creates idx_v, but read_write streaming + # cases run search before optimize, so hinting a not-yet-existing index + # would error every pre-optimize search. The optimizer picks idx_v once + # it exists; before that, the seq scan is the only valid plan anyway. + self.select_sql = ( + f"SELECT id FROM {self.db_name}.{self.table_name} ORDER BY {dist_func}(v, {vec_expr}) LIMIT %s" + ) + self.select_sql_with_filter = ( + f"SELECT id FROM {self.db_name}.{self.table_name} WHERE id >= %s ORDER BY " + f"{dist_func}(v, {vec_expr}) LIMIT %s" + ) + + yield + finally: + try: + if self.cursor is not None: + self.cursor.close() + if self.admin_cursor is not None: + self.admin_cursor.close() + if self.conn is not None: + self.conn.close() + finally: + self.cursor = None + self.admin_cursor = None + self.conn = None + + def optimize(self, data_size: int | None = None) -> None: + assert self.conn is not None, "Connection is not initialized" + assert self.admin_cursor is not None, "Admin cursor is not initialized" + + try: + log.info(f"{self.name} client create index : {self.table_name}") + self.admin_cursor.execute(f"USE {self.db_name}") + + # Build vector index attributes + index_param = self.case_config.index_param() + + attrs_json = _build_index_attrs_json(index_param) + + sql = f"CREATE VECTOR INDEX idx_v ON {self.table_name}(v) SECONDARY_ENGINE_ATTRIBUTE='{attrs_json}'" + log.info(f"{self.name} client execute create index: {sql}") + self.admin_cursor.execute(sql) + self.admin_cursor.execute("COMMIT") + except Exception as e: + log.warning(f"Failed to create index on {self.table_name}, error: {e}") + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception | None]: + """Insert a batch via LOAD DATA LOCAL INFILE with sorted PK. Uses the + hex-binary ``UNHEX`` path when the server supports it (probed in + :meth:`init`), otherwise the ``to_vector()`` text path. Requires + self.init() context. + """ + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + n = len(metadata) + if n == 0: + return 0, None + + tsv_path = Path(tempfile.gettempdir()) / ( + f"vdbb_volc_{self.db_name}_{self.table_name}_{os.getpid()}_{self._batch_counter}.tsv" + ) + self._batch_counter += 1 + + try: + _encode_batch_to_tsv(metadata, embeddings, self.dim, str(tsv_path), binary=self._binary_vec) + + tsv_literal = str(tsv_path).replace("'", "''") + set_clause = "(id, @h) SET v = UNHEX(@h)" if self._binary_vec else "(id, @v) SET v = to_vector(@v)" + load_sql = ( + f"LOAD DATA LOCAL INFILE '{tsv_literal}' " + f"INTO TABLE `{self.db_name}`.`{self.table_name}` " + "FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' " + f"{set_clause}" + ) + self.cursor.execute(load_sql) + self.conn.commit() + except Exception as e: + log.warning(f"Failed to LOAD DATA into Vector table ({self.table_name}), error: {e}") + return 0, e + else: + actual = self.cursor.rowcount + if actual != n: + # Return the real count so the runner's already_insert_count tracks + # what's actually on disk; on retry it will slice past the + # already-loaded prefix instead of duplicate-PK looping on the + # same batch. self.conn.commit() ran above, so those rows are + # durable. + msg = f"LOAD DATA wrote {actual} rows, expected {n}" + return actual, RuntimeError(msg) + return n, None + finally: + if tsv_path.exists(): + try: + tsv_path.unlink() + except OSError as e: + log.warning(f"Failed to unlink staging TSV {tsv_path}: {e}") + + def prepare_filter(self, filters: Filter): + """Select the filtered vs unfiltered search template for the case. + + Runners apply filters via this hook (called once per case inside the + init() context), not by passing ``filters`` to search_embedding. Store + the predicate value so search_embedding can bind it. + """ + if filters.type == FilterOp.NonFilter: + self._filtered = False + elif filters.type == FilterOp.NumGE: + self._filtered = True + self._filter_value = filters.int_value + else: + msg = f"Unsupported filter for VolcMySQL: {filters}" + raise ValueError(msg) + + def search_embedding( + self, + query: list[float], + k: int = 100, + **kwargs, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + # Binary path: raw LE float32 bytes (C-level struct.pack, ~us) consumed + # by the server as a binary vector via `_binary %s`; avoids the per-query + # str() formatting of 1536 floats and the server-side strtof text parse. + query_param = struct.pack(f"<{len(query)}f", *query) if self._binary_vec else str(query) + if self._filtered: + self.cursor.execute(self.select_sql_with_filter, (self._filter_value, query_param, k)) + else: + self.cursor.execute(self.select_sql, (query_param, k)) + return [row[0] for row in self.cursor.fetchall()] + + except mysql.Error: + log.exception("Failed to execute search query") + raise diff --git a/vectordb_bench/backend/runner/rate_runner.py b/vectordb_bench/backend/runner/rate_runner.py index 2387abfcb..91d0bb3ee 100644 --- a/vectordb_bench/backend/runner/rate_runner.py +++ b/vectordb_bench/backend/runner/rate_runner.py @@ -76,6 +76,18 @@ def _insert_embeddings(db: api.VectorDB, emb: list[list[float]], metadata: list[ log.debug("Failed to reset SeekDB connection on thread-local copy", exc_info=True) with db_copy.init(): _insert_embeddings(db_copy, emb, metadata, retry_idx=0) + elif db.name == "VolcMySQL": + # mysql.connector is not thread-safe; do not share one connection across workers. + # deepcopy() fails on an open conn (socket is not picklable / not copy-safe in spawn workers). + db_copy = copy(db) + try: + db_copy.conn = None + db_copy.cursor = None + db_copy.admin_cursor = None + except Exception: + log.debug("Failed to reset VolcMySQL connection on thread-local copy", exc_info=True) + with db_copy.init(): + _insert_embeddings(db_copy, emb, metadata, retry_idx=0) else: _insert_embeddings(db, emb, metadata, retry_idx=0) diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 7d6aa0031..e0cb98652 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -49,6 +49,7 @@ from ..backend.clients.turbopuffer.cli import TurboPuffer, TurboPufferUnpin from ..backend.clients.vectorchord.cli import VectorChordGraph, VectorChordRQ from ..backend.clients.vespa.cli import Vespa +from ..backend.clients.volc_mysql.cli import VolcMySQLHNSW from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex from ..backend.clients.zvec.cli import Zvec @@ -111,6 +112,7 @@ cli.add_command(PolarDBHNSWPQ) cli.add_command(PolarDBHNSWSQ) cli.add_command(SeekDBHNSW) +cli.add_command(VolcMySQLHNSW) if __name__ == "__main__": From cda6227206fe9ffaa742fe366de1fcac224c8018 Mon Sep 17 00:00:00 2001 From: Zijun Yang <37757768+zpatronus@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:57:39 +0800 Subject: [PATCH 43/49] fix(hologres): enable thread-safe concurrent inserts (#811) This addresses the functional failure in the Hologres client introduced by 7e251b6, which added the concurrent insert runner without setting thread_safe=False for Hologres. This implements per-thread connection management via threading.local() to enable safe parallel loading. --- .../backend/clients/hologres/hologres.py | 196 +++++++++++------- 1 file changed, 123 insertions(+), 73 deletions(-) diff --git a/vectordb_bench/backend/clients/hologres/hologres.py b/vectordb_bench/backend/clients/hologres/hologres.py index f5cbf698a..c396e23b0 100644 --- a/vectordb_bench/backend/clients/hologres/hologres.py +++ b/vectordb_bench/backend/clients/hologres/hologres.py @@ -3,6 +3,7 @@ import json import logging import struct +import threading from collections.abc import Generator from contextlib import contextmanager from io import StringIO @@ -51,8 +52,7 @@ def dump(self, obj: HoloFloat4Array) -> bytes: class Hologres(VectorDB): """Use psycopg instructions""" - conn: psycopg.Connection[Any] | None = None - cursor: psycopg.Cursor[Any] | None = None + thread_safe: bool = True # each thread gets its own connection via threading.local() _tg_name: str = "vdb_bench_tg_1" @@ -74,13 +74,20 @@ def __init__( self._primary_field = "id" self._vector_field = "embedding" - # construct basic units - self.conn, self.cursor = self._create_connection(**self.db_config) + # Thread-local storage for per-thread connections (thread-safe concurrent inserts) + self._local = threading.local() + self._all_conns: list[psycopg.Connection[Any]] = [] + self._conns_lock = threading.Lock() + + # Temporary connection for setup, closed at end of __init__ + conn, cursor = self._create_connection(**self.db_config) + self._local.conn = conn + self._local.cursor = cursor # create vector extension if self.case_config.is_proxima(): - self.cursor.execute("CREATE EXTENSION proxima;") - self.conn.commit() + cursor.execute("CREATE EXTENSION proxima;") + conn.commit() log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") if not any( @@ -102,10 +109,24 @@ def __init__( if self.case_config.create_index_before_load: self._create_index() - self.cursor.close() - self.conn.close() - self.cursor = None - self.conn = None + cursor.close() + conn.close() + self._local.cursor = None + self._local.conn = None + + def __getstate__(self): + # Exclude unpicklable threading objects; recreated in __setstate__ + state = self.__dict__.copy() + state.pop("_local", None) + state.pop("_conns_lock", None) + state.pop("_all_conns", None) + return state + + def __setstate__(self, state: dict) -> None: + self.__dict__.update(state) + self._local = threading.local() + self._conns_lock = threading.Lock() + self._all_conns = [] @staticmethod def _create_connection(**kwargs) -> tuple[Connection, Cursor]: @@ -118,18 +139,50 @@ def _create_connection(**kwargs) -> tuple[Connection, Cursor]: return conn, cursor + def _get_conn(self) -> Connection: + """Return this thread's connection, creating one if needed.""" + conn = getattr(self._local, "conn", None) + if conn is None or conn.closed: + conn, cursor = self._create_connection(**self.db_config) + self._local.conn = conn + self._local.cursor = cursor + self._set_search_guc_on(conn, cursor) + with self._conns_lock: + self._all_conns.append(conn) + return conn + + def _get_cursor(self) -> Cursor: + """Return this thread's cursor, creating connection if needed.""" + cursor = getattr(self._local, "cursor", None) + if cursor is None or cursor.closed: + self._get_conn() # creates both conn and cursor + cursor = self._local.cursor + return cursor + + def _set_search_guc_on(self, conn: Connection, cursor: Cursor) -> None: + """Set session-level search GUC on the given connection.""" + sql_guc = sql.SQL(f"SET hg_vector_ef_search = {self.case_config.ef_search};") + log.info(f"{self.name} client set search guc: {sql_guc.as_string()}") + cursor.execute(sql_guc) + conn.commit() + + def _set_search_guc(self) -> None: + """Set session-level search GUC on this thread's connection.""" + self._set_search_guc_on(self._get_conn(), self._get_cursor()) + @contextmanager def init(self) -> Generator[None, None, None]: - """ - Examples: - >>> with self.init(): - >>> self.insert_embeddings() - >>> self.search_embedding() - """ + """Open connection for calling thread, prepare for operations. - self.conn, self.cursor = self._create_connection(**self.db_config) - - self._set_search_guc() + Worker threads lazily create their own connections via _get_conn(). + On exit, closes all thread-local connections. + """ + conn, cursor = self._create_connection(**self.db_config) + self._local.conn = conn + self._local.cursor = cursor + self._set_search_guc_on(conn, cursor) + with self._conns_lock: + self._all_conns.append(conn) self._search_query_no_filter = sql.SQL(""" SELECT id @@ -159,35 +212,30 @@ def init(self) -> Generator[None, None, None]: try: yield finally: - self.cursor.close() - self.conn.close() - self.cursor = None - self.conn = None - - def _set_search_guc(self): - assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" - - sql_guc = sql.SQL(f"SET hg_vector_ef_search = {self.case_config.ef_search};") - log.info(f"{self.name} client set search guc: {sql_guc.as_string()}") - self.cursor.execute(sql_guc) - self.conn.commit() + with self._conns_lock: + conns_to_close = list(self._all_conns) + self._all_conns.clear() + for c in conns_to_close: + if not c.closed: + c.close() + self._local.conn = None + self._local.cursor = None def _drop_table(self): - assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" + conn = self._get_conn() + cursor = self._get_cursor() log.info(f"{self.name} client drop table : {self.table_name}") - self.cursor.execute( + cursor.execute( sql.SQL("DROP TABLE IF EXISTS {table_name};").format( table_name=sql.Identifier(self.table_name), ), ) - self.conn.commit() + conn.commit() try: log.info(f"{self.name} client purge table recycle bin: {self.table_name}") - self.cursor.execute( + cursor.execute( sql.SQL("purge TABLE {table_name};").format( table_name=sql.Identifier(self.table_name), ), @@ -195,23 +243,23 @@ def _drop_table(self): except Exception as e: log.info(f"{self.name} client purge table {self.table_name} recycle bin failed, error: {e}, ignore.") finally: - self.conn.commit() + conn.commit() try: log.info(f"{self.name} client drop table group : {self._tg_name}") - self.cursor.execute(sql.SQL(f"CALL HG_DROP_TABLE_GROUP('{self._tg_name}');")) + cursor.execute(sql.SQL(f"CALL HG_DROP_TABLE_GROUP('{self._tg_name}');")) except Exception as e: log.info(f"{self.name} client drop table group : {self._tg_name} failed, error: {e}, ignore.") finally: - self.conn.commit() + conn.commit() try: log.info(f"{self.name} client free cache") - self.cursor.execute("select hg_admin_command('freecache');") + cursor.execute("select hg_admin_command('freecache');") except Exception as e: log.info(f"{self.name} client free cache failed, error: {e}, ignore.") finally: - self.conn.commit() + conn.commit() def optimize(self, data_size: int | None = None): if self.case_config.create_index_after_load: @@ -220,12 +268,13 @@ def optimize(self, data_size: int | None = None): self._analyze() def _vacuum(self): + conn = self._get_conn() log.info(f"{self.name} client vacuum table : {self.table_name}") try: # VACUUM cannot run inside a transaction block # it's better to new a connection - self.conn.autocommit = True - with self.conn.cursor() as cursor: + conn.autocommit = True + with conn.cursor() as cursor: cursor.execute( sql.SQL(""" VACUUM {table_name}; @@ -238,16 +287,18 @@ def _vacuum(self): log.warning(f"Failed to vacuum table: {self.table_name} error: {e}") raise e from None finally: - self.conn.autocommit = True + conn.autocommit = True def _analyze(self): + cursor = self._get_cursor() log.info(f"{self.name} client analyze table : {self.table_name}") - self.cursor.execute(sql.SQL(f"ANALYZE {self.table_name};")) + cursor.execute(sql.SQL(f"ANALYZE {self.table_name};")) log.info(f"{self.name} client analyze table : {self.table_name} done") def _full_compact(self): + cursor = self._get_cursor() log.info(f"{self.name} client full compact table : {self.table_name}") - self.cursor.execute( + cursor.execute( sql.SQL(""" SELECT hologres.hg_full_compact_table( '{table_name}', @@ -261,8 +312,8 @@ def _full_compact(self): log.info(f"{self.name} client full compact table : {self.table_name} done") def _create_index(self): - assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" + conn = self._get_conn() + cursor = self._get_cursor() sql_index = sql.SQL(""" CALL set_table_property ('{table_name}', 'vectors', '{{ @@ -281,15 +332,15 @@ def _create_index(self): log.info(f"{self.name} client create index on table : {self.table_name}, with sql: {sql_index.as_string()}") try: - self.cursor.execute(sql_index) - self.conn.commit() + cursor.execute(sql_index) + conn.commit() except Exception as e: log.warning(f"Failed to create index on table: {self.table_name} error: {e}") raise e from None def _set_replica_count(self, replica_count: int = 2): - assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" + conn = self._get_conn() + cursor = self._get_cursor() try: # non-warehouse mode by default @@ -300,13 +351,13 @@ def _set_replica_count(self, replica_count: int = 2): # check warehouse mode sql_check = sql.SQL("select count(*) from hologres.hg_warehouses;") log.info(f"check warehouse mode with sql: {sql_check}") - self.cursor.execute(sql_check) - result_check = self.cursor.fetchone()[0] + cursor.execute(sql_check) + result_check = cursor.fetchone()[0] if result_check > 0: # get warehouse name sql_get_warehouse_name = sql.SQL("select current_warehouse();") log.info(f"get warehouse name with sql: {sql_get_warehouse_name}") - self.cursor.execute(sql_get_warehouse_name) + cursor.execute(sql_get_warehouse_name) sql_tg_replica = sql.SQL(""" CALL hg_table_group_set_warehouse_replica_count ( '{dbname}.{tg_name}', @@ -315,29 +366,29 @@ def _set_replica_count(self, replica_count: int = 2): ); """).format( tg_name=sql.SQL(self._tg_name), - warehouse_name=sql.SQL(self.cursor.fetchone()[0]), + warehouse_name=sql.SQL(cursor.fetchone()[0]), dbname=sql.SQL(self.db_config["dbname"]), replica_count=replica_count, ) log.info(f"{self.name} client set table group replica: {self._tg_name}, with sql: {sql_tg_replica}") - self.cursor.execute(sql_tg_replica) + cursor.execute(sql_tg_replica) except Exception as e: log.warning(f"Failed to set replica count, error: {e}, ignore") finally: - self.conn.commit() + conn.commit() def _create_table(self, dim: int): - assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" + conn = self._get_conn() + cursor = self._get_cursor() sql_tg = sql.SQL(f"CALL HG_CREATE_TABLE_GROUP ('{self._tg_name}', 1);") log.info(f"{self.name} client create table group : {self._tg_name}, with sql: {sql_tg}") try: - self.cursor.execute(sql_tg) + cursor.execute(sql_tg) except Exception as e: log.warning(f"Failed to create table group : {self._tg_name} error: {e}, ignore") finally: - self.conn.commit() + conn.commit() self._set_replica_count(replica_count=2) @@ -354,8 +405,8 @@ def _create_table(self, dim: int): ) log.info(f"{self.name} client create table : {self.table_name}, with sql: {sql_table.as_string()}") try: - self.cursor.execute(sql_table) - self.conn.commit() + cursor.execute(sql_table) + conn.commit() except Exception as e: log.warning(f"Failed to create table : {self.table_name} error: {e}") raise e from None @@ -366,8 +417,8 @@ def insert_embeddings( metadata: list[int], **kwargs: Any, ) -> tuple[int, Exception | None]: - assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" + conn = self._get_conn() + cursor = self._get_cursor() try: buffer = StringIO() @@ -375,11 +426,11 @@ def insert_embeddings( buffer.write("%d\t%s\n" % (metadata[i], "{" + ",".join("%f" % x for x in embeddings[i]) + "}")) buffer.seek(0) - with self.cursor.copy( + with cursor.copy( sql.SQL("COPY {table_name} FROM STDIN").format(table_name=sql.Identifier(self.table_name)) ) as copy: copy.write(buffer.getvalue()) - self.conn.commit() + conn.commit() return len(metadata), None except Exception as e: @@ -393,17 +444,16 @@ def search_embedding( filters: dict | None = None, timeout: int | None = None, ) -> list[int]: - assert self.conn is not None, "Connection is not initialized" - assert self.cursor is not None, "Cursor is not initialized" + cursor = self._get_cursor() ge = filters.get("id") if filters else None q = HoloFloat4Array(query) if ge is not None: params = (ge, q, k) - result = self.cursor.execute(self._search_query_with_filter, params, prepare=True, binary=True) + result = cursor.execute(self._search_query_with_filter, params, prepare=True, binary=True) else: params = (q, k) - result = self.cursor.execute(self._search_query_no_filter, params, prepare=True, binary=True) + result = cursor.execute(self._search_query_no_filter, params, prepare=True, binary=True) return [int(i[0]) for i in result.fetchall()] From 75628b4581d4778d348c68c7c170f17044354af8 Mon Sep 17 00:00:00 2001 From: Yuanzhan Gao Date: Thu, 30 Jul 2026 11:05:05 +0800 Subject: [PATCH 44/49] feat(cli): add common note support (#818) Signed-off-by: jamesgao-jpg --- README.md | 11 ++ tests/test_cli_note.py | 125 ++++++++++++++++++ vectordb_bench/backend/clients/endee/cli.py | 2 + .../backend/clients/pinecone/cli.py | 5 - vectordb_bench/cli/cli.py | 44 ++++++ 5 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 tests/test_cli_note.py diff --git a/README.md b/README.md index 847783545..f300f20d4 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,14 @@ Commands: ``` To list the options for each command, execute `vectordbbench [command] --help` +Use `--note` or `--note-file` to preserve deployment, resource, client, network, and constraint context in each result JSON under `task_config.db_config.note`. The options are mutually exclusive. Prefer `--note-file` for structured or multiline context, and never include credentials, tokens, or sensitive connection details. + +```shell +vectordbbench zillizautoindex \ + --note-file ./run-context.json \ + +``` + ```text $ vectordbbench pgvectorhnsw --help Usage: vectordbbench pgvectorhnsw [OPTIONS] @@ -118,6 +126,9 @@ Options: Case type --db-label TEXT Db label, default: date in ISO format [default: 2024-05-20T20:26:31.113290] + --note TEXT Run context stored with each result + [default: ""] + --note-file FILE Read run context from a UTF-8 text file --dry-run Print just the configuration and exit without running the tasks --k INTEGER K value for number of nearest neighbors to diff --git a/tests/test_cli_note.py b/tests/test_cli_note.py new file mode 100644 index 000000000..1c601686d --- /dev/null +++ b/tests/test_cli_note.py @@ -0,0 +1,125 @@ +import time +from pathlib import Path + +from click.testing import CliRunner +from pytest import MonkeyPatch + +from vectordb_bench.backend.clients.endee import cli as endee_cli +from vectordb_bench.backend.clients.test import cli as test_cli +from vectordb_bench.cli import cli as common_cli + + +def invoke_test_command(monkeypatch: MonkeyPatch, args: list[str]): + captured = {} + + def fake_run(tasks, task_label): + captured["task"] = tasks[0] + captured["task_label"] = task_label + + monkeypatch.setattr(common_cli.benchmark_runner, "run", fake_run) + monkeypatch.setattr(common_cli.benchmark_runner, "has_running", lambda: False) + result = CliRunner().invoke(test_cli.Test, args) + return result, captured + + +def invoke_endee_command(monkeypatch: MonkeyPatch, args: list[str]): + captured = {} + + def fake_run(tasks, task_label): + captured["task"] = tasks[0] + captured["task_label"] = task_label + + monkeypatch.setattr(endee_cli.benchmark_runner, "run", fake_run) + monkeypatch.setattr(endee_cli.benchmark_runner, "has_running", lambda: False) + monkeypatch.setattr(time, "sleep", lambda _: None) + result = CliRunner().invoke( + endee_cli.Endee, + ["--token", "secret", "--index-name", "test-index", *args], + ) + return result, captured + + +def test_common_cli_exposes_note_options() -> None: + result = CliRunner().invoke(test_cli.Test, ["--help"]) + + assert result.exit_code == 0, result.output + assert "--note TEXT" in result.output + assert "--note-file FILE" in result.output + + +def test_common_cli_stores_inline_note_in_task_config(monkeypatch: MonkeyPatch) -> None: + note = '{"schema":"vdbbench-context/v1","deployment":"local"}' + + result, captured = invoke_test_command(monkeypatch, ["--note", note]) + + assert result.exit_code == 0, result.output + assert captured["task"].db_config.note == note + + +def test_common_cli_reads_note_file_into_task_config(monkeypatch: MonkeyPatch, tmp_path: Path) -> None: + note = '{\n "schema": "vdbbench-context/v1",\n "deployment": "managed"\n}' + note_file = tmp_path / "run-context.json" + note_file.write_text(note + "\n", encoding="utf-8") + + result, captured = invoke_test_command(monkeypatch, ["--note-file", str(note_file)]) + + assert result.exit_code == 0, result.output + assert captured["task"].db_config.note == note + + +def test_endee_cli_reads_note_file_into_task_config(monkeypatch: MonkeyPatch, tmp_path: Path) -> None: + note = '{"schema":"vdbbench-context/v1","deployment":"endee-local"}' + note_file = tmp_path / "endee-run-context.json" + note_file.write_text(note, encoding="utf-8") + + result, captured = invoke_endee_command(monkeypatch, ["--note-file", str(note_file)]) + + assert result.exit_code == 0, result.output + assert captured["task"].db_config.note == note + + +def test_common_cli_rejects_inline_note_with_note_file(monkeypatch: MonkeyPatch, tmp_path: Path) -> None: + note_file = tmp_path / "run-context.json" + note_file.write_text("context", encoding="utf-8") + + result, _ = invoke_test_command( + monkeypatch, + ["--note", "inline", "--note-file", str(note_file)], + ) + + assert result.exit_code != 0 + assert "--note and --note-file cannot be used together" in result.output + + +def test_common_cli_rejects_empty_inline_note_with_note_file( + monkeypatch: MonkeyPatch, + tmp_path: Path, +) -> None: + note_file = tmp_path / "run-context.json" + note_file.write_text("context", encoding="utf-8") + + result, _ = invoke_test_command(monkeypatch, ["--note", "", "--note-file", str(note_file)]) + + assert result.exit_code != 0 + assert "--note and --note-file cannot be used together" in result.output + + +def test_common_cli_rejects_empty_note_file(monkeypatch: MonkeyPatch, tmp_path: Path) -> None: + note_file = tmp_path / "empty.txt" + note_file.write_text("", encoding="utf-8") + + result, _ = invoke_test_command(monkeypatch, ["--note-file", str(note_file)]) + + assert result.exit_code != 0 + assert "Note file is empty" in result.output + + +def test_common_cli_rejects_non_utf8_note_file(monkeypatch: MonkeyPatch, tmp_path: Path) -> None: + note_file = tmp_path / "invalid.txt" + with note_file.open("wb") as output: + output.write(b"\xff") + + result, _ = invoke_test_command(monkeypatch, ["--note-file", str(note_file)]) + + assert result.exit_code != 0 + assert "Note file is not valid UTF-8" in result.output diff --git a/vectordb_bench/backend/clients/endee/cli.py b/vectordb_bench/backend/clients/endee/cli.py index fb6e3c37c..5c9f0c9d3 100644 --- a/vectordb_bench/backend/clients/endee/cli.py +++ b/vectordb_bench/backend/clients/endee/cli.py @@ -10,6 +10,7 @@ click_parameter_decorators_from_typed_dict, get_custom_case_config, parse_task_stages, + resolve_db_note, ) from vectordb_bench.models import ( CaseConfig, @@ -94,6 +95,7 @@ def Endee(**parameters): # Filter out None values before creating config params_for_nd = {k: v for k, v in parameters.items() if v is not None} + params_for_nd["note"] = resolve_db_note(parameters["note"], parameters["note_file"]) db_config = EndeeConfig(**params_for_nd) custom_case_config = get_custom_case_config(parameters) diff --git a/vectordb_bench/backend/clients/pinecone/cli.py b/vectordb_bench/backend/clients/pinecone/cli.py index 03e1b5f1a..0529fc691 100644 --- a/vectordb_bench/backend/clients/pinecone/cli.py +++ b/vectordb_bench/backend/clients/pinecone/cli.py @@ -26,10 +26,6 @@ class PineconeTypedDict(TypedDict): str, click.option("--version", type=str, help="Database version", default="", show_default=True), ] - note: Annotated[ - str, - click.option("--note", type=str, help="Additional notes", default="", show_default=True), - ] class PineconeIndexTypedDict(CommonTypedDict, PineconeTypedDict): ... @@ -45,7 +41,6 @@ def Pinecone(**parameters: Unpack[PineconeIndexTypedDict]): db_config=PineconeConfig( db_label=parameters["db_label"], version=parameters["version"], - note=parameters["note"], api_key=SecretStr(parameters["api_key"]), index_name=parameters["index_name"], ), diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index a657effe9..d72340ac6 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -14,6 +14,7 @@ ) import click +from click.core import ParameterSource from yaml import load from .. import config @@ -66,6 +67,26 @@ def click_get_defaults_from_file(ctx, param, value): # noqa: ANN001, ARG001 return value +def resolve_db_note(note: str, note_file: Path | None) -> str: + ctx = click.get_current_context() + note_source = ctx.get_parameter_source("note") + note_file_source = ctx.get_parameter_source("note_file") + note_supplied = note_source not in {None, ParameterSource.DEFAULT} + note_file_supplied = note_file_source not in {None, ParameterSource.DEFAULT} + + if note_supplied and note_file_supplied: + raise click.UsageError("--note and --note-file cannot be used together") + if note_file is None: + return note + try: + content = note_file.read_text(encoding="utf-8").rstrip("\r\n") + except UnicodeDecodeError as e: + raise click.BadParameter("Note file is not valid UTF-8", param_hint="--note-file") from e + if not content.strip(): + raise click.BadParameter("Note file is empty", param_hint="--note-file") + return content + + def click_parameter_decorators_from_typed_dict( typed_dict: type, ) -> Callable[[click.decorators.FC], click.decorators.FC]: @@ -360,6 +381,25 @@ class CommonTypedDict(TypedDict): default=datetime.now().isoformat(), ), ] + note: Annotated[ + str, + click.option( + "--note", + type=str, + help="Run context stored with each result", + default="", + show_default=True, + ), + ] + note_file: Annotated[ + Path | None, + click.option( + "--note-file", + type=click.Path(exists=True, dir_okay=False, readable=True, path_type=Path), + help="Read run context from a UTF-8 text file", + default=None, + ), + ] dry_run: Annotated[ bool, click.option( @@ -829,6 +869,10 @@ def run( **parameters: expects keys from CommonTypedDict """ + db_config = db_config.model_copy( + update={"note": resolve_db_note(parameters["note"], parameters["note_file"])}, + ) + task = TaskConfig( db=db, db_config=db_config, From 5da3ad17dc7ee2bc110db4ffaf066d10c425b358 Mon Sep 17 00:00:00 2001 From: Yuanzhan Gao Date: Tue, 11 Aug 2026 10:55:02 +0800 Subject: [PATCH 45/49] feat: FTS: add OSS OpenSearch backend, semantic FTS recall, filtered FTS cases (#815) * Fix ES FTS CLI shard replica config Signed-off-by: jamesgao-jpg * Expose Milvus collection name in CLI Signed-off-by: jamesgao-jpg * Add semantic FTS recall metrics Signed-off-by: jamesgao-jpg * Add CLI BM25 controls for FTS Signed-off-by: jamesgao-jpg * Preserve Zilliz FTS search level from CLI Signed-off-by: jamesgao-jpg * Remove stale FTS math ground truth plumbing Signed-off-by: jamesgao-jpg * Add OSS OpenSearch FTS support Signed-off-by: jamesgao-jpg * Add OSS OpenSearch index name option Signed-off-by: jamesgao-jpg * Update FTS frontend semantic metrics Signed-off-by: jamesgao-jpg * Add FTS filtered dataset preparation Signed-off-by: jamesgao-jpg * Wire FTS filter data through runners Signed-off-by: jamesgao-jpg * Add FTS backend filter support Signed-off-by: jamesgao-jpg * Add FTS filter case controls Signed-off-by: jamesgao-jpg * fix: split fts filter recall from concurrency queries Signed-off-by: jamesgao-jpg * feat: add fts filtered search results Signed-off-by: jamesgao-jpg * fix: declare turbopuffer fts filter field schema Signed-off-by: jamesgao-jpg * refactor: defer filtered FTS support Remove filtered FTS dataset, runner, backend, frontend, test, and published result changes while retaining unfiltered OSS OpenSearch and semantic FTS support. Remove the task-local session progress file from the branch. Signed-off-by: jamesgao-jpg * chore: standardize OpenSearch FTS result filename Signed-off-by: jamesgao-jpg * fix: report OpenSearch bulk item failures Signed-off-by: jamesgao-jpg * feat(cli): add common note support Signed-off-by: jamesgao-jpg * feat(fts): restore filtered search backend support Restore the filtered full-text search cases, datasets, runners, backend clients, CLI controls, and regression tests removed by 7cd3046. Keep frontend support, generated result artifacts, session files, and documentation excluded from this backend migration. Signed-off-by: jamesgao-jpg * style(fts): satisfy current lint checks Apply current Black formatting to the restored filtered-FTS code, use direct integer filter attribute access, and document the intentional runtime error inside the serial-search exception boundary. Signed-off-by: jamesgao-jpg * fix(fts): scatter filter ids deterministically Assign FTS filter IDs through a versioned affine permutation so threshold filters preserve exact cardinality while spreading matches across corpus insertion order. Use the same mapping for semantic qrels and record its parameters in filter statistics for reproducibility. Signed-off-by: jamesgao-jpg * feat(fts): make filter id distribution configurable Expose sequential and permuted FTS filter-ID distributions through the common CLI and FTS case configuration, defaulting to the deterministic permutation. Propagate the selected mode into document insertion, semantic qrel filtering, and versioned result metadata. Changing modes requires reloading the target collection. Signed-off-by: jamesgao-jpg * chore: update FTS result publication Remove the published Vespa FTS result while preserving its backend. Restore the Zilliz Cloud filtered semantic result and keep filtered rows out of the standard FTS frontend. Signed-off-by: jamesgao-jpg * feat: publish permuted filtered FTS results Publish validated permuted concurrency results for Elasticsearch, OSS OpenSearch, and Zilliz Cloud. Add a Filtered QPS frontend view and remove the legacy sequential Zilliz artifact. Signed-off-by: jamesgao-jpg * results: publish Turbopuffer filtered FTS data Add ten permuted filtered FTS result artifacts for HotpotQA Large and MS MARCO Large across five filter rates. Extend the frontend coverage test to include Turbopuffer. The benchmark observed service-side 429 responses at some concurrency points; recorded QPS is successful request throughput. Signed-off-by: jamesgao-jpg * fix: align FTS semantic recall and filtered chart Signed-off-by: jamesgao-jpg * fix: group filtered FTS QPS by filter rate Signed-off-by: jamesgao-jpg * results: document FTS benchmark setup Signed-off-by: jamesgao-jpg * results: add filtered FTS serial metrics Merge validated serial latency and semantic metrics into the existing filtered concurrent result artifacts while preserving source provenance. Signed-off-by: jamesgao-jpg * chore: consolidate FTS result artifacts Signed-off-by: jamesgao-jpg * fix(fts): remove sequential filter mode Signed-off-by: jamesgao-jpg * fix(frontend): expose OpenSearch FTS cases Signed-off-by: jamesgao-jpg * fix(opensearch): build FTS CLI config directly Signed-off-by: jamesgao-jpg * fix(opensearch): bound FTS replica readiness Signed-off-by: jamesgao-jpg * fix(frontend): require Streamlit 1.61 Signed-off-by: jamesgao-jpg * docs(fts): list OSS OpenSearch coverage Signed-off-by: jamesgao-jpg * style(frontend): format filtered result selector Signed-off-by: jamesgao-jpg * fix(fts): unify document filter id assignment Signed-off-by: jamesgao-jpg * refactor(milvus): share CLI config builder Signed-off-by: jamesgao-jpg * refactor(fts): share Elasticsearch-compatible config Signed-off-by: jamesgao-jpg * chore(results): remove local TurboPuffer paths Signed-off-by: jamesgao-jpg * chore(results): refresh Zilliz Cloud FTS benchmarks Signed-off-by: jamesgao-jpg * fix(results): restore FTS load metrics and filter metadata Signed-off-by: jamesgao-jpg * chore(results): remove FTS text payload records Signed-off-by: jamesgao-jpg --------- Signed-off-by: jamesgao-jpg --- README.md | 2 +- docs/release/2026-06-full-text-search.md | 21 +- pyproject.toml | 2 +- tests/test_fts_backend_filters.py | 177 ++ tests/test_fts_cases.py | 47 + tests/test_fts_cli_user_control.py | 94 + tests/test_fts_dataset.py | 301 +++ tests/test_fts_filter_runner.py | 182 ++ tests/test_fts_format_results.py | 43 + tests/test_fts_frontend_results.py | 282 +++ tests/test_fts_metrics.py | 69 + tests/test_milvus_zilliz_cli.py | 25 + tests/test_oss_opensearch_fts.py | 340 +++ tests/test_turbopuffer_cli.py | 45 + vectordb_bench/backend/cases.py | 34 +- vectordb_bench/backend/clients/__init__.py | 4 + vectordb_bench/backend/clients/api.py | 29 - .../backend/clients/elastic_cloud/config.py | 43 +- .../clients/elastic_cloud/elastic_cloud.py | 48 +- .../clients/elasticsearch_compatible.py | 23 + vectordb_bench/backend/clients/milvus/cli.py | 223 +- .../backend/clients/milvus/config.py | 62 +- .../backend/clients/milvus/milvus.py | 29 +- .../backend/clients/oss_opensearch/cli.py | 35 +- .../backend/clients/oss_opensearch/config.py | 22 + .../clients/oss_opensearch/oss_opensearch.py | 236 +- .../clients/turbopuffer/turbopuffer.py | 47 +- .../backend/clients/vespa/config.py | 25 - vectordb_bench/backend/clients/vespa/vespa.py | 45 +- vectordb_bench/backend/dataset.py | 401 +++- .../backend/runner/concurrent_runner.py | 8 +- .../backend/runner/serial_runner.py | 22 +- vectordb_bench/backend/task_runner.py | 93 +- vectordb_bench/cli/cli.py | 92 +- .../frontend/config/dbCaseConfigs.py | 4 +- .../frontend/pages/full_text_search.py | 258 +- vectordb_bench/metric.py | 52 +- vectordb_bench/restful/format_res.py | 1 + ...lt_20260626_fts_standard_elasticcloud.json | 1648 +++++++++---- ...sult_20260708_fts_standard_opensearch.json | 2100 +++++++++++++++++ ...ult_20260626_fts_standard_turbopuffer.json | 1710 ++++++++++---- .../result_20260626_fts_standard_vespa.json | 1514 ------------ ...ult_20260626_fts_standard_zillizcloud.json | 1656 ++++++++----- 43 files changed, 8376 insertions(+), 3718 deletions(-) create mode 100644 tests/test_fts_backend_filters.py create mode 100644 tests/test_fts_cases.py create mode 100644 tests/test_fts_cli_user_control.py create mode 100644 tests/test_fts_dataset.py create mode 100644 tests/test_fts_filter_runner.py create mode 100644 tests/test_fts_format_results.py create mode 100644 tests/test_fts_frontend_results.py create mode 100644 tests/test_fts_metrics.py create mode 100644 tests/test_oss_opensearch_fts.py create mode 100644 vectordb_bench/backend/clients/elasticsearch_compatible.py create mode 100644 vectordb_bench/results/FullTextSearch/OpenSearch/result_20260708_fts_standard_opensearch.json delete mode 100644 vectordb_bench/results/FullTextSearch/Vespa/result_20260626_fts_standard_vespa.json diff --git a/README.md b/README.md index f300f20d4..7851ef81b 100644 --- a/README.md +++ b/README.md @@ -948,7 +948,7 @@ We've developed lots of comprehensive benchmark cases to test vector databases' #### Full Text Search Performance Case - **FullTextSearchPerformance:** Measures BM25-style text retrieval over raw text documents. The case inserts documents, runs the backend optimization or index-readiness step, then measures recall, latency, and QPS for text queries. - **Datasets:** The initial FTS benchmark uses MS MARCO and HotpotQA in small, medium, and large corpus sizes. -- **Ground truth:** Recall is computed against generated mathematical BM25 ground truth, not semantic relevance labels. +- **Ground truth:** Recall, MRR, and NDCG are computed against positive semantic relevance labels from `ir_datasets`. - **Payload profiles:** FTS supports IDs-only responses and text payload responses so users can compare pure retrieval throughput against response-size overhead. #### Streaming Cases - **Insertion-Under-Load Case:** Evaluates search performance while maintaining a constant insertion workload. VDBBench applies a steady stream of insert requests at a fixed rate to simulate real-world scenarios where search operations must perform reliably under continuous data ingestion. diff --git a/docs/release/2026-06-full-text-search.md b/docs/release/2026-06-full-text-search.md index 33e335eb1..6c7a35a7b 100644 --- a/docs/release/2026-06-full-text-search.md +++ b/docs/release/2026-06-full-text-search.md @@ -8,7 +8,7 @@ Full text search support adds BM25-style text retrieval workloads to VectorDBBen VectorDBBench has historically focused on dense vector search and vector-oriented cloud cases. That leaves a gap for systems that also expose native text retrieval, sparse search, or BM25 ranking. Users evaluating retrieval systems often need to compare text-only retrieval before deciding whether to use dense vector, sparse vector, hybrid, or reranking layers. -The Full Text Search benchmark covers that text-only layer. It measures end-to-end behavior for indexing raw text, optimizing the backend, searching with BM25-style ranking, and validating recall against mathematical ground truth. The benchmark intentionally separates this from semantic relevance labels: recall is computed against generated BM25 top-k ground truth, not human relevance judgments. +The Full Text Search benchmark covers that text-only layer. It measures end-to-end behavior for indexing raw text, optimizing the backend, searching with BM25-style ranking, and validating retrieval quality against semantic relevance labels from `ir_datasets`. The benchmark also records payload behavior. Some applications return only document IDs, while others return text fields in the search response. VectorDBBench models both paths through explicit payload profiles so throughput and latency can be interpreted together with the response shape. @@ -19,18 +19,19 @@ This round focuses on full text search support across the following backends: - Milvus, using its full text search BM25 path. - Zilliz Cloud, using the cloud full text search path. - Elasticsearch, using its BM25 text search path. +- OSS OpenSearch, using its BM25 text search path. - Vespa, using BM25 ranking over indexed text fields. - turbopuffer, using its full text search namespace path. -The benchmark is designed to keep the workload shape consistent while still recording backend-specific behavior. BM25 parameters and analyzer settings are read from dataset manifests when available, applied when the backend exposes matching controls, and recorded as unapplied parameters when a backend does not expose the same control. +The benchmark is designed to keep the workload shape consistent while still recording backend-specific behavior. BM25 parameters and analyzer settings are controlled through the backend case config or CLI flags, so users can run either product-default comparisons or explicitly tuned comparisons. ## The new tests we added ### FullTextSearchPerformance -**Purpose.** FullTextSearchPerformance measures BM25-style full text search as a first-class benchmark case. It answers the baseline question: after a backend indexes the same text corpus, what QPS, latency, and mathematical recall does it deliver for text queries? +**Purpose.** FullTextSearchPerformance measures BM25-style full text search as a first-class benchmark case. It answers the baseline question: after a backend indexes the same text corpus, what QPS, latency, recall, MRR, and NDCG does it deliver for text queries? -**How it works.** The case loads raw text documents, builds the backend text index, runs the backend optimize path, executes optional serial recall checks, and then runs concurrent search. The result metric records load duration, insert duration, optimize duration, QPS, serial latency, concurrent latency, recall, payload profile, inserted count, and additional parameters such as manifest BM25 settings. +**How it works.** The case loads raw text documents, builds the backend text index, runs the backend optimize path, executes optional serial quality checks, and then runs concurrent search. The result metric records load duration, insert duration, optimize duration, QPS, serial latency, concurrent latency, recall, MRR, NDCG, payload profile, inserted count, and explicit backend parameters. Example: run MS MARCO small on Milvus with IDs-only responses. @@ -45,11 +46,11 @@ vectordbbench milvusfts \ --task-label fts-milvus-msmarco-small-ids ``` -### Full text search datasets and math ground truth +### Full text search datasets and semantic ground truth -**Purpose.** Full text search recall should measure whether an implementation returns the mathematically expected BM25 neighbors for the indexed corpus. Human relevance labels are useful for IR evaluation, but they are not a direct correctness target for a database BM25 implementation. +**Purpose.** Full text search quality should measure whether a backend returns documents that are semantically relevant to the query, not only whether it reproduces one particular BM25 implementation's exact ranking. This aligns the benchmark with common IR evaluation practice and makes recall, MRR, and NDCG comparable across backend analyzers and ranking implementations. -**How it works.** FTS datasets provide raw text for document insertion and query execution. The ground-truth artifacts provide top-k neighbor IDs generated under a declared BM25/analyzer contract. Each dataset artifact can include a build manifest with BM25 parameters such as `k1`, `b`, and `avgdl`, plus analyzer settings. VectorDBBench loads those values before backend initialization so index construction can use the dataset contract where the backend supports it. +**How it works.** FTS datasets provide raw text for document insertion and query execution through `ir_datasets`. VectorDBBench loads positive qrels from the same source and uses them as semantic ground truth for serial quality checks. BM25 and analyzer settings are not loaded from dataset manifests; use backend config or CLI flags when a run needs explicit `k1`, `b`, analyzer, or backend-specific search parameters. Example: use a larger dataset while keeping the same FTS case type. @@ -109,9 +110,9 @@ This release note introduces the FTS benchmark path; it is not a complete benchm Important caveats: -- Mathematical BM25 ground truth is not the same as semantic relevance evaluation. A high recall score means the backend matched the declared BM25 ranking contract, not that it matched human relevance labels. -- Analyzer behavior can materially affect recall and ranking. Tokenization, lowercase filters, stop words, stemming, token length limits, and field normalization should be recorded with each dataset manifest and backend result. -- BM25 parameter support differs across products. Some backends expose `k1` and `b`, some expose average field length controls, and some compute or hide those values internally. VectorDBBench records applied and unapplied parameters so results can be interpreted correctly. +- Semantic relevance labels are not the same as exact BM25 implementation correctness. A high quality score means the backend retrieved judged-relevant documents, not that it matched a specific analyzer or scorer bit-for-bit. +- Analyzer behavior can materially affect recall and ranking. Tokenization, lowercase filters, stop words, stemming, token length limits, and field normalization should be recorded with the benchmark run when they are configured explicitly. +- BM25 parameter support differs across products. Some backends expose `k1` and `b`, some expose average field length controls, and some compute or hide those values internally. Use backend config or CLI flags for explicit comparisons; otherwise the run represents product-default behavior. - IDs-only and text payload runs answer different questions. IDs-only is the cleanest recall and throughput baseline; text payload runs expose response-size overhead. - Load duration includes backend-specific insert and optimize behavior. Products may differ in whether optimize means force merge, compaction, warmup, or index deployment readiness. - Result JSONs under `vectordb_bench/results/FullTextSearch` are curated examples for the frontend. They should not be treated as the full historical experiment archive. diff --git a/pyproject.toml b/pyproject.toml index 9039e8f70..223291721 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "click", "pyyaml", "pytz", - "streamlit>=1.47,<2", # 1.47 fixes streamlit#11660 + "streamlit>=1.61,<2", # Stateful tabs and stretch widths used by the FTS page "tornado>=6.0", "tqdm", "s3fs", diff --git a/tests/test_fts_backend_filters.py b/tests/test_fts_backend_filters.py new file mode 100644 index 000000000..830fa9c60 --- /dev/null +++ b/tests/test_fts_backend_filters.py @@ -0,0 +1,177 @@ +# ruff: noqa: ANN001, ARG001, E402 + +import io +import sys +import threading +import types + +import pytest + +sys.modules.setdefault("turbopuffer", types.SimpleNamespace(Turbopuffer=object)) + +from vectordb_bench.backend.clients.elastic_cloud.config import ElasticCloudFtsConfig +from vectordb_bench.backend.clients.elastic_cloud.elastic_cloud import ElasticCloud +from vectordb_bench.backend.clients.milvus.config import MilvusFtsConfig +from vectordb_bench.backend.clients.milvus.milvus import Milvus +from vectordb_bench.backend.clients.turbopuffer.turbopuffer import TurboPuffer +from vectordb_bench.backend.clients.vespa.config import VespaFtsConfig +from vectordb_bench.backend.clients.vespa.vespa import Vespa +from vectordb_bench.backend.filter import NewIntFilter + + +def test_milvus_fts_insert_and_search_use_filter_id(): + db = Milvus.__new__(Milvus) + db._is_fts = True + db.name = "Milvus" + db.collection_name = "col" + db.batch_size = 1000 + db.with_scalar_labels = False + db._primary_field = "doc_id" + db._text_field = "text" + db._filter_id_field = "filter_id" + db._sparse_field = "sparse" + db.case_config = MilvusFtsConfig() + calls = {} + + class Client: + def insert(self, collection_name, rows): + calls["insert"] = (collection_name, rows) + return {"insert_count": len(rows)} + + def search(self, **kwargs): + calls["search"] = kwargs + return [[{"entity": {"doc_id": "d1"}}]] + + db.client = Client() + + assert db.insert_documents(["alpha"], ["d1"], filter_ids=[12]) == (1, None) + db.prepare_filter(NewIntFilter(filter_rate=0.5, int_field="filter_id", int_value=10)) + assert db.search_documents("alpha") == ["d1"] + + assert calls["insert"] == ("col", [{"doc_id": "d1", "text": "alpha", "filter_id": 12}]) + assert calls["search"]["filter"] == "filter_id >= 10" + + +def test_elastic_cloud_fts_insert_and_search_use_filter_id(monkeypatch): + db = ElasticCloud.__new__(ElasticCloud) + db._is_fts = True + db.indice = "idx" + db.id_col_name = "doc_id" + db.text_col_name = "text" + db.filter_id_col_name = "filter_id" + db.filter = None + db.client = object() + bulk_calls = [] + + def fake_bulk(client, actions): + bulk_calls.append(actions) + return len(actions), [] + + monkeypatch.setattr("vectordb_bench.backend.clients.elastic_cloud.elastic_cloud.bulk", fake_bulk) + assert db.insert_documents(["alpha"], ["d1"], filter_ids=[3]) == (1, None) + assert bulk_calls == [ + [ + { + "_index": "idx", + "_id": "d1", + "_source": {"doc_id": "d1", "text": "alpha", "filter_id": 3}, + } + ] + ] + + search_calls = {} + + class Client: + def search(self, **kwargs): + search_calls.update(kwargs) + return {"hits": {"hits": [{"fields": {"doc_id": ["d1"]}}]}} + + db.client = Client() + db.prepare_filter(NewIntFilter(filter_rate=0.5, int_field="filter_id", int_value=3)) + assert db.search_documents("alpha") == ["d1"] + assert search_calls["query"] == { + "bool": { + "must": {"match": {"text": "alpha"}}, + "filter": {"range": {"filter_id": {"gte": 3}}}, + } + } + + +def test_elastic_cloud_fts_mapping_has_filter_id(): + assert ElasticCloudFtsConfig().index_param()["properties"]["filter_id"] == {"type": "long"} + + +def test_vespa_fts_feed_and_search_use_filter_id(): + db = Vespa.__new__(Vespa) + db._is_fts = True + db.schema_name = "schema" + db._text_field = "text" + db._filter_id_field = "filter_id" + db._filter_expr = None + db.case_config = VespaFtsConfig() + db._feed_lock = threading.Lock() + db._feed_written_count = 0 + output = io.StringIO() + + db._ensure_fts_feed_client = lambda: types.SimpleNamespace( + poll=lambda: None, + stdin=types.SimpleNamespace(write=output.write, flush=lambda: None), + ) + + db._write_fts_feed_batch(["alpha"], ["d1"], [9]) + assert '"filter_id":9' in output.getvalue() + + calls = {} + + class Result: + def get_json(self): + return {"root": {"children": [{"fields": {"id": "d1"}}]}} + + def query(payload): + calls["payload"] = payload + return Result() + + db.client = types.SimpleNamespace(query=query) + db.prepare_filter(NewIntFilter(filter_rate=0.5, int_field="filter_id", int_value=9)) + assert db.search_documents("alpha") == ["d1"] + assert calls["payload"]["yql"] == "select id from schema where userQuery() and filter_id >= 9" + + +def test_turbopuffer_fts_insert_and_search_use_filter_id(): + db = TurboPuffer.__new__(TurboPuffer) + db._is_fts = True + db._text_field = "text" + db._scalar_id_field = "id" + db._filter_id_field = "filter_id" + db.expr = None + db.db_case_config = types.SimpleNamespace(disable_backpressure=False) + write_calls = {} + query_calls = {} + + class Namespace: + def write(self, **kwargs): + write_calls.update(kwargs) + + def query(self, **kwargs): + query_calls.update(kwargs) + return types.SimpleNamespace(rows=[types.SimpleNamespace(id="d1")]) + + db.ns = Namespace() + + assert db.insert_documents(["alpha"], ["d1"], filter_ids=[5]) == (1, None) + db.prepare_filter(NewIntFilter(filter_rate=0.5, int_field="filter_id", int_value=5)) + assert db.search_documents("alpha") == ["d1"] + + assert write_calls["upsert_columns"]["filter_id"] == [5] + assert query_calls["filters"] == ("filter_id", "Gte", 5) + + +@pytest.mark.parametrize("db_cls", [Milvus, ElasticCloud, Vespa, TurboPuffer]) +def test_fts_filter_rejects_non_filter_id_field(db_cls): + db = db_cls.__new__(db_cls) + db._is_fts = True + db._filter_id_field = "filter_id" + db.filter_id_col_name = "filter_id" + + with pytest.raises(ValueError, match="filter_id"): + db.prepare_filter(NewIntFilter(filter_rate=0.5, int_field="id", int_value=1)) diff --git a/tests/test_fts_cases.py b/tests/test_fts_cases.py new file mode 100644 index 000000000..848bc3b45 --- /dev/null +++ b/tests/test_fts_cases.py @@ -0,0 +1,47 @@ +import pytest + +from vectordb_bench.backend.cases import FTS_FILTER_ID_FIELD, FTSBm25Performance +from vectordb_bench.backend.dataset import FtsDatasetWithSizeType +from vectordb_bench.backend.filter import FilterOp + + +def test_fts_filter_case_uses_filter_id_for_large_dataset(): + case = FTSBm25Performance( + dataset_with_size_type=FtsDatasetWithSizeType.MSMarcoLarge, + filter_rate=0.95, + ) + + filters = case.filters + + assert filters.type == FilterOp.NumGE + assert filters.int_field == FTS_FILTER_ID_FIELD + assert filters.int_value == int(8_841_823 * 0.95) + assert "Filter 95%" in case.name + + +def test_fts_filter_case_rejects_small_and_medium_datasets(): + with pytest.raises(ValueError, match="only supported"): + FTSBm25Performance( + dataset_with_size_type=FtsDatasetWithSizeType.MSMarcoSmall, + filter_rate=0.5, + ) + + with pytest.raises(ValueError, match="only supported"): + FTSBm25Performance( + dataset_with_size_type=FtsDatasetWithSizeType.HotpotQAMedium, + filter_rate=0.5, + ) + + +def test_fts_filter_case_rejects_unsupported_filter_rate(): + with pytest.raises(ValueError, match="must be one of"): + FTSBm25Performance( + dataset_with_size_type=FtsDatasetWithSizeType.HotpotQALarge, + filter_rate=0.8, + ) + + +def test_fts_filter_case_has_no_distribution_switch(): + case = FTSBm25Performance(dataset_with_size_type=FtsDatasetWithSizeType.MSMarcoLarge) + + assert not hasattr(case, "filter_id_distribution") diff --git a/tests/test_fts_cli_user_control.py b/tests/test_fts_cli_user_control.py new file mode 100644 index 000000000..1496b3bc8 --- /dev/null +++ b/tests/test_fts_cli_user_control.py @@ -0,0 +1,94 @@ +from typing import get_type_hints + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import IndexType +from vectordb_bench.backend.clients.elastic_cloud.config import ElasticCloudFtsConfig, ElasticCloudIndexConfig +from vectordb_bench.backend.clients.milvus.config import MilvusFtsConfig +from vectordb_bench.backend.clients.vespa.config import VespaFtsConfig, VespaHNSWConfig +from vectordb_bench.backend.dataset import FtsDatasetWithSizeType +from vectordb_bench.cli.cli import CommonTypedDict, get_custom_case_config, select_cli_db_case_config + + +def test_common_cli_exposes_optional_fts_bm25_overrides(): + hints = get_type_hints(CommonTypedDict, include_extras=True) + + assert "bm25_k1" in hints + assert "bm25_b" in hints + assert "fts_filter_rate" in hints + assert "fts_filter_id_distribution" not in hints + + +def test_cli_builds_fts_filter_case_config(): + custom_case = get_custom_case_config( + { + "case_type": "FTSBm25Performance", + "dataset_with_size_type": FtsDatasetWithSizeType.MSMarcoLarge.value, + "payload_profile": "ids_only", + "fts_filter_rate": 0.95, + } + ) + + assert custom_case == { + "dataset_with_size_type": FtsDatasetWithSizeType.MSMarcoLarge.value, + "payload_profile": "ids_only", + "filter_rate": 0.95, + } + + +def test_cli_applies_bm25_overrides_to_existing_fts_config(): + config = MilvusFtsConfig(drop_ratio_search=0.1) + + selected = select_cli_db_case_config( + DB.Milvus, + config, + "FTSBm25Performance", + {"bm25_k1": 1.4, "bm25_b": 0.6}, + ) + + assert isinstance(selected, MilvusFtsConfig) + assert selected.bm25_k1 == 1.4 + assert selected.bm25_b == 0.6 + assert selected.drop_ratio_search == 0.1 + assert selected.sparse_index_param()["params"] == { + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": 1.4, + "bm25_b": 0.6, + } + + +def test_cli_applies_bm25_overrides_after_routing_vector_config_to_fts(): + selected = select_cli_db_case_config( + DB.ElasticCloud, + ElasticCloudIndexConfig(index=IndexType.ES_HNSW, number_of_shards=3), + "FTSBm25Performance", + {"bm25_k1": 1.7, "bm25_b": 0.2}, + ) + + assert isinstance(selected, ElasticCloudFtsConfig) + assert selected.number_of_shards == 3 + assert selected.bm25_k1 == 1.7 + assert selected.bm25_b == 0.2 + assert selected.index_param()["properties"]["text"]["similarity"] == "vdbbench_bm25" + assert selected.similarity_settings() == { + "similarity": { + "vdbbench_bm25": { + "type": "BM25", + "k1": 1.7, + "b": 0.2, + } + } + } + + +def test_cli_leaves_fts_bm25_defaults_when_options_are_omitted(): + selected = select_cli_db_case_config( + DB.Vespa, + VespaHNSWConfig(), + "FTSBm25Performance", + {"bm25_k1": None, "bm25_b": None}, + ) + + assert isinstance(selected, VespaFtsConfig) + assert selected.bm25_k1 is None + assert selected.bm25_b is None + assert selected.rank_properties() == [] diff --git a/tests/test_fts_dataset.py b/tests/test_fts_dataset.py new file mode 100644 index 000000000..a5eaf9791 --- /dev/null +++ b/tests/test_fts_dataset.py @@ -0,0 +1,301 @@ +import math +from dataclasses import dataclass +from typing import ClassVar + +import pytest + +from vectordb_bench.backend.dataset import ( + FtsDataset, + FtsDatasetManager, + FtsDatasetWithSizeType, + FtsFilterIdPermutation, + HotpotQAFts, + HotpotQATranslator, + MSMarcoFts, + MSMarcoTranslator, + SizeLabel, +) +from vectordb_bench.backend.filter import NewIntFilter + + +@dataclass +class Query: + query_id: str + text: str + + +@dataclass +class Doc: + doc_id: str + text: str + title: str = "" + + +@dataclass +class Qrel: + query_id: str + doc_id: str + relevance: int + + +class FakeDataset: + def __init__(self): + self.queries = [Query("q1", "alpha query"), Query("q2", "beta query")] + self.docs = [ + Doc("d1", "alpha document", "A"), + Doc("d2", "beta document", "B"), + Doc("d3", "gamma document", "C"), + Doc("d4", "delta document", "D"), + ] + self.qrels = [Qrel("q1", "d3", 1), Qrel("q1", "d2", 0), Qrel("q2", "d1", 2)] + + def queries_iter(self): + yield from self.queries + + def docs_iter(self): + yield from self.docs + + def qrels_iter(self): + yield from self.qrels + + +class FakeDatasetWithMissingQrel(FakeDataset): + def __init__(self): + super().__init__() + self.qrels = [Qrel("q1", "missing", 1)] + + +class FakeDatasetWithLowPermutationQrels(FakeDataset): + def __init__(self): + super().__init__() + self.qrels = [Qrel("q1", "d3", 1), Qrel("q2", "d4", 2)] + + +def make_tiny_msmarco_manager(size: int = 3) -> FtsDatasetManager: + small_label = MSMarcoFts._size_label[100_000] + + class TinyMSMarcoFts(MSMarcoFts): + _size_label: ClassVar[dict[int, SizeLabel]] = { + **MSMarcoFts._size_label, + size: small_label._replace(size=size), + } + + return FtsDatasetManager(data=TinyMSMarcoFts(size=size)) + + +def test_msmarco_translator_uses_string_ids_and_qrels(): + translator = MSMarcoTranslator() + dataset = FakeDataset() + + query = translator.translate_query(Query("10", "hello")) + document = translator.translate_document(Doc("20", "hello\tworld\nagain")) + ground_truth = translator.load_ground_truth(dataset) + + assert query.query_id == "10" + assert document.doc_id == "20" + assert document.text == "hello world again" + assert ground_truth == {"q1": {"d3": 1}, "q2": {"d1": 2}} + + +def test_hotpotqa_translator_combines_title_and_text_and_uses_qrels(): + translator = HotpotQATranslator() + dataset = FakeDataset() + + document = translator.translate_document(Doc("doc-a", "body", "Title")) + ground_truth = translator.load_ground_truth(dataset) + + assert document.doc_id == "doc-a" + assert document.text == "Title body" + assert ground_truth == {"q1": {"d3": 1}, "q2": {"d1": 2}} + + +def test_fts_filter_id_permutation_has_stable_sequence(): + permutation = FtsFilterIdPermutation.for_size(8) + + assert permutation.algorithm == "affine_permutation_v1" + assert permutation.multiplier == 5 + assert permutation.offset == 3 + assert [permutation.map(i) for i in range(8)] == [3, 0, 5, 2, 7, 4, 1, 6] + assert permutation == FtsFilterIdPermutation.for_size(8) + + +@pytest.mark.parametrize("size", [1, 2, 3, 4, 8, 10, 97, 100, 1_000, 100_000, 1_000_000, 5_233_329, 8_841_823]) +def test_fts_filter_id_permutation_is_bijective(size: int): + permutation = FtsFilterIdPermutation.for_size(size) + + assert math.gcd(permutation.multiplier, size) == 1 + assert 0 <= permutation.offset < size + if size <= 1_000: + assert sorted(permutation.map(i) for i in range(size)) == list(range(size)) + + +@pytest.mark.parametrize(("size", "filter_rate"), [(8, 0.5), (100, 0.99), (101, 0.75)]) +def test_fts_filter_id_permutation_preserves_exact_selectivity(size: int, filter_rate: float): + permutation = FtsFilterIdPermutation.for_size(size) + filter_value = int(size * filter_rate) + + matched = sum(permutation.map(i) >= filter_value for i in range(size)) + + assert matched == size - filter_value + + +def test_fts_filter_id_permutation_scatters_matches_across_corpus_order(): + permutation = FtsFilterIdPermutation.for_size(100) + + matching_ordinals = [ordinal for ordinal in range(100) if permutation.map(ordinal) >= 90] + + assert matching_ordinals != list(range(90, 100)) + assert len({ordinal // 10 for ordinal in matching_ordinals}) >= 6 + + +def test_fts_filter_id_permutation_validates_bounds(): + with pytest.raises(ValueError, match="size must be positive"): + FtsFilterIdPermutation.for_size(0) + + permutation = FtsFilterIdPermutation.for_size(3) + with pytest.raises(ValueError, match="ordinal must be in"): + permutation.map(3) + + +def test_fts_iterator_preserves_qrel_docs_before_filler(): + manager = make_tiny_msmarco_manager() + manager._ir_dataset = FakeDataset() + manager.required_doc_ids = {"d4"} + manager.selected_doc_ids = manager._build_selected_doc_ids() + + assert manager.selected_doc_ids == {"d1", "d2", "d4"} + + batches = list(manager) + docs = [(doc.doc_id, doc.filter_id) for batch in batches for doc in batch] + + assert len(docs) == 3 + assert ("d4", 1) in docs + assert all(not doc_id.isdecimal() for doc_id, _ in docs) + assert [filter_id for _, filter_id in docs] == [2, 0, 1] + + +def test_fts_qrel_filter_ids_match_sparse_emitted_documents_when_one_is_skipped(): + class UnassignableDocument: + doc_id = "d3" + text = "malformed" + + @property + def filter_id(self) -> int | None: + return None + + @filter_id.setter + def filter_id(self, value: int | None) -> None: # noqa: ARG002 + raise ValueError("cannot assign filter_id") + + class SparseTranslator: + def iter_documents(self, dataset): # noqa: ARG002 + yield Doc("d1", "one") + yield Doc("d2", "two") + yield UnassignableDocument() + yield Doc("d4", "four") + yield Doc("d5", "five") + + manager = make_tiny_msmarco_manager(size=3) + manager._ir_dataset = object() + manager._translator = SparseTranslator() + manager.required_doc_ids = {"d1", "d5"} + manager.selected_doc_ids = {"d1", "d3", "d5"} + + qrel_filter_ids = manager._build_qrel_filter_ids() + emitted_filter_ids = {doc.doc_id: doc.filter_id for batch in manager for doc in batch} + + assert emitted_filter_ids == {"d1": 2, "d5": 0} + assert qrel_filter_ids == emitted_filter_ids + + +def test_fts_prepare_integer_filter_derives_filtered_qrels(monkeypatch: pytest.MonkeyPatch): + manager = make_tiny_msmarco_manager(size=4) + monkeypatch.setattr(manager._translator, "load", FakeDataset) + + filters = NewIntFilter(filter_rate=0.5, int_field="filter_id", int_value=2) + assert manager.prepare(source=None, filters=filters) + + assert [query.query_id for query in manager.queries_data] == ["q1", "q2"] + assert manager.gt_data == [{"d3": 1}, {"d1": 2}] + assert [query.query_id for query in manager.recall_queries_data] == ["q2"] + assert manager.recall_gt_data == [{"d1": 2}] + assert manager.recall_skipped is False + assert manager.recall_skip_reason is None + assert manager.qrel_filter_ids == {"d1": 3, "d3": 1} + assert manager.filter_stats == { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 2, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3, + "filter_id_offset": 3, + "matched_doc_count": 2, + "matched_doc_ratio": 0.5, + "original_query_count": 2, + "filtered_query_count": 1, + "filtered_query_ratio": 0.5, + "original_relevant_doc_count": 2, + "filtered_relevant_doc_count": 1, + } + + +def test_fts_prepare_integer_filter_skips_empty_filtered_qrels(monkeypatch: pytest.MonkeyPatch): + manager = make_tiny_msmarco_manager(size=4) + monkeypatch.setattr(manager._translator, "load", FakeDatasetWithLowPermutationQrels) + + filters = NewIntFilter(filter_rate=0.75, int_field="filter_id", int_value=3) + assert manager.prepare(source=None, filters=filters) + + assert [query.query_id for query in manager.queries_data] == ["q1", "q2"] + assert manager.gt_data == [{"d3": 1}, {"d4": 2}] + assert manager.recall_queries_data == [] + assert manager.recall_gt_data == [] + assert manager.recall_skipped is True + assert manager.recall_skip_reason == "no_positive_qrels_after_filter" + assert manager.filter_stats["filtered_query_count"] == 0 + assert manager.filter_stats["filtered_relevant_doc_count"] == 0 + + +def test_fts_prepare_integer_filter_requires_filter_id_field(monkeypatch: pytest.MonkeyPatch): + manager = make_tiny_msmarco_manager(size=4) + monkeypatch.setattr(manager._translator, "load", FakeDataset) + + filters = NewIntFilter(filter_rate=0.5, int_field="id", int_value=2) + with pytest.raises(ValueError, match="int_field='filter_id'"): + manager.prepare(source=None, filters=filters) + + +def test_fts_cap_rejects_required_qrel_docs_missing_from_corpus(): + manager = make_tiny_msmarco_manager() + manager._ir_dataset = FakeDataset() + manager.required_doc_ids = {"missing"} + + with pytest.raises(ValueError, match="missing from corpus"): + manager._build_selected_doc_ids() + + +def test_fts_prepare_propagates_missing_required_qrel_docs(monkeypatch: pytest.MonkeyPatch): + manager = FtsDatasetManager(data=MSMarcoFts(size=100_000)) + monkeypatch.setattr(manager._translator, "load", FakeDatasetWithMissingQrel) + + with pytest.raises(ValueError, match="missing from corpus"): + manager.prepare(source=None) + + +def test_fts_dataset_size_registry(): + assert FtsDataset.MSMARCO.manager(100_000).data.full_name == "MS MARCO FTS (SMALL)" + assert FtsDataset.MSMARCO.manager(1_000_000).data.full_name == "MS MARCO FTS (MEDIUM)" + assert FtsDataset.HOTPOTQA.manager(5_233_329).data.full_name == "HotpotQA FTS (LARGE)" + assert FtsDatasetWithSizeType.MSMarcoSmall.get_manager().data.size == 100_000 + assert FtsDatasetWithSizeType.HotpotQAMedium.get_manager().data.size == 1_000_000 + with pytest.raises(ValueError, match="not supported"): + FtsDataset.MSMARCO.manager(3) + + +def test_cap_smaller_than_required_qrels_is_invalid(): + manager = FtsDatasetManager(data=HotpotQAFts(size=100_000)) + manager.qrels_data = {"q1": {"a": 1, "b": 1, "c": 1}} + + with pytest.raises(ValueError, match="requires 3 qrel documents"): + manager._validate_cap(required_doc_ids={"a", "b", "c"}, target_size=2) diff --git a/tests/test_fts_filter_runner.py b/tests/test_fts_filter_runner.py new file mode 100644 index 000000000..fe7179334 --- /dev/null +++ b/tests/test_fts_filter_runner.py @@ -0,0 +1,182 @@ +import threading + +from vectordb_bench.backend.cases import CaseLabel +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import FtsDocument, FtsQuery +from vectordb_bench.backend.filter import non_filter +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.runner.concurrent_runner import ConcurrentInsertRunner +from vectordb_bench.backend.task_runner import CaseRunner +from vectordb_bench.models import TaskStage + + +def test_concurrent_insert_runner_fts_batch_includes_filter_ids(): + runner = ConcurrentInsertRunner.__new__(ConcurrentInsertRunner) + runner._deadline = None + runner._stop_event = None + runner._iter_lock = threading.Lock() + runner._prefetched_fts_batch = None + runner._dataset_iter = iter( + [ + [ + FtsDocument(doc_id="d1", text="alpha", filter_id=0), + FtsDocument(doc_id="d2", text="beta", filter_id=5), + ] + ] + ) + + assert runner._next_fts_batch() == { + "texts": ["alpha", "beta"], + "doc_ids": ["d1", "d2"], + "filter_ids": [0, 5], + } + + +def test_fts_pre_run_passes_filters_to_dataset(monkeypatch): + filter_obj = object() + + class Dataset: + def __init__(self): + self.calls = [] + + def prepare(self, source, filters=None): + self.calls.append((source, filters)) + + class Case: + label = CaseLabel.FullTextSearchPerformance + is_multitenant = False + dataset = Dataset() + filters = filter_obj + + config_obj = type("Config", (), {"stages": [TaskStage.LOAD]})() + runner = CaseRunner.construct(ca=Case(), config=config_obj, dataset_source=DatasetSource.S3) + init_calls = [] + monkeypatch.setattr(CaseRunner, "init_db", lambda self, drop_old=True: init_calls.append(drop_old)) + + runner._pre_run(drop_old=False) + + assert runner.ca.dataset.calls == [(DatasetSource.S3, filter_obj)] + assert init_calls == [False] + + +def test_fts_perf_metric_includes_dataset_filter_stats(): + class Dataset: + filter_stats = { + "filter_field": "filter_id", + "filter_value": 90, + "filtered_query_count": 12, + } + + class Case: + label = CaseLabel.FullTextSearchPerformance + dataset = Dataset() + + config_obj = type("Config", (), {"stages": []})() + runner = CaseRunner.construct(ca=Case(), config=config_obj) + + metric = runner._run_perf_case(drop_old=False) + + assert metric.additional_parameters["fts_filter"] == Dataset.filter_stats + + +class DummyDb: + name = "dummy" + + def supports_payload_profile(self, payload_profile): + return True + + def supports_document_payload_profile(self, payload_profile): + return True + + def search_documents(self, query, k, payload_profile=None): + return [] + + +class FtsConcurrencyConfig: + num_concurrency = [60, 80] + concurrency_duration = 30 + concurrency_timeout = 300 + serial_cooldown = 0 + + +class FtsCaseConfig: + k = 10 + concurrency_search_config = FtsConcurrencyConfig() + + +def test_fts_search_runners_use_full_queries_for_concurrency_and_filtered_queries_for_recall(): + class Dataset: + queries_data = [ + FtsQuery(query_id="q1", text="full query one"), + FtsQuery(query_id="q2", text="full query two"), + ] + gt_data = [{"d1": 1}, {"d2": 1}] + recall_queries_data = [FtsQuery(query_id="q2", text="full query two")] + recall_gt_data = [{"d2": 1}] + recall_skipped = False + recall_skip_reason = None + + class Case: + label = CaseLabel.FullTextSearchPerformance + dataset = Dataset() + filters = non_filter + payload_profile = PayloadProfile.IDS_ONLY + + config_obj = type( + "Config", + (), + { + "stages": [TaskStage.SEARCH_SERIAL, TaskStage.SEARCH_CONCURRENT], + "case_config": FtsCaseConfig(), + }, + )() + runner = CaseRunner.construct(ca=Case(), config=config_obj, db=DummyDb()) + + runner._init_fts_search_runner() + + assert runner.search_runner.test_data == ["full query one", "full query two"] + assert runner.serial_search_runner.test_data == ["full query two"] + assert runner.serial_search_runner.ground_truth == [{"d2": 1}] + + +def test_fts_perf_metric_marks_recall_skipped_without_blocking_case(monkeypatch): + class Dataset: + filter_stats = { + "filter_field": "filter_id", + "filter_value": 99, + "filtered_query_count": 0, + } + queries_data = [ + FtsQuery(query_id="q1", text="full query one"), + FtsQuery(query_id="q2", text="full query two"), + ] + recall_queries_data = [] + recall_skipped = True + recall_skip_reason = "no_positive_qrels_after_filter" + + class Case: + label = CaseLabel.FullTextSearchPerformance + dataset = Dataset() + + config_obj = type( + "Config", + (), + { + "stages": [TaskStage.SEARCH_SERIAL], + "case_config": FtsCaseConfig(), + }, + )() + runner = CaseRunner.construct(ca=Case(), config=config_obj) + monkeypatch.setattr(CaseRunner, "_init_search_runners", lambda self: None) + + metric = runner._run_perf_case(drop_old=False) + + assert metric.recall == 0.0 + assert metric.ndcg == 0.0 + assert metric.mrr == 0.0 + assert metric.additional_parameters["fts_recall"] == { + "skipped": True, + "reason": "no_positive_qrels_after_filter", + "serial_query_count": 0, + "full_query_count": 2, + } diff --git a/tests/test_fts_format_results.py b/tests/test_fts_format_results.py new file mode 100644 index 000000000..e46c5670a --- /dev/null +++ b/tests/test_fts_format_results.py @@ -0,0 +1,43 @@ +from vectordb_bench.backend.cases import CaseType +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.milvus.config import MilvusConfig, MilvusFtsConfig +from vectordb_bench.metric import Metric +from vectordb_bench.models import CaseConfig, CaseResult, TaskConfig, TestResult +from vectordb_bench.restful.format_res import format_results + + +def test_format_results_supports_fts_dataset_and_metrics(): + test_result = TestResult( + run_id="run-1", + task_label="fts-task", + timestamp=123, + results=[ + CaseResult( + task_config=TaskConfig( + db=DB.Milvus, + db_config=MilvusConfig(uri="http://localhost:19530"), + db_case_config=MilvusFtsConfig(), + case_config=CaseConfig(case_id=CaseType.FTSBm25Performance, k=10), + ), + metrics=Metric( + qps=12.5, + recall=0.7, + ndcg=0.8, + mrr=0.9, + serial_latency_p99=0.11, + serial_latency_p95=0.1, + conc_latency_p99_list=[0.2], + conc_latency_p95_list=[0.15], + conc_latency_avg_list=[0.12], + ), + ) + ], + ) + + [formatted] = format_results([test_result], "fts-task") + + assert formatted["dataset"] == "MS MARCO FTS (SMALL)" + assert formatted["dim"] == 0 + assert formatted["mrr"] == 0.9 + assert formatted["serial_latency_p95"] == 0.1 + assert formatted["conc_latency_p95_list"] == [0.15] diff --git a/tests/test_fts_frontend_results.py b/tests/test_fts_frontend_results.py new file mode 100644 index 000000000..5b802356b --- /dev/null +++ b/tests/test_fts_frontend_results.py @@ -0,0 +1,282 @@ +import json +import tomllib +from inspect import signature +from pathlib import Path +from typing import Any + +import pandas as pd +import streamlit as st + +from vectordb_bench.backend.clients import DB +from vectordb_bench.frontend.config.dbCaseConfigs import get_fts_case_items +from vectordb_bench.frontend.pages.full_text_search import ( + _concurrency_rows, + _draw_filtered_qps_tab, + _peak_filtered_qps_rows, + load_full_text_search_rows, +) + + +def test_oss_opensearch_fts_cases_are_selectable_in_run_test(): + assert all(item.supports_dbs([DB.OSSOpenSearch]) for item in get_fts_case_items()) + + +def test_streamlit_dependency_floor_supports_fts_tab_state_api(): + repo_root = Path(__file__).resolve().parents[1] + dependencies = tomllib.loads((repo_root / "pyproject.toml").read_text())["project"]["dependencies"] + + assert "streamlit>=1.61,<2" in dependencies + assert {"default", "key", "on_change"}.issubset(signature(st.tabs).parameters) + + +def test_frontend_separates_filtered_results_and_expands_concurrency_qps(tmp_path: Path): + result_file = tmp_path / "result_fts.json" + result_file.write_text( + json.dumps( + { + "task_label": "fts-results", + "results": [ + { + "metrics": { + "qps": 100.0, + "recall": 0.9, + "ndcg": 0.8, + "mrr": 0.7, + }, + "task_config": { + "db": "ZillizCloud", + "case_config": { + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + } + }, + }, + }, + { + "metrics": { + "qps": 200.0, + "recall": 0.95, + "ndcg": 0.85, + "mrr": 0.75, + "additional_parameters": { + "fts_filter": { + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + } + }, + "conc_num_list": [60, 80], + "conc_qps_list": [1200.0, 1500.0], + }, + "task_config": { + "db": "ZillizCloud", + "case_config": { + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.5, + } + }, + }, + }, + ], + } + ) + ) + + data = load_full_text_search_rows(tmp_path) + standard_data = data[~data["is_filtered"]] + filtered_data = data[data["is_filtered"]] + concurrency_data = _concurrency_rows(filtered_data) + peak_data = _peak_filtered_qps_rows(filtered_data) + + assert len(standard_data) == 1 + assert standard_data.iloc[0]["qps"] == 100.0 + assert len(filtered_data) == 1 + assert filtered_data.iloc[0]["filter_rate_label"] == "50%" + assert "filter_distribution" not in filtered_data.columns + assert concurrency_data[["concurrency", "qps"]].to_dict("records") == [ + {"concurrency": 60, "qps": 1200.0}, + {"concurrency": 80, "qps": 1500.0}, + ] + assert peak_data[["filter_rate_label", "concurrency", "qps"]].to_dict("records") == [ + {"filter_rate_label": "50%", "concurrency": 80, "qps": 1500.0} + ] + + +def test_checked_in_consolidated_permuted_results_expose_concurrency_qps_for_all_backends(): + repo_root = Path(__file__).resolve().parents[1] + result_dir = repo_root / "vectordb_bench" / "results" / "FullTextSearch" + + data = load_full_text_search_rows(result_dir) + filtered_data = data[data["is_filtered"]] + concurrency_data = _concurrency_rows(filtered_data) + peak_data = _peak_filtered_qps_rows(filtered_data) + + assert len(data) == 64 + assert len(filtered_data) == 40 + assert len(concurrency_data) == 90 + assert (pd.to_numeric(filtered_data["p99_s"]) > 0).all() + assert (pd.to_numeric(filtered_data["p95_s"]) > 0).all() + assert (pd.to_numeric(filtered_data["recall"]) > 0).all() + assert (pd.to_numeric(filtered_data["ndcg"]) > 0).all() + assert (pd.to_numeric(filtered_data["mrr"]) > 0).all() + assert len(peak_data) == 20 + assert set(filtered_data["backend"].astype(str)) == { + "ElasticSearch", + "OSSOpenSearch", + "TurboPuffer", + "ZillizCloud", + } + assert set(concurrency_data["concurrency"]) == {40, 60, 80} + assert set(peak_data["filter_rate_label"]) == {"50%", "75%", "90%", "95%", "99%"} + assert set(peak_data["concurrency"]).issubset({60, 80}) + assert set(pd.to_numeric(filtered_data["filter_rate"])) == {0.5, 0.75, 0.9, 0.95, 0.99} + assert not (result_dir / "ZillizCloud" / "result_20260709_fts_filtered_zillizcloud.json").exists() + + expected_result_files = { + "ElasticCloud": "result_20260626_fts_standard_elasticcloud.json", + "OpenSearch": "result_20260708_fts_standard_opensearch.json", + "TurboPuffer": "result_20260626_fts_standard_turbopuffer.json", + "ZillizCloud": "result_20260626_fts_standard_zillizcloud.json", + } + expected_result_counts = { + "ElasticCloud": 16, + "OpenSearch": 16, + "TurboPuffer": 16, + "ZillizCloud": 16, + } + result_files = sorted(result_dir.glob("*/result_*.json")) + assert len(result_files) == 4 + assert {path.parent.name: path.name for path in result_files} == expected_result_files + + filtered_results = [] + result_counts = {} + for result_file in result_files: + results = json.loads(result_file.read_text())["results"] + result_counts[result_file.parent.name] = len(results) + if result_file.parent.name == "ZillizCloud": + assert all( + "one persistent segment" + in json.loads(case_result["task_config"]["db_config"]["note"])["evidence"]["source"] + for case_result in results + ) + for case_result in results: + custom_case = case_result["task_config"]["case_config"].get("custom_case") or {} + fts_filter = case_result["metrics"].get("additional_parameters", {}).get("fts_filter") or {} + filter_rate = custom_case.get("filter_rate", fts_filter.get("filter_rate")) + if filter_rate is not None: + assert "filter_id_distribution" not in custom_case + assert fts_filter["filter_id_distribution"] == "affine_permutation_v1" + filtered_results.append(case_result) + + assert result_counts == expected_result_counts + assert len(filtered_results) == 40 + expected_serial_fields = { + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr", + } + for case_result in filtered_results: + stages = set(case_result["task_config"]["stages"]) + provenance = case_result["metrics"]["additional_parameters"].get("serial_measurement") + + assert {"search_concurrent", "search_serial"}.issubset(stages) + if provenance is None: + assert case_result["task_config"]["db"] == "ZillizCloud" + assert all(case_result["metrics"][field] > 0 for field in expected_serial_fields) + continue + assert provenance["composed_from_separate_run"] is True + assert set(provenance["metric_fields"]) == expected_serial_fields + assert provenance["source_file"].startswith("result_") + assert len(provenance["source_sha256"]) == 64 + assert "search_serial" in provenance["source_stages"] + assert provenance["source_note"] + + +def test_filtered_qps_chart_groups_backend_bars_by_filter_rate(): + repo_root = Path(__file__).resolve().parents[1] + result_dir = repo_root / "vectordb_bench" / "results" / "FullTextSearch" + data = load_full_text_search_rows(result_dir) + filtered_data = data[data["is_filtered"] & (data["dataset_family"].astype(str) == "MS MARCO")] + + class PlotRecorder: + figure = None + + def info(self, _message: str) -> None: + raise AssertionError("Expected filtered QPS chart data") + + def plotly_chart(self, figure: Any, **_kwargs) -> None: + self.figure = figure + + recorder = PlotRecorder() + _draw_filtered_qps_tab(recorder, filtered_data) + + assert recorder.figure is not None + assert recorder.figure.layout.barmode == "group" + assert {trace.type for trace in recorder.figure.data} == {"bar"} + assert {trace.name for trace in recorder.figure.data} == { + "ElasticSearch", + "OSSOpenSearch", + "TurboPuffer", + "ZillizCloud", + } + expected_filter_rates = {"50%", "75%", "90%", "95%", "99%"} + assert all(set(trace.x) == expected_filter_rates for trace in recorder.figure.data) + assert recorder.figure.layout.xaxis.type == "category" + assert tuple(recorder.figure.layout.xaxis.categoryarray) == ("50%", "75%", "90%", "95%", "99%") + + +def test_checked_in_zilliz_standard_results_use_semantic_metrics(): + repo_root = Path(__file__).resolve().parents[1] + result_dir = repo_root / "vectordb_bench" / "results" / "FullTextSearch" + + data = load_full_text_search_rows(result_dir) + zilliz_data = data[ + (~data["is_filtered"]) & (data["backend"].astype(str) == "ZillizCloud") & (data["payload"] == "ids_only") + ] + expected = { + "MS MARCO Small": (0.9157, 0.7206, 0.6713), + "MS MARCO Medium": (0.8261, 0.5298, 0.4572), + "MS MARCO Large": (0.6283, 0.2763, 0.1889), + "HotpotQA Small": (0.9225, 0.8459, 0.9485), + "HotpotQA Medium": (0.8437, 0.7322, 0.8636), + "HotpotQA Large": (0.7674, 0.6265, 0.7574), + } + + assert len(zilliz_data) == len(expected) + for row in zilliz_data.to_dict("records"): + assert (row["recall"], row["ndcg"], row["mrr"]) == expected[str(row["dataset"])] + + +def test_checked_in_zilliz_large_standard_results_retain_load_metrics(): + repo_root = Path(__file__).resolve().parents[1] + result_file = ( + repo_root + / "vectordb_bench" + / "results" + / "FullTextSearch" + / "ZillizCloud" + / "result_20260626_fts_standard_zillizcloud.json" + ) + expected = { + "HotpotQA Large (5.2M documents)": (5_233_329, 92.5443, 81.9515, 174.4958), + "MS MARCO Large (8.8M documents)": (8_841_823, 183.6571, 98.8113, 282.4684), + } + + large_results = {} + for case_result in json.loads(result_file.read_text())["results"]: + custom_case = case_result["task_config"]["case_config"].get("custom_case") or {} + dataset = custom_case.get("dataset_with_size_type") + if dataset in expected and custom_case.get("filter_rate") is None: + metrics = case_result["metrics"] + large_results[dataset] = ( + metrics["inserted_count"], + metrics["insert_duration"], + metrics["optimize_duration"], + metrics["load_duration"], + ) + + assert large_results == expected diff --git a/tests/test_fts_metrics.py b/tests/test_fts_metrics.py new file mode 100644 index 000000000..fbbb8552a --- /dev/null +++ b/tests/test_fts_metrics.py @@ -0,0 +1,69 @@ +from types import TracebackType +from typing import Any + +import pytest + +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.backend.runner.serial_runner import SerialSearchRunner +from vectordb_bench.backend.workload import WorkloadKind +from vectordb_bench.metric import calc_mrr_fts, calc_ndcg_fts, calc_recall_fts + + +class FakeSearchDB: + name = "FakeSearchDB" + + def init(self): + return self + + def __enter__(self): + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> bool: + return False + + def prepare_filter(self, filters: Any) -> None: + return None + + def supports_document_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return payload_profile == PayloadProfile.IDS_ONLY + + def supports_payload_profile(self, payload_profile: PayloadProfile) -> bool: + return True + + def search_documents(self, query: str, k: int) -> list[str]: + assert query == "query" + assert k == 3 + return ["missing", "d2", "d1"] + + +def test_semantic_fts_metrics_use_relevance_grades(): + qrels = {"d1": 3, "d2": 1} + got = ["missing", "d2", "d1"] + + assert calc_recall_fts(2, qrels, got) == 0.5 + assert calc_recall_fts(3, qrels, got) == 1.0 + assert calc_mrr_fts(3, qrels, got) == 0.5 + assert calc_ndcg_fts(3, qrels, got) == pytest.approx(0.5869, abs=1e-4) + + +def test_serial_runner_returns_semantic_fts_metrics(): + runner = SerialSearchRunner( + db=FakeSearchDB(), + test_data=["query"], + ground_truth=[{"d1": 3, "d2": 1}], + k=3, + workload_kind=WorkloadKind.FULL_TEXT, + ) + + recall, ndcg, mrr, p99, p95 = runner.search((runner.test_data, runner.ground_truth)) + + assert recall == 1.0 + assert ndcg == 0.5869 + assert mrr == 0.5 + assert p99 >= 0 + assert p95 >= 0 diff --git a/tests/test_milvus_zilliz_cli.py b/tests/test_milvus_zilliz_cli.py index a028c535b..4177c8898 100644 --- a/tests/test_milvus_zilliz_cli.py +++ b/tests/test_milvus_zilliz_cli.py @@ -5,6 +5,31 @@ from vectordb_bench.backend.clients.zilliz_cloud import cli as zilliz_cli +def test_milvus_cli_builds_shared_connection_config() -> None: + parameters = { + "db_label": "milvus-test", + "uri": "http://localhost:19530", + "user_name": "root", + "password": "secret", + "num_shards": "2", + "replica_number": "3", + "collection_name": "bench_collection", + } + + config = milvus_cli._build_milvus_config(parameters) + + assert config.db_label == "milvus-test" + assert config.uri.get_secret_value() == "http://localhost:19530" + assert config.user == "root" + assert config.password.get_secret_value() == "secret" + assert config.num_shards == 2 + assert config.replica_number == 3 + assert config.collection_name == "bench_collection" + + parameters["password"] = None + assert milvus_cli._build_milvus_config(parameters).password is None + + def test_milvus_autoindex_cli_enables_partition_key_for_multitenant_case( monkeypatch: MonkeyPatch, ) -> None: diff --git a/tests/test_oss_opensearch_fts.py b/tests/test_oss_opensearch_fts.py new file mode 100644 index 000000000..25b5b295c --- /dev/null +++ b/tests/test_oss_opensearch_fts.py @@ -0,0 +1,340 @@ +# ruff: noqa: E402 + +import sys +import types + +import pytest +from click.testing import CliRunner + + +class _FakeOpenSearch: + pass + + +sys.modules.setdefault("opensearchpy", types.SimpleNamespace(OpenSearch=_FakeOpenSearch)) + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import IndexType +from vectordb_bench.backend.clients.elastic_cloud.config import ElasticCloudFtsConfig +from vectordb_bench.backend.clients.oss_opensearch import cli as oss_opensearch_cli +from vectordb_bench.backend.clients.oss_opensearch.config import OSSOpenSearchFtsConfig, OSSOpenSearchIndexConfig +from vectordb_bench.backend.clients.oss_opensearch.oss_opensearch import OSSOpenSearch, OpenSearchError +from vectordb_bench.backend.filter import NewIntFilter +from vectordb_bench.backend.payload import PayloadProfile +from vectordb_bench.cli.cli import select_cli_db_case_config + + +def make_fts_db(): + db = OSSOpenSearch.__new__(OSSOpenSearch) + db.index_name = "idx" + db.id_col_name = "doc_id" + db.text_col_name = "text" + db.filter_id_col_name = "filter_id" + db.filter = None + db._is_fts = True + db.client = object() + return db + + +def test_oss_opensearch_fts_config_defaults(): + config = OSSOpenSearchFtsConfig() + + assert config.index_param() == ElasticCloudFtsConfig().index_param() + assert config.index_param()["properties"]["doc_id"] == {"type": "keyword"} + assert config.index_param()["properties"]["filter_id"] == {"type": "long"} + assert config.index_param()["properties"]["text"] == {"type": "text"} + assert config.search_param() == {} + + +def test_oss_opensearch_fts_config_supports_bm25_similarity(): + config = OSSOpenSearchFtsConfig(bm25_k1=1.2, bm25_b=0.75) + elastic_config = ElasticCloudFtsConfig(bm25_k1=1.2, bm25_b=0.75) + + assert config.similarity_settings() == elastic_config.similarity_settings() + assert config.index_param()["properties"]["text"]["similarity"] == "vdbbench_bm25" + assert config.similarity_settings() == { + "similarity": { + "vdbbench_bm25": { + "type": "BM25", + "k1": 1.2, + "b": 0.75, + } + } + } + + +def test_oss_opensearch_declares_full_text_support(): + assert OSSOpenSearch.supports_full_text_search() is True + assert DB.OSSOpenSearch.case_config_cls(IndexType.FTS) is OSSOpenSearchFtsConfig + + +def test_oss_opensearch_fts_cli_does_not_require_vector_index_options(monkeypatch: pytest.MonkeyPatch): + captured = {} + monkeypatch.setattr(oss_opensearch_cli, "run", lambda **kwargs: captured.update(kwargs)) + + result = CliRunner().invoke( + oss_opensearch_cli.OSSOpenSearch, + ["--host", "localhost", "--case-type", "FTSBm25Performance", "--dry-run"], + ) + + assert result.exit_code == 0, result.output + assert isinstance(captured["db_case_config"], OSSOpenSearchFtsConfig) + + +def test_oss_opensearch_fts_cli_uses_single_node_defaults_and_force_merge_flag(monkeypatch: pytest.MonkeyPatch): + captured = {} + monkeypatch.setattr(oss_opensearch_cli, "run", lambda **kwargs: captured.update(kwargs)) + + result = CliRunner().invoke( + oss_opensearch_cli.OSSOpenSearch, + [ + "--host", + "localhost", + "--case-type", + "FTSBm25Performance", + "--force-merge-enabled", + "false", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["db_case_config"].number_of_replicas == 0 + assert captured["db_case_config"].force_merge_enabled is False + + +def test_oss_opensearch_generic_fts_routing_preserves_force_merge(): + selected = select_cli_db_case_config( + DB.OSSOpenSearch, + OSSOpenSearchIndexConfig(force_merge_enabled=False), + "FTSBm25Performance", + ) + + assert isinstance(selected, OSSOpenSearchFtsConfig) + assert selected.force_merge_enabled is False + + +def test_oss_opensearch_green_health_wait_is_bounded_and_diagnostic(): + db = make_fts_db() + db.case_config = OSSOpenSearchFtsConfig(number_of_replicas=1) + calls = {} + + class Cluster: + def health(self, **kwargs): + calls.update(kwargs) + return { + "status": "yellow", + "timed_out": True, + "unassigned_shards": 1, + "number_of_nodes": 1, + } + + db.client = types.SimpleNamespace(cluster=Cluster()) + + with pytest.raises(OpenSearchError, match="configured_replicas=1.*unassigned_shards=1.*number_of_nodes=1"): + db._wait_till_green() + + assert calls == { + "index": "idx", + "wait_for_status": "green", + "timeout": "30m", + } + + +def test_oss_opensearch_create_index_fts_uses_text_mappings_and_settings(): + db = OSSOpenSearch.__new__(OSSOpenSearch) + db._is_fts = True + db.case_config = OSSOpenSearchFtsConfig( + number_of_shards=2, + number_of_replicas=1, + refresh_interval="10s", + ) + db.index_name = "idx" + calls = {} + + class Indices: + def create(self, **kwargs): + calls.update(kwargs) + + class Client: + indices = Indices() + + db._create_index(Client()) + + assert calls == { + "index": "idx", + "body": { + "settings": { + "index": { + "number_of_shards": 2, + "number_of_replicas": 1, + "refresh_interval": "10s", + } + }, + "mappings": { + "properties": { + "doc_id": {"type": "keyword"}, + "filter_id": {"type": "long"}, + "text": {"type": "text"}, + } + }, + }, + } + + +def test_oss_opensearch_insert_documents_builds_bulk_body(): + db = make_fts_db() + captured = {} + + class Client: + def bulk(self, **kwargs): + captured.update(kwargs) + return {"errors": False} + + db.client = Client() + + assert db.insert_documents(["alpha", "beta"], ["d1", "d2"], filter_ids=[0, 7]) == (2, None) + assert captured["body"] == [ + {"index": {"_index": "idx", "_id": "d1"}}, + {"doc_id": "d1", "text": "alpha", "filter_id": 0}, + {"index": {"_index": "idx", "_id": "d2"}}, + {"doc_id": "d2", "text": "beta", "filter_id": 7}, + ] + + +def test_oss_opensearch_insert_documents_reports_partial_bulk_failure(): + db = make_fts_db() + + class Client: + def bulk(self, **kwargs): + return { + "errors": True, + "items": [ + {"index": {"_id": "d1", "status": 201}}, + { + "index": { + "_id": "d2", + "status": 429, + "error": { + "type": "rejected_execution_exception", + "reason": "indexing queue is full", + }, + } + }, + ], + } + + db.client = Client() + + insert_count, error = db.insert_documents(["alpha", "beta"], ["d1", "d2"]) + + assert insert_count == 1 + assert isinstance(error, RuntimeError) + assert "failed for 1/2 documents" in str(error) + assert "successful=1" in str(error) + assert "id=d2" in str(error) + assert "rejected_execution_exception: indexing queue is full" in str(error) + + +def test_oss_opensearch_insert_documents_rejects_malformed_bulk_error_response(): + db = make_fts_db() + + class Client: + def bulk(self, **kwargs): + return {"errors": True} + + db.client = Client() + + insert_count, error = db.insert_documents(["alpha"], ["d1"]) + + assert insert_count == 0 + assert isinstance(error, RuntimeError) + assert str(error) == "OpenSearch FTS bulk response reported errors without an items list" + + +def test_oss_opensearch_insert_documents_validates_lengths(): + db = make_fts_db() + + class Client: + def bulk(self, **kwargs): + raise AssertionError("bulk should not be called") + + db.client = Client() + + with pytest.raises(ValueError, match=r"Mismatch between texts .* and doc_ids .* lengths"): + db.insert_documents(["alpha", "beta"], ["d1"]) + + +def test_oss_opensearch_search_documents_builds_match_query(): + db = make_fts_db() + calls = {} + + class Client: + def search(self, **kwargs): + calls.update(kwargs) + return {"hits": {"hits": [{"fields": {"doc_id": ["d1"]}}]}} + + db.client = Client() + + assert db.search_documents("hello world", k=3) == ["d1"] + assert calls["index"] == "idx" + assert calls["body"] == {"query": {"match": {"text": "hello world"}}} + assert calls["size"] == 3 + assert calls["stored_fields"] == "_none_" + assert calls["filter_path"] == ["hits.hits._id", "hits.hits.fields.doc_id"] + + +def test_oss_opensearch_search_documents_applies_fts_filter(): + db = make_fts_db() + calls = {} + + class Client: + def search(self, **kwargs): + calls.update(kwargs) + return {"hits": {"hits": [{"fields": {"doc_id": ["d1"]}}]}} + + db.client = Client() + db.prepare_filter(NewIntFilter(filter_rate=0.5, int_field="filter_id", int_value=10)) + + assert db.search_documents("hello world", k=3) == ["d1"] + assert calls["body"] == { + "query": { + "bool": { + "must": {"match": {"text": "hello world"}}, + "filter": {"range": {"filter_id": {"gte": 10}}}, + } + } + } + + +def test_oss_opensearch_search_documents_requests_text_payload(): + db = make_fts_db() + calls = {} + + class Client: + def search(self, **kwargs): + calls.update(kwargs) + return {"hits": {"hits": [{"_id": "d1", "_source": {"text": "hello"}}]}} + + db.client = Client() + + assert db.search_documents("hello world", k=3, payload_profile=PayloadProfile.TEXT) == ["d1"] + assert calls["_source"] == ["text"] + assert "stored_fields" not in calls + assert calls["filter_path"] == [ + "hits.hits._id", + "hits.hits.fields.doc_id", + "hits.hits._source.text", + ] + + +def test_oss_opensearch_document_methods_guard_non_fts_mode(): + db = OSSOpenSearch.__new__(OSSOpenSearch) + db._is_fts = False + db.client = object() + + with pytest.raises(RuntimeError, match="OSSOpenSearch full-text insert requires OSSOpenSearchFtsConfig"): + db.insert_documents(["alpha"], ["d1"]) + + with pytest.raises(RuntimeError, match="OSSOpenSearch full-text search requires OSSOpenSearchFtsConfig"): + db.search_documents("alpha") diff --git a/tests/test_turbopuffer_cli.py b/tests/test_turbopuffer_cli.py index b167a319e..46db473ca 100644 --- a/tests/test_turbopuffer_cli.py +++ b/tests/test_turbopuffer_cli.py @@ -291,3 +291,48 @@ def fake_wait_for_pinning(api_key, region, namespace, replicas, api_base_url=Non ("PATCH", {"pinning": None}, "secret", "aws-us-west-2", "laion100m", None), ("WAIT", None, "secret", "aws-us-west-2", "laion100m", None, 7200), ] + + +def test_turbopuffer_fts_insert_declares_filter_id_filterable() -> None: + writes = [] + + class FakeNamespace: + def write(self, **kwargs): + writes.append(kwargs) + + db = object.__new__(TurboPuffer) + db.ns = FakeNamespace() + db._is_fts = True + db._text_field = "text" + db._scalar_id_field = "id" + db._filter_id_field = "filter_id" + db.db_case_config = SimpleNamespace(disable_backpressure=False) + + count, error = db.insert_documents( + texts=["alpha", "beta"], + doc_ids=["doc-1", "doc-2"], + filter_ids=[10, 20], + ) + + assert error is None + assert count == 2 + assert writes == [ + { + "upsert_columns": { + "id": ["doc-1", "doc-2"], + "text": ["alpha", "beta"], + "filter_id": [10, 20], + }, + "schema": { + "text": { + "type": "string", + "full_text_search": True, + }, + "filter_id": { + "type": "int", + "filterable": True, + }, + }, + "disable_backpressure": False, + } + ] diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 0253168e0..24eaf768e 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -20,6 +20,17 @@ log = logging.getLogger(__name__) +FTS_FILTER_ID_FIELD = "filter_id" +FTS_FILTER_RATES = (0.5, 0.75, 0.9, 0.95, 0.99) + + +def _format_filter_rate(filter_rate: float) -> str: + return f"{filter_rate * 100:g}%" + + +def _is_supported_fts_filter_rate(filter_rate: float) -> bool: + return any(abs(filter_rate - supported_rate) < 1e-9 for supported_rate in FTS_FILTER_RATES) + class CaseType(Enum): """ @@ -920,7 +931,10 @@ class FtsPerformanceCase(Case): @property def filters(self) -> Filter: - return non_filter + if self.filter_rate is None: + return non_filter + int_value = int(self.dataset.data.size * self.filter_rate) + return NewIntFilter(filter_rate=self.filter_rate, int_field=FTS_FILTER_ID_FIELD, int_value=int_value) def estimated_payload_bytes_per_query(self, k: int | None) -> int: if k is None: @@ -935,21 +949,37 @@ class FTSBm25Performance(FtsPerformanceCase): def __init__( self, dataset_with_size_type: FtsDatasetWithSizeType | str = FtsDatasetWithSizeType.MSMarcoSmall, + filter_rate: float | None = None, **kwargs, ): if not isinstance(dataset_with_size_type, FtsDatasetWithSizeType): dataset_with_size_type = FtsDatasetWithSizeType(dataset_with_size_type) + if filter_rate is not None: + if not dataset_with_size_type.is_advanced: + msg = "FTS filter cases are only supported for MS MARCO Large and HotpotQA Large" + raise ValueError(msg) + if not _is_supported_fts_filter_rate(filter_rate): + supported_rates = ", ".join(_format_filter_rate(rate) for rate in FTS_FILTER_RATES) + msg = f"FTS filter_rate must be one of: {supported_rates}" + raise ValueError(msg) dataset = dataset_with_size_type.get_manager() - name = f"FTS BM25 Performance - {dataset_with_size_type.value}" + filter_suffix = f", Filter {_format_filter_rate(filter_rate)}" if filter_rate is not None else "" + name = f"FTS BM25 Performance - {dataset_with_size_type.value}{filter_suffix}" description = ( f"This case tests native BM25 full-text search performance on {dataset_with_size_type.value}. " "It measures index building time, recall, serial latency, and search QPS." ) + if filter_rate is not None: + description += ( + f" The FTS filter case searches only documents with {FTS_FILTER_ID_FIELD} >= " + f"int(dataset_size * {filter_rate})." + ) super().__init__( name=name, description=description, dataset=dataset, dataset_with_size_type=dataset_with_size_type, + filter_rate=filter_rate, load_timeout=dataset_with_size_type.get_load_timeout(), optimize_timeout=dataset_with_size_type.get_optimize_timeout(), **kwargs, diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index bf3d36312..beacb37af 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -560,6 +560,10 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return AWSOpenSearchIndexConfig if self == DB.OSSOpenSearch: + if index_type == IndexType.FTS: + from .oss_opensearch.config import OSSOpenSearchFtsConfig + + return OSSOpenSearchFtsConfig from .oss_opensearch.config import OSSOpenSearchIndexConfig return OSSOpenSearchIndexConfig diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index dcb1921c6..bac90b7be 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -151,35 +151,6 @@ def index_param(self) -> dict: def search_param(self) -> dict: raise NotImplementedError - def apply_fts_manifest( - self, - bm25_params: dict[str, float], - analyzer_params: dict, - ) -> tuple["DBCaseConfig", dict]: - """Apply FTS dataset manifest parameters to this case config. - - Full-text search datasets may provide BM25 and analyzer settings used to - build the mathematical ground truth. Backends that can reproduce those - settings should return an updated config with supported parameters - applied. Unsupported parameters must be reported in the returned metadata - instead of being silently ignored. - - Args: - bm25_params(dict[str, float]): BM25 parameters from the dataset - manifest, such as k1, b, and avgdl. - analyzer_params(dict): analyzer settings from the dataset manifest. - - Returns: - tuple[DBCaseConfig, dict]: updated config and a report describing - applied and unapplied BM25/analyzer parameters. - """ - return self, { - "applied_bm25_params": {}, - "unapplied_bm25_params": dict(bm25_params), - "applied_analyzer_params": {}, - "unapplied_analyzer_params": dict(analyzer_params), - } - class EmptyDBCaseConfig(BaseModel, DBCaseConfig): """EmptyDBCaseConfig will be used if the vector database has no case specific configs""" diff --git a/vectordb_bench/backend/clients/elastic_cloud/config.py b/vectordb_bench/backend/clients/elastic_cloud/config.py index 5b04e3366..9f9961efc 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/config.py +++ b/vectordb_bench/backend/clients/elastic_cloud/config.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, SecretStr, model_validator from ..api import DBCaseConfig, DBConfig, IndexType, MetricType +from ..elasticsearch_compatible import build_bm25_similarity_settings, build_fts_index_param class ElasticCloudConfig(DBConfig, BaseModel): @@ -145,49 +146,11 @@ class ElasticCloudFtsConfig(BaseModel, DBCaseConfig): bm25_k1: float | None = None bm25_b: float | None = None - def apply_fts_manifest( - self, - bm25_params: dict[str, float], - analyzer_params: dict, - ) -> tuple[DBCaseConfig, dict]: - updates = {} - applied_bm25_params = {} - - if "k1" in bm25_params: - updates["bm25_k1"] = bm25_params["k1"] - applied_bm25_params["k1"] = bm25_params["k1"] - if "b" in bm25_params: - updates["bm25_b"] = bm25_params["b"] - applied_bm25_params["b"] = bm25_params["b"] - - return self.model_copy(update=updates), { - "applied_bm25_params": applied_bm25_params, - "unapplied_bm25_params": {k: v for k, v in bm25_params.items() if k not in applied_bm25_params}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": dict(analyzer_params), - } - def index_param(self) -> dict: - text_mapping = {"type": "text"} - if self.bm25_k1 is not None or self.bm25_b is not None: - text_mapping["similarity"] = "vdbbench_bm25" - return { - "properties": { - "doc_id": {"type": "keyword"}, - "text": text_mapping, - }, - } + return build_fts_index_param(self.bm25_k1, self.bm25_b) def search_param(self) -> dict: return {} def similarity_settings(self) -> dict: - if self.bm25_k1 is None and self.bm25_b is None: - return {} - - bm25_settings = {"type": "BM25"} - if self.bm25_k1 is not None: - bm25_settings["k1"] = self.bm25_k1 - if self.bm25_b is not None: - bm25_settings["b"] = self.bm25_b - return {"similarity": {"vdbbench_bm25": bm25_settings}} + return build_bm25_similarity_settings(self.bm25_k1, self.bm25_b) diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index 552965cfb..c930dc057 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -50,6 +50,7 @@ def __init__( self.with_scalar_labels = with_scalar_labels self._is_fts = isinstance(db_case_config, ElasticCloudFtsConfig) self.text_col_name = "text" + self.filter_id_col_name = "filter_id" if self._is_fts: self.id_col_name = "doc_id" @@ -187,17 +188,26 @@ def insert_documents( if len(docs) != len(doc_ids): msg = f"Mismatch between texts ({len(docs)}) and doc_ids ({len(doc_ids)}) lengths" raise ValueError(msg) - actions = [ - { - "_index": self.indice, - "_id": str(doc_ids[i]), - "_source": { - self.id_col_name: str(doc_ids[i]), - self.text_col_name: docs[i], - }, + filter_ids = kwargs.get("filter_ids") + if filter_ids is not None and len(filter_ids) != len(docs): + msg = f"Mismatch between texts ({len(docs)}) and filter_ids ({len(filter_ids)}) lengths" + raise ValueError(msg) + + actions = [] + for i, doc in enumerate(docs): + source = { + self.id_col_name: str(doc_ids[i]), + self.text_col_name: doc, } - for i in range(len(docs)) - ] + if filter_ids is not None: + source[self.filter_id_col_name] = int(filter_ids[i]) + actions.append( + { + "_index": self.indice, + "_id": str(doc_ids[i]), + "_source": source, + } + ) try: result = bulk(self.client, actions) return result[0], None @@ -208,10 +218,19 @@ def insert_documents( def prepare_filter(self, filters: Filter): self.routing_key = None if filters.type == FilterOp.NonFilter: - self.filter = [] + self.filter = None if self._is_fts else [] elif filters.type == FilterOp.NumGE: - self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} + if self._is_fts: + if getattr(filters, "int_field", None) != self.filter_id_col_name: + msg = f"ElasticCloud FTS filters only support int_field='{self.filter_id_col_name}'" + raise ValueError(msg) + self.filter = {"range": {self.filter_id_col_name: {"gte": filters.int_value}}} + else: + self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} elif filters.type == FilterOp.StrEqual: + if self._is_fts: + msg = f"Not support Filter for ElasticCloud FTS - {filters}" + raise ValueError(msg) self.filter = {"term": {self.label_col_name: filters.label_value}} if self.case_config.use_routing: self.routing_key = filters.label_value @@ -277,9 +296,12 @@ def search_documents( filter_path = ["hits.hits._id", f"hits.hits.fields.{self.id_col_name}"] if payload_profile == PayloadProfile.TEXT: filter_path.append(f"hits.hits._source.{self.text_col_name}") + query_clause = {"match": {self.text_col_name: query}} + if getattr(self, "filter", None): + query_clause = {"bool": {"must": query_clause, "filter": self.filter}} search_kwargs = { "index": self.indice, - "query": {"match": {self.text_col_name: query}}, + "query": query_clause, "size": k, "_source": source, "docvalue_fields": [self.id_col_name], diff --git a/vectordb_bench/backend/clients/elasticsearch_compatible.py b/vectordb_bench/backend/clients/elasticsearch_compatible.py new file mode 100644 index 000000000..6e040b3c6 --- /dev/null +++ b/vectordb_bench/backend/clients/elasticsearch_compatible.py @@ -0,0 +1,23 @@ +def build_fts_index_param(bm25_k1: float | None, bm25_b: float | None) -> dict: + text_mapping = {"type": "text"} + if bm25_k1 is not None or bm25_b is not None: + text_mapping["similarity"] = "vdbbench_bm25" + return { + "properties": { + "doc_id": {"type": "keyword"}, + "filter_id": {"type": "long"}, + "text": text_mapping, + }, + } + + +def build_bm25_similarity_settings(bm25_k1: float | None, bm25_b: float | None) -> dict: + if bm25_k1 is None and bm25_b is None: + return {} + + bm25_settings = {"type": "BM25"} + if bm25_k1 is not None: + bm25_settings["k1"] = bm25_k1 + if bm25_b is not None: + bm25_settings["b"] = bm25_b + return {"similarity": {"vdbbench_bm25": bm25_settings}} diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 48ad45608..738248ac3 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -27,6 +27,20 @@ def _with_partition_key(db_case_config: BaseModel, parameters: dict) -> BaseMode return db_case_config.model_copy(update={"use_partition_key": _use_partition_key(parameters)}) +def _build_milvus_config(parameters: dict) -> BaseModel: + from .config import MilvusConfig + + return MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + collection_name=parameters["collection_name"], + ) + + class MilvusTypedDict(TypedDict): uri: Annotated[ str, @@ -62,6 +76,17 @@ class MilvusTypedDict(TypedDict): show_default=True, ), ] + collection_name: Annotated[ + str, + click.option( + "--collection-name", + type=str, + help="Collection name for Milvus", + required=False, + default="VDBBench", + show_default=True, + ), + ] use_partition_key: Annotated[ bool | None, click.option( @@ -81,18 +106,11 @@ class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): ... @cli.command() @click_parameter_decorators_from_typed_dict(MilvusAutoIndexTypedDict) def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): - from .config import AutoIndexConfig, MilvusConfig + from .config import AutoIndexConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key(AutoIndexConfig(), parameters), **parameters, ) @@ -101,18 +119,11 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusAutoIndexTypedDict) def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): - from .config import FLATConfig, MilvusConfig + from .config import FLATConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key(FLATConfig(), parameters), **parameters, ) @@ -124,18 +135,11 @@ class MilvusHNSWTypedDict(CommonTypedDict, MilvusTypedDict, HNSWFlavor3): ... @cli.command() @click_parameter_decorators_from_typed_dict(MilvusHNSWTypedDict) def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): - from .config import HNSWConfig, MilvusConfig + from .config import HNSWConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( HNSWConfig( M=parameters["m"], @@ -192,18 +196,11 @@ class MilvusHNSWPQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusHNSWTypedDic @cli.command() @click_parameter_decorators_from_typed_dict(MilvusHNSWPQTypedDict) def MilvusHNSWPQ(**parameters: Unpack[MilvusHNSWPQTypedDict]): - from .config import HNSWPQConfig, MilvusConfig + from .config import HNSWPQConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( HNSWPQConfig( M=parameters["m"], @@ -239,18 +236,11 @@ class MilvusHNSWPRQTypedDict( @cli.command() @click_parameter_decorators_from_typed_dict(MilvusHNSWPRQTypedDict) def MilvusHNSWPRQ(**parameters: Unpack[MilvusHNSWPRQTypedDict]): - from .config import HNSWPRQConfig, MilvusConfig + from .config import HNSWPRQConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( HNSWPRQConfig( M=parameters["m"], @@ -283,18 +273,11 @@ class MilvusHNSWSQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusHNSWTypedDic @cli.command() @click_parameter_decorators_from_typed_dict(MilvusHNSWSQTypedDict) def MilvusHNSWSQ(**parameters: Unpack[MilvusHNSWSQTypedDict]): - from .config import HNSWSQConfig, MilvusConfig + from .config import HNSWSQConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( HNSWSQConfig( M=parameters["m"], @@ -317,18 +300,11 @@ class MilvusIVFFlatTypedDict(CommonTypedDict, MilvusTypedDict, IVFFlatTypedDictN @cli.command() @click_parameter_decorators_from_typed_dict(MilvusIVFFlatTypedDict) def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): - from .config import IVFFlatConfig, MilvusConfig + from .config import IVFFlatConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( IVFFlatConfig( nlist=parameters["nlist"], @@ -343,18 +319,11 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusIVFFlatTypedDict) def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): - from .config import IVFSQ8Config, MilvusConfig + from .config import IVFSQ8Config run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( IVFSQ8Config( nlist=parameters["nlist"], @@ -408,18 +377,11 @@ class MilvusIVFRABITQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusIVFFlatTy @cli.command() @click_parameter_decorators_from_typed_dict(MilvusIVFRABITQTypedDict) def MilvusIVFRabitQ(**parameters: Unpack[MilvusIVFRABITQTypedDict]): - from .config import IVFRABITQConfig, MilvusConfig + from .config import IVFRABITQConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( IVFRABITQConfig( nlist=parameters["nlist"], @@ -442,18 +404,11 @@ class MilvusDISKANNTypedDict(CommonTypedDict, MilvusTypedDict): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusDISKANNTypedDict) def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): - from .config import DISKANNConfig, MilvusConfig + from .config import DISKANNConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( DISKANNConfig( search_list=parameters["search_list"], @@ -475,18 +430,11 @@ class MilvusGPUIVFTypedDict(CommonTypedDict, MilvusTypedDict, MilvusIVFFlatTyped @cli.command() @click_parameter_decorators_from_typed_dict(MilvusGPUIVFTypedDict) def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): - from .config import GPUIVFFlatConfig, MilvusConfig + from .config import GPUIVFFlatConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( GPUIVFFlatConfig( nlist=parameters["nlist"], @@ -514,18 +462,11 @@ class MilvusGPUBruteForceTypedDict(CommonTypedDict, MilvusTypedDict): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusGPUBruteForceTypedDict) def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): - from .config import GPUBruteForceConfig, MilvusConfig + from .config import GPUBruteForceConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( GPUBruteForceConfig( metric_type=parameters["metric_type"], @@ -607,18 +548,11 @@ class MilvusSVSVamanaTypedDict(CommonTypedDict, MilvusTypedDict): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusSVSVamanaTypedDict) def MilvusSVSVamana(**parameters: Unpack[MilvusSVSVamanaTypedDict]): - from .config import MilvusConfig, SVSVamanaConfig + from .config import SVSVamanaConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( SVSVamanaConfig( svs_graph_max_degree=parameters["svs_graph_max_degree"], @@ -637,18 +571,11 @@ def MilvusSVSVamana(**parameters: Unpack[MilvusSVSVamanaTypedDict]): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusSVSVamanaTypedDict) def MilvusSVSVamanaLVQ(**parameters: Unpack[MilvusSVSVamanaTypedDict]): - from .config import MilvusConfig, SVSVamanaLVQConfig + from .config import SVSVamanaLVQConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( SVSVamanaLVQConfig( svs_graph_max_degree=parameters["svs_graph_max_degree"], @@ -681,18 +608,11 @@ class MilvusSVSVamanaLeanVecTypedDict(MilvusSVSVamanaTypedDict): @cli.command() @click_parameter_decorators_from_typed_dict(MilvusSVSVamanaLeanVecTypedDict) def MilvusSVSVamanaLeanVec(**parameters: Unpack[MilvusSVSVamanaLeanVecTypedDict]): - from .config import MilvusConfig, SVSVamanaLeanVecConfig + from .config import SVSVamanaLeanVecConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( SVSVamanaLeanVecConfig( svs_graph_max_degree=parameters["svs_graph_max_degree"], @@ -722,18 +642,11 @@ class MilvusGPUIVFPQTypedDict( @cli.command() @click_parameter_decorators_from_typed_dict(MilvusGPUIVFPQTypedDict) def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): - from .config import GPUIVFPQConfig, MilvusConfig + from .config import GPUIVFPQConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( GPUIVFPQConfig( nlist=parameters["nlist"], @@ -766,18 +679,11 @@ class MilvusGPUCAGRATypedDict(CommonTypedDict, MilvusTypedDict, MilvusGPUIVFType @cli.command() @click_parameter_decorators_from_typed_dict(MilvusGPUCAGRATypedDict) def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): - from .config import GPUCAGRAConfig, MilvusConfig + from .config import GPUCAGRAConfig run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=_with_partition_key( GPUCAGRAConfig( intermediate_graph_degree=parameters["intermediate_graph_degree"], @@ -819,7 +725,7 @@ def MilvusFTS(**parameters: Unpack[MilvusFTSTypedDict]): This command uses the MS MARCO dev/small dataset for FTS testing. """ - from .config import MilvusConfig, MilvusFtsConfig + from .config import MilvusFtsConfig # Set default case_type to large dataset if not specified if parameters.get("case_type") == "Performance1536D50K": # Default from CommonTypedDict @@ -827,14 +733,7 @@ def MilvusFTS(**parameters: Unpack[MilvusFTSTypedDict]): run( db=DBTYPE, - db_config=MilvusConfig( - db_label=parameters["db_label"], - uri=SecretStr(parameters["uri"]), - user=parameters["user_name"], - password=SecretStr(parameters["password"]) if parameters["password"] else None, - num_shards=int(parameters["num_shards"]), - replica_number=int(parameters["replica_number"]), - ), + db_config=_build_milvus_config(parameters), db_case_config=MilvusFtsConfig( drop_ratio_search=parameters.get("drop_ratio_search"), ), diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index b069685fa..99d5ac8cf 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -13,6 +13,7 @@ class MilvusConfig(DBConfig): password: SecretStr | None = None num_shards: int = 1 replica_number: int = 1 + collection_name: str = "VDBBench" def to_dict(self) -> dict: return { @@ -21,6 +22,7 @@ def to_dict(self) -> dict: "password": self.password.get_secret_value() if self.password else None, "num_shards": self.num_shards, "replica_number": self.replica_number, + "collection_name": self.collection_name, } @@ -536,66 +538,6 @@ class MilvusFtsConfig(BaseModel, DBCaseConfig): analyzer_stop_words: str | None = None drop_ratio_search: float | None = None - @staticmethod - def _manifest_filter_list(analyzer_params: dict) -> list: - filters = analyzer_params.get("filter") or [] - if isinstance(filters, list): - return filters - return [filters] - - def _analyzer_manifest_updates(self, analyzer_params: dict) -> dict: - if not analyzer_params: - return {} - - updates = {} - tokenizer = analyzer_params.get("tokenizer") - if tokenizer: - updates["analyzer_tokenizer"] = tokenizer - - filters = self._manifest_filter_list(analyzer_params) - updates["analyzer_enable_lowercase"] = "lowercase" in filters - - length_max = None - stop_words = None - for item in filters: - if not isinstance(item, dict): - continue - if item.get("type") == "length": - length_max = item.get("max") - elif item.get("type") == "stop": - configured_stop_words = item.get("stop_words") - if isinstance(configured_stop_words, list): - stop_words = ",".join(str(word) for word in configured_stop_words) - elif configured_stop_words: - stop_words = str(configured_stop_words) - - updates["analyzer_max_token_length"] = length_max - updates["analyzer_stop_words"] = stop_words - return updates - - def apply_fts_manifest( - self, - bm25_params: dict[str, float], - analyzer_params: dict, - ) -> tuple[DBCaseConfig, dict]: - updates = {} - applied_bm25_params = {} - - if "k1" in bm25_params: - updates["bm25_k1"] = bm25_params["k1"] - applied_bm25_params["k1"] = bm25_params["k1"] - if "b" in bm25_params: - updates["bm25_b"] = bm25_params["b"] - applied_bm25_params["b"] = bm25_params["b"] - updates.update(self._analyzer_manifest_updates(analyzer_params)) - - return self.model_copy(update=updates), { - "applied_bm25_params": applied_bm25_params, - "unapplied_bm25_params": {k: v for k, v in bm25_params.items() if k not in applied_bm25_params}, - "applied_analyzer_params": dict(analyzer_params), - "unapplied_analyzer_params": {}, - } - def analyzer_param(self) -> dict: analyzer_params = {} if self.analyzer_tokenizer: diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 6ac1f5817..0ed09aaea 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -64,9 +64,11 @@ def __init__( # noqa: PLR0915 self.batch_size = fts_batch_size or MILVUS_FTS_BATCH_SIZE self._primary_field = "doc_id" self._text_field = "text" + self._filter_id_field = "filter_id" self._sparse_field = "sparse_vector" self._sparse_index_name = "sparse_vector_idx" self._doc_id_sort_index_name = "doc_id_sort_idx" + self._filter_id_sort_index_name = "filter_id_sort_idx" self._main_index_name = self._sparse_index_name self._sort_index_name = self._doc_id_sort_index_name self._sort_index_field = self._primary_field @@ -108,6 +110,7 @@ def __init__( # noqa: PLR0915 else self.case_config.index_param().get("analyzer_params", {"type": "english"}) ) schema.add_field(self._primary_field, DataType.VARCHAR, max_length=512, is_primary=True) + schema.add_field(self._filter_id_field, DataType.INT64) schema.add_field( self._text_field, DataType.VARCHAR, @@ -200,6 +203,12 @@ def _build_index_params(self): index_name=self._sort_index_name, index_type="STL_SORT", ) + if self._is_fts: + index_params.add_index( + field_name=self._filter_id_field, + index_name=self._filter_id_sort_index_name, + index_type="STL_SORT", + ) if self.with_scalar_labels: index_params.add_index( field_name=self._scalar_payload_label_field, @@ -398,6 +407,10 @@ def insert_documents( batch_size = kwargs.get("batch_size", self.batch_size) labels_data = kwargs.get("labels_data") + filter_ids = kwargs.get("filter_ids") + if filter_ids is not None and len(filter_ids) != len(docs): + msg = f"Mismatch between texts ({len(docs)}) and filter_ids ({len(filter_ids)}) lengths" + raise ValueError(msg) insert_count = 0 try: @@ -409,6 +422,8 @@ def insert_documents( self._primary_field: str(doc_ids[i]), self._text_field: docs[i], } + if filter_ids is not None: + row[self._filter_id_field] = int(filter_ids[i]) if self.with_scalar_labels: row[self._scalar_label_field] = labels_data[i] if labels_data is not None else "" rows.append(row) @@ -427,8 +442,17 @@ def insert_documents( def prepare_filter(self, filters: Filter): if self._is_fts: - self.expr = "" - return + if filters.type == FilterOp.NonFilter: + self.expr = "" + return + if filters.type == FilterOp.NumGE: + if getattr(filters, "int_field", None) != self._filter_id_field: + msg = f"Milvus FTS filters only support int_field='{self._filter_id_field}'" + raise ValueError(msg) + self.expr = f"{self._filter_id_field} >= {filters.int_value}" + return + msg = f"Not support Filter for Milvus FTS - {filters}" + raise ValueError(msg) if filters.type == FilterOp.NonFilter: self.expr = "" elif filters.type == FilterOp.NumGE: @@ -520,6 +544,7 @@ def search_documents( anns_field=self._sparse_field, search_params=self.case_config.search_param(), limit=k, + filter=getattr(self, "expr", ""), output_fields=output_fields, ) diff --git a/vectordb_bench/backend/clients/oss_opensearch/cli.py b/vectordb_bench/backend/clients/oss_opensearch/cli.py index 0c4b694ea..03da18fdb 100644 --- a/vectordb_bench/backend/clients/oss_opensearch/cli.py +++ b/vectordb_bench/backend/clients/oss_opensearch/cli.py @@ -11,6 +11,7 @@ click_parameter_decorators_from_typed_dict, run, ) +from ...cases import CaseType from .. import DB from .config import OSSOpenSearchQuantization, OSSOS_Engine @@ -18,6 +19,7 @@ class OSSOpenSearchTypedDict(TypedDict): + index_name: Annotated[str, click.option("--index-name", type=str, help="Db index name", default="vdb_bench_index")] host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] port: Annotated[int, click.option("--port", type=int, default=80, help="Db Port")] user: Annotated[str, click.option("--user", type=str, help="Db User")] @@ -29,7 +31,7 @@ class OSSOpenSearchTypedDict(TypedDict): number_of_replicas: Annotated[ int, click.option( - "--number-of-replicas", type=int, help="Number of replica copies for each primary shard", default=1 + "--number-of-replicas", type=int, help="Number of replica copies for each primary shard", default=0 ), ] index_thread_qty: Annotated[ @@ -147,17 +149,17 @@ class OSSOpenSearchHNSWTypedDict(CommonTypedDict, OSSOpenSearchTypedDict, HNSWFl @cli.command() @click_parameter_decorators_from_typed_dict(OSSOpenSearchHNSWTypedDict) def OSSOpenSearch(**parameters: Unpack[OSSOpenSearchHNSWTypedDict]): - from .config import OSSOpenSearchConfig, OSSOpenSearchIndexConfig + from .config import OSSOpenSearchConfig, OSSOpenSearchFtsConfig, OSSOpenSearchIndexConfig - run( - db=DB.OSSOpenSearch, - db_config=OSSOpenSearchConfig( - host=parameters["host"], - port=parameters["port"], - user=parameters["user"], - password=SecretStr(parameters["password"]), - ), - db_case_config=OSSOpenSearchIndexConfig( + if parameters["case_type"] == CaseType.FTSBm25Performance.name: + db_case_config = OSSOpenSearchFtsConfig( + number_of_shards=parameters["number_of_shards"], + number_of_replicas=parameters["number_of_replicas"], + refresh_interval=parameters["refresh_interval"], + force_merge_enabled=parameters["force_merge_enabled"], + ) + else: + db_case_config = OSSOpenSearchIndexConfig( number_of_shards=parameters["number_of_shards"], number_of_replicas=parameters["number_of_replicas"], index_thread_qty=parameters["index_thread_qty"], @@ -174,6 +176,17 @@ def OSSOpenSearch(**parameters: Unpack[OSSOpenSearchHNSWTypedDict]): quantization_type=OSSOpenSearchQuantization(parameters["quantization_type"]), confidence_interval=parameters["confidence_interval"], clip=parameters["clip"], + ) + + run( + db=DB.OSSOpenSearch, + db_config=OSSOpenSearchConfig( + index_name=parameters["index_name"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], + password=SecretStr(parameters["password"]), ), + db_case_config=db_case_config, **parameters, ) diff --git a/vectordb_bench/backend/clients/oss_opensearch/config.py b/vectordb_bench/backend/clients/oss_opensearch/config.py index 7a8b1d98e..05f8f584a 100644 --- a/vectordb_bench/backend/clients/oss_opensearch/config.py +++ b/vectordb_bench/backend/clients/oss_opensearch/config.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, SecretStr, field_validator, model_validator from ..api import DBCaseConfig, DBConfig, MetricType +from ..elasticsearch_compatible import build_bm25_similarity_settings, build_fts_index_param log = logging.getLogger(__name__) @@ -12,6 +13,7 @@ class OSSOpenSearchConfig(DBConfig, BaseModel): _extra_empty_skip: ClassVar[frozenset[str]] = frozenset({"user", "password", "host"}) + index_name: str = "vdb_bench_index" host: str = "" port: int = 80 user: str | None = None @@ -25,6 +27,7 @@ def to_dict(self) -> dict: else () ) return { + "index_name": self.index_name, "hosts": [{"host": self.host, "port": self.port}], "http_auth": http_auth, "use_ssl": use_ssl, @@ -247,3 +250,22 @@ def index_param(self) -> dict: def search_param(self) -> dict: return {"ef_search": self.efSearch} + + +class OSSOpenSearchFtsConfig(BaseModel, DBCaseConfig): + number_of_shards: int = 1 + number_of_replicas: int = 0 + refresh_interval: str = "30s" + force_merge_enabled: bool = True + metric_type: MetricType = MetricType.BM25 + bm25_k1: float | None = None + bm25_b: float | None = None + + def index_param(self) -> dict: + return build_fts_index_param(self.bm25_k1, self.bm25_b) + + def search_param(self) -> dict: + return {} + + def similarity_settings(self) -> dict: + return build_bm25_similarity_settings(self.bm25_k1, self.bm25_b) diff --git a/vectordb_bench/backend/clients/oss_opensearch/oss_opensearch.py b/vectordb_bench/backend/clients/oss_opensearch/oss_opensearch.py index f71850a17..d26514497 100644 --- a/vectordb_bench/backend/clients/oss_opensearch/oss_opensearch.py +++ b/vectordb_bench/backend/clients/oss_opensearch/oss_opensearch.py @@ -9,15 +9,16 @@ from packaging.version import parse as parse_version from vectordb_bench.backend.filter import Filter, FilterOp +from vectordb_bench.backend.payload import PayloadProfile from ..api import VectorDB -from .config import OSSOpenSearchIndexConfig, OSSOS_Engine +from .config import OSSOpenSearchFtsConfig, OSSOpenSearchIndexConfig, OSSOS_Engine log = logging.getLogger(__name__) WAITING_FOR_REFRESH_SEC: Final[int] = 30 WAITING_FOR_FORCE_MERGE_SEC: Final[int] = 30 -SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC: Final[int] = 30 +REPLICA_HEALTH_TIMEOUT: Final[str] = "30m" # Central registry for version-dependent OpenSearch index settings. # Add new rules here to automatically support future versions. @@ -194,7 +195,7 @@ def __init__( self, dim: int, db_config: dict[str, Any], - db_case_config: OSSOpenSearchIndexConfig, + db_case_config: OSSOpenSearchIndexConfig | OSSOpenSearchFtsConfig, index_name: str = "vdb_bench_index", # must be lowercase id_col_name: str = "_id", label_col_name: str = "label", @@ -205,13 +206,19 @@ def __init__( ) -> None: """Initialize the OpenSearch client.""" self.dim = dim - self.db_config = db_config + self.db_config = dict(db_config) + configured_index_name = self.db_config.pop("index_name", None) self.case_config = db_case_config - self.index_name = index_name + self.index_name = configured_index_name or index_name self.id_col_name = id_col_name self.label_col_name = label_col_name self.vector_col_name = vector_col_name self.with_scalar_labels = with_scalar_labels + self._is_fts = isinstance(db_case_config, OSSOpenSearchFtsConfig) + self.text_col_name = "text" + self.filter_id_col_name = "filter_id" + if self._is_fts: + self.id_col_name = "doc_id" # Initialize client state self.client: OpenSearch | None = None @@ -236,13 +243,21 @@ def _handle_index_initialization(self, client: OpenSearch, drop_old: bool) -> No if not is_existed: self._create_index(client) log.info(f"OSS_OpenSearch client create index: {self.index_name}") - self._update_ef_search_before_search(client) - self._load_graphs_to_memory(client) + if not self._is_fts: + self._update_ef_search_before_search(client) + self._load_graphs_to_memory(client) def need_normalize_cosine(self) -> bool: """Whether this database needs to normalize dataset to support COSINE metric.""" return True + @classmethod + def supports_full_text_search(cls) -> bool: + return True + + def has_text_field(self) -> bool: + return bool(getattr(self, "_is_fts", False) and getattr(self, "text_col_name", None)) + def _get_cluster_version(self, client: OpenSearch) -> Version: """ Return the OpenSearch cluster version as a comparable Version object. @@ -307,7 +322,31 @@ def _get_bulk_manager(self, client: OpenSearch) -> BulkInsertManager: """Get bulk insert manager for the given client.""" return BulkInsertManager(client, self.index_name, self.case_config) + def _create_fts_index(self, client: OpenSearch) -> None: + mappings = self.case_config.index_param() + index_settings = { + "number_of_shards": self.case_config.number_of_shards, + "number_of_replicas": self.case_config.number_of_replicas, + "refresh_interval": self.case_config.refresh_interval, + } + index_settings.update(self.case_config.similarity_settings()) + settings = {"index": index_settings} + try: + log.info(f"Creating FTS index with settings: {settings}") + log.info(f"Creating FTS index with mappings: {mappings}") + client.indices.create( + index=self.index_name, + body={"settings": settings, "mappings": mappings}, + ) + except Exception as e: + log.warning(f"Failed to create FTS index: {self.index_name} error: {e!s}") + raise e from None + def _create_index(self, client: OpenSearch) -> None: + if self._is_fts: + self._create_fts_index(client) + return + cluster_version = self._get_cluster_version(client) if self.case_config.on_disk and cluster_version < Version("2.17"): @@ -420,6 +459,94 @@ def insert_embeddings( log.info(f"Using {num_clients} parallel clients for data insertion") return self._insert_with_multiple_clients(embeddings, metadata, num_clients, labels_data) + def insert_documents( + self, + texts: Iterable[str], + doc_ids: list[str], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + if not getattr(self, "_is_fts", False): + msg = "OSSOpenSearch full-text insert requires OSSOpenSearchFtsConfig" + raise RuntimeError(msg) + assert self.client is not None, "should self.init() first" + docs = list(texts) + if len(docs) != len(doc_ids): + msg = f"Mismatch between texts ({len(docs)}) and doc_ids ({len(doc_ids)}) lengths" + raise ValueError(msg) + filter_ids = kwargs.get("filter_ids") + if filter_ids is not None and len(filter_ids) != len(docs): + msg = f"Mismatch between texts ({len(docs)}) and filter_ids ({len(filter_ids)}) lengths" + raise ValueError(msg) + + insert_data: list[dict[str, Any]] = [] + for i, doc in enumerate(docs): + doc_id = str(doc_ids[i]) + source = {self.id_col_name: doc_id, self.text_col_name: doc} + if filter_ids is not None: + source[self.filter_id_col_name] = int(filter_ids[i]) + insert_data.append({"index": {"_index": self.index_name, "_id": doc_id}}) + insert_data.append(source) + + try: + response = self.client.bulk(body=insert_data) + except Exception as e: + log.warning(f"Failed to insert FTS docs: {self.index_name} error: {e!s}") + return 0, e + insert_count, error = self._parse_fts_bulk_response(response, len(docs)) + if error is not None: + log.warning("FTS bulk insert failed: %s", error) + return insert_count, error + + @staticmethod + def _parse_fts_bulk_response(response: dict[str, Any], expected_count: int) -> tuple[int, Exception | None]: + if not response.get("errors"): + return expected_count, None + + items = response.get("items") + if not isinstance(items, list): + error = RuntimeError("OpenSearch FTS bulk response reported errors without an items list") + return 0, error + + success_count = 0 + failure_samples = [] + for position, item in enumerate(items): + if not isinstance(item, dict) or len(item) != 1: + failure_samples.append(f"item[{position}]=malformed") + continue + + operation, result = next(iter(item.items())) + if not isinstance(result, dict): + failure_samples.append(f"item[{position}] {operation}=malformed") + continue + + status = result.get("status") + if isinstance(status, int) and 200 <= status < 300 and "error" not in result: + success_count += 1 + continue + + error_detail = result.get("error") + if isinstance(error_detail, dict): + error_type = error_detail.get("type", "unknown") + reason = error_detail.get("reason", "unknown") + error_summary = f"{error_type}: {reason}" + else: + error_summary = str(error_detail or "unknown") + failure_samples.append( + f"item[{position}] {operation} id={result.get('_id', 'unknown')} " + f"status={status or 'unknown'} error={error_summary}" + ) + + success_count = min(success_count, expected_count) + failed_count = max(expected_count - success_count, 1) + sample_summary = "; ".join(failure_samples[:3]) or "no failed item details" + if len(items) != expected_count: + sample_summary = f"response_items={len(items)}, expected_items={expected_count}; {sample_summary}" + error = RuntimeError( + f"OpenSearch FTS bulk insert failed for {failed_count}/{expected_count} documents; " + f"successful={success_count}; {sample_summary}" + ) + return success_count, error + def _insert_with_single_client( self, embeddings: Iterable[list[float]], @@ -569,14 +696,67 @@ def search_embedding( log.warning(f"Failed to search: {self.index_name} error: {e!s}") raise e from None + def search_documents( + self, + query: str, + k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + **kwargs: Any, + ) -> list[str]: + if not getattr(self, "_is_fts", False): + msg = "OSSOpenSearch full-text search requires OSSOpenSearchFtsConfig" + raise RuntimeError(msg) + if not self.supports_document_payload_profile(payload_profile): + msg = f"OSSOpenSearch does not support document payload_profile={payload_profile.value}" + raise NotImplementedError(msg) + assert self.client is not None, "should self.init() first" + + source = [self.text_col_name] if payload_profile == PayloadProfile.TEXT else False + filter_path = ["hits.hits._id", f"hits.hits.fields.{self.id_col_name}"] + if payload_profile == PayloadProfile.TEXT: + filter_path.append(f"hits.hits._source.{self.text_col_name}") + query_clause: dict[str, Any] = {"match": {self.text_col_name: query}} + if self.filter: + query_clause = {"bool": {"must": query_clause, "filter": self.filter}} + search_kwargs: dict[str, Any] = { + "index": self.index_name, + "body": {"query": query_clause}, + "size": k, + "_source": source, + "docvalue_fields": [self.id_col_name], + "filter_path": filter_path, + } + if payload_profile != PayloadProfile.TEXT: + search_kwargs["stored_fields"] = "_none_" + response = self.client.search(**search_kwargs) + + doc_ids = [] + for hit in response.get("hits", {}).get("hits", []): + if hit.get("_id") is not None: + doc_ids.append(str(hit["_id"])) + continue + values = hit.get("fields", {}).get(self.id_col_name, []) + if values: + doc_ids.append(str(values[0])) + return doc_ids + def prepare_filter(self, filters: Filter) -> None: """Prepare filter conditions for search operations.""" self.routing_key = None if filters.type == FilterOp.NonFilter: self.filter = None elif filters.type == FilterOp.NumGE: - self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} + if self._is_fts: + if getattr(filters, "int_field", None) != self.filter_id_col_name: + msg = f"OSSOpenSearch FTS filters only support int_field='{self.filter_id_col_name}'" + raise ValueError(msg) + self.filter = {"range": {self.filter_id_col_name: {"gte": filters.int_value}}} + else: + self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} elif filters.type == FilterOp.StrEqual: + if self._is_fts: + msg = f"Not support Filter for OSSOpenSearch FTS - {filters}" + raise ValueError(msg) self.filter = {"term": {self.label_col_name: filters.label_value}} if self.case_config.use_routing: self.routing_key = filters.label_value @@ -587,6 +767,14 @@ def prepare_filter(self, filters: Filter) -> None: def optimize(self, data_size: int | None = None) -> None: """Optimize the index for better search performance.""" + if self._is_fts: + self._refresh_index() + if self.case_config.force_merge_enabled: + self._do_fts_force_merge() + self._refresh_index() + self._update_replicas() + self._refresh_index() + return self._update_ef_search() # Call refresh first to ensure that all segments are created self._refresh_index() @@ -621,13 +809,20 @@ def _update_replicas(self): def _wait_till_green(self): log.info("Wait for index to become green..") - while True: - res = self.client.cat.indices(index=self.index_name, h="health", format="json") - health = res[0]["health"] - if health == "green": - break - log.info(f"The index {self.index_name} has health : {health} and is not green. Retrying") - time.sleep(SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC) + response = self.client.cluster.health( + index=self.index_name, + wait_for_status="green", + timeout=REPLICA_HEALTH_TIMEOUT, + ) + health = response.get("status", "unknown") + if response.get("timed_out") or health != "green": + msg = ( + f"Index {self.index_name} did not reach green health within {REPLICA_HEALTH_TIMEOUT}: " + f"status={health}, configured_replicas={self.case_config.number_of_replicas}, " + f"unassigned_shards={response.get('unassigned_shards', 'unknown')}, " + f"number_of_nodes={response.get('number_of_nodes', 'unknown')}" + ) + raise OpenSearchError(msg) log.info(f"Index {self.index_name} is green..") def _refresh_index(self): @@ -645,6 +840,17 @@ def _refresh_index(self): continue log.debug(f"Completed refresh for index {self.index_name}") + def _do_fts_force_merge(self): + log.info(f"Starting FTS force merge for index {self.index_name}") + force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false" + force_merge_task_id = self.client.transport.perform_request("POST", force_merge_endpoint)["task"] + while True: + time.sleep(WAITING_FOR_FORCE_MERGE_SEC) + task_status = self.client.tasks.get(task_id=force_merge_task_id) + if task_status["completed"]: + break + log.info(f"Completed FTS force merge for index {self.index_name}") + def _do_force_merge(self): log.info(f"Updating the Index thread qty to {self.case_config.index_thread_qty_during_force_merge}.") diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py index 38103f1a1..918112ede 100644 --- a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -119,6 +119,7 @@ def __init__( self._scalar_label_field = "label" self._scalar_payload_label_field = db_config.get("scalar_payload_label_field", self._scalar_label_field) self._text_field = "text" + self._filter_id_field = "filter_id" self.with_scalar_labels = with_scalar_labels self.expr = None @@ -387,20 +388,33 @@ def insert_documents( if len(docs) != len(doc_ids): msg = f"Mismatch between texts ({len(docs)}) and doc_ids ({len(doc_ids)}) lengths" raise ValueError(msg) + filter_ids = kwargs.get("filter_ids") + if filter_ids is not None and len(filter_ids) != len(docs): + msg = f"Mismatch between texts ({len(docs)}) and filter_ids ({len(filter_ids)}) lengths" + raise ValueError(msg) text_field = self._text_field + upsert_columns = { + self._scalar_id_field: [str(doc_id) for doc_id in doc_ids], + text_field: docs, + } + if filter_ids is not None: + upsert_columns[self._filter_id_field] = [int(filter_id) for filter_id in filter_ids] + schema = { + text_field: { + "type": "string", + "full_text_search": True, + } + } + if filter_ids is not None: + schema[self._filter_id_field] = { + "type": "int", + "filterable": True, + } try: self.ns.write( - upsert_columns={ - self._scalar_id_field: [str(doc_id) for doc_id in doc_ids], - text_field: docs, - }, - schema={ - text_field: { - "type": "string", - "full_text_search": True, - } - }, + upsert_columns=upsert_columns, + schema=schema, disable_backpressure=self.db_case_config.disable_backpressure, ) except Exception as e: @@ -447,6 +461,8 @@ def search_documents( "rank_by": (self._text_field, "BM25", query), "top_k": k, } + if self.expr is not None: + query_kwargs["filters"] = self.expr if payload_profile == PayloadProfile.TEXT: query_kwargs["include_attributes"] = [self._text_field] res = self.ns.query(**query_kwargs) @@ -457,8 +473,17 @@ def prepare_filter(self, filters: Filter): if filters.type == FilterOp.NonFilter: self.expr = None elif filters.type == FilterOp.NumGE: - self.expr = (self._scalar_id_field, "Gte", filters.int_value) + if self._is_fts: + if getattr(filters, "int_field", None) != self._filter_id_field: + msg = f"TurboPuffer FTS filters only support int_field='{self._filter_id_field}'" + raise ValueError(msg) + self.expr = (self._filter_id_field, "Gte", filters.int_value) + else: + self.expr = (self._scalar_id_field, "Gte", filters.int_value) elif filters.type == FilterOp.StrEqual: + if self._is_fts: + msg = f"Not support Filter for TurboPuffer FTS - {filters}" + raise ValueError(msg) self.expr = (self._scalar_payload_label_field, "Eq", filters.label_value) else: msg = f"Not support Filter for TurboPuffer - {filters}" diff --git a/vectordb_bench/backend/clients/vespa/config.py b/vectordb_bench/backend/clients/vespa/config.py index 006160a0b..5eeced0f6 100644 --- a/vectordb_bench/backend/clients/vespa/config.py +++ b/vectordb_bench/backend/clients/vespa/config.py @@ -59,31 +59,6 @@ class VespaFtsConfig(BaseModel, DBCaseConfig): feed_client_command: str = "vespa" feed_client_connections: int | None = None - def apply_fts_manifest( - self, - bm25_params: dict[str, float], - analyzer_params: dict, - ) -> tuple[DBCaseConfig, dict]: - updates = {} - applied_bm25_params = {} - - if "k1" in bm25_params: - updates["bm25_k1"] = bm25_params["k1"] - applied_bm25_params["k1"] = bm25_params["k1"] - if "b" in bm25_params: - updates["bm25_b"] = bm25_params["b"] - applied_bm25_params["b"] = bm25_params["b"] - if "avgdl" in bm25_params: - updates["bm25_avgdl"] = bm25_params["avgdl"] - applied_bm25_params["avgdl"] = bm25_params["avgdl"] - - return self.model_copy(update=updates), { - "applied_bm25_params": applied_bm25_params, - "unapplied_bm25_params": {k: v for k, v in bm25_params.items() if k not in applied_bm25_params}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": dict(analyzer_params), - } - def index_param(self) -> dict: return {} diff --git a/vectordb_bench/backend/clients/vespa/vespa.py b/vectordb_bench/backend/clients/vespa/vespa.py index d10d906a8..12fa65d5e 100644 --- a/vectordb_bench/backend/clients/vespa/vespa.py +++ b/vectordb_bench/backend/clients/vespa/vespa.py @@ -13,6 +13,7 @@ from vespa import application +from vectordb_bench.backend.filter import Filter, FilterOp from vectordb_bench.backend.payload import PayloadProfile from ..api import VectorDB @@ -46,6 +47,11 @@ def _tail_text(value: str, limit: int = VESPA_FEED_OUTPUT_TAIL_CHARS) -> str: class Vespa(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + def __init__( self, dim: int, @@ -61,6 +67,8 @@ def __init__( self._is_fts = isinstance(self.case_config, VespaFtsConfig) self.schema_name = collection_name self._text_field = "text" + self._filter_id_field = "filter_id" + self._filter_expr: str | None = None client = self.deploy_http() client.wait_for_application_up() @@ -168,9 +176,13 @@ def insert_documents( if len(texts) != len(doc_ids): msg = f"Mismatch between texts ({len(texts)}) and doc_ids ({len(doc_ids)}) lengths" raise ValueError(msg) + filter_ids = kwargs.get("filter_ids") + if filter_ids is not None and len(filter_ids) != len(texts): + msg = f"Mismatch between texts ({len(texts)}) and filter_ids ({len(filter_ids)}) lengths" + raise ValueError(msg) try: - self._write_fts_feed_batch(texts, doc_ids) + self._write_fts_feed_batch(texts, doc_ids, filter_ids) except Exception as exc: log.warning("Vespa feed failed for schema %s", self.schema_name, exc_info=True) return 0, exc @@ -223,12 +235,15 @@ def _ensure_fts_feed_client(self) -> subprocess.Popen: ) return self._feed_proc - def _write_fts_feed_batch(self, texts: list[str], doc_ids: list[str]) -> None: + def _write_fts_feed_batch(self, texts: list[str], doc_ids: list[str], filter_ids: list[int] | None = None) -> None: lines = [] - for doc_id, text in zip(doc_ids, texts, strict=True): + for i, (doc_id, text) in enumerate(zip(doc_ids, texts, strict=True)): + fields = {"id": str(doc_id), "text": text} + if filter_ids is not None: + fields[self._filter_id_field] = int(filter_ids[i]) operation = { "put": self._vespa_document_id(str(doc_id)), - "fields": {"id": str(doc_id), "text": text}, + "fields": fields, } lines.append(json.dumps(operation, ensure_ascii=False, separators=(",", ":"))) @@ -349,9 +364,12 @@ def search_embedding( f"nearestNeighbor({embedding_field}, query_embedding)" ) + prepared_filter = getattr(self, "_filter_expr", None) if filters: id_filter = filters.get("id") - yql += f" and id >= {id_filter}" + prepared_filter = f"id >= {id_filter}" + if prepared_filter: + yql += f" and {prepared_filter}" query_embedding = query if self.case_config.quantization_type == "none" else util.binarize_tensor(query) @@ -379,6 +397,8 @@ def search_documents( if payload_profile == PayloadProfile.TEXT: selected_fields = f"id, {self._text_field}" yql = f"select {selected_fields} from {self.schema_name} where userQuery()" + if self._filter_expr: + yql += f" and {self._filter_expr}" result = self.client.query( { "yql": yql, @@ -399,6 +419,20 @@ def search_documents( ids.append(doc_id) return ids + def prepare_filter(self, filters: Filter) -> None: + if filters.type == FilterOp.NonFilter: + self._filter_expr = None + return + if filters.type == FilterOp.NumGE: + expected_field = self._filter_id_field if self._is_fts else "id" + if getattr(filters, "int_field", None) != expected_field: + msg = f"Vespa filters only support int_field='{expected_field}' in this mode" + raise ValueError(msg) + self._filter_expr = f"{expected_field} >= {filters.int_value}" + return + msg = f"Not support Filter for Vespa - {filters}" + raise ValueError(msg) + def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases. @@ -461,6 +495,7 @@ def _create_application_package(self): if self._is_fts: fields = [ Field("id", "string", indexing=["summary", "attribute"]), + Field("filter_id", "int", indexing=["summary", "attribute"]), Field( "text", "string", diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index f905f469a..176c08e17 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -4,8 +4,8 @@ >>> Dataset.Cohere.get(100_000) """ -import json import logging +import math import pathlib import types import typing @@ -559,11 +559,50 @@ class FtsDocument: doc_id: str text: str + filter_id: int | None = None -FTS_GT_FILE = "neighbors.parquet" -FTS_BUILD_MANIFEST_FILE = "build_manifest.json" -FTS_MATH_GT_FILES = (FTS_GT_FILE, FTS_BUILD_MANIFEST_FILE, "manifest.json") +_FTS_FILTER_GOLDEN_RATIO_64 = 0x9E3779B97F4A7C15 +_FTS_FILTER_OFFSET_SEED = 0xD1B54A32D192ED03 + + +@dataclass(frozen=True) +class FtsFilterIdPermutation: + """Deterministic bijection that scatters FTS filter IDs across corpus order.""" + + size: int + multiplier: int + offset: int + + @property + def algorithm(self) -> str: + return "affine_permutation_v1" + + @classmethod + def for_size(cls, size: int) -> "FtsFilterIdPermutation": + if size <= 0: + msg = f"FTS filter ID permutation size must be positive, got {size}" + raise ValueError(msg) + if size == 1: + return cls(size=1, multiplier=1, offset=0) + + multiplier = max(1, (size * _FTS_FILTER_GOLDEN_RATIO_64) >> 64) + while math.gcd(multiplier, size) != 1: + multiplier += 1 + if multiplier >= size: + multiplier = 1 + + return cls( + size=size, + multiplier=multiplier, + offset=_FTS_FILTER_OFFSET_SEED % size, + ) + + def map(self, ordinal: int) -> int: + if ordinal < 0 or ordinal >= self.size: + msg = f"FTS filter ID ordinal must be in [0, {self.size}), got {ordinal}" + raise ValueError(msg) + return (self.multiplier * ordinal + self.offset) % self.size class FtsDatasetTranslator(ABC): @@ -603,6 +642,25 @@ def iter_documents(self, dataset: typing.Any) -> Iterator[FtsDocument]: for doc in dataset.docs_iter(): yield self.translate_document(doc) + def load_ground_truth(self, dataset: typing.Any) -> dict[str, dict[str, int]]: + """Load positive semantic qrels keyed by query id. + + ir_datasets qrels may contain non-positive judgments. Those are not + relevant documents for recall/MRR/NDCG, so they are ignored here. + """ + qrels: dict[str, dict[str, int]] = {} + for qrel in dataset.qrels_iter(): + relevance = int(getattr(qrel, "relevance", 0)) + if relevance <= 0: + continue + query_id = str(qrel.query_id) + doc_id = str(qrel.doc_id) + qrels.setdefault(query_id, {})[doc_id] = max( + relevance, + qrels.get(query_id, {}).get(doc_id, 0), + ) + return qrels + class MSMarcoTranslator(FtsDatasetTranslator): """Translator for MS MARCO passage retrieval dataset.""" @@ -711,6 +769,8 @@ class FtsDatasetManager(BaseModel): Similar to DatasetManager, but for text-based FTS datasets: - queries_data: loaded queries (similar to test_data in vectors) - gt_data: loaded ground truth (similar to gt_data in vectors) + - recall_queries_data: recall-valid queries after optional FTS filter + - recall_gt_data: recall-valid ground truth after optional FTS filter - translator: dataset-specific translator for schema conversion - _ir_dataset: ir_datasets dataset object for direct access """ @@ -719,9 +779,16 @@ class FtsDatasetManager(BaseModel): _translator: typing.Any = PrivateAttr() queries_data: list[FtsQuery] | None = None - gt_data: list[list[str]] | None = None - bm25_params: dict[str, float] = PydanticField(default_factory=dict) - analyzer_params: dict[str, typing.Any] = PydanticField(default_factory=dict) + gt_data: list[dict[str, int]] | None = None + recall_queries_data: list[FtsQuery] | None = None + recall_gt_data: list[dict[str, int]] | None = None + recall_skipped: bool = False + recall_skip_reason: str | None = None + qrels_data: dict[str, dict[str, int]] = PydanticField(default_factory=dict) + required_doc_ids: set[str] = PydanticField(default_factory=set) + selected_doc_ids: set[str] | None = None + qrel_filter_ids: dict[str, int] = PydanticField(default_factory=dict) + filter_stats: dict[str, int | float | str] = PydanticField(default_factory=dict) _ir_dataset: typing.Any = PrivateAttr(default=None) def __init__(self, **data): @@ -752,68 +819,172 @@ def data_dir(self) -> pathlib.Path: self.data.dir_name, ) - def _download_math_gt_files(self) -> None: - DatasetSource.S3.reader().read( - dataset=self.data.dir_name.lower(), - files=list(FTS_MATH_GT_FILES), - local_ds_root=self.data_dir, - ) + def _validate_cap(self, required_doc_ids: set[str], target_size: int) -> None: + if len(required_doc_ids) > target_size: + msg = ( + f"{self.data.full_name} size={target_size} is too small for semantic qrels; " + f"requires {len(required_doc_ids)} qrel documents" + ) + raise ValueError(msg) - def _load_math_gt_data(self) -> list[list[str]]: - p = pathlib.Path(self.data_dir, FTS_GT_FILE) - if not p.exists(): - msg = f"No such file: {p}" - raise FileNotFoundError(msg) - gt_rows = pl.read_parquet(p)[self.data.gt_neighbors_field].to_list() - # FTS math GT stores dense document row IDs, not original ir_datasets doc IDs. - # FtsDocumentIterator assigns these same row IDs during insertion. - return [[str(doc_id) for doc_id in row if str(doc_id) != "-1"] for row in gt_rows] - - def _load_build_manifest(self) -> dict[str, typing.Any]: - p = pathlib.Path(self.data_dir, FTS_BUILD_MANIFEST_FILE) - if not p.exists(): - msg = f"No such file: {p}" - raise FileNotFoundError(msg) - manifest = json.loads(p.read_text(encoding="utf-8")) - if not isinstance(manifest, dict): - msg = f"Invalid FTS build manifest: {p}" - raise TypeError(msg) - return manifest + def _build_selected_doc_ids(self) -> set[str]: + """Select the capped corpus while preserving every positive qrel doc.""" + if self._ir_dataset is None: + msg = "ir_datasets dataset not loaded. Call prepare() first." + raise RuntimeError(msg) + + required_doc_ids = set(self.required_doc_ids) + self._validate_cap(required_doc_ids=required_doc_ids, target_size=self.data.size) + + selected_doc_ids = set(required_doc_ids) + found_required_doc_ids: set[str] = set() + for doc in self._translator.iter_documents(self._ir_dataset): + doc_id = str(doc.doc_id) + if doc_id in required_doc_ids: + found_required_doc_ids.add(doc_id) + + if doc_id not in selected_doc_ids and len(selected_doc_ids) < self.data.size: + selected_doc_ids.add(doc_id) - def _validate_build_manifest(self, manifest: dict[str, typing.Any]) -> None: - source_ir_dataset = manifest.get("source_ir_dataset") - if source_ir_dataset is not None and source_ir_dataset != self._translator.ir_datasets_name: + if len(selected_doc_ids) >= self.data.size and found_required_doc_ids == required_doc_ids: + break + + missing_doc_ids = required_doc_ids - found_required_doc_ids + if missing_doc_ids: + preview = ", ".join(sorted(missing_doc_ids)[:10]) msg = ( - f"{self.data.full_name} manifest source_ir_dataset={source_ir_dataset!r} " - f"does not match {self._translator.ir_datasets_name!r}" + f"{self.data.full_name} semantic qrel docs missing from corpus: {preview}" + f"{'...' if len(missing_doc_ids) > 10 else ''}" ) raise ValueError(msg) - for field_name in ("doc_limit", "indexed_doc_count"): - value = manifest.get(field_name) - if value is None: + return selected_doc_ids + + def _iter_selected_documents_with_filter_ids(self) -> Iterator[FtsDocument]: + """Yield selected documents with the exact filter IDs used for insertion and qrels.""" + if self._ir_dataset is None: + msg = "ir_datasets dataset not loaded. Call prepare() first." + raise RuntimeError(msg) + + permutation = FtsFilterIdPermutation.for_size(self.data.size) + documents = iter(self._translator.iter_documents(self._ir_dataset)) + emitted_count = 0 + while emitted_count < self.data.size: + try: + doc = next(documents) + doc.doc_id = str(doc.doc_id) + if self.selected_doc_ids is not None and doc.doc_id not in self.selected_doc_ids: + continue + doc.filter_id = permutation.map(emitted_count) + except StopIteration: + break + except Exception as e: + log.debug(f"Skipping malformed document: {e}") continue - if int(value) != self.data.size: - msg = f"{self.data.full_name} manifest {field_name}={value} does not match size={self.data.size}" - raise ValueError(msg) - query_count = manifest.get("query_count") - if query_count is not None and self.queries_data is not None and int(query_count) != len(self.queries_data): + emitted_count += 1 + yield doc + + def _build_qrel_filter_ids(self) -> dict[str, int]: + """Map qrel doc IDs to their deterministic permuted FTS filter ID.""" + if self.selected_doc_ids is None: + msg = "selected_doc_ids is required before building FTS filter IDs" + raise RuntimeError(msg) + + qrel_doc_ids = set(self.required_doc_ids) + qrel_filter_ids: dict[str, int] = {} + for doc in self._iter_selected_documents_with_filter_ids(): + doc_id = doc.doc_id + if doc_id in qrel_doc_ids: + qrel_filter_ids[doc_id] = doc.filter_id + + missing_doc_ids = qrel_doc_ids - set(qrel_filter_ids) + if missing_doc_ids: + preview = ", ".join(sorted(missing_doc_ids)[:10]) msg = ( - f"{self.data.full_name} manifest query_count={query_count} " - f"does not match loaded query count={len(self.queries_data)}" + f"{self.data.full_name} semantic qrel docs missing filter_id assignment: {preview}" + f"{'...' if len(missing_doc_ids) > 10 else ''}" ) raise ValueError(msg) + return qrel_filter_ids + + def _apply_integer_filter_to_qrels( + self, + queries: list[FtsQuery], + ground_truth: list[dict[str, int]], + filters: Filter, + ) -> tuple[list[FtsQuery], list[dict[str, int]]]: + filter_field = getattr(filters, "int_field", "filter_id") + if filter_field != "filter_id": + msg = f"FTS integer filters require int_field='filter_id', got {filter_field!r}" + raise ValueError(msg) + + filter_value = int(filters.int_value) + if filter_value < 0 or filter_value > self.data.size: + msg = f"FTS filter_id threshold must be in [0, {self.data.size}], got {filter_value}" + raise ValueError(msg) - def _load_manifest_params(self) -> None: - manifest = self._load_build_manifest() - self._validate_build_manifest(manifest) - bm25 = manifest.get("bm25") or {} - analyzer = manifest.get("analyzer") or {} - self.bm25_params = { - key: float(bm25[key]) for key in ("k1", "b", "avgdl") if key in bm25 and bm25[key] is not None + self.qrel_filter_ids = self._build_qrel_filter_ids() + filtered_queries: list[FtsQuery] = [] + filtered_gt: list[dict[str, int]] = [] + for query, qrels in zip(queries, ground_truth, strict=True): + filtered_qrels = { + doc_id: rel for doc_id, rel in qrels.items() if self.qrel_filter_ids.get(doc_id, -1) >= filter_value + } + if not filtered_qrels: + continue + filtered_queries.append(query) + filtered_gt.append(filtered_qrels) + + matched_doc_count = self.data.size - filter_value + filtered_relevant_doc_ids = {doc_id for qrels in filtered_gt for doc_id in qrels} + permutation = FtsFilterIdPermutation.for_size(self.data.size) + self.filter_stats = { + "filter_type": filters.type.value, + "filter_field": filter_field, + "filter_value": filter_value, + "filter_rate": filters.filter_rate, + "filter_id_distribution": permutation.algorithm, + "filter_id_multiplier": permutation.multiplier, + "filter_id_offset": permutation.offset, + "matched_doc_count": matched_doc_count, + "matched_doc_ratio": round(matched_doc_count / self.data.size, 6), + "original_query_count": len(queries), + "filtered_query_count": len(filtered_queries), + "filtered_query_ratio": round(len(filtered_queries) / len(queries), 6), + "original_relevant_doc_count": len(self.required_doc_ids), + "filtered_relevant_doc_count": len(filtered_relevant_doc_ids), } - self.analyzer_params = analyzer if isinstance(analyzer, dict) else {} + log.info( + "Applied FTS integer filter %s >= %s: queries %s/%s, relevant docs %s/%s", + filter_field, + filter_value, + len(filtered_queries), + len(queries), + len(filtered_relevant_doc_ids), + len(self.required_doc_ids), + ) + if not filtered_queries: + self.recall_skipped = True + self.recall_skip_reason = "no_positive_qrels_after_filter" + return filtered_queries, filtered_gt + + def _apply_filters_to_qrels( + self, + queries: list[FtsQuery], + ground_truth: list[dict[str, int]], + filters: Filter | None, + ) -> tuple[list[FtsQuery], list[dict[str, int]]]: + self.filter_stats = {} + self.qrel_filter_ids = {} + self.recall_skipped = False + self.recall_skip_reason = None + if filters is None or filters.type == FilterOp.NonFilter: + return queries, ground_truth + if filters.type == FilterOp.NumGE: + return self._apply_integer_filter_to_qrels(queries, ground_truth, filters) + msg = f"FTS dataset filtering does not support filter type {filters.type}" + raise ValueError(msg) def prepare( self, @@ -825,11 +996,12 @@ def prepare( Directly uses ir_datasets API without generating TSV files: 1. Downloads dataset using ir_datasets (if needed) 2. Loads dataset object using translator - 3. Loads queries from ir_datasets and mathematical ground truth from S3 + 3. Loads queries and semantic qrels from ir_datasets Args: source: Data source to download from (should be IR_DATASETS for FTS) - filters: Optional filters (not used for FTS) + filters: Optional filters. FTS supports natural semantic GT + filtering for integer filter_id cases. Returns: bool: True if preparation successful, False otherwise @@ -849,27 +1021,53 @@ def prepare( self._ir_dataset = self._translator.load() log.info(f"Successfully loaded ir_datasets dataset: {self._translator.ir_datasets_name}") - # Force ir_datasets lazy document cache work before timed insert. - for idx, _ in enumerate(self._translator.iter_documents(self._ir_dataset), start=1): - if idx >= self.data.size: - break - - # Load queries from ir_datasets and mathematical ground truth artifacts by row order. + # Load queries from ir_datasets and semantic ground truth by query id. if self.data.with_gt: - # Load queries using translator - self.queries_data = list(self._translator.iter_queries(self._ir_dataset)) - log.info(f"Loaded {len(self.queries_data)} queries into memory") - - self._download_math_gt_files() - self._load_manifest_params() - self.gt_data = self._load_math_gt_data() - if len(self.queries_data) != len(self.gt_data): - msg = ( - f"{self.data.full_name} query count {len(self.queries_data)} " - f"does not match ground truth row count {len(self.gt_data)}" + all_queries = list(self._translator.iter_queries(self._ir_dataset)) + log.info(f"Loaded {len(all_queries)} queries into memory") + + self.qrels_data = self._translator.load_ground_truth(self._ir_dataset) + self.queries_data = [] + self.gt_data = [] + for query in all_queries: + qrels = self.qrels_data.get(query.query_id) + if not qrels: + continue + self.queries_data.append( + FtsQuery( + query_id=query.query_id, + text=query.text, + ) ) + self.gt_data.append(qrels) + + if not self.queries_data: + msg = f"{self.data.full_name} has no queries with positive semantic qrels" raise ValueError(msg) # noqa: TRY301 - log.info(f"Loaded mathematical ground truth for {len(self.gt_data)} queries into memory") + + self.required_doc_ids = {doc_id for qrels in self.gt_data for doc_id in qrels} + self.selected_doc_ids = self._build_selected_doc_ids() + self.recall_queries_data, self.recall_gt_data = self._apply_filters_to_qrels( + self.queries_data, + self.gt_data, + filters, + ) + log.info( + "Loaded semantic qrels for %s queries; recall uses %s queries; " + "selected %s corpus docs including %s qrel docs", + len(self.gt_data), + len(self.recall_gt_data), + len(self.selected_doc_ids), + len(self.required_doc_ids), + ) + else: + self.selected_doc_ids = None + self.qrel_filter_ids = {} + self.filter_stats = {} + self.recall_queries_data = None + self.recall_gt_data = None + self.recall_skipped = False + self.recall_skip_reason = None except (TypeError, ValueError): log.exception("Invalid FTS dataset configuration") @@ -913,7 +1111,6 @@ def __init__(self, dataset: FtsDatasetManager, batch_size: int = config.NUM_PER_ self._ds = dataset self._batch_size = batch_size self._finished = False - self._doc_count = 0 # Track total documents processed self._docs_iter = None def __iter__(self): @@ -931,47 +1128,19 @@ def __next__(self) -> list[FtsDocument]: if self._finished: raise StopIteration - # Initialize iterator on first call if self._docs_iter is None: - if self._ds._ir_dataset is None: - error_msg = "ir_datasets dataset not loaded. Call prepare() first." - log.error(error_msg) - raise RuntimeError(error_msg) - - log.info("Starting to iterate documents using translator") - self._docs_iter = self._ds._translator.iter_documents(self._ds._ir_dataset) - - # Read batch with proper error handling - try: - batch = [] - for _ in range(self._batch_size): - if self._doc_count >= self._ds.data.size: - self._finished = True - if batch: - return batch - raise StopIteration # noqa: TRY301 - try: - doc = next(self._docs_iter) - doc.doc_id = str(self._doc_count) - batch.append(doc) - self._doc_count += 1 - except StopIteration: - self._finished = True - if batch: - return batch - raise - except Exception as e: - log.debug(f"Skipping malformed document: {e}") - continue + self._docs_iter = self._ds._iter_selected_documents_with_filter_ids() - except StopIteration: - self._finished = True - raise - except Exception: - log.exception("Error reading documents from translator") - raise - else: - return batch + batch = [] + while len(batch) < self._batch_size: + try: + batch.append(next(self._docs_iter)) + except StopIteration: + self._finished = True + if batch: + return batch + raise + return batch def __enter__(self): """Enter context manager.""" diff --git a/vectordb_bench/backend/runner/concurrent_runner.py b/vectordb_bench/backend/runner/concurrent_runner.py index 194fbf53c..e41af22ee 100644 --- a/vectordb_bench/backend/runner/concurrent_runner.py +++ b/vectordb_bench/backend/runner/concurrent_runner.py @@ -224,10 +224,16 @@ def _next_fts_batch(self) -> dict | None: doc_ids = [] texts = [] + filter_ids = [] for doc in batch: doc_ids.append(doc.doc_id if hasattr(doc, "doc_id") else str(doc["doc_id"])) texts.append(doc.text if hasattr(doc, "text") else doc["text"]) - return {"texts": texts, "doc_ids": doc_ids} + filter_id = doc.filter_id if hasattr(doc, "filter_id") else doc.get("filter_id", None) + filter_ids.append(filter_id) + insert_kwargs = {"texts": texts, "doc_ids": doc_ids} + if any(filter_id is not None for filter_id in filter_ids): + insert_kwargs["filter_ids"] = filter_ids + return insert_kwargs def _worker_loop(self) -> int: """Worker loop: pull batches from the shared iterator and insert them.""" diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index fc46aed23..f671d381a 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -14,7 +14,7 @@ from vectordb_bench.backend.workload import WorkloadKind from ... import config -from ...metric import calc_ndcg, calc_recall, calc_recall_fts, get_ideal_dcg +from ...metric import calc_mrr_fts, calc_ndcg, calc_ndcg_fts, calc_recall, calc_recall_fts, get_ideal_dcg from ...models import LoadTimeoutError from .. import utils from ..clients import api @@ -133,7 +133,7 @@ def __init__( self, db: api.VectorDB, test_data: list, - ground_truth: list[list[int]], + ground_truth: list[list[int]] | list[dict[str, int]], k: int = 100, filters: Filter = non_filter, payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, @@ -205,7 +205,7 @@ def _get_db_search_res( return results - def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, ...]: + def search(self, args: tuple[list, list[list[int]] | list[dict[str, int]]]) -> tuple[float, ...]: log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") with self.db.init(): self.db.prepare_filter(self.filters) @@ -215,7 +215,7 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, ...]: log.debug(f"test dataset size: {len(test_data)}") log.debug(f"ground truth size: {len(ground_truth) if ground_truth is not None else 0}") - latencies, recalls, ndcgs = [], [], [] + latencies, recalls, ndcgs, mrrs = [], [], [], [] tenant_rng = random.Random(0) for idx, emb in enumerate(test_data): tenant = ( @@ -236,13 +236,16 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, ...]: gt = ground_truth[idx] if self._use_fts_metrics: recalls.append(calc_recall_fts(self.k, gt, results)) + ndcgs.append(calc_ndcg_fts(self.k, gt, results)) + mrrs.append(calc_mrr_fts(self.k, gt, results)) else: recalls.append(calc_recall(self.k, gt[: self.k], results)) ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) else: recalls.append(0) - if not self._use_fts_metrics: - ndcgs.append(0) + ndcgs.append(0) + if self._use_fts_metrics: + mrrs.append(0) if len(latencies) % 100 == 0: log.debug( @@ -252,22 +255,25 @@ def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, ...]: avg_latency = round(np.mean(latencies), 4) avg_recall = round(np.mean(recalls), 4) + avg_ndcg = round(np.mean(ndcgs), 4) cost = round(np.sum(latencies), 4) p99 = round(np.percentile(latencies, 99), 4) p95 = round(np.percentile(latencies, 95), 4) if self._use_fts_metrics: + avg_mrr = round(np.mean(mrrs), 4) log.info( f"{mp.current_process().name:14} search entire test_data: " f"cost={cost}s, " f"queries={len(latencies)}, " f"avg_recall={avg_recall}, " + f"avg_ndcg={avg_ndcg}, " + f"avg_mrr={avg_mrr}, " f"avg_latency={avg_latency}, " f"p99={p99}, " f"p95={p95}" ) - return (avg_recall, p99, p95) + return (avg_recall, avg_ndcg, avg_mrr, p99, p95) - avg_ndcg = round(np.mean(ndcgs), 4) log.info( f"{mp.current_process().name:14} search entire test_data: " f"cost={cost}s, " diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 483e4a941..d66d6f454 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -7,7 +7,6 @@ from enum import Enum, auto import numpy as np -from pydantic import PrivateAttr from .. import config from ..base import BaseModel @@ -64,8 +63,6 @@ class CaseRunner(BaseModel): read_write_runner: ReadWriteRunner | None = None cold_warm_search_runner: ColdWarmSearchRunner | None = None - _fts_manifest_report: dict = PrivateAttr(default_factory=dict) - def __eq__(self, obj: any): if isinstance(obj, CaseRunner): key = self.load_reuse_key() @@ -213,24 +210,6 @@ def init_db(self, drop_old: bool = True) -> None: **extra_db_kwargs, ) - def _apply_fts_manifest_params(self) -> None: - bm25_params = dict(getattr(self.ca.dataset, "bm25_params", {}) or {}) - analyzer_params = dict(getattr(self.ca.dataset, "analyzer_params", {}) or {}) - self.config.db_case_config, manifest_report = self.config.db_case_config.apply_fts_manifest( - bm25_params=bm25_params, - analyzer_params=analyzer_params, - ) - self._fts_manifest_report = { - "fts_manifest": { - "bm25": bm25_params, - "analyzer": analyzer_params, - }, - **manifest_report, - } - - def _fts_manifest_additional_parameters(self) -> dict: - return dict(self._fts_manifest_report) - def _pre_run(self, drop_old: bool = True): try: self._validate_cloud_cold_latency_config(drop_old) @@ -247,8 +226,10 @@ def _pre_run(self, drop_old: bool = True): raise ValueError(msg) if self.is_fts: - self.ca.dataset.prepare(self.dataset_source) - self._apply_fts_manifest_params() + self.ca.dataset.prepare( + self.dataset_source, + filters=self.ca.filters, + ) self.init_db(drop_old) return @@ -336,6 +317,14 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: log.info("Start performance case") try: m = Metric() + if self.is_fts and getattr(self.ca.dataset, "filter_stats", None): + m.additional_parameters["fts_filter"] = dict(self.ca.dataset.filter_stats) + m.additional_parameters["fts_recall"] = { + "skipped": bool(getattr(self.ca.dataset, "recall_skipped", False)), + "reason": getattr(self.ca.dataset, "recall_skip_reason", None), + "serial_query_count": len(getattr(self.ca.dataset, "recall_queries_data", []) or []), + "full_query_count": len(getattr(self.ca.dataset, "queries_data", []) or []), + } if drop_old: if TaskStage.LOAD in self.config.stages: count, load_dur = self._load_data() @@ -378,7 +367,7 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: time.sleep(cooldown) search_results = self._serial_search() if self.is_fts: - m.recall, m.serial_latency_p99, m.serial_latency_p95 = search_results + m.recall, m.ndcg, m.mrr, m.serial_latency_p99, m.serial_latency_p95 = search_results else: m.recall, m.ndcg, m.serial_latency_p99, m.serial_latency_p95 = search_results if hasattr(self.ca, "payload_profile"): @@ -386,9 +375,6 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: m.payload_estimated_bytes_per_query = self.ca.estimated_payload_bytes_per_query( self.config.case_config.k ) - if self.is_fts: - m.additional_parameters.update(self._fts_manifest_additional_parameters()) - except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") traceback.print_exc() @@ -544,6 +530,15 @@ def _serial_search(self) -> tuple[float, ...]: FTS cases return recall, p99, p95. """ try: + if self.serial_search_runner is None: + if self.is_fts and getattr(self.ca.dataset, "recall_skipped", False): + log.warning( + "Skipping FTS serial recall: %s", + getattr(self.ca.dataset, "recall_skip_reason", "unknown"), + ) + return (0.0, 0.0, 0.0, 0.0, 0.0) + msg = "serial search runner is not initialized" + raise RuntimeError(msg) # noqa: TRY301 results, _ = self.serial_search_runner.run() except Exception as e: log.warning(f"search error: {e!s}, {e}") @@ -643,24 +638,42 @@ def _init_fts_search_runner(self): msg = "FTS dataset is missing queries or ground truth. Call prepare() before initializing search." raise ValueError(msg) test_texts = [q.text for q in fts_dataset.queries_data] - ground_truth = fts_dataset.gt_data - if len(test_texts) != len(ground_truth): - msg = f"FTS query count {len(test_texts)} does not match ground truth row count {len(ground_truth)}" + if len(test_texts) != len(fts_dataset.gt_data): + msg = f"FTS query count {len(test_texts)} does not match ground truth row count {len(fts_dataset.gt_data)}" raise ValueError(msg) - log.info(f"FTS test will use {len(test_texts)} queries for testing") + log.info(f"FTS concurrent test will use {len(test_texts)} queries for testing") self.test_texts = test_texts if TaskStage.SEARCH_SERIAL in self.config.stages: - self.serial_search_runner = SerialSearchRunner( - db=self.db, - test_data=test_texts, - ground_truth=ground_truth, - filters=self.ca.filters, - k=self.config.case_config.k, - payload_profile=self.ca.payload_profile, - workload_kind=WorkloadKind.FULL_TEXT, - ) + recall_queries = fts_dataset.recall_queries_data + recall_ground_truth = fts_dataset.recall_gt_data + if recall_queries is None or recall_ground_truth is None: + msg = ( + "FTS dataset is missing recall queries or ground truth. Call prepare() before initializing search." + ) + raise ValueError(msg) + if len(recall_queries) != len(recall_ground_truth): + msg = ( + f"FTS recall query count {len(recall_queries)} does not match " + f"ground truth row count {len(recall_ground_truth)}" + ) + raise ValueError(msg) + if fts_dataset.recall_skipped: + log.warning("FTS serial recall will be skipped: %s", fts_dataset.recall_skip_reason) + self.serial_search_runner = None + else: + recall_test_texts = [q.text for q in recall_queries] + log.info(f"FTS serial recall will use {len(recall_test_texts)} queries") + self.serial_search_runner = SerialSearchRunner( + db=self.db, + test_data=recall_test_texts, + ground_truth=recall_ground_truth, + filters=self.ca.filters, + k=self.config.case_config.k, + payload_profile=self.ca.payload_profile, + workload_kind=WorkloadKind.FULL_TEXT, + ) if TaskStage.SEARCH_CONCURRENT in self.config.stages: self.search_runner = MultiProcessingSearchRunner( db=self.db, diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index d72340ac6..abcb91607 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -18,6 +18,7 @@ from yaml import load from .. import config +from ..backend.cases import FTS_FILTER_RATES from ..backend.clients import DB from ..backend.clients.api import IndexType, MetricType from ..backend.dataset import DatasetWithSizeType, FtsDatasetWithSizeType @@ -40,6 +41,8 @@ DEFAULT_DATASET_WITH_SIZE_TYPE = DatasetWithSizeType.CohereMedium.value SUPPORTED_DATASET_WITH_SIZE_TYPES = "|".join(dataset.value for dataset in DatasetWithSizeType) +SUPPORTED_FTS_DATASET_WITH_SIZE_TYPES = "|".join(dataset.value for dataset in FtsDatasetWithSizeType) +SUPPORTED_FTS_FILTER_RATES = "|".join(f"{rate:g}" for rate in FTS_FILTER_RATES) def copy_if_not_none( @@ -280,20 +283,60 @@ def get_custom_case_config(parameters: dict) -> dict: "dataset_with_size_type": dataset_with_size_type, "payload_profile": parameters.get("payload_profile", PayloadProfile.IDS_ONLY.value), } + copy_if_not_none(custom_case_config, parameters, "fts_filter_rate", "filter_rate") return custom_case_config -def select_cli_db_case_config(db: DB, db_case_config: DBCaseConfig, case_type: str) -> DBCaseConfig: +def copy_fts_compatible_db_case_fields(source: DBCaseConfig, target: DBCaseConfig) -> DBCaseConfig: + """Copy CLI fields that remain meaningful when routing a backend to its FTS config.""" + preserved_fields = ( + "number_of_shards", + "number_of_replicas", + "refresh_interval", + "use_force_merge", + "force_merge_enabled", + "disable_backpressure", + "level", + ) + updates = { + field: getattr(source, field) for field in preserved_fields if hasattr(source, field) and hasattr(target, field) + } + if not updates: + return target + return target.model_copy(update=updates) + + +def apply_fts_cli_db_case_params( + db_case_config: DBCaseConfig, + parameters: dict[str, Any] | None, +) -> DBCaseConfig: + if not parameters: + return db_case_config + + updates = { + field: parameters[field] + for field in ("bm25_k1", "bm25_b") + if parameters.get(field) is not None and hasattr(db_case_config, field) + } + if not updates: + return db_case_config + return db_case_config.model_copy(update=updates) + + +def select_cli_db_case_config( + db: DB, + db_case_config: DBCaseConfig, + case_type: str, + parameters: dict[str, Any] | None = None, +) -> DBCaseConfig: if case_type != CaseType.FTSBm25Performance.name: return db_case_config fts_case_config_cls = db.case_config_cls(IndexType.FTS) if isinstance(db_case_config, fts_case_config_cls): - return db_case_config - fts_db_case_config = fts_case_config_cls() - if hasattr(db_case_config, "disable_backpressure") and hasattr(fts_db_case_config, "disable_backpressure"): - fts_db_case_config.disable_backpressure = db_case_config.disable_backpressure - return fts_db_case_config + return apply_fts_cli_db_case_params(db_case_config, parameters) + fts_db_case_config = copy_fts_compatible_db_case_fields(db_case_config, fts_case_config_cls()) + return apply_fts_cli_db_case_params(fts_db_case_config, parameters) log = logging.getLogger(__name__) @@ -572,9 +615,8 @@ class CommonTypedDict(TypedDict): help="Dataset with size type. When omitted, filter/insert cases use Medium Cohere (768dim, 1M), " "CloudPayloadSearchCase and CloudColdLatencyCase use LAION 100M, and CloudMultiTenantSearchCase " f"uses Large Cohere (768dim, 10M). Supported vector values include " - f"{SUPPORTED_DATASET_WITH_SIZE_TYPES}. For FTSBm25Performance, supported default UI datasets include " - f"{FtsDatasetWithSizeType.MSMarcoSmall.value}|{FtsDatasetWithSizeType.MSMarcoMedium.value}|" - f"{FtsDatasetWithSizeType.HotpotQASmall.value}|{FtsDatasetWithSizeType.HotpotQAMedium.value}.", + f"{SUPPORTED_DATASET_WITH_SIZE_TYPES}. For FTSBm25Performance, supported datasets include " + f"{SUPPORTED_FTS_DATASET_WITH_SIZE_TYPES}.", default=None, ), ] @@ -606,6 +648,36 @@ class CommonTypedDict(TypedDict): show_default=True, ), ] + bm25_k1: Annotated[ + float | None, + click.option( + "--bm25-k1", + type=float, + default=None, + help="Optional BM25 k1 override for FTS cases. Omit to use the backend default.", + ), + ] + bm25_b: Annotated[ + float | None, + click.option( + "--bm25-b", + type=float, + default=None, + help="Optional BM25 b override for FTS cases. Omit to use the backend default.", + ), + ] + fts_filter_rate: Annotated[ + float | None, + click.option( + "--fts-filter-rate", + type=float, + default=None, + help=( + "Optional FTS integer filter rate for FTSBm25Performance. " + f"Only valid for large FTS datasets. Supported values: {SUPPORTED_FTS_FILTER_RATES}." + ), + ), + ] cloud_filter_rate: Annotated[ float | None, click.option( @@ -876,7 +948,7 @@ def run( task = TaskConfig( db=db, db_config=db_config, - db_case_config=select_cli_db_case_config(db, db_case_config, parameters["case_type"]), + db_case_config=select_cli_db_case_config(db, db_case_config, parameters["case_type"], parameters), case_config=CaseConfig( case_id=CaseType[parameters["case_type"]], k=parameters["k"], diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index ac1703dbe..daacae26a 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -12,7 +12,7 @@ MAX_STREAMLIT_INT = (1 << 53) - 1 DB_LIST = [d for d in DB if d != DB.Test] -FTS_SUPPORTED_DBS = {DB.Milvus, DB.ZillizCloud, DB.ElasticCloud, DB.Vespa, DB.TurboPuffer} +FTS_SUPPORTED_DBS = {DB.Milvus, DB.ZillizCloud, DB.ElasticCloud, DB.OSSOpenSearch, DB.Vespa, DB.TurboPuffer} class Delimiter(Enum): @@ -2326,6 +2326,7 @@ class CaseConfigInput(BaseModel): ElasticCloudFtsConfig = [] VespaFtsConfig = [] +OSSOpenSearchFtsConfig = [] TurboPufferFtsConfig = [] WeaviateLoadConfig = [ @@ -3275,6 +3276,7 @@ class FilterType(Enum): DB.OSSOpenSearch: { CaseLabel.Load: OSSOpensearchLoadingConfig, CaseLabel.Performance: OSSOpenSearchPerformanceConfig, + CaseLabel.FullTextSearchPerformance: OSSOpenSearchFtsConfig, }, DB.PgVector: { CaseLabel.Load: PgVectorLoadingConfig, diff --git a/vectordb_bench/frontend/pages/full_text_search.py b/vectordb_bench/frontend/pages/full_text_search.py index ea20076c3..d6023c0c8 100644 --- a/vectordb_bench/frontend/pages/full_text_search.py +++ b/vectordb_bench/frontend/pages/full_text_search.py @@ -23,14 +23,17 @@ "HotpotQA Large", ] # Published FTS results currently cover this cloud/service backend subset. -BACKEND_ORDER = ["ZillizCloud", "ElasticSearch", "Vespa", "TurboPuffer"] +BACKEND_ORDER = ["ZillizCloud", "ElasticSearch", "OSSOpenSearch", "TurboPuffer"] BACKEND_COLORS = { "ZillizCloud": "#0D6EFD", "ElasticSearch": "#04D6C8", - "Vespa": "#61D790", + "OSSOpenSearch": "#61D790", "TurboPuffer": "#FF6B2C", } SIZE_ORDER = ["Small", "Medium", "Large"] +FILTER_RATE_LABEL_ORDER = ["50%", "75%", "90%", "95%", "99%"] +CHART_TABS = ["QPS", "Recall", "NDCG", "MRR", "Load", "Filtered QPS"] +FILTERED_TAB = "Filtered QPS" def _normalize_backend(db: str, result_file: Path) -> str: @@ -71,6 +74,12 @@ def _dataset_axis_order(data: pd.DataFrame) -> list[str]: return labels +def _filter_rate_label(value: Any) -> str: + if value is None or pd.isna(value): + return "Unfiltered" + return f"{float(value):.0%}" + + def _backend_metric_order(data: pd.DataFrame, metric: str, ascending: bool) -> list[str]: if data.empty or metric not in data: return BACKEND_ORDER @@ -102,12 +111,16 @@ def _parse_result_file(result_file: Path) -> list[dict[str, Any]]: task_config = case_result.get("task_config", {}) case_config = task_config.get("case_config", {}) custom_case = case_config.get("custom_case") or {} + additional_parameters = metrics.get("additional_parameters") or {} + fts_filter = additional_parameters.get("fts_filter") or {} + filter_rate = fts_filter.get("filter_rate", custom_case.get("filter_rate")) dataset_label = custom_case.get("dataset_with_size_type", "") dataset_family, dataset_size, dataset_key = _dataset_parts(dataset_label) dataset_doc_count = _dataset_doc_count(dataset_label) dataset_axis_label = _dataset_axis_label(dataset_key, dataset_doc_count) backend = _normalize_backend(task_config.get("db", ""), result_file) payload = metrics.get("payload_profile") or custom_case.get("payload_profile") or "ids_only" + row_task_label = task_config.get("task_label") or task_label rows.append( { @@ -119,10 +132,15 @@ def _parse_result_file(result_file: Path) -> list[dict[str, Any]]: "dataset_axis_label": dataset_axis_label, "payload": payload, "context": _run_context(task_label), - "task_label": task_label, + "task_label": row_task_label, + "filter_rate": filter_rate, + "filter_rate_label": _filter_rate_label(filter_rate), + "is_filtered": filter_rate is not None, "load_s": metrics.get("load_duration", 0.0), "qps": metrics.get("qps", 0.0), "recall": metrics.get("recall", 0.0), + "ndcg": metrics.get("ndcg", 0.0), + "mrr": metrics.get("mrr", 0.0), "p95_s": metrics.get("serial_latency_p95", 0.0), "p99_s": metrics.get("serial_latency_p99", 0.0), "concurrency": metrics.get("conc_num_list") or [], @@ -133,14 +151,8 @@ def _parse_result_file(result_file: Path) -> list[dict[str, Any]]: return rows -def _latest_backend_result_files(result_dir: Path) -> list[Path]: - result_files = [] - backend_dirs = sorted(path for path in result_dir.iterdir() if path.is_dir()) - for backend_dir in backend_dirs: - backend_files = sorted(backend_dir.glob("result_*.json")) - if backend_files: - result_files.append(backend_files[-1]) - return result_files +def _result_files(result_dir: Path) -> list[Path]: + return sorted(result_dir.rglob("result_*.json")) def load_full_text_search_rows(result_dir: Path = RESULT_DIR) -> pd.DataFrame: @@ -148,7 +160,7 @@ def load_full_text_search_rows(result_dir: Path = RESULT_DIR) -> pd.DataFrame: return pd.DataFrame() rows = [] - for result_file in _latest_backend_result_files(result_dir): + for result_file in _result_files(result_dir): rows.extend(_parse_result_file(result_file)) data = pd.DataFrame(rows) @@ -162,7 +174,8 @@ def load_full_text_search_rows(result_dir: Path = RESULT_DIR) -> pd.DataFrame: data["dataset"] = pd.Categorical(data["dataset"], DATASET_ORDER, ordered=True) data["backend"] = pd.Categorical(data["backend"], BACKEND_ORDER, ordered=True) data["dataset_size"] = pd.Categorical(data["dataset_size"], SIZE_ORDER, ordered=True) - return data.sort_values(["dataset", "backend", "payload"]).reset_index(drop=True) + data["filter_rate"] = pd.to_numeric(data["filter_rate"], errors="coerce") + return data.sort_values(["is_filtered", "dataset", "filter_rate", "backend", "payload"]).reset_index(drop=True) def _filter_data(st: Any, data: pd.DataFrame) -> pd.DataFrame: @@ -172,12 +185,15 @@ def _filter_data(st: Any, data: pd.DataFrame) -> pd.DataFrame: "Dataset", [dataset for dataset in DATASET_ORDER if dataset in set(data["dataset"].astype(str))], default=[dataset for dataset in DATASET_ORDER if dataset in set(data["dataset"].astype(str))], + key="fts-standard-datasets", ) backend_options = [backend for backend in BACKEND_ORDER if backend in set(data["backend"].astype(str))] - selected_backends = st.multiselect("Backend", backend_options, default=backend_options) + selected_backends = st.multiselect( + "Backend", backend_options, default=backend_options, key="fts-standard-backends" + ) payloads = sorted(data["payload"].dropna().unique().tolist()) default_payloads = ["ids_only"] if "ids_only" in payloads else payloads - selected_payloads = st.multiselect("Payload", payloads, default=default_payloads) + selected_payloads = st.multiselect("Payload", payloads, default=default_payloads, key="fts-standard-payloads") filters = ( data["dataset"].astype(str).isin(selected_datasets) @@ -188,6 +204,26 @@ def _filter_data(st: Any, data: pd.DataFrame) -> pd.DataFrame: return data[filters].copy() +def _filter_filtered_data(st: Any, data: pd.DataFrame) -> pd.DataFrame: + with st.sidebar: + st.header("Filters") + dataset_options = [ + family for family in ["MS MARCO", "HotpotQA"] if family in set(data["dataset_family"].astype(str)) + ] + selected_dataset = st.selectbox("Dataset", dataset_options, key="fts-filtered-dataset-family") + backend_options = [backend for backend in BACKEND_ORDER if backend in set(data["backend"].astype(str))] + selected_backends = st.multiselect( + "Backend", + backend_options, + default=backend_options, + key="fts-filtered-backends", + ) + filters = data["dataset_family"].astype(str).eq(selected_dataset) & data["backend"].astype(str).isin( + selected_backends + ) + return data[filters].copy() + + def _draw_summary_table(st: Any, data: pd.DataFrame) -> None: columns = [ "dataset", @@ -196,6 +232,8 @@ def _draw_summary_table(st: Any, data: pd.DataFrame) -> None: "load_s", "qps", "recall", + "ndcg", + "mrr", "p95_s", "p99_s", ] @@ -207,12 +245,59 @@ def _draw_summary_table(st: Any, data: pd.DataFrame) -> None: "load_s": st.column_config.NumberColumn("Load s", format="%.4f"), "qps": st.column_config.NumberColumn("QPS", format="%.4f"), "recall": st.column_config.NumberColumn("Recall", format="%.4f"), + "ndcg": st.column_config.NumberColumn("NDCG", format="%.4f"), + "mrr": st.column_config.NumberColumn("MRR", format="%.4f"), "p95_s": st.column_config.NumberColumn("p95 s", format="%.4f"), "p99_s": st.column_config.NumberColumn("p99 s", format="%.4f"), }, ) +def _draw_filtered_summary_table(st: Any, data: pd.DataFrame) -> None: + concurrency_data = _concurrency_rows(data) + if concurrency_data.empty: + st.info("No filtered concurrency QPS rows found.") + return + + columns = [ + "dataset", + "backend", + "filter_rate_label", + "concurrency", + "qps", + "task_label", + ] + st.dataframe( + concurrency_data[columns], + hide_index=True, + width="stretch", + column_config={ + "filter_rate_label": "Filter", + "concurrency": st.column_config.NumberColumn("Concurrency", format="%d"), + "qps": st.column_config.NumberColumn("QPS", format="%.4f"), + }, + ) + + +def _chart_label(value: Any, metric: str) -> str: + if pd.isna(value): + return "" + + value = float(value) + abs_value = abs(value) + if metric in {"recall", "ndcg", "mrr"}: + return f"{value:.3f}" + if metric == "qps": + return f"{value / 1000:.1f}k" if abs_value >= 1000 else f"{value:.0f}" + if metric == "load_s": + if abs_value >= 3600: + return f"{value / 3600:.1f}h" + if abs_value >= 60: + return f"{value / 60:.1f}m" + return f"{value:.1f}s" + return f"{value:.1f}" + + def _draw_metric_chart( st: Any, data: pd.DataFrame, @@ -224,8 +309,12 @@ def _draw_metric_chart( backend_order = BACKEND_ORDER show_text = data["payload"].nunique() <= 1 + chart_data = data.copy() + if show_text: + chart_data["_chart_label"] = chart_data[metric].apply(lambda value: _chart_label(value, metric)) + fig = px.bar( - data, + chart_data, x="dataset_axis_label", y=metric, color="backend", @@ -234,16 +323,15 @@ def _draw_metric_chart( category_orders={"dataset_axis_label": _dataset_axis_order(data), "backend": backend_order}, color_discrete_map=BACKEND_COLORS, hover_data=["dataset_doc_count", "payload", "context", "task_label"], - text_auto=".4g" if show_text else False, + text="_chart_label" if show_text else None, title=title, ) if show_text: - text_template = "%{y:.4f}" if metric == "recall" else "%{y:.1f}" fig.update_traces( - texttemplate=text_template, + texttemplate="%{text}", textposition="outside", textangle=0, - textfont={"size": 11}, + textfont={"size": 10}, cliponaxis=False, ) fig.update_layout( @@ -263,11 +351,14 @@ def _concurrency_rows(data: pd.DataFrame) -> pd.DataFrame: rows.append( { "dataset": row["dataset"], + "dataset_family": row["dataset_family"], "dataset_axis_label": row["dataset_axis_label"], "dataset_doc_count": row["dataset_doc_count"], "backend": row["backend"], "payload": row["payload"], "context": row["context"], + "filter_rate": row["filter_rate"], + "filter_rate_label": row["filter_rate_label"], "concurrency": concurrency, "qps": qps, "task_label": row["task_label"], @@ -276,6 +367,20 @@ def _concurrency_rows(data: pd.DataFrame) -> pd.DataFrame: return pd.DataFrame(rows) +def _peak_filtered_qps_rows(data: pd.DataFrame) -> pd.DataFrame: + concurrency_data = _concurrency_rows(data) + if concurrency_data.empty: + return concurrency_data + + concurrency_data["qps"] = pd.to_numeric(concurrency_data["qps"], errors="coerce") + concurrency_data = concurrency_data.dropna(subset=["qps"]) + if concurrency_data.empty: + return concurrency_data + + peak_indices = concurrency_data.groupby(["backend", "filter_rate_label"], sort=False, observed=True)["qps"].idxmax() + return concurrency_data.loc[peak_indices].reset_index(drop=True) + + def _draw_concurrency_chart(st: Any, data: pd.DataFrame) -> None: concurrency_data = _concurrency_rows(data) if concurrency_data.empty: @@ -303,6 +408,57 @@ def _draw_concurrency_chart(st: Any, data: pd.DataFrame) -> None: st.plotly_chart(fig, width="stretch", key="fts-concurrency-qps") +def _draw_filtered_qps_tab(st: Any, data: pd.DataFrame) -> None: + filtered_data = data[(data["payload"] == "ids_only") & data["filter_rate"].notna()].copy() + peak_data = _peak_filtered_qps_rows(filtered_data) + if peak_data.empty: + st.info("No filtered concurrency QPS rows found.") + return + + selected_family = str(peak_data["dataset_family"].iloc[0]) + peak_data["_chart_label"] = peak_data["qps"].apply(lambda value: _chart_label(value, "qps")) + backend_order = _backend_metric_order(peak_data, "qps", ascending=False) + filter_rate_order = [label for label in FILTER_RATE_LABEL_ORDER if label in set(peak_data["filter_rate_label"])] + + fig = px.bar( + peak_data, + x="filter_rate_label", + y="qps", + color="backend", + barmode="group", + category_orders={ + "filter_rate_label": filter_rate_order, + "backend": backend_order, + }, + color_discrete_map=BACKEND_COLORS, + hover_data=[ + "dataset", + "dataset_doc_count", + "payload", + "concurrency", + "task_label", + ], + text="_chart_label", + title=f"{selected_family} Peak Filtered Concurrent Search QPS", + ) + fig.update_traces( + texttemplate="%{text}", + textposition="outside", + textangle=0, + textfont={"size": 10}, + cliponaxis=False, + ) + fig.update_layout( + margin={"l": 0, "r": 0, "t": 56, "b": 12, "pad": 8}, + legend={"orientation": "h", "yanchor": "bottom", "y": 1, "xanchor": "right", "x": 1, "title": ""}, + xaxis_title="Filter rate", + yaxis_title="QPS", + uniformtext={"minsize": 10, "mode": "show"}, + ) + fig.update_xaxes(type="category", categoryorder="array", categoryarray=filter_rate_order) + st.plotly_chart(fig, width="stretch", key=f"fts-filtered-concurrency-qps-{selected_family}") + + def main(): st.set_page_config( page_title="Full Text Search Cloud Results", @@ -314,7 +470,7 @@ def main(): NavToPages(st) st.title("Full Text Search Cloud Results") - st.caption("Published FTS results for Zilliz Cloud, ElasticSearch, Vespa, and TurboPuffer.") + st.caption("Published FTS results for Zilliz Cloud, ElasticSearch, OpenSearch, and TurboPuffer.") data = load_full_text_search_rows() if data.empty: @@ -322,33 +478,67 @@ def main(): footer(st.container()) return - shown_data = _filter_data(st, data) - if shown_data.empty: + normal_data = data[~data["is_filtered"]].copy() + filtered_data = data[data["is_filtered"]].copy() + active_tab = st.session_state.get("fts-chart-tabs", CHART_TABS[0]) + if active_tab == FILTERED_TAB: + shown_data = normal_data + shown_filtered_data = _filter_filtered_data(st, filtered_data) if not filtered_data.empty else filtered_data + else: + shown_data = _filter_data(st, normal_data) if not normal_data.empty else normal_data + shown_filtered_data = filtered_data + + if shown_data.empty and shown_filtered_data.empty: st.warning("No rows match the selected filters.") footer(st.container()) return - _draw_summary_table(st, shown_data) - chart_tabs = st.tabs(["QPS", "Recall", "Load"]) + if active_tab == FILTERED_TAB and not shown_filtered_data.empty: + _draw_filtered_summary_table(st, shown_filtered_data) + elif not shown_data.empty: + _draw_summary_table(st, shown_data) + + chart_tabs = st.tabs(CHART_TABS, default=active_tab, key="fts-chart-tabs", on_change="rerun") with chart_tabs[0]: qps_data = shown_data - _draw_metric_chart( - st, - qps_data, - "qps", - "Search QPS", - _backend_metric_order(qps_data, "qps", ascending=False), - ) + if qps_data.empty: + st.info("No standard FTS rows match the selected filters.") + else: + _draw_metric_chart( + st, + qps_data, + "qps", + "Search QPS", + _backend_metric_order(qps_data, "qps", ascending=False), + ) with chart_tabs[1]: recall_data = shown_data[shown_data["payload"] == "ids_only"] _draw_metric_chart( st, recall_data, "recall", - "Math-GT Recall", + "Semantic Recall", _backend_metric_order(recall_data, "recall", ascending=False), ) with chart_tabs[2]: + ndcg_data = shown_data[shown_data["payload"] == "ids_only"] + _draw_metric_chart( + st, + ndcg_data, + "ndcg", + "NDCG", + _backend_metric_order(ndcg_data, "ndcg", ascending=False), + ) + with chart_tabs[3]: + mrr_data = shown_data[shown_data["payload"] == "ids_only"] + _draw_metric_chart( + st, + mrr_data, + "mrr", + "MRR", + _backend_metric_order(mrr_data, "mrr", ascending=False), + ) + with chart_tabs[4]: load_data = shown_data[shown_data["payload"] == "ids_only"] _draw_metric_chart( st, @@ -357,6 +547,8 @@ def main(): "Load Duration", _backend_metric_order(load_data, "load_s", ascending=True), ) + with chart_tabs[5]: + _draw_filtered_qps_tab(st, shown_filtered_data) footer(st.container()) diff --git a/vectordb_bench/metric.py b/vectordb_bench/metric.py index f442b2e97..540cefd28 100644 --- a/vectordb_bench/metric.py +++ b/vectordb_bench/metric.py @@ -24,6 +24,7 @@ class Metric: serial_latency_p95: float = 0.0 recall: float = 0.0 ndcg: float = 0.0 + mrr: float = 0.0 conc_num_list: list[int] = field(default_factory=list) conc_qps_list: list[float] = field(default_factory=list) conc_latency_p99_list: list[float] = field(default_factory=list) @@ -121,9 +122,50 @@ def calc_ndcg(ground_truth: list[int], got: list[int], ideal_dcg: float) -> floa return dcg / ideal_dcg -def calc_recall_fts(k: int, ground_truth: list[int], got: list[int]) -> float: - if not ground_truth or k <= 0: +def _positive_fts_qrels(ground_truth: dict[str, int] | list[int] | list[str]) -> dict[str, int]: + if isinstance(ground_truth, dict): + return {str(doc_id): int(rel) for doc_id, rel in ground_truth.items() if int(rel) > 0} + return {str(doc_id): 1 for doc_id in ground_truth} + + +def calc_recall_fts(k: int, ground_truth: dict[str, int] | list[int] | list[str], got: list[int] | list[str]) -> float: + gt = _positive_fts_qrels(ground_truth) + if not gt or k <= 0: + return 0.0 + got_set = {str(doc_id) for doc_id in got[:k]} + return len(set(gt) & got_set) / len(gt) + + +def calc_mrr_fts(k: int, ground_truth: dict[str, int] | list[int] | list[str], got: list[int] | list[str]) -> float: + gt = _positive_fts_qrels(ground_truth) + if not gt or k <= 0: return 0.0 - gt_set = set(ground_truth) - hits = gt_set & set(got[:k]) - return calc_recall(len(gt_set), gt_set, hits) + for rank, doc_id in enumerate(got[:k], start=1): + if str(doc_id) in gt: + return 1 / rank + return 0.0 + + +def calc_ndcg_fts(k: int, ground_truth: dict[str, int] | list[int] | list[str], got: list[int] | list[str]) -> float: + gt = _positive_fts_qrels(ground_truth) + if not gt or k <= 0: + return 0.0 + + dcg = 0.0 + seen = set() + for rank, raw_doc_id in enumerate(got[:k], start=1): + doc_id = str(raw_doc_id) + if doc_id in seen: + continue + seen.add(doc_id) + rel = gt.get(doc_id, 0) + if rel > 0: + dcg += rel / np.log2(rank + 1) + + ideal_dcg = 0.0 + for rank, rel in enumerate(sorted(gt.values(), reverse=True)[:k], start=1): + ideal_dcg += rel / np.log2(rank + 1) + + if ideal_dcg == 0: + return 0.0 + return dcg / ideal_dcg diff --git a/vectordb_bench/restful/format_res.py b/vectordb_bench/restful/format_res.py index a7f8cfe9d..c4af0d579 100644 --- a/vectordb_bench/restful/format_res.py +++ b/vectordb_bench/restful/format_res.py @@ -34,6 +34,7 @@ class FormatResult(BaseModel): serial_latency_p95: float = 0 recall: float = 0 ndcg: float = 0 + mrr: float = 0 conc_num_list: list[int] = [] conc_qps_list: list[float] = [] conc_latency_p99_list: list[float] = [] diff --git a/vectordb_bench/results/FullTextSearch/ElasticCloud/result_20260626_fts_standard_elasticcloud.json b/vectordb_bench/results/FullTextSearch/ElasticCloud/result_20260626_fts_standard_elasticcloud.json index f93240e37..1ac7e9773 100644 --- a/vectordb_bench/results/FullTextSearch/ElasticCloud/result_20260626_fts_standard_elasticcloud.json +++ b/vectordb_bench/results/FullTextSearch/ElasticCloud/result_20260626_fts_standard_elasticcloud.json @@ -8,30 +8,30 @@ "insert_duration": 92.7146, "optimize_duration": 90.3815, "load_duration": 183.0961, - "qps": 674.5161, - "serial_latency_p99": 0.0807, - "serial_latency_p95": 0.054, - "recall": 0.9191, - "ndcg": 0.0, + "qps": 1353.3032, + "serial_latency_p99": 0.0303, + "serial_latency_p95": 0.0232, + "recall": 0.7637, + "ndcg": 0.6244, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 646.8905, - 674.5161 + 1344.6128, + 1353.3032 ], "conc_latency_p99_list": [ - 0.18089975638085887, - 0.2406859093923414 + 0.08339079822006171, + 0.09601655204241978 ], "conc_latency_p95_list": [ - 0.12103873724954597, - 0.17825506660083193 + 0.06611652700194098, + 0.08258865799944035 ], "conc_latency_avg_list": [ - 0.06169217704263551, - 0.11838266990017098 + 0.04456428650467425, + 0.05904593880422133 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, @@ -84,14 +84,15 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.755 }, "task_config": { "db": "ElasticCloud", "db_config": { "db_label": "2026-06-23T17:49:51.080771", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -140,37 +141,37 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 92.7146, - "optimize_duration": 90.3815, - "load_duration": 183.0961, - "qps": 593.5741, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 17.887, + "optimize_duration": 35.1281, + "load_duration": 53.015, + "qps": 4489.7666, + "serial_latency_p99": 0.0086, + "serial_latency_p95": 0.0067, + "recall": 0.8378, + "ndcg": 0.7287, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 550.5964, - 593.5741 + 4481.0331, + 4489.7666 ], "conc_latency_p99_list": [ - 0.2036623409306056, - 0.24277012127848122 + 0.025020206998306094, + 0.029247877919842704 ], "conc_latency_p95_list": [ - 0.13283362570000462, - 0.19686596379851826 + 0.02024304099904839, + 0.02491419739963021 ], "conc_latency_avg_list": [ - 0.07247916053391205, - 0.13447533944632156 + 0.013372686260277478, + 0.01778988601007296 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 5233329, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, @@ -182,7 +183,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 47.43239876568051 + "avgdl": 55.803761 }, "analyzer": { "filter": [ @@ -196,7 +197,7 @@ "b": 0.75 }, "unapplied_bm25_params": { - "avgdl": 47.43239876568051 + "avgdl": 55.803761 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -219,14 +220,15 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.8598 }, "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:57:42.938491", + "db_label": "2026-06-23T17:43:25.431349", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -249,8 +251,8 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Large (5.2M documents)", - "payload_profile": "text" + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { @@ -265,6 +267,7 @@ "stages": [ "drop_old", "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -274,37 +277,37 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 17.887, - "optimize_duration": 35.1281, - "load_duration": 53.015, - "qps": 1985.7794, - "serial_latency_p99": 0.0245, - "serial_latency_p95": 0.0172, - "recall": 0.9241, - "ndcg": 0.0, + "insert_duration": 1.9662, + "optimize_duration": 31.09, + "load_duration": 33.0562, + "qps": 12590.5736, + "serial_latency_p99": 0.0023, + "serial_latency_p95": 0.002, + "recall": 0.9203, + "ndcg": 0.842, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 1866.3556, - 1985.7794 + 12534.1131, + 12590.5736 ], "conc_latency_p99_list": [ - 0.06111374244974294, - 0.07737018370979049 + 0.008597700440295735, + 0.010427862039068704 ], "conc_latency_p95_list": [ - 0.04278086074828025, - 0.060899814298318235 + 0.007032485200761586, + 0.008721868197972072 ], "conc_latency_avg_list": [ - 0.021414252878386827, - 0.04022542792055797 + 0.0047639834292315195, + 0.006304263635102981 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 1000000, + "inserted_count": 100000, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, @@ -316,7 +319,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 55.803761 + "avgdl": 56.88413 }, "analyzer": { "filter": [ @@ -330,7 +333,7 @@ "b": 0.75 }, "unapplied_bm25_params": { - "avgdl": 55.803761 + "avgdl": 56.88413 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -353,14 +356,15 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.9446 }, "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:43:25.431349", + "db_label": "2026-06-23T17:38:23.654768", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -383,7 +387,7 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "dataset_with_size_type": "HotpotQA Small (100K documents)", "payload_profile": "ids_only" }, "k": 100, @@ -409,37 +413,37 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 17.887, - "optimize_duration": 35.1281, - "load_duration": 53.015, - "qps": 1674.5226, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 160.1365, + "optimize_duration": 92.5894, + "load_duration": 252.7259, + "qps": 2686.9741, + "serial_latency_p99": 0.0222, + "serial_latency_p95": 0.0157, + "recall": 0.6228, + "ndcg": 0.2732, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 1605.1991, - 1674.5226 + 2686.9741, + 2663.4012 ], "conc_latency_p99_list": [ - 0.06576651665181996, - 0.08476530473002608 + 0.05383890143901231, + 0.06195318944053717 ], "conc_latency_p95_list": [ - 0.04643557349754701, - 0.0706571103990427 + 0.039473479198204584, + 0.04770255699986588 ], "conc_latency_avg_list": [ - 0.024898764599791963, - 0.047717378295758305 + 0.02230479944928826, + 0.029977797620890846 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 1000000, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, @@ -451,7 +455,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 55.803761 + "avgdl": 57.83363736188793 }, "analyzer": { "filter": [ @@ -465,7 +469,7 @@ "b": 0.75 }, "unapplied_bm25_params": { - "avgdl": 55.803761 + "avgdl": 57.83363736188793 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -488,14 +492,15 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.1862 }, "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:47:10.387803", + "db_label": "2026-06-23T17:25:04.982818", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -518,8 +523,8 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Medium (1M documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { @@ -534,6 +539,7 @@ "stages": [ "drop_old", "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -543,37 +549,37 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1.9662, - "optimize_duration": 31.09, - "load_duration": 33.0562, - "qps": 6719.6031, - "serial_latency_p99": 0.0053, - "serial_latency_p95": 0.0043, - "recall": 0.9367, - "ndcg": 0.0, + "insert_duration": 17.8893, + "optimize_duration": 34.2775, + "load_duration": 52.1668, + "qps": 10152.153, + "serial_latency_p99": 0.0043, + "serial_latency_p95": 0.0034, + "recall": 0.8028, + "ndcg": 0.5222, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 6293.0808, - 6719.6031 + 10134.0539, + 10152.153 ], "conc_latency_p99_list": [ - 0.021160466300316292, - 0.033499905739445265 + 0.014198277799223388, + 0.016789430999779142 ], "conc_latency_p95_list": [ - 0.014183212001444187, - 0.024311021799803705 + 0.01101246209909732, + 0.013366064998990623 ], "conc_latency_avg_list": [ - 0.0063481626425471735, - 0.011883786454976971 + 0.005904385368471612, + 0.007849020531480228 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 100000, + "inserted_count": 1000000, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, @@ -585,7 +591,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 56.88413 + "avgdl": 57.242324 }, "analyzer": { "filter": [ @@ -599,7 +605,7 @@ "b": 0.75 }, "unapplied_bm25_params": { - "avgdl": 56.88413 + "avgdl": 57.242324 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -622,14 +628,15 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.4526 }, "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:38:23.654768", + "db_label": "2026-06-23T17:19:23.319688", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -652,7 +659,7 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Small (100K documents)", + "dataset_with_size_type": "MS MARCO Medium (1M documents)", "payload_profile": "ids_only" }, "k": 100, @@ -678,36 +685,36 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1.9662, - "optimize_duration": 31.09, - "load_duration": 33.0562, - "qps": 3471.0893, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 1.8862, + "optimize_duration": 31.0352, + "load_duration": 32.9214, + "qps": 12530.9805, + "serial_latency_p99": 0.0016, + "serial_latency_p95": 0.0014, + "recall": 0.9116, + "ndcg": 0.7158, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 3245.7069, - 3471.0893 + 12508.8898, + 12530.9805 ], "conc_latency_p99_list": [ - 0.03633042563960771, - 0.04837336179916747 + 0.00861286495841341, + 0.00828596436098451 ], "conc_latency_p95_list": [ - 0.025955834001797476, - 0.038765777499065734 + 0.005912762001389639, + 0.007557337400794491 ], "conc_latency_avg_list": [ - 0.01231323081025481, - 0.023020162052078588 + 0.004776813722617175, + 0.006322142056532246 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, "inserted_count": 100000, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, @@ -720,7 +727,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 56.88413 + "avgdl": 55.01342 }, "analyzer": { "filter": [ @@ -734,7 +741,7 @@ "b": 0.75 }, "unapplied_bm25_params": { - "avgdl": 56.88413 + "avgdl": 55.01342 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -757,14 +764,15 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.6665 }, "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:41:05.239861", + "db_label": "2026-06-23T17:14:30.502248", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -787,8 +795,8 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Small (100K documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { @@ -803,6 +811,7 @@ "stages": [ "drop_old", "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -812,70 +821,88 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 160.1365, - "optimize_duration": 92.5894, - "load_duration": 252.7259, - "qps": 1582.6793, - "serial_latency_p99": 0.0502, - "serial_latency_p95": 0.0306, - "recall": 0.9352, - "ndcg": 0.0, + "insert_duration": 210.0847, + "optimize_duration": 93.2572, + "load_duration": 303.3419, + "qps": 721.3662, + "serial_latency_p99": 0.07, + "serial_latency_p95": 0.0475, + "recall": 0.7966, + "ndcg": 0.6175, + "mrr": 0.6258, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 1485.1078, - 1582.6793 + 709.952, + 721.3662 ], "conc_latency_p99_list": [ - 0.10130014419868526, - 0.1204227190394886 + 0.1806982008819614, + 0.2100805627823138 ], "conc_latency_p95_list": [ - 0.06413858909909301, - 0.08464620050290249 + 0.13791442114634248, + 0.16302671675002783 ], "conc_latency_avg_list": [ - 0.026891243904133805, - 0.05046188340033532 + 0.08436115583290123, + 0.11065719012112213 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 8841823, + "inserted_count": 5233329, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 2616664, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 2616665, + "matched_doc_ratio": 0.5, + "original_query_count": 7405, + "filtered_query_count": 5547, + "filtered_query_ratio": 0.749088, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 6846 }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75 - }, - "unapplied_bm25_params": { - "avgdl": 57.83363736188793 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 5547, + "full_query_count": 7405 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "num_per_batch": 1000, + "load_concurrency": 4, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" + "source_run_id": "6d07ef1559b6411b9d0afa54df3785aa", + "source_task_label": "fts_filtered_serial_elasticsearch_hotpotqa-large_r50_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_hotpotqa-large_r50_permuted_elasticcloud.json", + "source_sha256": "91b664746a916c20e5a0af5fde40bfcc30ccc8125a2dfaece86511f68ea94583", + "source_db_label": "2026-08-03T16:47:58.795789", + "source_stages": [ + "drop_old", + "load", + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" } }, "st_ideal_insert_duration": 0, @@ -896,9 +923,9 @@ "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:25:04.982818", + "db_label": "2026-07-16T19:33:09.977867", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -915,102 +942,118 @@ "refresh_interval": "30s", "use_force_merge": true, "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75 + "bm25_k1": null, + "bm25_b": null }, "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Large (8.8M documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.5 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ "drop_old", "load", - "search_serial", - "search_concurrent" + "search_concurrent", + "search_serial" ], - "load_concurrency": 0 + "load_concurrency": 4 }, "label": ":)" }, { "metrics": { "max_load_count": 0, - "insert_duration": 160.1365, - "optimize_duration": 92.5894, - "load_duration": 252.7259, - "qps": 1204.033, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 750.5232, + "serial_latency_p99": 0.0637, + "serial_latency_p95": 0.0436, + "recall": 0.8263, + "ndcg": 0.6439, + "mrr": 0.6188, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 1131.3448, - 1204.033 + 742.2813, + 750.5232 ], "conc_latency_p99_list": [ - 0.10636549504153667, - 0.13718578399857506 + 0.17308506104236582, + 0.19856040821949153 ], "conc_latency_p95_list": [ - 0.0707971137984714, - 0.10115620599754038 + 0.12934453199850396, + 0.1560941719049879 ], "conc_latency_avg_list": [ - 0.03531235997717238, - 0.06635058395138439 + 0.08063411026564973, + 0.10631068993652631 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 8841823, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75 + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 3924996, + "filter_rate": 0.75, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 1308333, + "matched_doc_ratio": 0.25, + "original_query_count": 7405, + "filtered_query_count": 3175, + "filtered_query_ratio": 0.428764, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 3379 }, - "unapplied_bm25_params": { - "avgdl": 57.83363736188793 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 3175, + "full_query_count": 7405 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" + "source_run_id": "17bc3bc82c7a4367b901f21e8415867a", + "source_task_label": "fts_filtered_serial_elasticsearch_hotpotqa-large_r75_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_hotpotqa-large_r75_permuted_elasticcloud.json", + "source_sha256": "0ed262bb98569e2c29ffda21c498767d25ff895a23fe02a78db6b40b2f0191cc", + "source_db_label": "2026-08-03T16:56:16.769773", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" } }, "st_ideal_insert_duration": 0, @@ -1031,9 +1074,9 @@ "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:32:21.485921", + "db_label": "2026-07-16T19:41:37.525743", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -1050,29 +1093,30 @@ "refresh_interval": "30s", "use_force_merge": true, "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75 + "bm25_k1": null, + "bm25_b": null }, "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Large (8.8M documents)", - "payload_profile": "text" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.75 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", - "search_concurrent" + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, @@ -1081,70 +1125,84 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 17.8893, - "optimize_duration": 34.2775, - "load_duration": 52.1668, - "qps": 5702.2528, - "serial_latency_p99": 0.0092, - "serial_latency_p95": 0.0065, - "recall": 0.9437, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 770.9499, + "serial_latency_p99": 0.0724, + "serial_latency_p95": 0.0508, + "recall": 0.8665, + "ndcg": 0.7004, + "mrr": 0.6627, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 5383.2674, - 5702.2528 + 759.173, + 770.9499 ], "conc_latency_p99_list": [ - 0.02630764468107369, - 0.03738318958105083 + 0.15933862284160563, + 0.18434771904256195 ], "conc_latency_p95_list": [ - 0.017680951600959793, - 0.02877839460124961 + 0.12385570425285548, + 0.14966088689725432 ], "conc_latency_avg_list": [ - 0.007422129557916056, - 0.014006341815361298 + 0.07888747691351794, + 0.10353734658255655 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 1000000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75 + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4709996, + "filter_rate": 0.9, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 523333, + "matched_doc_ratio": 0.1, + "original_query_count": 7405, + "filtered_query_count": 1367, + "filtered_query_ratio": 0.184605, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 1335 }, - "unapplied_bm25_params": { - "avgdl": 57.242324 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 1367, + "full_query_count": 7405 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" + "source_run_id": "b4641cb14b1644279c5f35fee2458dfb", + "source_task_label": "fts_filtered_serial_elasticsearch_hotpotqa-large_r90_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_hotpotqa-large_r90_permuted_elasticcloud.json", + "source_sha256": "9a52e707376a127c8d538bab1a67883b6c5521df0d866d1a72aaab1cca0ddf2c", + "source_db_label": "2026-08-03T16:59:14.726191", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" } }, "st_ideal_insert_duration": 0, @@ -1165,9 +1223,9 @@ "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:19:23.319688", + "db_label": "2026-07-16T19:45:01.768660", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -1184,30 +1242,30 @@ "refresh_interval": "30s", "use_force_merge": true, "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75 + "bm25_k1": null, + "bm25_b": null }, "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Medium (1M documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.9 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, @@ -1216,70 +1274,84 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 17.8893, - "optimize_duration": 34.2775, - "load_duration": 52.1668, - "qps": 3677.9597, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 774.5308, + "serial_latency_p99": 0.0542, + "serial_latency_p95": 0.0383, + "recall": 0.884, + "ndcg": 0.7266, + "mrr": 0.6865, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 3473.3114, - 3677.9597 + 774.5308, + 772.2879 ], "conc_latency_p99_list": [ - 0.0357281443210376, - 0.04769462314165139 + 0.15306021131982533, + 0.17734859948323 ], "conc_latency_p95_list": [ - 0.025091530600184316, - 0.03831459170251036 + 0.12027961940184463, + 0.14829936109745176 ], "conc_latency_avg_list": [ - 0.011505997403950962, - 0.021721806785628715 + 0.07731039411178581, + 0.10336622360207741 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 1000000, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75 + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4971662, + "filter_rate": 0.95, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 261667, + "matched_doc_ratio": 0.05, + "original_query_count": 7405, + "filtered_query_count": 698, + "filtered_query_ratio": 0.094261, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 664 }, - "unapplied_bm25_params": { - "avgdl": 57.242324 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 698, + "full_query_count": 7405 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" + "source_run_id": "a72a9198404847c696aa094dc2ecebce", + "source_task_label": "fts_filtered_serial_elasticsearch_hotpotqa-large_r95_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_hotpotqa-large_r95_permuted_elasticcloud.json", + "source_sha256": "cd51a79edd64d94df8814bbfb745ddb555e251c338a2ddafae49c6cf989e5ea2", + "source_db_label": "2026-08-03T17:01:44.955740", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" } }, "st_ideal_insert_duration": 0, @@ -1300,9 +1372,9 @@ "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:22:25.552010", + "db_label": "2026-07-16T19:48:27.444308", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -1319,29 +1391,30 @@ "refresh_interval": "30s", "use_force_merge": true, "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75 + "bm25_k1": null, + "bm25_b": null }, "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Medium (1M documents)", - "payload_profile": "text" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.95 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", - "search_concurrent" + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, @@ -1350,70 +1423,237 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1.8862, - "optimize_duration": 31.0352, - "load_duration": 32.9214, - "qps": 12479.8332, - "serial_latency_p99": 0.0029, - "serial_latency_p95": 0.0023, - "recall": 0.9497, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 936.4393, + "serial_latency_p99": 0.0359, + "serial_latency_p95": 0.0298, + "recall": 0.898, + "ndcg": 0.7825, + "mrr": 0.7494, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 11315.3177, - 12479.8332 + 919.3657, + 936.4393 ], "conc_latency_p99_list": [ - 0.012597403119434608, - 0.024557725561316985 + 0.11650546061813662, + 0.13283855842935735 ], "conc_latency_p95_list": [ - 0.00729868780035758, - 0.01442482639795344 + 0.09433838454860961, + 0.11730172135248722 ], "conc_latency_avg_list": [ - 0.003526799491246593, - 0.00638580620427849 + 0.06513757521688086, + 0.08528450502047526 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 100000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 5180995, + "filter_rate": 0.99, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 52334, + "matched_doc_ratio": 0.01, + "original_query_count": 7405, + "filtered_query_count": 147, + "filtered_query_ratio": 0.019851, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 136 }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 147, + "full_query_count": 7405 }, - "unapplied_bm25_params": { - "avgdl": 55.01342 + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "482bc4f2cf3a4c1692ff65756cd33ea5", + "source_task_label": "fts_filtered_serial_elasticsearch_hotpotqa-large_r99_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_hotpotqa-large_r99_permuted_elasticcloud.json", + "source_sha256": "2f644813b64d937af334d5c52d120a4d89a096927034adb6f11c7cfd63bb95d5", + "source_db_label": "2026-08-03T17:03:52.164545", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-07-16T19:51:49.557699", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.99 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 ], - "tokenizer": "standard" + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 305.6984, + "optimize_duration": 93.6347, + "load_duration": 399.3331, + "qps": 1670.2698, + "serial_latency_p99": 0.0427, + "serial_latency_p95": 0.0268, + "recall": 0.6974, + "ndcg": 0.3455, + "mrr": 0.256, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1613.9683, + 1670.2698 + ], + "conc_latency_p99_list": [ + 0.10500255563529215, + 0.10993121209095964 + ], + "conc_latency_p95_list": [ + 0.068664093100233, + 0.07755282529615215 + ], + "conc_latency_avg_list": [ + 0.03712288018409116, + 0.04780773417065931 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4420911, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 4420912, + "matched_doc_ratio": 0.5, + "original_query_count": 6980, + "filtered_query_count": 3658, + "filtered_query_ratio": 0.524069, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 3758 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 3658, + "full_query_count": 6980 + }, + "num_per_batch": 1000, + "load_concurrency": 4, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "44d302a866ef41f487cd50bab4cd6ee1", + "source_task_label": "fts_filtered_serial_elasticsearch_msmarco-large_r50_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_msmarco-large_r50_permuted_elasticcloud.json", + "source_sha256": "0573762c9a2d93cd37248a6e6c63d2a0f6e245844e1f18ef83fa4368bea1ee57", + "source_db_label": "2026-08-03T16:36:18.370955", + "source_stages": [ + "drop_old", + "load", + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" } }, "st_ideal_insert_duration": 0, @@ -1434,9 +1674,9 @@ "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:14:30.502248", + "db_label": "2026-07-16T18:34:54.790049", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -1453,30 +1693,181 @@ "refresh_interval": "30s", "use_force_merge": true, "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75 + "bm25_k1": null, + "bm25_b": null }, "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Small (100K documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.5 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ "drop_old", "load", - "search_serial", - "search_concurrent" + "search_concurrent", + "search_serial" + ], + "load_concurrency": 4 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1760.9692, + "serial_latency_p99": 0.0375, + "serial_latency_p95": 0.0237, + "recall": 0.7618, + "ndcg": 0.4234, + "mrr": 0.3358, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1760.9692, + 1748.2651 + ], + "conc_latency_p99_list": [ + 0.08974919100000989, + 0.10211793256872619 + ], + "conc_latency_p95_list": [ + 0.060971413004153874, + 0.0733459422524902 + ], + "conc_latency_avg_list": [ + 0.0339761071088688, + 0.04568243171278137 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 6631367, + "filter_rate": 0.75, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 2210456, + "matched_doc_ratio": 0.25, + "original_query_count": 6980, + "filtered_query_count": 1818, + "filtered_query_ratio": 0.260458, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 1839 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 1818, + "full_query_count": 6980 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "e798f27d27574fb4a070b632d6a39fd3", + "source_task_label": "fts_filtered_serial_elasticsearch_msmarco-large_r75_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_msmarco-large_r75_permuted_elasticcloud.json", + "source_sha256": "2633cc2c75fdae118a8be7dd7cde4faa8ee705a35a1d6812d2355784c395c46a", + "source_db_label": "2026-08-03T16:44:18.348174", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-07-16T18:43:49.325484", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.75 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, @@ -1485,70 +1876,233 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1.8862, - "optimize_duration": 31.0352, - "load_duration": 32.9214, - "qps": 5037.2489, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1727.6187, + "serial_latency_p99": 0.0349, + "serial_latency_p95": 0.0232, + "recall": 0.8383, + "ndcg": 0.5152, + "mrr": 0.4301, "conc_num_list": [ - 40, + 60, 80 ], "conc_qps_list": [ - 4701.0004, - 5037.2489 + 1727.6187, + 1712.5496 ], "conc_latency_p99_list": [ - 0.026061404299944115, - 0.03860565954051707 + 0.08606930284106058, + 0.09963348716031763 ], "conc_latency_p95_list": [ - 0.017935610502172503, - 0.030578274452818733 + 0.06112156380477239, + 0.07310520800238009 ], "conc_latency_avg_list": [ - 0.008499634080099454, - 0.01585666237961175 + 0.03468250891671445, + 0.04662555647302419 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 100000, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 7957640, + "filter_rate": 0.9, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 884183, + "matched_doc_ratio": 0.1, + "original_query_count": 6980, + "filtered_query_count": 739, + "filtered_query_ratio": 0.105874, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 740 }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 739, + "full_query_count": 6980 }, - "unapplied_bm25_params": { - "avgdl": 55.01342 + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "3db387eca78a41b988db0fa5d8653078", + "source_task_label": "fts_filtered_serial_elasticsearch_msmarco-large_r90_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_msmarco-large_r90_permuted_elasticcloud.json", + "source_sha256": "323dda4c7f6f14141ee1ff1b1df6b7d6dc20269bc1d01f90f36d22c143780f49", + "source_db_label": "2026-08-03T16:45:23.353032", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-07-16T18:46:08.382474", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.9 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 ], - "tokenizer": "standard" + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1640.447, + "serial_latency_p99": 0.0328, + "serial_latency_p95": 0.022, + "recall": 0.8559, + "ndcg": 0.5826, + "mrr": 0.5105, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1637.2394, + 1640.447 + ], + "conc_latency_p99_list": [ + 0.08551632568014624, + 0.09862579727101546 + ], + "conc_latency_p95_list": [ + 0.06308142885354755, + 0.07469368650054091 + ], + "conc_latency_avg_list": [ + 0.036597530028333033, + 0.048665543477514936 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 8399731, + "filter_rate": 0.95, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 442092, + "matched_doc_ratio": 0.05, + "original_query_count": 6980, + "filtered_query_count": 347, + "filtered_query_ratio": 0.049713, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 347 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 347, + "full_query_count": 6980 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "8a211aced09a403486d088c32fd068e2", + "source_task_label": "fts_filtered_serial_elasticsearch_msmarco-large_r95_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_msmarco-large_r95_permuted_elasticcloud.json", + "source_sha256": "c6d3dc57545fd1e5b9c44ae31f26e985c80ad3eebddd838c559228c52338170c", + "source_db_label": "2026-08-03T16:46:18.769194", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" } }, "st_ideal_insert_duration": 0, @@ -1569,9 +2123,9 @@ "task_config": { "db": "ElasticCloud", "db_config": { - "db_label": "2026-06-23T17:17:03.106278", + "db_label": "2026-07-16T18:48:23.449047", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", "cloud_id": null, "scheme": "http", "host": "**********", @@ -1588,29 +2142,179 @@ "refresh_interval": "30s", "use_force_merge": true, "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75 + "bm25_k1": null, + "bm25_b": null }, "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Small (100K documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.95 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", - "search_concurrent" + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1760.9558, + "serial_latency_p99": 0.0221, + "serial_latency_p95": 0.0191, + "recall": 0.9403, + "ndcg": 0.7033, + "mrr": 0.6373, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1754.6975, + 1760.9558 + ], + "conc_latency_p99_list": [ + 0.06734019880532284, + 0.07769490615988613 + ], + "conc_latency_p95_list": [ + 0.052860249001241755, + 0.06493684799788751 + ], + "conc_latency_avg_list": [ + 0.03414497660724748, + 0.04535883119607527 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 8753404, + "filter_rate": 0.99, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 88419, + "matched_doc_ratio": 0.01, + "original_query_count": 6980, + "filtered_query_count": 67, + "filtered_query_ratio": 0.009599, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 67 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 67, + "full_query_count": 6980 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "c459c22a67544caaa467bacec9626531", + "source_task_label": "fts_filtered_serial_elasticsearch_msmarco-large_r99_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_elasticsearch_msmarco-large_r99_permuted_elasticcloud.json", + "source_sha256": "39cc6a5202b5daa079b35cd28d86e379866dced531bc59f50a2ac3a2ba30f78a", + "source_db_label": "2026-08-03T16:47:10.714140", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "2026-07-16T18:50:39.525006", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"elasticsearch/v1\",\n \"backend_version\": \"9.4.3\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"elasticsearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "cloud_id": null, + "scheme": "http", + "host": "**********", + "port": 9200, + "user": "elastic", + "user_name": null, + "password": "**********", + "use_ssl": false, + "verify_certs": true + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "use_force_merge": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.99 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, diff --git a/vectordb_bench/results/FullTextSearch/OpenSearch/result_20260708_fts_standard_opensearch.json b/vectordb_bench/results/FullTextSearch/OpenSearch/result_20260708_fts_standard_opensearch.json new file mode 100644 index 000000000..d30aa6227 --- /dev/null +++ b/vectordb_bench/results/FullTextSearch/OpenSearch/result_20260708_fts_standard_opensearch.json @@ -0,0 +1,2100 @@ +{ + "run_id": "fb9d4d97c0fd414f8b462556670c3105", + "task_label": "opensearch37_semantic_ids_only", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 21.4087, + "optimize_duration": 30.5468, + "load_duration": 51.9554, + "qps": 12691.5259, + "serial_latency_p99": 0.0016, + "serial_latency_p95": 0.0014, + "recall": 0.9115, + "ndcg": 0.7158, + "mrr": 0.6664, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 12490.6378, + 12691.5259 + ], + "conc_latency_p99_list": [ + 0.009508526180288754, + 0.008180375498341164 + ], + "conc_latency_p95_list": [ + 0.006050063249676896, + 0.007465197501005605 + ], + "conc_latency_avg_list": [ + 0.004786182434018657, + 0.006250595112561167 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0 + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_os37_msmarco_small", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36.5969, + "optimize_duration": 30.0643, + "load_duration": 66.6612, + "qps": 9716.9805, + "serial_latency_p99": 0.0045, + "serial_latency_p95": 0.0035, + "recall": 0.8028, + "ndcg": 0.5222, + "mrr": 0.4526, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 9679.3154, + 9716.9805 + ], + "conc_latency_p99_list": [ + 0.014330312998936279, + 0.016602437859110065 + ], + "conc_latency_p95_list": [ + 0.01117035200149985, + 0.01337789999961386 + ], + "conc_latency_avg_list": [ + 0.0061837669065991155, + 0.008203579954480796 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0 + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_os37_msmarco_medium", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 241.1154, + "optimize_duration": 64.3404, + "load_duration": 305.4557, + "qps": 2765.6385, + "serial_latency_p99": 0.0232, + "serial_latency_p95": 0.0159, + "recall": 0.6228, + "ndcg": 0.2731, + "mrr": 0.186, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 2765.6385, + 2703.4414 + ], + "conc_latency_p99_list": [ + 0.054147378889720287, + 0.06431732813121925 + ], + "conc_latency_p95_list": [ + 0.03906455834985536, + 0.04768981275119586 + ], + "conc_latency_avg_list": [ + 0.021667828285827452, + 0.029549008838550658 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0 + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_os37_msmarco_large", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 15.3955, + "optimize_duration": 30.623, + "load_duration": 46.0185, + "qps": 12605.0846, + "serial_latency_p99": 0.0024, + "serial_latency_p95": 0.0021, + "recall": 0.9203, + "ndcg": 0.842, + "mrr": 0.9446, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 12191.1982, + 12605.0846 + ], + "conc_latency_p99_list": [ + 0.010707631260593202, + 0.013225803580644424 + ], + "conc_latency_p95_list": [ + 0.00833549370181572, + 0.010521701102152285 + ], + "conc_latency_avg_list": [ + 0.004901968566726041, + 0.0063033020004832005 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0 + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_os37_hotpotqa_small", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 33.1365, + "optimize_duration": 30.857, + "load_duration": 63.9935, + "qps": 4279.073, + "serial_latency_p99": 0.009, + "serial_latency_p95": 0.0071, + "recall": 0.8378, + "ndcg": 0.7287, + "mrr": 0.8598, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 4279.073, + 4277.3037 + ], + "conc_latency_p99_list": [ + 0.025680999840842586, + 0.03080206583093969 + ], + "conc_latency_p95_list": [ + 0.02089815480067045, + 0.02596823259973461 + ], + "conc_latency_avg_list": [ + 0.01400255309733968, + 0.018675460777613128 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0 + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_os37_hotpotqa_medium", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 132.2746, + "optimize_duration": 61.683, + "load_duration": 193.9576, + "qps": 1336.4625, + "serial_latency_p99": 0.0316, + "serial_latency_p95": 0.0241, + "recall": 0.7637, + "ndcg": 0.6243, + "mrr": 0.7549, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1335.4086, + 1336.4625 + ], + "conc_latency_p99_list": [ + 0.08600163375929698, + 0.10016711670032244 + ], + "conc_latency_p95_list": [ + 0.06740941459793247, + 0.08448904250144551 + ], + "conc_latency_avg_list": [ + 0.044878777739965396, + 0.0597764434350997 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 0 + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_os37_hotpotqa_large", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only" + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 150.1223, + "optimize_duration": 34.1788, + "load_duration": 184.3012, + "qps": 667.304, + "serial_latency_p99": 0.0803, + "serial_latency_p95": 0.0553, + "recall": 0.7965, + "ndcg": 0.6174, + "mrr": 0.6257, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 667.304, + 666.4777 + ], + "conc_latency_p99_list": [ + 0.1921520423081529, + 0.22862113459021222 + ], + "conc_latency_p95_list": [ + 0.14732917190267472, + 0.17784609899790668 + ], + "conc_latency_avg_list": [ + 0.08955421200884714, + 0.11974453346600444 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 2616664, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 2616665, + "matched_doc_ratio": 0.5, + "original_query_count": 7405, + "filtered_query_count": 5547, + "filtered_query_ratio": 0.749088, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 6846 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 5547, + "full_query_count": 7405 + }, + "num_per_batch": 1000, + "load_concurrency": 4, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "75d27b07a19d488e9dd9d111eb2fd30e", + "source_task_label": "fts_filtered_serial_opensearch_hotpotqa-large_r50_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_hotpotqa-large_r50_permuted_ossopensearch.json", + "source_sha256": "81f6b480b6063c23cc4b61d58b7471b8170fa6bbe3e267753bdff3d6cc6e55cf", + "source_db_label": "", + "source_stages": [ + "drop_old", + "load", + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent", + "search_serial" + ], + "load_concurrency": 4 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 731.8949, + "serial_latency_p99": 0.0724, + "serial_latency_p95": 0.0511, + "recall": 0.8263, + "ndcg": 0.6439, + "mrr": 0.6187, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 720.3874, + 731.8949 + ], + "conc_latency_p99_list": [ + 0.1764921818590665, + 0.20388233807985673 + ], + "conc_latency_p95_list": [ + 0.1350544641034503, + 0.16113367400394052 + ], + "conc_latency_avg_list": [ + 0.08302399005114736, + 0.10901853523287403 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 3924996, + "filter_rate": 0.75, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 1308333, + "matched_doc_ratio": 0.25, + "original_query_count": 7405, + "filtered_query_count": 3175, + "filtered_query_ratio": 0.428764, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 3379 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 3175, + "full_query_count": 7405 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "eb07a1bea78a41778047b3f591b54b6a", + "source_task_label": "fts_filtered_serial_opensearch_hotpotqa-large_r75_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_hotpotqa-large_r75_permuted_ossopensearch.json", + "source_sha256": "fe28e8d8cc38f6bc7da35157b17f1419f3ae1bde6cbde25f3b734ff829e0d15a", + "source_db_label": "", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.75 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 774.4857, + "serial_latency_p99": 0.063, + "serial_latency_p95": 0.0455, + "recall": 0.8665, + "ndcg": 0.7004, + "mrr": 0.6627, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 774.0823, + 774.4857 + ], + "conc_latency_p99_list": [ + 0.15563926174960205, + 0.18285104553018763 + ], + "conc_latency_p95_list": [ + 0.12185666524928812, + 0.15047706774530525 + ], + "conc_latency_avg_list": [ + 0.07737237257563531, + 0.10304668327704196 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4709996, + "filter_rate": 0.9, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 523333, + "matched_doc_ratio": 0.1, + "original_query_count": 7405, + "filtered_query_count": 1367, + "filtered_query_ratio": 0.184605, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 1335 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 1367, + "full_query_count": 7405 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "bbd65a96f4b94f63859e386d8b479262", + "source_task_label": "fts_filtered_serial_opensearch_hotpotqa-large_r90_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_hotpotqa-large_r90_permuted_ossopensearch.json", + "source_sha256": "5e4b7a9ec8eea54738159a4c1ea8b7e4e5339420e9a30146dc3921ced18484fd", + "source_db_label": "", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.9 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 822.775, + "serial_latency_p99": 0.0559, + "serial_latency_p95": 0.0403, + "recall": 0.884, + "ndcg": 0.7266, + "mrr": 0.6865, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 822.775, + 822.129 + ], + "conc_latency_p99_list": [ + 0.1412675818782008, + 0.1656299286980356 + ], + "conc_latency_p95_list": [ + 0.11175147520334575, + 0.13831253509706584 + ], + "conc_latency_avg_list": [ + 0.07280328828906549, + 0.09711557214002686 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4971662, + "filter_rate": 0.95, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 261667, + "matched_doc_ratio": 0.05, + "original_query_count": 7405, + "filtered_query_count": 698, + "filtered_query_ratio": 0.094261, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 664 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 698, + "full_query_count": 7405 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "cafbf08c0a1b4340aeffdd5c17a98e8a", + "source_task_label": "fts_filtered_serial_opensearch_hotpotqa-large_r95_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_hotpotqa-large_r95_permuted_ossopensearch.json", + "source_sha256": "d347d5d0d87058dc524a0f64457fc34dbd52fe7eab60fd50f1e381a61aa8e723", + "source_db_label": "", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.95 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1001.7318, + "serial_latency_p99": 0.0382, + "serial_latency_p95": 0.0313, + "recall": 0.898, + "ndcg": 0.7825, + "mrr": 0.7494, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1001.074, + 1001.7318 + ], + "conc_latency_p99_list": [ + 0.10869441425413238, + 0.1255568589524046 + ], + "conc_latency_p95_list": [ + 0.08671272930077974, + 0.10970903755223843 + ], + "conc_latency_avg_list": [ + 0.05986702706436918, + 0.07976030199309388 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 5180995, + "filter_rate": 0.99, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 52334, + "matched_doc_ratio": 0.01, + "original_query_count": 7405, + "filtered_query_count": 147, + "filtered_query_ratio": 0.019851, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 136 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 147, + "full_query_count": 7405 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "f540cef0cce545a2b37e3666c93c2ac0", + "source_task_label": "fts_filtered_serial_opensearch_hotpotqa-large_r99_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_hotpotqa-large_r99_permuted_ossopensearch.json", + "source_sha256": "c40d03752bfaa9281670ea8acbb2ff9525f9a093e8fe92246ccc89ff1fc7ad33", + "source_db_label": "", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.99 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 235.9506, + "optimize_duration": 63.7682, + "load_duration": 299.7188, + "qps": 1531.2932, + "serial_latency_p99": 0.0625, + "serial_latency_p95": 0.04, + "recall": 0.6974, + "ndcg": 0.3452, + "mrr": 0.2556, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1397.028, + 1531.2932 + ], + "conc_latency_p99_list": [ + 0.12297188279801045, + 0.11898466303653542 + ], + "conc_latency_p95_list": [ + 0.07994781309971585, + 0.08496303739520955 + ], + "conc_latency_avg_list": [ + 0.042852920943267835, + 0.05207596939025791 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4420911, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 4420912, + "matched_doc_ratio": 0.5, + "original_query_count": 6980, + "filtered_query_count": 3658, + "filtered_query_ratio": 0.524069, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 3758 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 3658, + "full_query_count": 6980 + }, + "num_per_batch": 1000, + "load_concurrency": 4, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "d137f84a931c420387dcf03ee9a1fac1", + "source_task_label": "fts_filtered_serial_opensearch_msmarco-large_r50_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_msmarco-large_r50_permuted_ossopensearch.json", + "source_sha256": "03db427c5251728b1b5a515d2c6428d1f69ad8f7511ab3dad76ce9a693fad889", + "source_db_label": "", + "source_stages": [ + "drop_old", + "load", + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent", + "search_serial" + ], + "load_concurrency": 4 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1606.6222, + "serial_latency_p99": 0.0418, + "serial_latency_p95": 0.0275, + "recall": 0.7618, + "ndcg": 0.423, + "mrr": 0.3352, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1606.6222, + 1590.4299 + ], + "conc_latency_p99_list": [ + 0.09791465523405339, + 0.11049418440234161 + ], + "conc_latency_p95_list": [ + 0.06849266580247786, + 0.08139459499943769 + ], + "conc_latency_avg_list": [ + 0.03727788527657192, + 0.050209997870867544 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 6631367, + "filter_rate": 0.75, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 2210456, + "matched_doc_ratio": 0.25, + "original_query_count": 6980, + "filtered_query_count": 1818, + "filtered_query_ratio": 0.260458, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 1839 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 1818, + "full_query_count": 6980 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "5a5f2ed5c49e4f4996ba8aa5569549cd", + "source_task_label": "fts_filtered_serial_opensearch_msmarco-large_r75_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_msmarco-large_r75_permuted_ossopensearch.json", + "source_sha256": "ef2e679c0369c5c6d3d00b762073bbc522c148b997c149bd887b5cc634bfc648", + "source_db_label": "", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.75 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1695.751, + "serial_latency_p99": 0.036, + "serial_latency_p95": 0.0245, + "recall": 0.8383, + "ndcg": 0.5152, + "mrr": 0.43, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1640.8756, + 1695.751 + ], + "conc_latency_p99_list": [ + 0.09178599175880653, + 0.1001698837034927 + ], + "conc_latency_p95_list": [ + 0.06526272699993568, + 0.07415721294491956 + ], + "conc_latency_avg_list": [ + 0.0365111554621457, + 0.047110976089449794 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 7957640, + "filter_rate": 0.9, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 884183, + "matched_doc_ratio": 0.1, + "original_query_count": 6980, + "filtered_query_count": 739, + "filtered_query_ratio": 0.105874, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 740 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 739, + "full_query_count": 6980 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "84995532aa774083b82a5ee1871c9524", + "source_task_label": "fts_filtered_serial_opensearch_msmarco-large_r90_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_msmarco-large_r90_permuted_ossopensearch.json", + "source_sha256": "273610a85902345c8ac567f192050bcda7f7668bcf9d4ed6da1b1b67710352c0", + "source_db_label": "", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.9 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1667.8223, + "serial_latency_p99": 0.0333, + "serial_latency_p95": 0.0223, + "recall": 0.8559, + "ndcg": 0.5826, + "mrr": 0.5105, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1667.8223, + 1631.7832 + ], + "conc_latency_p99_list": [ + 0.08383034317885171, + 0.09970359897662995 + ], + "conc_latency_p95_list": [ + 0.061932306094968095, + 0.07524768869843683 + ], + "conc_latency_avg_list": [ + 0.03594126598367885, + 0.04894108378701258 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 8399731, + "filter_rate": 0.95, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 442092, + "matched_doc_ratio": 0.05, + "original_query_count": 6980, + "filtered_query_count": 347, + "filtered_query_ratio": 0.049713, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 347 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 347, + "full_query_count": 6980 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "4cac0590f7104d73abb22181858b51be", + "source_task_label": "fts_filtered_serial_opensearch_msmarco-large_r95_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_msmarco-large_r95_permuted_ossopensearch.json", + "source_sha256": "958c326fb20b13a2446655f1ba77fd141ed4daa98a84bcc016e398472bb18b4e", + "source_db_label": "", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.95 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1758.8721, + "serial_latency_p99": 0.0237, + "serial_latency_p95": 0.0205, + "recall": 0.9403, + "ndcg": 0.7033, + "mrr": 0.6373, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 1726.0672, + 1758.8721 + ], + "conc_latency_p99_list": [ + 0.0674230600016017, + 0.07780777791922446 + ], + "conc_latency_p95_list": [ + 0.05345191999731469, + 0.06511080019990913 + ], + "conc_latency_avg_list": [ + 0.03472604073317141, + 0.04543198008032014 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 8753404, + "filter_rate": 0.99, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 88419, + "matched_doc_ratio": 0.01, + "original_query_count": 6980, + "filtered_query_count": 67, + "filtered_query_ratio": 0.009599, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 67 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 67, + "full_query_count": 6980 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "7a432f6e12d34390a787e5ea7727e51b", + "source_task_label": "fts_filtered_serial_opensearch_msmarco-large_r99_permuted", + "source_timestamp": 1785715200.0, + "source_file": "result_20260803_fts_filtered_serial_opensearch_msmarco-large_r99_permuted_ossopensearch.json", + "source_sha256": "48dffb64b34cb1500e65e45183ec2b590158d9df84b20da2249cfe3003356321", + "source_db_label": "", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "OSSOpenSearch", + "db_config": { + "db_label": "", + "version": "", + "note": "{\n \"schema\": \"vdbbench-run-context/v1\",\n \"profile\": \"opensearch/v1\",\n \"backend_version\": \"3.7.0\",\n \"deployment\": {\n \"method\": \"docker\",\n \"config\": {\n \"mode\": \"single_node\"\n }\n },\n \"server_groups\": [\n {\n \"role\": \"opensearch-data-master\",\n \"count\": 1,\n \"jvm_heap_gib\": 30,\n \"hardware\": {\n \"machine_type\": \"i8g.4xlarge\",\n \"cpu_count\": 16,\n \"memory_gib\": 123.54,\n \"storage\": {\n \"type\": \"local_nvme\",\n \"device_count\": 1,\n \"total_gib\": 3492.46,\n \"model\": \"Amazon EC2 NVMe Instance Storage\"\n }\n }\n }\n ],\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_gib\": 61.65,\n \"storage\": {\n \"type\": \"ebs\",\n \"device_count\": 1,\n \"total_gib\": 500,\n \"model\": \"Amazon Elastic Block Store\"\n }\n }\n}", + "index_name": "vdbbench_fts_total_series", + "host": "10.15.9.42", + "port": 9200, + "user": "", + "password": "" + }, + "db_case_config": { + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "force_merge_enabled": true, + "metric_type": "BM25", + "bm25_k1": null, + "bm25_b": null + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.99 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1783468800.0 +} diff --git a/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json b/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json index a063a9b83..00c60972a 100644 --- a/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json +++ b/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json @@ -11,8 +11,8 @@ "qps": 1005.2819, "serial_latency_p99": 0.044, "serial_latency_p95": 0.0329, - "recall": 0.8665, - "ndcg": 0.0, + "recall": 0.7745, + "ndcg": 0.6371, "conc_num_list": [ 40, 60, @@ -73,8 +73,7 @@ "tokenizer": "standard" }, "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_large\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-large-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + "load_concurrency": 0 }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -89,14 +88,15 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.7666 }, "task_config": { "db": "TurboPuffer", "db_config": { "db_label": "2026-06-24T12:21:56.394316", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\"\n },\n \"provenance\": {\n \"capture\": \"retrospective\",\n \"basis\": \"User confirmed filtered and unfiltered FTS cases used the same backend setup.\"\n }\n}", "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, @@ -142,42 +142,42 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1464.6515, - "optimize_duration": 60.0477, - "load_duration": 1524.6993, - "qps": 979.5781, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 72.6864, + "optimize_duration": 60.0531, + "load_duration": 132.7395, + "qps": 1470.4418, + "serial_latency_p99": 0.0481, + "serial_latency_p95": 0.0263, + "recall": 0.8388, + "ndcg": 0.7277, "conc_num_list": [ 40, 60, 80 ], "conc_qps_list": [ - 979.5781, - 969.5597, - 964.3595 + 1470.4418, + 1430.4126, + 1302.5586 ], "conc_latency_p99_list": [ - 1.035232208402449, - 1.0622234767588816, - 1.068804296057642 + 0.196402730199046, + 1.0448004564788427, + 1.0593402567028534 ], "conc_latency_p95_list": [ - 0.047188996000477344, - 0.054698529998131545, - 1.008040709995839 + 0.030726359249456436, + 0.03609789424808695, + 0.04879993350186851 ], "conc_latency_avg_list": [ - 0.04051427541809371, - 0.06074978477290257, - 0.08095703978218151 + 0.026958557542264637, + 0.04139604535845175, + 0.05992647821000913 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 5233329, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, @@ -187,7 +187,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 47.43239876568051 + "avgdl": 55.803761 }, "analyzer": { "filter": [ @@ -200,7 +200,7 @@ "unapplied_bm25_params": { "k1": 1.2, "b": 0.75, - "avgdl": 47.43239876568051 + "avgdl": 55.803761 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -210,8 +210,7 @@ "tokenizer": "standard" }, "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_large\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-large-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + "load_concurrency": 0 }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -226,18 +225,19 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.8579 }, "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:43:28.084741", + "db_label": "2026-06-24T12:15:28.328411", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\"\n },\n \"provenance\": {\n \"capture\": \"retrospective\",\n \"basis\": \"User confirmed filtered and unfiltered FTS cases used the same backend setup.\"\n }\n}", "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-hotpotqa-large-c20-b1000-20260624T105757Z", + "namespace": "vdbbench-hotpotqa-medium-c20-b1000-20260624T105757Z", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -254,8 +254,8 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Large (5.2M documents)", - "payload_profile": "text" + "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { @@ -269,6 +269,7 @@ } }, "stages": [ + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -278,42 +279,42 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 72.6864, - "optimize_duration": 60.0531, - "load_duration": 132.7395, - "qps": 1470.4418, - "serial_latency_p99": 0.0481, - "serial_latency_p95": 0.0263, - "recall": 0.9224, - "ndcg": 0.0, + "insert_duration": 7.1608, + "optimize_duration": 60.0462, + "load_duration": 67.207, + "qps": 1589.0248, + "serial_latency_p99": 0.0413, + "serial_latency_p95": 0.0222, + "recall": 0.9211, + "ndcg": 0.8425, "conc_num_list": [ 40, 60, 80 ], "conc_qps_list": [ - 1470.4418, - 1430.4126, - 1302.5586 + 1298.648, + 1556.9237, + 1589.0248 ], "conc_latency_p99_list": [ - 0.196402730199046, - 1.0448004564788427, - 1.0593402567028534 + 1.0122341442467582, + 1.040216220889779, + 1.0511543367576086 ], "conc_latency_p95_list": [ - 0.030726359249456436, - 0.03609789424808695, - 0.04879993350186851 + 0.0409361274978437, + 0.03276492429940844, + 0.03599922600260471 ], "conc_latency_avg_list": [ - 0.026958557542264637, - 0.04139604535845175, - 0.05992647821000913 + 0.030516342558019813, + 0.03799870032145427, + 0.0491667296798954 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 1000000, + "inserted_count": 100000, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, @@ -323,7 +324,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 55.803761 + "avgdl": 56.88413 }, "analyzer": { "filter": [ @@ -336,7 +337,7 @@ "unapplied_bm25_params": { "k1": 1.2, "b": 0.75, - "avgdl": 55.803761 + "avgdl": 56.88413 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -346,8 +347,7 @@ "tokenizer": "standard" }, "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_medium\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-medium-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + "load_concurrency": 0 }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -362,18 +362,19 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.9445 }, "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:15:28.328411", + "db_label": "2026-06-24T12:09:49.389682", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\"\n },\n \"provenance\": {\n \"capture\": \"retrospective\",\n \"basis\": \"User confirmed filtered and unfiltered FTS cases used the same backend setup.\"\n }\n}", "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-hotpotqa-medium-c20-b1000-20260624T105757Z", + "namespace": "vdbbench-hotpotqa-small-c20-b1000-20260624T105757Z", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -390,7 +391,7 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "dataset_with_size_type": "HotpotQA Small (100K documents)", "payload_profile": "ids_only" }, "k": 100, @@ -415,43 +416,43 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 72.6864, - "optimize_duration": 60.0531, - "load_duration": 132.7395, - "qps": 1540.7809, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 832.0622, + "optimize_duration": 60.0494, + "load_duration": 892.1117, + "qps": 1316.9277, + "serial_latency_p99": 0.042, + "serial_latency_p95": 0.026, + "recall": 0.6245, + "ndcg": 0.2747, "conc_num_list": [ 40, 60, 80 ], "conc_qps_list": [ - 1540.7809, - 1473.7761, - 1361.0409 + 1316.9277, + 1297.3733, + 1284.6661 ], "conc_latency_p99_list": [ - 0.1373698992803109, - 1.0427753794664023, - 1.0594265741333946 + 1.0073482914027407, + 1.0495113781692633, + 1.0606270659984147 ], "conc_latency_p95_list": [ - 0.02906204179889753, - 0.03409058889992593, - 0.04557989979657569 + 0.035672420999617316, + 0.04159055870040899, + 0.04908532199988258 ], "conc_latency_avg_list": [ - 0.025724693255668438, - 0.04015530633661177, - 0.05740395977107171 + 0.03006028460855096, + 0.045569930162918985, + 0.061195468189531976 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 1000000, - "insert_rows_per_second": 0.0, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 10626.3967, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, @@ -460,7 +461,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 55.803761 + "avgdl": 57.83363736188793 }, "analyzer": { "filter": [ @@ -473,7 +474,7 @@ "unapplied_bm25_params": { "k1": 1.2, "b": 0.75, - "avgdl": 55.803761 + "avgdl": 57.83363736188793 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -483,8 +484,7 @@ "tokenizer": "standard" }, "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_medium\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-medium-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + "load_concurrency": 0 }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -499,18 +499,19 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.1879 }, "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:39:57.159706", + "db_label": "2026-06-24T12:05:15.433938", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\"\n },\n \"provenance\": {\n \"capture\": \"retrospective\",\n \"basis\": \"User confirmed filtered and unfiltered FTS cases used the same backend setup.\"\n }\n}", "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-hotpotqa-medium-c20-b1000-20260624T105757Z", + "namespace": "vdbbench-fts-tpuf-msmarco-large-c20-b1000-sdkretry-20260624T094413Z", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -527,8 +528,8 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Medium (1M documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { @@ -542,6 +543,7 @@ } }, "stages": [ + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -551,43 +553,43 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 7.1608, - "optimize_duration": 60.0462, - "load_duration": 67.207, - "qps": 1589.0248, - "serial_latency_p99": 0.0413, - "serial_latency_p95": 0.0222, - "recall": 0.9382, - "ndcg": 0.0, + "insert_duration": 70.9455, + "optimize_duration": 60.0465, + "load_duration": 130.9921, + "qps": 1506.0323, + "serial_latency_p99": 0.0363, + "serial_latency_p95": 0.0227, + "recall": 0.8038, + "ndcg": 0.5226, "conc_num_list": [ 40, 60, 80 ], "conc_qps_list": [ - 1298.648, - 1556.9237, - 1589.0248 + 1506.0323, + 1501.8061, + 1494.903 ], "conc_latency_p99_list": [ - 1.0122341442467582, - 1.040216220889779, - 1.0511543367576086 + 0.1576661457991571, + 1.040742779001448, + 1.0539034581720625 ], "conc_latency_p95_list": [ - 0.0409361274978437, - 0.03276492429940844, - 0.03599922600260471 + 0.0298914346043603, + 0.03358310699695721, + 0.03905020675083514 ], "conc_latency_avg_list": [ - 0.030516342558019813, - 0.03799870032145427, - 0.0491667296798954 + 0.026345458167461474, + 0.03912392624068874, + 0.05247619187674705 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 100000, - "insert_rows_per_second": 0.0, + "inserted_count": 1000000, + "insert_rows_per_second": 14095.3267, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, @@ -596,7 +598,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 56.88413 + "avgdl": 57.242324 }, "analyzer": { "filter": [ @@ -609,7 +611,7 @@ "unapplied_bm25_params": { "k1": 1.2, "b": 0.75, - "avgdl": 56.88413 + "avgdl": 57.242324 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -619,8 +621,7 @@ "tokenizer": "standard" }, "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_small\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-small-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + "load_concurrency": 0 }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -635,18 +636,19 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.4529 }, "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:09:49.389682", + "db_label": "2026-06-24T12:00:41.474589", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\"\n },\n \"provenance\": {\n \"capture\": \"retrospective\",\n \"basis\": \"User confirmed filtered and unfiltered FTS cases used the same backend setup.\"\n }\n}", "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-hotpotqa-small-c20-b1000-20260624T105757Z", + "namespace": "vdbbench-fts-tpuf-msmarco-medium-c20-b1000-cli-20260624T085600Z", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -663,7 +665,7 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Small (100K documents)", + "dataset_with_size_type": "MS MARCO Medium (1M documents)", "payload_profile": "ids_only" }, "k": 100, @@ -688,43 +690,43 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 7.1608, - "optimize_duration": 60.0462, - "load_duration": 67.207, - "qps": 1732.5275, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 7.6649, + "optimize_duration": 60.0529, + "load_duration": 67.7177, + "qps": 1588.1696, + "serial_latency_p99": 0.0379, + "serial_latency_p95": 0.0204, + "recall": 0.9125, + "ndcg": 0.7156, "conc_num_list": [ 40, 60, 80 ], "conc_qps_list": [ - 1732.5275, - 1573.2425, - 1644.7612 + 1588.1696, + 1437.2379, + 1542.8857 ], "conc_latency_p99_list": [ - 0.10047229536954584, - 1.0371929118002297, - 1.050296281276096 + 0.14128930038219492, + 1.043362593751226, + 1.0526617984978657 ], "conc_latency_p95_list": [ - 0.02665470994543283, - 0.032170177746593254, - 0.03446190499817008 + 0.03021625500296065, + 0.03545568099798402, + 0.03865519299870357 ], "conc_latency_avg_list": [ - 0.022886237167794053, - 0.037611734150211816, - 0.04739313356214065 + 0.024957518324872457, + 0.041209095071141, + 0.050988707258252025 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, "inserted_count": 100000, - "insert_rows_per_second": 0.0, + "insert_rows_per_second": 13046.4846, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, @@ -733,7 +735,7 @@ "bm25": { "k1": 1.2, "b": 0.75, - "avgdl": 56.88413 + "avgdl": 55.01342 }, "analyzer": { "filter": [ @@ -746,7 +748,7 @@ "unapplied_bm25_params": { "k1": 1.2, "b": 0.75, - "avgdl": 56.88413 + "avgdl": 55.01342 }, "applied_analyzer_params": {}, "unapplied_analyzer_params": { @@ -756,8 +758,7 @@ "tokenizer": "standard" }, "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_hotpotqa_c20_b1000_cli_sdkretry_20260624T105757Z\/hotpotqa_small\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-hotpotqa-small-c20-b1000-sdkretry-20260624T105757Z_turbopuffer.json" + "load_concurrency": 0 }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -772,18 +773,19 @@ "st_conc_qps_list_list": [], "st_conc_latency_p99_list_list": [], "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] + "st_conc_latency_avg_list_list": [], + "mrr": 0.666 }, "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:36:19.214107", + "db_label": "2026-06-24T11:56:16.498129", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\"\n },\n \"provenance\": {\n \"capture\": \"retrospective\",\n \"basis\": \"User confirmed filtered and unfiltered FTS cases used the same backend setup.\"\n }\n}", "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-hotpotqa-small-c20-b1000-20260624T105757Z", + "namespace": "vdbbench-fts-tpuf-msmarco-small-c20-b1000-cli-20260624T084228Z", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -800,8 +802,8 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Small (100K documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Small (100K documents)", + "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { @@ -815,6 +817,7 @@ } }, "stages": [ + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -824,76 +827,679 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 832.0622, - "optimize_duration": 60.0494, - "load_duration": 892.1117, - "qps": 1316.9277, - "serial_latency_p99": 0.042, - "serial_latency_p95": 0.026, - "recall": 0.9395, - "ndcg": 0.0, + "insert_duration": 364.2029, + "optimize_duration": 60.0409, + "load_duration": 424.2438, + "qps": 86.5254, + "serial_latency_p99": 0.3979, + "serial_latency_p95": 0.3663, + "recall": 0.8087, + "ndcg": 0.6319, + "mrr": 0.6398, "conc_num_list": [ - 40, 60, 80 ], "conc_qps_list": [ - 1316.9277, - 1297.3733, - 1284.6661 + 37.7946, + 86.5254 ], "conc_latency_p99_list": [ - 1.0073482914027407, - 1.0495113781692633, - 1.0606270659984147 + 7.258740147238386, + 6.517188084549162 ], "conc_latency_p95_list": [ - 0.035672420999617316, - 0.04159055870040899, - 0.04908532199988258 + 4.526965558199117, + 2.5522155794988066 ], "conc_latency_avg_list": [ - 0.03006028460855096, - 0.045569930162918985, - 0.061195468189531976 + 1.2428941904958388, + 0.837548937249788 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 8841823, - "insert_rows_per_second": 10626.3967, + "inserted_count": 5233329, + "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 2616664, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 2616665, + "matched_doc_ratio": 0.5, + "original_query_count": 7405, + "filtered_query_count": 5547, + "filtered_query_ratio": 0.749088, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 6846 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 5547, + "full_query_count": 7405 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "num_per_batch": 1000, + "load_concurrency": 4, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" + "source_run_id": "8c0d2eabf9594bdfb5d9d1957f10ace6", + "source_task_label": "fts_filtered_serial_turbopuffer_hotpotqa-large_r50_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_hotpotqa-large_r50_permuted_turbopuffer.json", + "source_sha256": "529edcaeea08b03e39532fa8ab30025d640c9834774ed1c16cee2c5a1901243a", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "drop_old", + "load", + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 1\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"failed_partial_namespace_exists_and_will_be_reset_once\",\n \"target_namespaces_absent\": false,\n \"verified_at\": \"2026-08-04T03:55:01Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\"\n ]\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", + "version": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "drop_old", + "load", + "search_concurrent", + "search_serial" + ], + "load_concurrency": 4 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 147.1902, + "serial_latency_p99": 0.2358, + "serial_latency_p95": 0.2075, + "recall": 0.8378, + "ndcg": 0.658, + "mrr": 0.6328, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 147.1902, + 143.3594 + ], + "conc_latency_p99_list": [ + 2.4505238299202756, + 4.068990539201332 + ], + "conc_latency_p95_list": [ + 1.1879939647995341, + 2.368760299999849 + ], + "conc_latency_avg_list": [ + 0.3934921818363348, + 0.5292828738100664 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 3924996, + "filter_rate": 0.75, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 1308333, + "matched_doc_ratio": 0.25, + "original_query_count": 7405, + "filtered_query_count": 3175, + "filtered_query_ratio": 0.428764, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 3379 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 3175, + "full_query_count": 7405 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "02e928d4d3794fa7a63130805ed5db3b", + "source_task_label": "fts_filtered_serial_turbopuffer_hotpotqa-large_r75_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_hotpotqa-large_r75_permuted_turbopuffer.json", + "source_sha256": "ac19277d3979b54ad1f290d7be34b85dfbfa240739b7c47e5a6cc0702e5e51f4", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 1\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"failed_partial_namespace_exists_and_will_be_reset_once\",\n \"target_namespaces_absent\": false,\n \"verified_at\": \"2026-08-04T03:55:01Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\"\n ]\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", + "version": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.75 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 296.0612, + "serial_latency_p99": 0.1359, + "serial_latency_p95": 0.1139, + "recall": 0.8764, + "ndcg": 0.7161, + "mrr": 0.6797, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 296.0612, + 295.9694 + ], + "conc_latency_p99_list": [ + 2.3035320769305687, + 2.355359453583151 + ], + "conc_latency_p95_list": [ + 1.0913702148993252, + 1.1043454573995404 + ], + "conc_latency_avg_list": [ + 0.1996025803722017, + 0.2650406779299211 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4709996, + "filter_rate": 0.9, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 523333, + "matched_doc_ratio": 0.1, + "original_query_count": 7405, + "filtered_query_count": 1367, + "filtered_query_ratio": 0.184605, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 1335 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 1367, + "full_query_count": 7405 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "5136de1fe98b49af853b4a0af228658b", + "source_task_label": "fts_filtered_serial_turbopuffer_hotpotqa-large_r90_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_hotpotqa-large_r90_permuted_turbopuffer.json", + "source_sha256": "2cad3733352755ac5c977993cffe703cad9b0412358ae9f6c9c73cb80e6014b0", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 1\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"failed_partial_namespace_exists_and_will_be_reset_once\",\n \"target_namespaces_absent\": false,\n \"verified_at\": \"2026-08-04T03:55:01Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\"\n ]\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", + "version": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.9 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 441.3432, + "serial_latency_p99": 0.0853, + "serial_latency_p95": 0.0728, + "recall": 0.8911, + "ndcg": 0.7417, + "mrr": 0.703, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 441.3432, + 437.3237 + ], + "conc_latency_p99_list": [ + 1.1061684508192775, + 2.2820966475379825 + ], + "conc_latency_p95_list": [ + 1.0580634986525184, + 1.0726680879986816 + ], + "conc_latency_avg_list": [ + 0.13406016079070324, + 0.17815990107083907 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4971662, + "filter_rate": 0.95, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 261667, + "matched_doc_ratio": 0.05, + "original_query_count": 7405, + "filtered_query_count": 698, + "filtered_query_ratio": 0.094261, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 664 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 698, + "full_query_count": 7405 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "10852825e3a84b69ab828b13a1353b97", + "source_task_label": "fts_filtered_serial_turbopuffer_hotpotqa-large_r95_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_hotpotqa-large_r95_permuted_turbopuffer.json", + "source_sha256": "6398c61fbc97046d03dda2265d76c7acd288c4ba016f6c100757847c6b1e5c3c", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 1\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"failed_partial_namespace_exists_and_will_be_reset_once\",\n \"target_namespaces_absent\": false,\n \"verified_at\": \"2026-08-04T03:55:01Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\"\n ]\n }\n}" + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "TurboPuffer", + "db_config": { + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", + "version": "", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", + "region": "aws-us-west-2", + "api_base_url": null, + "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", + "multitenant_namespace_prefix": "vdbbench_mt_", + "scalar_payload_label_field": "label", + "pin_namespace": false, + "pin_namespace_requested": false, + "pin_replicas": 1, + "pin_timeout": 2700, + "pin_target_namespace_count": 0 + }, + "db_case_config": { + "metric_type": "BM25", + "time_wait_warmup": 60, + "disable_backpressure": false + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.95 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_concurrent", + "search_serial" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 780.4231, + "serial_latency_p99": 0.0391, + "serial_latency_p95": 0.0347, + "recall": 0.9048, + "ndcg": 0.8018, + "mrr": 0.7724, + "conc_num_list": [ + 60, + 80 + ], + "conc_qps_list": [ + 774.6021, + 780.4231 + ], + "conc_latency_p99_list": [ + 1.069292031079094, + 1.0759943062603998 + ], + "conc_latency_p95_list": [ + 0.085047684600795, + 1.0324142765017315 + ], + "conc_latency_avg_list": [ + 0.07620351681519844, + 0.10069744977102481 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 5180995, + "filter_rate": 0.99, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 52334, + "matched_doc_ratio": 0.01, + "original_query_count": 7405, + "filtered_query_count": 147, + "filtered_query_ratio": 0.019851, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 136 }, - "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_large_c20_b1000_cli_sdkretry_20260624T094413Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-large-c20-b1000-sdkretry-20260624T094413Z_turbopuffer.json" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 147, + "full_query_count": 7405 + }, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "054d1a9c83ec421ea21cd0a0c759639b", + "source_task_label": "fts_filtered_serial_turbopuffer_hotpotqa-large_r99_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_hotpotqa-large_r99_permuted_turbopuffer.json", + "source_sha256": "36898441804921398245c9a62813ac064c0e5f23c703189bc3d23060c1499bf3", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 1\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"failed_partial_namespace_exists_and_will_be_reset_once\",\n \"target_namespaces_absent\": false,\n \"verified_at\": \"2026-08-04T03:55:01Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\"\n ]\n }\n}" + } }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -913,13 +1519,13 @@ "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:05:15.433938", + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", - "note": "", - "api_key": "**********", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-fts-tpuf-msmarco-large-c20-b1000-sdkretry-20260624T094413Z", + "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -936,23 +1542,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Large (8.8M documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.99 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "search_serial", - "search_concurrent" + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, @@ -961,76 +1568,89 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 832.0622, - "optimize_duration": 60.0494, - "load_duration": 892.1117, - "qps": 1302.7749, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 1259.9243, + "optimize_duration": 60.0441, + "load_duration": 1319.9684, + "qps": 122.3297, + "serial_latency_p99": 0.2339, + "serial_latency_p95": 0.2155, + "recall": 0.6981, + "ndcg": 0.3471, + "mrr": 0.2577, "conc_num_list": [ - 40, 60, 80 ], "conc_qps_list": [ - 1302.7749, - 1164.7122, - 1273.9904 + 114.6727, + 122.3297 ], "conc_latency_p99_list": [ - 1.0048391348012942, - 1.0523817389193573, - 1.0604548653973325 + 3.9483368308206264, + 4.207485043439037 ], "conc_latency_p95_list": [ - 0.03545594900060678, - 0.04788232199934996, - 0.04981473534498943 + 2.292980255803195, + 2.3655117287480607 ], "conc_latency_avg_list": [ - 0.030413566662682186, - 0.0503768273187017, - 0.061305946983837616 + 0.5048540764116053, + 0.6185035208870814 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, "inserted_count": 8841823, - "insert_rows_per_second": 10626.3967, + "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4420911, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 4420912, + "matched_doc_ratio": 0.5, + "original_query_count": 6980, + "filtered_query_count": 3658, + "filtered_query_ratio": 0.524069, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 3758 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 3658, + "full_query_count": 6980 }, "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_large_c20_b1000_cli_sdkretry_20260624T094413Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-large-c20-b1000-sdkretry-20260624T094413Z_turbopuffer.json" + "load_concurrency": 4, + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" + ], + "source_run_id": "61ecd49ecda54ad68fe5310dc850c556", + "source_task_label": "fts_filtered_serial_turbopuffer_msmarco-large_r50_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_msmarco-large_r50_permuted_turbopuffer.json", + "source_sha256": "9a5a1ad4e8fae210e9561e467f262f4cefbff58b500045c113a76213a1599f8a", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "drop_old", + "load", + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 4\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"must_not_exist\",\n \"target_namespaces_absent\": true,\n \"verified_at\": \"2026-08-03T12:05:22Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ]\n }\n}" + } }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1050,13 +1670,13 @@ "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:33:35.259663", + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", - "note": "", - "api_key": "**********", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-fts-tpuf-msmarco-large-c20-b1000-sdkretry-20260624T094413Z", + "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -1074,99 +1694,112 @@ "case_id": 503, "custom_case": { "dataset_with_size_type": "MS MARCO Large (8.8M documents)", - "payload_profile": "text" + "payload_profile": "ids_only", + "filter_rate": 0.5 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "search_concurrent" + "drop_old", + "load", + "search_concurrent", + "search_serial" ], - "load_concurrency": 0 + "load_concurrency": 4 }, "label": ":)" }, { "metrics": { "max_load_count": 0, - "insert_duration": 70.9455, - "optimize_duration": 60.0465, - "load_duration": 130.9921, - "qps": 1506.0323, - "serial_latency_p99": 0.0363, - "serial_latency_p95": 0.0227, - "recall": 0.9488, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 113.5809, + "serial_latency_p99": 0.1376, + "serial_latency_p95": 0.1236, + "recall": 0.7629, + "ndcg": 0.4256, + "mrr": 0.3382, "conc_num_list": [ - 40, 60, 80 ], "conc_qps_list": [ - 1506.0323, - 1501.8061, - 1494.903 + 108.4603, + 113.5809 ], "conc_latency_p99_list": [ - 0.1576661457991571, - 1.040742779001448, - 1.0539034581720625 + 4.06640162431162, + 4.172310605399252 ], "conc_latency_p95_list": [ - 0.0298914346043603, - 0.03358310699695721, - 0.03905020675083514 + 2.3489085682997026, + 2.480539664000389 ], "conc_latency_avg_list": [ - 0.026345458167461474, - 0.03912392624068874, - 0.05247619187674705 + 0.5278105149102582, + 0.6535439539011187 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 1000000, - "insert_rows_per_second": 14095.3267, + "inserted_count": 0, + "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 6631367, + "filter_rate": 0.75, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 2210456, + "matched_doc_ratio": 0.25, + "original_query_count": 6980, + "filtered_query_count": 1818, + "filtered_query_ratio": 0.260458, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 1839 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 1818, + "full_query_count": 6980 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" - }, - "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_medium_c20_b1000_cli_20260624T085600Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-medium-c20-b1000-cli-20260624T085600Z_turbopuffer.json" + "source_run_id": "75ec2473225f4497b49bdec6e36b657e", + "source_task_label": "fts_filtered_serial_turbopuffer_msmarco-large_r75_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_msmarco-large_r75_permuted_turbopuffer.json", + "source_sha256": "76ca2498ee6b124e74c424f03e2f8adb317d376c30e6ea5e8d62162fddb63a58", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 4\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"must_not_exist\",\n \"target_namespaces_absent\": true,\n \"verified_at\": \"2026-08-03T12:05:22Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ]\n }\n}" + } }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1186,13 +1819,13 @@ "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:00:41.474589", + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", - "note": "", - "api_key": "**********", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-fts-tpuf-msmarco-medium-c20-b1000-cli-20260624T085600Z", + "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -1209,23 +1842,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Medium (1M documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.75 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "search_serial", - "search_concurrent" + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, @@ -1234,76 +1868,85 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 70.9455, - "optimize_duration": 60.0465, - "load_duration": 130.9921, - "qps": 1573.3805, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 174.0195, + "serial_latency_p99": 0.0761, + "serial_latency_p95": 0.0678, + "recall": 0.8369, + "ndcg": 0.5165, + "mrr": 0.4319, "conc_num_list": [ - 40, 60, 80 ], "conc_qps_list": [ - 1573.3805, - 1444.0492, - 1421.5596 + 172.6959, + 174.0195 ], "conc_latency_p99_list": [ - 0.12313141820966869, - 1.0423146819185058, - 1.0560591126610959 + 2.431309359999432, + 3.965449629778485 ], "conc_latency_p95_list": [ - 0.02848500000254715, - 0.0345763255034399, - 0.04240300605015367 + 1.165103798997734, + 2.323448423299851 ], "conc_latency_avg_list": [ - 0.025195208364014388, - 0.04078606566304797, - 0.05524199400874184 + 0.33913986293953957, + 0.4434340066370408 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 1000000, - "insert_rows_per_second": 14095.3267, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 7957640, + "filter_rate": 0.9, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 884183, + "matched_doc_ratio": 0.1, + "original_query_count": 6980, + "filtered_query_count": 739, + "filtered_query_ratio": 0.105874, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 740 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 739, + "full_query_count": 6980 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" - }, - "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_medium_c20_b1000_cli_20260624T085600Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-medium-c20-b1000-cli-20260624T085600Z_turbopuffer.json" + "source_run_id": "10fc45c381124295bd4a2dcee19d4f50", + "source_task_label": "fts_filtered_serial_turbopuffer_msmarco-large_r90_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_msmarco-large_r90_permuted_turbopuffer.json", + "source_sha256": "7b47b06c4a1fbb699fb2fc39a6185410383ac2d511393bc2942cdb3b3d14c9d3", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 4\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"must_not_exist\",\n \"target_namespaces_absent\": true,\n \"verified_at\": \"2026-08-03T12:05:22Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ]\n }\n}" + } }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1323,13 +1966,13 @@ "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:30:55.307994", + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", - "note": "", - "api_key": "**********", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-fts-tpuf-msmarco-medium-c20-b1000-cli-20260624T085600Z", + "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -1346,22 +1989,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Medium (1M documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.9 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "search_concurrent" + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, @@ -1370,76 +2015,85 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 7.6649, - "optimize_duration": 60.0529, - "load_duration": 67.7177, - "qps": 1588.1696, - "serial_latency_p99": 0.0379, - "serial_latency_p95": 0.0204, - "recall": 0.9537, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 237.8194, + "serial_latency_p99": 0.0562, + "serial_latency_p95": 0.053, + "recall": 0.853, + "ndcg": 0.5835, + "mrr": 0.512, "conc_num_list": [ - 40, 60, 80 ], "conc_qps_list": [ - 1588.1696, - 1437.2379, - 1542.8857 + 237.8194, + 232.9519 ], "conc_latency_p99_list": [ - 0.14128930038219492, - 1.043362593751226, - 1.0526617984978657 + 2.3618834851501744, + 2.3980035969008893 ], "conc_latency_p95_list": [ - 0.03021625500296065, - 0.03545568099798402, - 0.03865519299870357 + 1.116876252649854, + 1.1324147257482764 ], "conc_latency_avg_list": [ - 0.024957518324872457, - 0.041209095071141, - 0.050988707258252025 + 0.24743568345607964, + 0.3360648585017976 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 100000, - "insert_rows_per_second": 13046.4846, + "inserted_count": 0, + "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 8399731, + "filter_rate": 0.95, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 442092, + "matched_doc_ratio": 0.05, + "original_query_count": 6980, + "filtered_query_count": 347, + "filtered_query_ratio": 0.049713, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 347 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 347, + "full_query_count": 6980 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" - }, - "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_small_c20_b1000_cli_20260624T084228Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-small-c20-b1000-cli-20260624T084228Z_turbopuffer.json" + "source_run_id": "6a289aff37334e4b8146e6279ed4e0e0", + "source_task_label": "fts_filtered_serial_turbopuffer_msmarco-large_r95_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_msmarco-large_r95_permuted_turbopuffer.json", + "source_sha256": "4ba5221ba7b9294eef13e5adde96d3d16f65e278938e3b44e6b160cd2bd7b25f", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 4\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"must_not_exist\",\n \"target_namespaces_absent\": true,\n \"verified_at\": \"2026-08-03T12:05:22Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ]\n }\n}" + } }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1459,13 +2113,13 @@ "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T11:56:16.498129", + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", - "note": "", - "api_key": "**********", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-fts-tpuf-msmarco-small-c20-b1000-cli-20260624T084228Z", + "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -1482,23 +2136,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Small (100K documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.95 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "search_serial", - "search_concurrent" + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, @@ -1507,76 +2162,85 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 7.6649, - "optimize_duration": 60.0529, - "load_duration": 67.7177, - "qps": 1565.7761, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 292.0949, + "serial_latency_p99": 0.0592, + "serial_latency_p95": 0.0491, + "recall": 0.9403, + "ndcg": 0.7061, + "mrr": 0.6404, "conc_num_list": [ - 40, 60, 80 ], "conc_qps_list": [ - 1565.7761, - 1296.9044, - 1395.2693 + 292.0949, + 288.0392 ], "conc_latency_p99_list": [ - 0.1240674191407033, - 1.049226711880474, - 1.055339480610637 + 2.2982839281494307, + 2.3517081442788914 ], "conc_latency_p95_list": [ - 0.028216213650011923, - 0.04351905840157994, - 0.04549482634683953 + 1.097375179249866, + 1.1113469616489966 ], "conc_latency_avg_list": [ - 0.0253025184556393, - 0.0456011056513432, - 0.056052650601209515 + 0.20162277745392373, + 0.26459405379449835 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 100000, - "insert_rows_per_second": 13046.4846, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 8753404, + "filter_rate": 0.99, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 88419, + "matched_doc_ratio": 0.01, + "original_query_count": 6980, + "filtered_query_count": 67, + "filtered_query_ratio": 0.009599, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 67 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 67, + "full_query_count": 6980 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "serial_measurement": { + "composed_from_separate_run": true, + "metric_fields": [ + "serial_latency_p99", + "serial_latency_p95", + "recall", + "ndcg", + "mrr" ], - "tokenizer": "standard" - }, - "num_per_batch": 1000, - "load_concurrency": 0, - "load_metrics_source_result_json": "\/home\/ubuntu\/bench-runs\/turbopuffer_msmarco_small_c20_b1000_cli_20260624T084228Z\/runtime_results\/TurboPuffer\/result_20260624_turbopuffer-msmarco-small-c20-b1000-cli-20260624T084228Z_turbopuffer.json" + "source_run_id": "753531e38a8b4fae91339d21ba00d7eb", + "source_task_label": "fts_filtered_serial_turbopuffer_msmarco-large_r99_permuted", + "source_timestamp": 1785801600.0, + "source_file": "result_20260804_fts_filtered_serial_turbopuffer_msmarco-large_r99_permuted_turbopuffer.json", + "source_sha256": "fa8e826ecb4e227835341e63ca2fc006ba68df6f6e2721887ba680db935ba319", + "source_db_label": "turbopuffer-aws-us-west-2-filtered-permuted-serial", + "source_stages": [ + "search_serial" + ], + "source_note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"service_version\": \"not_exposed\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"235ac46886e58d48f9561fb8c39ef913fa3e71d7\",\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"working_tree_clean\": false,\n \"working_tree_scope\": \"pre-existing FullTextSearch result JSON note edits only\"\n },\n \"execution\": {\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\",\n \"namespace_strategy\": \"create_new_per_dataset_and_reuse_across_filter_rates\",\n \"retain_namespace_after_run\": true,\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"sdk_max_retries\": 4,\n \"write_backpressure_enabled\": true,\n \"warmup_seconds_after_load\": 60,\n \"insert_batch_size\": 1000,\n \"load_concurrency\": 4\n },\n \"validation\": {\n \"credentials_stored\": false,\n \"namespace_list_http_status\": 200,\n \"target_namespace_precondition\": \"must_not_exist\",\n \"target_namespaces_absent\": true,\n \"verified_at\": \"2026-08-03T12:05:22Z\",\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ]\n }\n}" + } }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], @@ -1596,13 +2260,13 @@ "task_config": { "db": "TurboPuffer", "db_config": { - "db_label": "2026-06-24T12:28:10.345065", + "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", - "note": "", - "api_key": "**********", + "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", + "api_key": "", "region": "aws-us-west-2", "api_base_url": null, - "namespace": "vdbbench-fts-tpuf-msmarco-small-c20-b1000-cli-20260624T084228Z", + "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", "multitenant_namespace_prefix": "vdbbench_mt_", "scalar_payload_label_field": "label", "pin_namespace": false, @@ -1619,22 +2283,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Small (100K documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.99 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ - 40, 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "search_concurrent" + "search_concurrent", + "search_serial" ], "load_concurrency": 0 }, diff --git a/vectordb_bench/results/FullTextSearch/Vespa/result_20260626_fts_standard_vespa.json b/vectordb_bench/results/FullTextSearch/Vespa/result_20260626_fts_standard_vespa.json deleted file mode 100644 index eb7160a3e..000000000 --- a/vectordb_bench/results/FullTextSearch/Vespa/result_20260626_fts_standard_vespa.json +++ /dev/null @@ -1,1514 +0,0 @@ -{ - "run_id": "fts_standard_vespa", - "task_label": "fts_standard", - "results": [ - { - "metrics": { - "max_load_count": 0, - "insert_duration": 326.0195, - "optimize_duration": 0.0467, - "load_duration": 326.0662, - "qps": 178.292, - "serial_latency_p99": 0.445, - "serial_latency_p95": 0.4445, - "recall": 0.7532, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 92.1252, - 178.292 - ], - "conc_latency_p99_list": [ - 0.4649717587499981, - 0.47357734852928834 - ], - "conc_latency_p95_list": [ - 0.45766577114909524, - 0.4641533440004423 - ], - "conc_latency_avg_list": [ - 0.4314541248062237, - 0.4440924808065595 - ], - "payload_profile": "ids_only", - "payload_estimated_bytes_per_query": 2000, - "inserted_count": 5233329, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 47.43239876568051 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 47.43239876568051 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 47.43239876568051, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "HotpotQA Large (5.2M documents)", - "payload_profile": "ids_only" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 326.0195, - "optimize_duration": 0.0467, - "load_duration": 326.0662, - "qps": 172.1563, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 88.631, - 172.1563 - ], - "conc_latency_p99_list": [ - 0.4895644657585399, - 0.4961875680503364 - ], - "conc_latency_p95_list": [ - 0.4765562847500405, - 0.4800053587003276 - ], - "conc_latency_avg_list": [ - 0.44739083606485935, - 0.45469620353127227 - ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 5233329, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 47.43239876568051 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 47.43239876568051 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 47.43239876568051, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "HotpotQA Large (5.2M documents)", - "payload_profile": "text" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 87.4509, - "optimize_duration": 0.0146, - "load_duration": 87.4655, - "qps": 188.1674, - "serial_latency_p99": 0.2691, - "serial_latency_p95": 0.2192, - "recall": 0.9134, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 129.2045, - 188.1674 - ], - "conc_latency_p99_list": [ - 0.4561221865990228, - 0.47219445047965564 - ], - "conc_latency_p95_list": [ - 0.44640964640020686, - 0.46283690039963404 - ], - "conc_latency_avg_list": [ - 0.3080734912494232, - 0.4224045833953113 - ], - "payload_profile": "ids_only", - "payload_estimated_bytes_per_query": 2000, - "inserted_count": 1000000, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.803761 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.803761 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 55.803761, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "HotpotQA Medium (1M documents)", - "payload_profile": "ids_only" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 87.4509, - "optimize_duration": 0.0146, - "load_duration": 87.4655, - "qps": 184.839, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 124.568, - 184.839 - ], - "conc_latency_p99_list": [ - 0.470236903199766, - 0.4859163774794434 - ], - "conc_latency_p95_list": [ - 0.45786149219966316, - 0.47220526239943866 - ], - "conc_latency_avg_list": [ - 0.3197571245104845, - 0.42871789511037967 - ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 1000000, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.803761 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.803761 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 55.803761, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "HotpotQA Medium (1M documents)", - "payload_profile": "text" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 9.4358, - "optimize_duration": 0.0081, - "load_duration": 9.4439, - "qps": 903.9248, - "serial_latency_p99": 0.0311, - "serial_latency_p95": 0.0264, - "recall": 0.9262, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 903.9248, - 210.3829 - ], - "conc_latency_p99_list": [ - 0.10201139999971932, - 23.7793526906097 - ], - "conc_latency_p95_list": [ - 0.07956749100012528, - 0.1337348711004779 - ], - "conc_latency_avg_list": [ - 0.04422038388924561, - 0.37968395195953486 - ], - "payload_profile": "ids_only", - "payload_estimated_bytes_per_query": 2000, - "inserted_count": 100000, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 56.88413 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 56.88413 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 56.88413, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "HotpotQA Small (100K documents)", - "payload_profile": "ids_only" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 9.4358, - "optimize_duration": 0.0081, - "load_duration": 9.4439, - "qps": 788.0643, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 788.0643, - 342.891 - ], - "conc_latency_p99_list": [ - 0.116296334949675, - 1.4236666194401957 - ], - "conc_latency_p95_list": [ - 0.089655852249507, - 0.14479435460016243 - ], - "conc_latency_avg_list": [ - 0.0506907555806803, - 0.22207524820958327 - ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 100000, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 56.88413 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 56.88413 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 56.88413, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "HotpotQA Small (100K documents)", - "payload_profile": "text" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 702.0398, - "optimize_duration": 0.074, - "load_duration": 702.1138, - "qps": 201.2276, - "serial_latency_p99": 0.4449, - "serial_latency_p95": 0.4444, - "recall": 0.9057, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 108.7747, - 201.2276 - ], - "conc_latency_p99_list": [ - 0.46301382789988565, - 0.47322508225033744 - ], - "conc_latency_p95_list": [ - 0.45525903649968313, - 0.4619954339998458 - ], - "conc_latency_avg_list": [ - 0.36484632344246043, - 0.39432222691413743 - ], - "payload_profile": "ids_only", - "payload_estimated_bytes_per_query": 2000, - "inserted_count": 8841823, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 57.83363736188793, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "MS MARCO Large (8.8M documents)", - "payload_profile": "ids_only" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 702.0398, - "optimize_duration": 0.074, - "load_duration": 702.1138, - "qps": 193.9787, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 108.0544, - 193.9787 - ], - "conc_latency_p99_list": [ - 0.48343497323963674, - 0.4929514833995654 - ], - "conc_latency_p95_list": [ - 0.4710945042004823, - 0.4792746837499635 - ], - "conc_latency_avg_list": [ - 0.36689615236970297, - 0.4052692454002725 - ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 8841823, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 57.83363736188793, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "MS MARCO Large (8.8M documents)", - "payload_profile": "text" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 81.6544, - "optimize_duration": 0.0155, - "load_duration": 81.67, - "qps": 317.5358, - "serial_latency_p99": 0.1402, - "serial_latency_p95": 0.1062, - "recall": 0.9859, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 283.1602, - 317.5358 - ], - "conc_latency_p99_list": [ - 0.38076332676982494, - 0.4561359607801206 - ], - "conc_latency_p95_list": [ - 0.3054511334998549, - 0.44553361209977993 - ], - "conc_latency_avg_list": [ - 0.1408606566523405, - 0.2508771823668165 - ], - "payload_profile": "ids_only", - "payload_estimated_bytes_per_query": 2000, - "inserted_count": 1000000, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 57.242324, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "MS MARCO Medium (1M documents)", - "payload_profile": "ids_only" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 81.6544, - "optimize_duration": 0.0155, - "load_duration": 81.67, - "qps": 299.9899, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 276.9879, - 299.9899 - ], - "conc_latency_p99_list": [ - 0.38208317819017773, - 0.4668930329600153 - ], - "conc_latency_p95_list": [ - 0.3130642607502523, - 0.4508931028001825 - ], - "conc_latency_avg_list": [ - 0.14397322018103537, - 0.2652584571078699 - ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 1000000, - "insert_rows_per_second": 0.0, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 57.242324, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "MS MARCO Medium (1M documents)", - "payload_profile": "text" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 8.7233, - "optimize_duration": 0.0095, - "load_duration": 8.7328, - "qps": 643.7175, - "serial_latency_p99": 0.019, - "serial_latency_p95": 0.0155, - "recall": 0.9857, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 643.7175, - 553.2639 - ], - "conc_latency_p99_list": [ - 0.06595422097017949, - 0.5136447451604238 - ], - "conc_latency_p95_list": [ - 0.049367965999931575, - 0.0805305323999846 - ], - "conc_latency_avg_list": [ - 0.02392549430111337, - 0.07046851805299408 - ], - "payload_profile": "ids_only", - "payload_estimated_bytes_per_query": 2000, - "inserted_count": 100000, - "insert_rows_per_second": 11463.5516, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 55.01342, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "MS MARCO Small (100K documents)", - "payload_profile": "ids_only" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_serial", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "insert_duration": 8.7233, - "optimize_duration": 0.0095, - "load_duration": 8.7328, - "qps": 601.592, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, - "conc_num_list": [ - 40, - 80 - ], - "conc_qps_list": [ - 601.592, - 551.9681 - ], - "conc_latency_p99_list": [ - 0.07571082960002969, - 0.7508284560699189 - ], - "conc_latency_p95_list": [ - 0.053776488000039535, - 0.09836964239973439 - ], - "conc_latency_avg_list": [ - 0.030178424329577746, - 0.10644925614934629 - ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 100000, - "insert_rows_per_second": 11463.5516, - "insert_completion_seconds": 0.0, - "searchable_after_insert_seconds": 0.0, - "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "unapplied_bm25_params": {}, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "st_ideal_insert_duration": 0, - "st_search_stage_list": [], - "st_search_time_list": [], - "st_max_qps_list_list": [], - "st_recall_list": [], - "st_ndcg_list": [], - "st_serial_latency_p99_list": [], - "st_serial_latency_p95_list": [], - "st_conc_failed_rate_list": [], - "st_conc_num_list_list": [], - "st_conc_qps_list_list": [], - "st_conc_latency_p99_list_list": [], - "st_conc_latency_p95_list_list": [], - "st_conc_latency_avg_list_list": [] - }, - "task_config": { - "db": "Vespa", - "db_config": { - "db_label": "", - "version": "", - "note": "", - "url": "**********", - "port": 8080 - }, - "db_case_config": { - "metric_type": "BM25", - "bm25_k1": 1.2, - "bm25_b": 0.75, - "bm25_avgdl": 55.01342, - "feed_client_command": "vespa", - "feed_client_connections": null - }, - "case_config": { - "case_id": 503, - "custom_case": { - "dataset_with_size_type": "MS MARCO Small (100K documents)", - "payload_profile": "text" - }, - "k": 100, - "concurrency_search_config": { - "num_concurrency": [ - 40, - 80 - ], - "concurrency_duration": 30, - "concurrency_timeout": 3600 - } - }, - "stages": [ - "drop_old", - "load", - "search_concurrent" - ], - "load_concurrency": 0 - }, - "label": ":)" - } - ], - "file_fmt": "result_{}_{}_{}.json", - "timestamp": 1782259200.0 -} diff --git a/vectordb_bench/results/FullTextSearch/ZillizCloud/result_20260626_fts_standard_zillizcloud.json b/vectordb_bench/results/FullTextSearch/ZillizCloud/result_20260626_fts_standard_zillizcloud.json index 032a97292..6bf69df97 100644 --- a/vectordb_bench/results/FullTextSearch/ZillizCloud/result_20260626_fts_standard_zillizcloud.json +++ b/vectordb_bench/results/FullTextSearch/ZillizCloud/result_20260626_fts_standard_zillizcloud.json @@ -8,30 +8,36 @@ "insert_duration": 92.5443, "optimize_duration": 81.9515, "load_duration": 174.4958, - "qps": 1291.365, - "serial_latency_p99": 0.02, - "serial_latency_p95": 0.0136, - "recall": 0.9935, - "ndcg": 0.0, + "qps": 1763.3502, + "serial_latency_p99": 0.0331, + "serial_latency_p95": 0.0222, + "recall": 0.7674, + "ndcg": 0.6265, + "mrr": 0.7574, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 1291.365, - 1163.3626 + 1738.3381, + 1760.0975, + 1763.3502 ], "conc_latency_p99_list": [ - 0.04671075918056886, - 0.09800235276106833 + 0.05301692362816539, + 0.06365443888615119, + 0.07476101434847807 ], "conc_latency_p95_list": [ - 0.038866160500401745, - 0.08286118600008194 + 0.03960793959267899, + 0.05068928290565963, + 0.061392078457720343 ], "conc_latency_avg_list": [ - 0.027654020284034526, - 0.06254546518180665 + 0.02283351154340615, + 0.0336278670182451, + 0.044479376343180034 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, @@ -40,36 +46,122 @@ "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, - "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 47.43239876568051 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 47.43239876568051 + "additional_parameters": {}, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "task15919-perf-8cu-oneseg-26-retest", + "version": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "vdbbench_fts_serial_hotpotqa_permuted_20260803" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only" }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 ], - "tokenizer": "standard" + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 33.6028, + "optimize_duration": 171.902, + "load_duration": 205.5048, + "qps": 4913.8932, + "serial_latency_p99": 0.0107, + "serial_latency_p95": 0.0079, + "recall": 0.8421, + "ndcg": 0.7319, + "mrr": 0.8636, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 4444.7978, + 4823.2308, + 4913.8932 + ], + "conc_latency_p99_list": [ + 0.018735603306413395, + 0.024391331560909748, + 0.02949425330734808 + ], + "conc_latency_p95_list": [ + 0.01459028425233555, + 0.019086839698138645, + 0.023468223997042514 + ], + "conc_latency_avg_list": [ + 0.00892458813977406, + 0.012230295266112242, + 0.01590514856506513 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 1000000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 4 + }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], "st_search_time_list": [], @@ -88,15 +180,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T16:12:30.162495", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_15919_hotpotqa_1m" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -114,17 +206,19 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "dataset_with_size_type": "HotpotQA Medium (1M documents)", "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ @@ -133,78 +227,172 @@ "search_serial", "search_concurrent" ], - "load_concurrency": 0 + "load_concurrency": 4 }, "label": ":)" }, { "metrics": { "max_load_count": 0, - "insert_duration": 92.5443, - "optimize_duration": 81.9515, - "load_duration": 174.4958, - "qps": 1311.121, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 15.6627, + "optimize_duration": 31.7788, + "load_duration": 47.4414, + "qps": 9577.4013, + "serial_latency_p99": 0.0038, + "serial_latency_p95": 0.0034, + "recall": 0.9225, + "ndcg": 0.8459, + "mrr": 0.9485, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 1252.5852, - 1311.121 + 9125.0541, + 9577.4013, + 9299.4642 ], "conc_latency_p99_list": [ - 0.04848522670017702, - 0.08333016816090094 + 0.0077219682798022405, + 0.011414849478169344, + 0.014094987509161003 ], "conc_latency_p95_list": [ - 0.04233008824876378, - 0.0747867592010152 + 0.006174615799682212, + 0.009074159996816888, + 0.011642935798590763 ], "conc_latency_avg_list": [ - 0.03172382021377068, - 0.05988630893483264 + 0.004340849293562033, + 0.006164715776924379, + 0.008391938989513186 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 5233329, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 47.43239876568051 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 47.43239876568051 + "load_concurrency": 4 + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "task15919-perf-8cu-oneseg-26-retest", + "version": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "vdbbench_fts_15919_hotpotqa_100k" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Small (100K documents)", + "payload_profile": "ids_only" }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 ], - "tokenizer": "standard" + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 4 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 183.6571, + "optimize_duration": 98.8113, + "load_duration": 282.4684, + "qps": 4252.6411, + "serial_latency_p99": 0.0171, + "serial_latency_p95": 0.0116, + "recall": 0.6293, + "ndcg": 0.2765, + "mrr": 0.1889, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 3780.8848, + 4101.5114, + 4252.6411 + ], + "conc_latency_p99_list": [ + 0.02807033602366573, + 0.03447981245903066, + 0.03956317615084118 + ], + "conc_latency_p95_list": [ + 0.01986048770704655, + 0.025059146400599273, + 0.029465016446192746 + ], + "conc_latency_avg_list": [ + 0.010491047885415203, + 0.014407923132888786, + 0.01838277799370675 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 8841823, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": {}, "st_ideal_insert_duration": 0, "st_search_stage_list": [], "st_search_time_list": [], @@ -223,15 +411,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T16:21:46.167469", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_msmarco_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -249,22 +437,23 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Large (5.2M documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -274,33 +463,39 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 19.2043, - "optimize_duration": 84.4117, - "load_duration": 103.616, - "qps": 5024.8673, - "serial_latency_p99": 0.0099, - "serial_latency_p95": 0.0074, - "recall": 0.9348, - "ndcg": 0.0, + "insert_duration": 37.9759, + "optimize_duration": 116.3291, + "load_duration": 154.305, + "qps": 9734.7457, + "serial_latency_p99": 0.0052, + "serial_latency_p95": 0.0041, + "recall": 0.8091, + "ndcg": 0.5272, + "mrr": 0.4571, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 4539.6507, - 5024.8673 + 8342.2067, + 9230.3084, + 9734.7457 ], "conc_latency_p99_list": [ - 0.02007044190080706, - 0.03161907671967124 + 0.009523463060322682, + 0.012660344481118952, + 0.015673157188284638 ], "conc_latency_p95_list": [ - 0.015511998999681963, - 0.02536790084959648 + 0.007388512603938574, + 0.0100790631986456, + 0.012480728703667407 ], "conc_latency_avg_list": [ - 0.009547202163125637, - 0.01752803334968866 + 0.004753512925593361, + 0.006390420499957776, + 0.008009355143216565 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, @@ -311,34 +506,125 @@ "indexed_after_searchable_seconds": 0.0, "additional_parameters": { "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.803761 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.803761 + "load_concurrency": 4 + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "task15919-perf-8cu-oneseg-26-retest", + "version": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "vdbbench_fts_15919_msmarco_1m" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "MS MARCO Medium (1M documents)", + "payload_profile": "ids_only" }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 ], - "tokenizer": "standard" + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ], + "load_concurrency": 4 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 21.2111, + "optimize_duration": 39.6819, + "load_duration": 60.893, + "qps": 12134.2059, + "serial_latency_p99": 0.0035, + "serial_latency_p95": 0.003, + "recall": 0.9157, + "ndcg": 0.7206, + "mrr": 0.6713, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 11108.1281, + 12134.2059, + 11328.5193 + ], + "conc_latency_p99_list": [ + 0.00618511842898442, + 0.009046302261704121, + 0.011947811315185387 + ], + "conc_latency_p95_list": [ + 0.0047809927520575, + 0.007048962705448503, + 0.009702080397983082 + ], + "conc_latency_avg_list": [ + 0.0035620799904098437, + 0.0048648263825587625, + 0.006883590285044699 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 100000, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "num_per_batch": 1000, + "load_concurrency": 4 + }, "st_ideal_insert_duration": 0, "st_search_stage_list": [], "st_search_time_list": [], @@ -357,15 +643,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T16:04:10.277433", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_15919_msmarco_100k" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -383,17 +669,19 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Medium (1M documents)", + "dataset_with_size_type": "MS MARCO Small (100K documents)", "payload_profile": "ids_only" }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ @@ -402,76 +690,76 @@ "search_serial", "search_concurrent" ], - "load_concurrency": 0 + "load_concurrency": 4 }, "label": ":)" }, { "metrics": { "max_load_count": 0, - "insert_duration": 19.2043, - "optimize_duration": 84.4117, - "load_duration": 103.616, - "qps": 4094.1935, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1492.7432, + "serial_latency_p99": 0.0353, + "serial_latency_p95": 0.0253, + "recall": 0.7975, + "ndcg": 0.6176, + "mrr": 0.6255, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 3772.8914, - 4094.1935 + 1310.191, + 1469.5941, + 1492.7432 ], "conc_latency_p99_list": [ - 0.021262989360184278, - 0.03570330930069758 + 0.06350544799934141, + 0.07389547589700673, + 0.08604409836596456 ], "conc_latency_p95_list": [ - 0.01692326859993045, - 0.028539174500110676 + 0.049314529009279795, + 0.05927793249429669, + 0.07131427159474693 ], "conc_latency_avg_list": [ - 0.010518937966150272, - 0.01910471804373547 + 0.030282339088862433, + 0.04027559123208463, + 0.05258652822193272 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 1000000, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.803761 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.803761 + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 2616664, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 2616665, + "matched_doc_ratio": 0.5, + "original_query_count": 7405, + "filtered_query_count": 5547, + "filtered_query_ratio": 0.749088, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 6846 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 5547, + "full_query_count": 7405 } }, "st_ideal_insert_duration": 0, @@ -492,15 +780,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T16:09:07.352737", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_hotpotqa_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -518,22 +806,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Medium (1M documents)", - "payload_profile": "text" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.5 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -543,69 +833,69 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1.9379, - "optimize_duration": 25.6039, - "load_duration": 27.5418, - "qps": 11840.6594, - "serial_latency_p99": 0.0036, - "serial_latency_p95": 0.0031, - "recall": 0.9987, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 1925.4864, + "serial_latency_p99": 0.0261, + "serial_latency_p95": 0.0193, + "recall": 0.8291, + "ndcg": 0.6445, + "mrr": 0.6195, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 9511.1365, - 11840.6594 + 1704.6319, + 1874.4532, + 1925.4864 ], "conc_latency_p99_list": [ - 0.00816284980028285, - 0.014502515860076526 + 0.04659453985688739, + 0.056227957597002394, + 0.06380992440390398 ], "conc_latency_p95_list": [ - 0.006357606800520441, - 0.01137593650018971 + 0.03717728605115552, + 0.0455952929914929, + 0.053582699001708534 ], "conc_latency_avg_list": [ - 0.004235700703938657, - 0.007192490032664778 + 0.023269752517795814, + 0.03155254825551937, + 0.04073994229261837 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 100000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 56.88413 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 56.88413 + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 3924996, + "filter_rate": 0.75, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 1308333, + "matched_doc_ratio": 0.25, + "original_query_count": 7405, + "filtered_query_count": 3175, + "filtered_query_ratio": 0.428764, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 3379 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 3175, + "full_query_count": 7405 } }, "st_ideal_insert_duration": 0, @@ -626,15 +916,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T15:59:30.355088", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_hotpotqa_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -652,22 +942,23 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Small (100K documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.75 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ], @@ -678,69 +969,69 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1.9379, - "optimize_duration": 25.6039, - "load_duration": 27.5418, - "qps": 8475.056, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2484.2534, + "serial_latency_p99": 0.0193, + "serial_latency_p95": 0.0144, + "recall": 0.8687, + "ndcg": 0.7021, + "mrr": 0.6648, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 7722.3517, - 8475.056 + 2200.1066, + 2447.8999, + 2484.2534 ], "conc_latency_p99_list": [ - 0.009759001410220668, - 0.018094878880474424 + 0.03526244850218063, + 0.04156497399890213, + 0.04930016008423986 ], "conc_latency_p95_list": [ - 0.007649297249099612, - 0.013853047499105738 + 0.028342882498691324, + 0.03412441574255354, + 0.04170981489733093 ], "conc_latency_avg_list": [ - 0.005133058755745676, - 0.009210371172970354 + 0.018032777527778057, + 0.024189374054596657, + 0.03155972211155846 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 100000, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 56.88413 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4709996, + "filter_rate": 0.9, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 523333, + "matched_doc_ratio": 0.1, + "original_query_count": 7405, + "filtered_query_count": 1367, + "filtered_query_ratio": 0.184605, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 1335 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 56.88413 - }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 1367, + "full_query_count": 7405 } }, "st_ideal_insert_duration": 0, @@ -761,15 +1052,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T16:02:03.206355", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_hotpotqa_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -787,22 +1078,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "HotpotQA Small (100K documents)", - "payload_profile": "text" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.9 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -812,69 +1105,69 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 183.6571, - "optimize_duration": 98.8113, - "load_duration": 282.4684, - "qps": 2859.4474, - "serial_latency_p99": 0.008, - "serial_latency_p95": 0.0058, - "recall": 0.9636, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2771.399, + "serial_latency_p99": 0.0163, + "serial_latency_p95": 0.0123, + "recall": 0.884, + "ndcg": 0.7273, + "mrr": 0.6873, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 2397.9818, - 2859.4474 + 2474.7168, + 2727.7045, + 2771.399 ], "conc_latency_p99_list": [ - 0.025391751239949356, - 0.038862020799206254 + 0.030666672199731695, + 0.037067809136351555, + 0.04432674863608554 ], "conc_latency_p95_list": [ - 0.02053890500028501, - 0.032976875500025926 + 0.02494138489928446, + 0.030652639707841444, + 0.0372560238080041 ], "conc_latency_avg_list": [ - 0.013524379689949012, - 0.02457546268064057 + 0.016034226178701837, + 0.02163131208297352, + 0.02828691272543025 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 8841823, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4971662, + "filter_rate": 0.95, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 261667, + "matched_doc_ratio": 0.05, + "original_query_count": 7405, + "filtered_query_count": 698, + "filtered_query_ratio": 0.094261, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 664 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 698, + "full_query_count": 7405 } }, "st_ideal_insert_duration": 0, @@ -895,15 +1188,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T15:37:30.224815", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_hotpotqa_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -921,22 +1214,23 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Large (8.8M documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.95 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ], @@ -947,69 +1241,205 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 183.6571, - "optimize_duration": 98.8113, - "load_duration": 282.4684, - "qps": 2997.5483, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2887.0488, + "serial_latency_p99": 0.0147, + "serial_latency_p95": 0.0108, + "recall": 0.898, + "ndcg": 0.774, + "mrr": 0.7382, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 2764.3614, - 2997.5483 + 2611.7941, + 2842.4523, + 2887.0488 ], "conc_latency_p99_list": [ - 0.027363914300149103, - 0.0428042954213015 + 0.028886324879422363, + 0.034578777490824, + 0.04206452721206003 ], "conc_latency_p95_list": [ - 0.022107690500342867, - 0.03582373630015354 + 0.023465643404051658, + 0.028878978548164017, + 0.0356531626493961 ], "conc_latency_avg_list": [ - 0.014345869193528543, - 0.026168574204900128 + 0.015185543958564984, + 0.02079020171914545, + 0.027100502311054302 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 8841823, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 5180995, + "filter_rate": 0.99, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 3234376, + "filter_id_offset": 1082441, + "matched_doc_count": 52334, + "matched_doc_ratio": 0.01, + "original_query_count": 7405, + "filtered_query_count": 147, + "filtered_query_ratio": 0.019851, + "original_relevant_doc_count": 13783, + "filtered_relevant_doc_count": 136 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.83363736188793 + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 147, + "full_query_count": 7405 + } + }, + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_serial_latency_p95_list": [], + "st_conc_failed_rate_list": [], + "st_conc_num_list_list": [], + "st_conc_qps_list_list": [], + "st_conc_latency_p99_list_list": [], + "st_conc_latency_p95_list_list": [], + "st_conc_latency_avg_list_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "task15919-perf-8cu-oneseg-26-retest", + "version": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", + "uri": "**********", + "user": "root", + "password": "**********", + "token": "", + "num_shards": 1, + "collection_name": "vdbbench_fts_serial_hotpotqa_permuted_20260803" + }, + "db_case_config": { + "index_type": "AUTOINDEX", + "metric_type": "BM25", + "inverted_index_algo": "DAAT_MAXSCORE", + "bm25_k1": null, + "bm25_b": null, + "analyzer_tokenizer": "standard", + "analyzer_enable_lowercase": true, + "analyzer_max_token_length": null, + "analyzer_stop_words": null, + "drop_ratio_search": null, + "level": 1 + }, + "case_config": { + "case_id": 503, + "custom_case": { + "dataset_with_size_type": "HotpotQA Large (5.2M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.99 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 40, + 60, + 80 ], - "tokenizer": "standard" + "concurrency_duration": 30, + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 + } + }, + "stages": [ + "search_serial", + "search_concurrent" + ], + "load_concurrency": 0 + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2157.5384, + "serial_latency_p99": 0.0251, + "serial_latency_p95": 0.0189, + "recall": 0.7044, + "ndcg": 0.3475, + "mrr": 0.2565, + "conc_num_list": [ + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1602.7776, + 1965.32, + 2157.5384 + ], + "conc_latency_p99_list": [ + 0.04808061543459191, + 0.05590218743600418, + 0.06283525501348776 + ], + "conc_latency_p95_list": [ + 0.038264308209181766, + 0.044974703500338366, + 0.05120332330261589 + ], + "conc_latency_avg_list": [ + 0.024761475757823056, + 0.03012121981174715, + 0.036368580013719426 + ], + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, + "insert_rows_per_second": 0.0, + "insert_completion_seconds": 0.0, + "searchable_after_insert_seconds": 0.0, + "indexed_after_searchable_seconds": 0.0, + "additional_parameters": { + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 4420911, + "filter_rate": 0.5, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 4420912, + "matched_doc_ratio": 0.5, + "original_query_count": 6980, + "filtered_query_count": 3658, + "filtered_query_ratio": 0.524069, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 3758 + }, + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 3658, + "full_query_count": 6980 } }, "st_ideal_insert_duration": 0, @@ -1030,15 +1460,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T15:50:48.791703", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_msmarco_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -1057,21 +1487,23 @@ "case_id": 503, "custom_case": { "dataset_with_size_type": "MS MARCO Large (8.8M documents)", - "payload_profile": "text" + "payload_profile": "ids_only", + "filter_rate": 0.5 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -1081,69 +1513,69 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 20.5452, - "optimize_duration": 119.3876, - "load_duration": 139.9328, - "qps": 10440.9225, - "serial_latency_p99": 0.0045, - "serial_latency_p95": 0.0037, - "recall": 0.9194, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 2761.6694, + "serial_latency_p99": 0.0192, + "serial_latency_p95": 0.0146, + "recall": 0.7668, + "ndcg": 0.4252, + "mrr": 0.3365, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 8403.6721, - 10440.9225 + 2105.0956, + 2509.0089, + 2761.6694 ], "conc_latency_p99_list": [ - 0.010635902699323196, - 0.01738944091935992 + 0.03667745854880194, + 0.043968265774310566, + 0.04997027571342189 ], "conc_latency_p95_list": [ - 0.008483375000650994, - 0.01400671859992144 + 0.029508575000363628, + 0.03553523525770286, + 0.04100487310497556 ], "conc_latency_avg_list": [ - 0.005399416001780447, - 0.00889505466866669 + 0.018847058349541947, + 0.023569575725606327, + 0.028409127438431932 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 1000000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 6631367, + "filter_rate": 0.75, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 2210456, + "matched_doc_ratio": 0.25, + "original_query_count": 6980, + "filtered_query_count": 1818, + "filtered_query_ratio": 0.260458, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 1839 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 1818, + "full_query_count": 6980 } }, "st_ideal_insert_duration": 0, @@ -1164,15 +1596,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T15:28:10.214232", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_msmarco_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -1190,22 +1622,23 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Medium (1M documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.75 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ], @@ -1216,69 +1649,69 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 20.5452, - "optimize_duration": 119.3876, - "load_duration": 139.9328, - "qps": 7477.8788, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3467.3872, + "serial_latency_p99": 0.0139, + "serial_latency_p95": 0.0113, + "recall": 0.8383, + "ndcg": 0.5192, + "mrr": 0.435, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 6408.2792, - 7477.8788 + 2695.8315, + 3198.641, + 3467.3872 ], "conc_latency_p99_list": [ - 0.012072665600589968, - 0.02024130038942528 + 0.028783008844475263, + 0.03443616359960289, + 0.03963516379910289 ], "conc_latency_p95_list": [ - 0.009671015750882361, - 0.01634627664943764 + 0.023086678193067197, + 0.02811312419653404, + 0.032976010798302015 ], "conc_latency_avg_list": [ - 0.006184244148888284, - 0.01044489949882264 + 0.014714017594863122, + 0.018505583353593157, + 0.022566899350545157 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 1000000, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 57.242324 + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 7957640, + "filter_rate": 0.9, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 884183, + "matched_doc_ratio": 0.1, + "original_query_count": 6980, + "filtered_query_count": 739, + "filtered_query_ratio": 0.105874, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 740 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 739, + "full_query_count": 6980 } }, "st_ideal_insert_duration": 0, @@ -1299,15 +1732,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T15:31:42.678098", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_msmarco_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -1325,22 +1758,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Medium (1M documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.9 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -1350,69 +1785,69 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1.8223, - "optimize_duration": 18.0832, - "load_duration": 19.9055, - "qps": 14523.9158, - "serial_latency_p99": 0.003, - "serial_latency_p95": 0.0028, - "recall": 0.9854, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 3866.9025, + "serial_latency_p99": 0.0126, + "serial_latency_p95": 0.01, + "recall": 0.8473, + "ndcg": 0.5792, + "mrr": 0.5077, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 11465.34, - 14523.9158 + 3009.9118, + 3519.8953, + 3866.9025 ], "conc_latency_p99_list": [ - 0.006605165030141507, - 0.013721053038825629 + 0.02564647176885045, + 0.03126390196281136, + 0.03594922040065286 ], "conc_latency_p95_list": [ - 0.005089374199815211, - 0.009782408600040073 + 0.020645123507711104, + 0.02540512894993297, + 0.02950239399797283 ], "conc_latency_avg_list": [ - 0.003510005176568982, - 0.006679762620736736 + 0.013172088723884543, + 0.01679307632555934, + 0.02024829934249553 ], "payload_profile": "ids_only", "payload_estimated_bytes_per_query": 2000, - "inserted_count": 100000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } - }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 8399731, + "filter_rate": 0.95, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 442092, + "matched_doc_ratio": 0.05, + "original_query_count": 6980, + "filtered_query_count": 347, + "filtered_query_ratio": 0.049713, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 347 }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 347, + "full_query_count": 6980 } }, "st_ideal_insert_duration": 0, @@ -1433,15 +1868,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T15:23:49.914938", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_msmarco_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -1459,22 +1894,23 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Small (100K documents)", - "payload_profile": "ids_only" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.95 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", "search_serial", "search_concurrent" ], @@ -1485,69 +1921,69 @@ { "metrics": { "max_load_count": 0, - "insert_duration": 1.8223, - "optimize_duration": 18.0832, - "load_duration": 19.9055, - "qps": 9382.1707, - "serial_latency_p99": 0.0, - "serial_latency_p95": 0.0, - "recall": 0.0, - "ndcg": 0.0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 4028.3996, + "serial_latency_p99": 0.0246, + "serial_latency_p95": 0.0098, + "recall": 0.9403, + "ndcg": 0.6958, + "mrr": 0.628, "conc_num_list": [ 40, + 60, 80 ], "conc_qps_list": [ - 9382.1707, - 9299.0682 + 3205.3504, + 3736.9223, + 4028.3996 ], "conc_latency_p99_list": [ - 0.007831546979778047, - 0.014077205400753908 + 0.023615233003511128, + 0.02917098919919226, + 0.034491611427802125 ], "conc_latency_p95_list": [ - 0.006045342150900976, - 0.011566458601373598 + 0.019226595002692193, + 0.02385359159961807, + 0.028303666404826795 ], "conc_latency_avg_list": [ - 0.004228128073816301, - 0.008412116105405435 + 0.012365652090037205, + 0.015849408703119124, + 0.01946939073238616 ], - "payload_profile": "text", - "payload_estimated_bytes_per_query": 53200, - "inserted_count": 100000, + "payload_profile": "ids_only", + "payload_estimated_bytes_per_query": 2000, + "inserted_count": 0, "insert_rows_per_second": 0.0, "insert_completion_seconds": 0.0, "searchable_after_insert_seconds": 0.0, "indexed_after_searchable_seconds": 0.0, "additional_parameters": { - "num_per_batch": 1000, - "load_concurrency": 0, - "fts_manifest": { - "bm25": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "analyzer": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" - } + "fts_filter": { + "filter_type": "NumGE", + "filter_field": "filter_id", + "filter_value": 8753404, + "filter_rate": 0.99, + "filter_id_distribution": "affine_permutation_v1", + "filter_id_multiplier": 5464547, + "filter_id_offset": 1684563, + "matched_doc_count": 88419, + "matched_doc_ratio": 0.01, + "original_query_count": 6980, + "filtered_query_count": 67, + "filtered_query_ratio": 0.009599, + "original_relevant_doc_count": 7433, + "filtered_relevant_doc_count": 67 }, - "applied_bm25_params": {}, - "unapplied_bm25_params": { - "k1": 1.2, - "b": 0.75, - "avgdl": 55.01342 - }, - "applied_analyzer_params": {}, - "unapplied_analyzer_params": { - "filter": [ - "lowercase" - ], - "tokenizer": "standard" + "fts_recall": { + "skipped": false, + "reason": null, + "serial_query_count": 67, + "full_query_count": 6980 } }, "st_ideal_insert_duration": 0, @@ -1568,15 +2004,15 @@ "task_config": { "db": "ZillizCloud", "db_config": { - "db_label": "2026-06-23T15:26:11.226516", + "db_label": "task15919-perf-8cu-oneseg-26-retest", "version": "", - "note": "", + "note": "{\n \"schema\": \"vdbbench-zilliz-cloud-context/v1\",\n \"backend\": {\n \"name\": \"ZillizCloud\",\n \"region\": \"aws-us-west-2\",\n \"cu\": 8,\n \"replica_count\": 1,\n \"instance_type\": \"performance\",\n \"deployment_method\": \"kubernetes\",\n \"deployment_mode\": \"standalone\",\n \"server_role\": \"milvus-standalone-r1\",\n \"server_count\": 1\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\"\n },\n \"evidence\": {\n \"source\": \"User-confirmed Task 15919 deployment shape; one persistent segment per benchmark collection validated by run preflight and post-run inventory\",\n \"credential_storage\": \"external mode-600 secret file removed by runner cleanup\"\n }\n}", "uri": "**********", "user": "root", "password": "**********", "token": "", "num_shards": 1, - "collection_name": "ZillizCloudFTSBench" + "collection_name": "vdbbench_fts_serial_msmarco_permuted_20260803" }, "db_case_config": { "index_type": "AUTOINDEX", @@ -1594,22 +2030,24 @@ "case_config": { "case_id": 503, "custom_case": { - "dataset_with_size_type": "MS MARCO Small (100K documents)", - "payload_profile": "text" + "dataset_with_size_type": "MS MARCO Large (8.8M documents)", + "payload_profile": "ids_only", + "filter_rate": 0.99 }, "k": 100, "concurrency_search_config": { "num_concurrency": [ 40, + 60, 80 ], "concurrency_duration": 30, - "concurrency_timeout": 3600 + "concurrency_timeout": 3600, + "serial_cooldown": 0.0 } }, "stages": [ - "drop_old", - "load", + "search_serial", "search_concurrent" ], "load_concurrency": 0 @@ -1618,5 +2056,5 @@ } ], "file_fmt": "result_{}_{}_{}.json", - "timestamp": 1782172800.0 + "timestamp": 1786060800.0 } From cba0049bcc06b1f2ff48241c9bb66bffe8234c35 Mon Sep 17 00:00:00 2001 From: hiimivantang Date: Tue, 11 Aug 2026 16:22:48 +0800 Subject: [PATCH 46/49] fix: mask empty api_key in TurboPuffer FTS result file (#845) 10 of 16 case results in result_20260626_fts_standard_turbopuffer.json were written with "api_key": "", which fails DBConfig's not_empty_field validator when TestResult.read_file re-instantiates TurboPufferConfig, crashing the whole results page with a ValidationError. Replace the empty strings with the standard "**********" placeholder used by the redaction logic. Co-authored-by: Claude Fable 5 --- ...ult_20260626_fts_standard_turbopuffer.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json b/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json index 00c60972a..76dd16e30 100644 --- a/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json +++ b/vectordb_bench/results/FullTextSearch/TurboPuffer/result_20260626_fts_standard_turbopuffer.json @@ -932,7 +932,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", @@ -1081,7 +1081,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", @@ -1228,7 +1228,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", @@ -1375,7 +1375,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", @@ -1522,7 +1522,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-hotpotqa-large-permuted-20260802", @@ -1673,7 +1673,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", @@ -1822,7 +1822,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", @@ -1969,7 +1969,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", @@ -2116,7 +2116,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", @@ -2263,7 +2263,7 @@ "db_label": "turbopuffer-aws-us-west-2-filtered-permuted", "version": "", "note": "{\n \"schema\": \"vdbbench-turbopuffer-context/v1\",\n \"backend\": {\n \"name\": \"TurboPuffer\",\n \"deployment\": \"managed_service\",\n \"service_version\": \"not_exposed\",\n \"region\": \"aws-us-west-2\",\n \"api_base_url\": \"https://aws-us-west-2.turbopuffer.com\",\n \"sdk_version\": \"2.6.0\"\n },\n \"client\": {\n \"machine_type\": \"i8g.2xlarge\",\n \"cpu_count\": 8,\n \"memory_kib\": 64648904,\n \"branch\": \"fts_v2_backend\",\n \"commit\": \"07bcefdbd5b04f51faebfc857219061d46148b6b\",\n \"working_tree_clean\": true\n },\n \"execution\": {\n \"pin_namespace\": false,\n \"pin_replicas\": 1,\n \"write_backpressure_enabled\": true,\n \"sdk_max_retries\": 4,\n \"warmup_seconds_after_load\": 60,\n \"credential_delivery\": \"ephemeral mode-600 config outside logs and report bundle; removed by trap\"\n },\n \"validation\": {\n \"namespace_list_http_status\": 200,\n \"existing_namespace_metadata_http_status\": 200,\n \"historical_capacity_evidence\": {\n \"datasets\": [\n \"HotpotQA Large (5.2M documents)\",\n \"MS MARCO Large (8.8M documents)\"\n ],\n \"batch_size\": 1000\n },\n \"credentials_stored\": false\n }\n}", - "api_key": "", + "api_key": "**********", "region": "aws-us-west-2", "api_base_url": null, "namespace": "vdbbench-fts-filter-tpuf-msmarco-large-permuted-20260802", From 5d0d3148ef7a7c39879bcda2a97f38e0b16f06f4 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Tue, 11 Aug 2026 17:46:32 +0800 Subject: [PATCH 47/49] fix: ensure Milvus force merge covers compacting segments (#840) Milvus force merge is best-effort and may skip segments already compacting. After flushing, wait for segment sorting and index readiness, run one normal manual compaction, then retry force merge with a fresh persistent segment snapshot until every segment visible for an attempt is included in its generated plans. Bound force merge to 10 attempts with a 30-second retry interval. Add regressions for compaction ordering, partial-plan and no-plan retries, fresh per-attempt snapshots, and retry exhaustion. See also: #784 Signed-off-by: yangxuan --- tests/test_milvus.py | 204 +++++++++++++++++- .../backend/clients/milvus/milvus.py | 53 ++++- 2 files changed, 247 insertions(+), 10 deletions(-) diff --git a/tests/test_milvus.py b/tests/test_milvus.py index 8dcef4f1c..ef131ef87 100644 --- a/tests/test_milvus.py +++ b/tests/test_milvus.py @@ -5,7 +5,7 @@ import logging from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, call import pytest from pydantic import SecretStr @@ -37,7 +37,7 @@ def _milvus( milvus.case_config = SimpleNamespace(is_gpu_index=is_gpu_index) milvus.client = MagicMock() milvus.client.compact.side_effect = compact_side_effect - milvus.client.compact.return_value = 0 + milvus.client.compact.return_value = 42 milvus._wait_for_segments_sorted = MagicMock() milvus._wait_for_index = MagicMock() milvus._wait_for_compaction = MagicMock() @@ -48,7 +48,7 @@ def test_optimize_compact_uses_safe_force_merge_target_size(self): milvus._optimize() - milvus.client.compact.assert_called_once_with("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) + milvus.client.compact.assert_any_call("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) milvus.client.refresh_load.assert_called_once_with("test_collection") def test_optimize_compacts_fts_collections(self): @@ -56,9 +56,205 @@ def test_optimize_compacts_fts_collections(self): milvus._optimize() - milvus.client.compact.assert_called_once_with("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) + milvus.client.compact.assert_any_call("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) milvus.client.refresh_load.assert_called_once_with("test_collection") + def test_optimize_flushes_and_runs_normal_compaction_before_force_merge(self): + milvus = self._milvus(compact_side_effect=[41, 42]) + + milvus._optimize() + + milvus.client.flush.assert_called_once_with("test_collection") + assert milvus.client.compact.call_args_list == [ + call("test_collection"), + call("test_collection", target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB), + ] + assert milvus._wait_for_segments_sorted.call_count == 2 + assert milvus._wait_for_index.call_count == 3 + assert milvus._wait_for_compaction.call_args_list == [call(41), call(42)] + milvus.client.refresh_load.assert_called_once_with("test_collection") + + def test_optimize_retries_when_compacting_segments_are_missing_from_force_merge_plan( + self, monkeypatch: pytest.MonkeyPatch + ): + class FakeMilvusClient: + def __init__(self): + self.segment_ids = [1, 2, 3] + self.force_merge_completed = False + + def flush(self, _collection_name: str): + pass + + def list_persistent_segments(self, _collection_name: str): + return [ + SimpleNamespace( + segment_id=segment_id, + is_sorted=True, + state_name="Flushed", + level_name="L1", + ) + for segment_id in self.segment_ids + ] + + def describe_index(self, _collection_name: str, _index_name: str): + return {"pending_index_rows": 0} + + def compact(self, _collection_name: str, *, target_size: int | None = None): + if target_size is None: + return 90 + assert target_size == MILVUS_FORCE_MERGE_TARGET_SIZE_MB + if self.segment_ids == [1, 2, 3]: + return 100 + if self.segment_ids == [10, 20]: + return 101 + message = f"unexpected force merge input: {self.segment_ids}" + raise AssertionError(message) + + def get_compaction_state(self, compaction_id: int): + if compaction_id == 90: + pass + elif compaction_id == 100: + self.segment_ids = [10, 20] + elif compaction_id == 101: + self.segment_ids = [30] + self.force_merge_completed = True + else: + message = f"unexpected compaction id: {compaction_id}" + raise AssertionError(message) + return "Completed" + + def get_compaction_plans(self, compaction_id: int): + sources = [1] if compaction_id == 100 else [10, 20] + return SimpleNamespace(plans=[SimpleNamespace(sources=sources)]) + + def refresh_load(self, _collection_name: str): + assert self.force_merge_completed, "refresh started after only a partial force merge" + + milvus = Milvus.__new__(Milvus) + milvus.name = "Milvus" + milvus.collection_name = "test_collection" + milvus._is_fts = False + milvus._main_index_name = "vector_idx" + milvus.case_config = SimpleNamespace(is_gpu_index=False) + milvus.client = FakeMilvusClient() + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.time.sleep", lambda _seconds: None) + + milvus.optimize(data_size=500_000) + + assert milvus.client.force_merge_completed + + def test_optimize_retries_when_all_segments_are_compacting(self, monkeypatch: pytest.MonkeyPatch): + class FakeMilvusClient: + def __init__(self): + self.segment_ids = [1, 2] + self.compact_attempts = 0 + self.force_merge_completed = False + + def flush(self, _collection_name: str): + pass + + def list_persistent_segments(self, _collection_name: str): + return [ + SimpleNamespace( + segment_id=segment_id, + is_sorted=True, + state_name="Flushed", + level_name="L1", + ) + for segment_id in self.segment_ids + ] + + def describe_index(self, _collection_name: str, _index_name: str): + return {"pending_index_rows": 0} + + def compact(self, _collection_name: str, *, target_size: int | None = None): + if target_size is None: + return 90 + assert target_size == MILVUS_FORCE_MERGE_TARGET_SIZE_MB + self.compact_attempts += 1 + if self.compact_attempts == 1: + self.segment_ids = [10] + return -1 + return 101 + + def get_compaction_state(self, compaction_id: int): + if compaction_id == 90: + return "Completed" + assert compaction_id == 101 + self.segment_ids = [20] + self.force_merge_completed = True + return "Completed" + + def get_compaction_plans(self, compaction_id: int): + assert compaction_id == 101 + return SimpleNamespace(plans=[SimpleNamespace(sources=[10])]) + + def refresh_load(self, _collection_name: str): + assert self.force_merge_completed, "refresh started without a force merge job" + + milvus = Milvus.__new__(Milvus) + milvus.name = "Milvus" + milvus.collection_name = "test_collection" + milvus._is_fts = False + milvus._main_index_name = "vector_idx" + milvus.case_config = SimpleNamespace(is_gpu_index=False) + milvus.client = FakeMilvusClient() + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.time.sleep", lambda _seconds: None) + + milvus.optimize(data_size=500_000) + + assert milvus.client.force_merge_completed + + def test_force_merge_uses_fresh_snapshots_and_stops_after_max_attempts(self, monkeypatch: pytest.MonkeyPatch): + class FakeMilvusClient: + def __init__(self): + self.segment_ids = [1, 2] + self.compact_attempts = 0 + + def list_persistent_segments(self, _collection_name: str): + return [ + SimpleNamespace( + segment_id=segment_id, + is_sorted=True, + state_name="Flushed", + level_name="L1", + ) + for segment_id in self.segment_ids + ] + + def describe_index(self, _collection_name: str, _index_name: str): + return {"pending_index_rows": 0} + + def compact(self, _collection_name: str, *, target_size: int): + assert target_size == MILVUS_FORCE_MERGE_TARGET_SIZE_MB + self.compact_attempts += 1 + return 100 + self.compact_attempts + + def get_compaction_state(self, _compaction_id: int): + next_segment_ids = { + 1: [10, 20], + 2: [30, 40], + 3: [50, 60], + } + self.segment_ids = next_segment_ids[self.compact_attempts] + return "Completed" + + def get_compaction_plans(self, _compaction_id: int): + planned_source = {1: 1, 2: 10, 3: 30}[self.compact_attempts] + return SimpleNamespace(plans=[SimpleNamespace(sources=[planned_source])]) + + milvus = Milvus.__new__(Milvus) + milvus.name = "Milvus" + milvus.collection_name = "test_collection" + milvus._main_index_name = "vector_idx" + milvus.client = FakeMilvusClient() + monkeypatch.setattr("vectordb_bench.backend.clients.milvus.milvus.time.sleep", lambda _seconds: None) + + with pytest.raises(RuntimeError, match=r"after 3 attempts.*missing segments: \[40\]"): + milvus._force_merge(max_attempts=3) + + assert milvus.client.compact_attempts == 3 + def test_optimize_skips_gpu_index_compaction(self): milvus = self._milvus(is_gpu_index=True) diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index 0ed09aaea..73c3c242e 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -19,6 +19,8 @@ MILVUS_LOAD_REQS_SIZE = 1.5 * 1024 * 1024 MILVUS_FTS_BATCH_SIZE = 1000 MILVUS_FORCE_MERGE_TARGET_SIZE_MB = ((1 << 63) - 1) // (1024**2) +MILVUS_FORCE_MERGE_MAX_ATTEMPTS = 10 +MILVUS_FORCE_MERGE_RETRY_INTERVAL_SECONDS = 30 class Milvus(VectorDB): @@ -301,6 +303,10 @@ def _wait_for_index(self): break time.sleep(5) + def _wait_for_compaction_ready(self): + self._wait_for_segments_sorted() + self._wait_for_index() + def _wait_for_compaction(self, compaction_id: int): while True: state = self.client.get_compaction_state(compaction_id) @@ -308,6 +314,44 @@ def _wait_for_compaction(self, compaction_id: int): break time.sleep(0.5) + def _force_merge_source_segment_ids(self) -> set[int]: + # isCompacting is not exposed by MilvusClient, so verify plan coverage + # against every persistent data segment that force merge should consume. + segments = self.client.list_persistent_segments(self.collection_name) + return { + segment.segment_id + for segment in segments + if segment.state_name == "Flushed" and segment.level_name not in {"L0", "L2"} + } + + def _force_merge(self, max_attempts: int = MILVUS_FORCE_MERGE_MAX_ATTEMPTS): + if max_attempts <= 0: + message = "force merge max_attempts must be greater than zero" + raise ValueError(message) + + for attempt in range(1, max_attempts + 1): + self._wait_for_compaction_ready() + expected_source_ids = self._force_merge_source_segment_ids() + compaction_id = self.client.compact(self.collection_name, target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB) + if compaction_id <= 0: + failure_detail = f"generated no plan for snapshot {sorted(expected_source_ids)}" + else: + self._wait_for_compaction(compaction_id) + plans = self.client.get_compaction_plans(compaction_id) + actual_source_ids = {source_id for plan in plans.plans for source_id in plan.sources} + attempt_missing_source_ids = expected_source_ids - actual_source_ids + if not attempt_missing_source_ids: + return + + failure_detail = f"missing segments: {sorted(attempt_missing_source_ids)}" + + if attempt == max_attempts: + message = f"{self.name} force merge failed after {max_attempts} attempts; {failure_detail}" + raise RuntimeError(message) + + log.info(f"{self.name} force merge attempt {attempt}/{max_attempts} {failure_detail}; retrying...") + time.sleep(MILVUS_FORCE_MERGE_RETRY_INTERVAL_SECONDS) + def _optimize(self): log.info(f"{self.name} optimizing before search") try: @@ -317,14 +361,11 @@ def _optimize(self): log.debug("skip force merge compaction for gpu index type.") else: try: - # wait for sort, index, compact - self._wait_for_segments_sorted() - self._wait_for_index() - compaction_id = self.client.compact( - self.collection_name, target_size=MILVUS_FORCE_MERGE_TARGET_SIZE_MB - ) + self._wait_for_compaction_ready() + compaction_id = self.client.compact(self.collection_name) if compaction_id > 0: self._wait_for_compaction(compaction_id) + self._force_merge() log.info(f"{self.name} force merge compaction completed.") except Exception as e: log.warning(f"{self.name} compact or list segments error: {e}") From 4ea181055b11af7d75161179ef2feff9539c3b33 Mon Sep 17 00:00:00 2001 From: HUANG XIAO <33706975+norrishuang@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:00:46 +0800 Subject: [PATCH 48/49] Make AOSS insert batch size configurable (#843) * Make AOSS insert batch size configurable Signed-off-by: norrishuang <12380647@qq.com> * Validate AOSS insert batch size --------- Signed-off-by: norrishuang <12380647@qq.com> --- README.md | 5 +- tests/test_aws_opensearch.py | 46 +++++++++++++++++++ .../clients/aws_opensearch/aws_opensearch.py | 9 ++-- 3 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 tests/test_aws_opensearch.py diff --git a/README.md b/README.md index 7851ef81b..85d34d76d 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,7 @@ OpenSearch Serverless (AOSS) is a serverless deployment option for Amazon OpenSe **Example: Run performance test on OpenSearch Serverless** ```shell -vectordbbench awsopensearch --db-label aoss \ +NUM_PER_BATCH=100 vectordbbench awsopensearch --db-label aoss \ --serverless --aws-region us-east-1 \ --host .aoss.us-east-1.on.aws --port 443 \ --case-type Performance768D1M \ @@ -303,12 +303,13 @@ OpenSearch Serverless-specific options: |--------|-------------| | `--serverless` | Enable OpenSearch Serverless mode (uses AWS SigV4 auth) | | `--aws-region` | AWS region for the AOSS collection (default: `us-east-1`) | +| `NUM_PER_BATCH` | Number of vectors per Serverless bulk request (default: `100`) | > **Notes:** > - `--user` and `--password` are not needed for Serverless mode > - `--engine` is accepted but ignored internally (AOSS manages the engine) > - `--force-merge-enabled`, `--refresh-interval`, `--flush-threshold-size`, and `--cb-threshold` are ignored for Serverless -> - Data insertion uses smaller batch sizes (100) for Serverless API compatibility +> - Keep `NUM_PER_BATCH` small enough for the Serverless bulk API request limits ### Run Elastic Cloud from command line diff --git a/tests/test_aws_opensearch.py b/tests/test_aws_opensearch.py new file mode 100644 index 000000000..2e2930338 --- /dev/null +++ b/tests/test_aws_opensearch.py @@ -0,0 +1,46 @@ +from types import SimpleNamespace + +import pytest + +from vectordb_bench import config +from vectordb_bench.backend.clients.aws_opensearch.aws_opensearch import AWSOpenSearch + + +def test_serverless_insert_uses_configured_batch_size(monkeypatch) -> None: + bulk_requests = [] + + def bulk(*, body): + bulk_requests.append(body) + + monkeypatch.setattr(config, "NUM_PER_BATCH", 2) + + db = object.__new__(AWSOpenSearch) + db.client = SimpleNamespace(bulk=bulk) + db._is_serverless = True + db.index_name = "test-index" + db.vector_col_name = "embedding" + db.with_scalar_labels = False + + inserted, error = db._insert_with_single_client( + embeddings=[[0.1], [0.2], [0.3], [0.4], [0.5]], + metadata=[1, 2, 3, 4, 5], + ) + + assert inserted == 5 + assert error is None + assert [len(request) // 2 for request in bulk_requests] == [2, 2, 1] + assert [document["id"] for request in bulk_requests for document in request[1::2]] == [1, 2, 3, 4, 5] + + +@pytest.mark.parametrize("batch_size", [0, -1]) +def test_serverless_insert_rejects_non_positive_batch_size(monkeypatch, batch_size: int) -> None: + monkeypatch.setattr(config, "NUM_PER_BATCH", batch_size) + + db = object.__new__(AWSOpenSearch) + db._is_serverless = True + + with pytest.raises(ValueError, match="NUM_PER_BATCH must be greater than 0"): + db._insert_with_single_client( + embeddings=[[0.1]], + metadata=[1], + ) diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index 034793f4b..eb0195b88 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -5,6 +5,7 @@ from opensearchpy import OpenSearch +from vectordb_bench import config from vectordb_bench.backend.filter import Filter, FilterOp from ..api import VectorDB @@ -237,8 +238,8 @@ def insert_embeddings( num_clients = self.case_config.number_of_indexing_clients or 1 log.info(f"Number of indexing clients from case_config: {num_clients}") - # OpenSearch Serverless requires the single-client path: it does not support - # custom _id and needs the benchmark id stored in _source with small batches. + # OpenSearch Serverless requires the single-client path because it does not + # support custom _id and needs the benchmark id stored in _source. if self._is_serverless: log.info("Using single client for data insertion (OpenSearch Serverless)") return self._insert_with_single_client(embeddings, metadata, labels_data) @@ -256,7 +257,9 @@ def _insert_with_single_client( labels_data: list[str] | None = None, ) -> tuple[int, Exception]: embeddings_list = list(embeddings) - batch_size = 100 if self._is_serverless else len(embeddings_list) + batch_size = config.NUM_PER_BATCH if self._is_serverless else len(embeddings_list) + if self._is_serverless and batch_size <= 0: + raise ValueError("NUM_PER_BATCH must be greater than 0 for OpenSearch Serverless") total_inserted = 0 for i in range(0, len(embeddings_list), batch_size): From b01605c32e7d708e11a84660ad8371fea644d455 Mon Sep 17 00:00:00 2001 From: Rowan Copley Date: Mon, 17 Aug 2026 18:40:01 -0700 Subject: [PATCH 49/49] Sync fork with Zilliz main --- pyproject.toml | 5 +- tests/test_antfly.py | 107 +++ tests/test_filtered_adapters.py | 79 +++ vectordb_bench/backend/clients/__init__.py | 16 + .../backend/clients/antfly/__init__.py | 0 .../backend/clients/antfly/antfly.py | 618 ++++++++++++++++++ vectordb_bench/backend/clients/antfly/cli.py | 107 +++ .../backend/clients/antfly/config.py | 49 ++ .../backend/clients/chroma/chroma.py | 29 +- vectordb_bench/backend/clients/chroma/cli.py | 1 + .../clients/elastic_cloud/elastic_cloud.py | 9 +- .../backend/clients/qdrant_local/cli.py | 5 +- .../clients/qdrant_local/qdrant_local.py | 39 +- .../clients/weaviate_cloud/weaviate_cloud.py | 32 +- vectordb_bench/cli/vectordbbench.py | 2 + 15 files changed, 1064 insertions(+), 34 deletions(-) create mode 100644 tests/test_antfly.py create mode 100644 tests/test_filtered_adapters.py create mode 100644 vectordb_bench/backend/clients/antfly/__init__.py create mode 100644 vectordb_bench/backend/clients/antfly/antfly.py create mode 100644 vectordb_bench/backend/clients/antfly/cli.py create mode 100644 vectordb_bench/backend/clients/antfly/config.py diff --git a/pyproject.toml b/pyproject.toml index 223291721..ddbd3425b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,8 +55,8 @@ test = [ restful = [ "flask" ] qdrant = [ "qdrant-client" ] pinecone = [ "pinecone" ] -weaviate = [ "weaviate-client" ] -elastic = [ "elasticsearch" ] +weaviate = [ "weaviate-client>=3.26.7,<4" ] +elastic = [ "elasticsearch>=8,<9" ] # For elastic and aliyun_elasticsearch pgvector = [ "psycopg", "psycopg-binary", "pgvector" ] @@ -87,6 +87,7 @@ seekdb = [ "mysql-connector-python" ] volc_mysql = [ "mysql-connector-python" ] pinot = [ "requests" ] adbpg = [ "psycopg", "psycopg-binary", "pgvector" ] +antfly = [ "httpx" ] [project.urls] Repository = "https://github.com/zilliztech/VectorDBBench" diff --git a/tests/test_antfly.py b/tests/test_antfly.py new file mode 100644 index 000000000..024a3b14f --- /dev/null +++ b/tests/test_antfly.py @@ -0,0 +1,107 @@ +import base64 +import struct +from typing import Any + +import pytest + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.antfly.antfly import Antfly +from vectordb_bench.backend.clients.antfly.config import AntflyConfig, AntflyIndexConfig +from vectordb_bench.backend.filter import IntFilter, LabelFilter, non_filter +from vectordb_bench.backend.payload import PayloadProfile + + +class _CaseConfig: + def index_param(self): + return {"distance_metric": "cosine"} + + def search_param(self): + return {"search_effort": 0.6} + + +class _Response: + is_success = True + status_code = 200 + text = "" + + def raise_for_status(self): + return None + + +class _Client: + def __init__(self): + self.posts = [] + + def post(self, path: str, *, json: dict[str, Any]): + self.posts.append((path, json)) + return _Response() + + +def _adapter(*, with_scalar_labels: bool = False): + adapter = Antfly.__new__(Antfly) + adapter.case_config = _CaseConfig() + adapter.collection_name = "vdbbench" + adapter.with_scalar_labels = with_scalar_labels + adapter._filter_query = None + adapter._pack_query_vectors = True + adapter._legacy_api = False + adapter._write_sync_level = "write" + adapter._bench_status_last_log = 0.0 + adapter.client = _Client() + return adapter + + +def test_antfly_is_registered(): + assert DB.Antfly.config_cls is AntflyConfig + assert DB.Antfly.case_config_cls() is AntflyIndexConfig + assert DB.Antfly.init_cls is Antfly + + +def test_antfly_uses_native_cosine_and_ids_only_queries(): + adapter = _adapter() + + assert adapter.need_normalize_cosine() is False + body = adapter._metadata_query_body([1.0, 2.0], 10) + assert body["fields"] == [] + assert body["search_effort"] == 0.6 + + adapter.prepare_filter(IntFilter(filter_rate=0.99, int_field="id", int_value=49_500)) + assert adapter._metadata_query_body([1.0, 2.0], 10)["filter_query"] == { + "numeric_range": {"field": "id", "min": 49_500, "inclusive_min": True} + } + + adapter.prepare_filter(LabelFilter(label_percentage=0.01)) + assert adapter._filter_query == {"term": {"labels": "label_1p"}} + adapter.prepare_filter(non_filter) + assert "filter_query" not in adapter._metadata_query_body([1.0, 2.0], 10) + + with pytest.raises(NotImplementedError): + adapter.search_embedding([1.0, 2.0], payload_profile=PayloadProfile.VECTOR) + + +def test_antfly_insert_preserves_vectors_and_labels(): + adapter = _adapter(with_scalar_labels=True) + + inserted, error = adapter.insert_embeddings([[3.0, 4.0]], [7], labels_data=["bucket_007"]) + + assert error is None + assert inserted == 1 + _, payload = adapter.client.posts[0] + row = payload["inserts"]["key:7"] + vector = base64.b64decode(row["_embeddings"]["vec"]) + assert struct.unpack("<2f", vector) == pytest.approx((3.0, 4.0)) + assert row["labels"] == "bucket_007" + assert payload["sync_level"] == "write" + + +def test_write_readiness_probe_does_not_create_a_tombstone(): + adapter = _adapter() + client = _Client() + + adapter._wait_for_write_ready(client) + + assert len(client.posts) == 1 + assert client.posts[0][1] == { + "inserts": {"key:__circus_write_probe__": {"id": -1}}, + "sync_level": "write", + } diff --git a/tests/test_filtered_adapters.py b/tests/test_filtered_adapters.py new file mode 100644 index 000000000..a5d0d713f --- /dev/null +++ b/tests/test_filtered_adapters.py @@ -0,0 +1,79 @@ +from importlib import import_module +from typing import Any + +import pytest + +from vectordb_bench.backend.filter import FilterOp, IntFilter, non_filter + + +@pytest.fixture +def numeric_filter() -> IntFilter: + return IntFilter(filter_rate=0.99, int_field="id", int_value=49_500) + + +def _adapter_class(dependency: str, module: str, name: str) -> Any: + pytest.importorskip(dependency) + return getattr(import_module(module), name) + + +def test_qdrant_numeric_filter_is_inclusive(numeric_filter: IntFilter): + adapter_cls = _adapter_class( + "qdrant_client", + "vectordb_bench.backend.clients.qdrant_local.qdrant_local", + "QdrantLocal", + ) + + assert FilterOp.NumGE in adapter_cls.supported_filter_types + adapter = adapter_cls.__new__(adapter_cls) + adapter._primary_field = "pk" + adapter.prepare_filter(numeric_filter) + numeric_range = adapter._query_filter.must[0].range + assert numeric_range.gte == 49_500 + assert numeric_range.gt is None + adapter.prepare_filter(non_filter) + assert adapter._query_filter is None + + +def test_weaviate_numeric_filter_is_inclusive(numeric_filter: IntFilter): + adapter_cls = _adapter_class( + "weaviate", + "vectordb_bench.backend.clients.weaviate_cloud.weaviate_cloud", + "WeaviateCloud", + ) + + assert FilterOp.NumGE in adapter_cls.supported_filter_types + adapter = adapter_cls.__new__(adapter_cls) + adapter._scalar_field = "key" + adapter.prepare_filter(numeric_filter) + assert adapter._where_filter == { + "path": ["key"], + "operator": "GreaterThanEqual", + "valueInt": 49_500, + } + + +def test_chroma_numeric_filter_is_inclusive(numeric_filter: IntFilter): + adapter_cls = _adapter_class( + "chromadb", + "vectordb_bench.backend.clients.chroma.chroma", + "ChromaClient", + ) + + assert FilterOp.NumGE in adapter_cls.supported_filter_types + adapter = adapter_cls.__new__(adapter_cls) + adapter.prepare_filter(numeric_filter) + assert adapter._where_filter == {"index": {"$gte": 49_500}} + + +def test_elasticsearch_numeric_filter_is_inclusive(numeric_filter: IntFilter): + adapter_cls = _adapter_class( + "elasticsearch", + "vectordb_bench.backend.clients.elastic_cloud.elastic_cloud", + "ElasticCloud", + ) + + assert FilterOp.NumGE in adapter_cls.supported_filter_types + adapter = adapter_cls.__new__(adapter_cls) + adapter.id_col_name = "id" + adapter.prepare_filter(numeric_filter) + assert adapter.filter == {"range": {"id": {"gte": 49_500}}} diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index beacb37af..81c65c4e0 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -65,6 +65,7 @@ class DB(Enum): SeekDB = "SeekDB" VolcMySQL = "VolcMySQL" Adbpg = "AnalyticDB for PostgreSQL" + Antfly = "Antfly" @property def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 @@ -281,6 +282,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 return Adbpg + if self == DB.Antfly: + from .antfly.antfly import Antfly + + return Antfly + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -499,6 +505,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 return AdbpgConfig + if self == DB.Antfly: + from .antfly.config import AntflyConfig + + return AntflyConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @@ -719,6 +730,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return AdbpgIndexConfig + if self == DB.Antfly: + from .antfly.config import AntflyIndexConfig + + return AntflyIndexConfig + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/antfly/__init__.py b/vectordb_bench/backend/clients/antfly/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/backend/clients/antfly/antfly.py b/vectordb_bench/backend/clients/antfly/antfly.py new file mode 100644 index 000000000..be9a99c5b --- /dev/null +++ b/vectordb_bench/backend/clients/antfly/antfly.py @@ -0,0 +1,618 @@ +import base64 +import json +import logging +import os +import struct +import time +from contextlib import contextmanager +from typing import Any + +import httpx + +from ...filter import Filter, FilterOp +from ...payload import PayloadProfile +from ..api import DBCaseConfig, VectorDB + +log = logging.getLogger(__name__) + +BATCH_CHUNK_SIZE = 500 +TABLE_READY_TIMEOUT = 30 +TABLE_READY_POLL_INTERVAL = 2 +INDEX_READY_TIMEOUT = 7200 +INDEX_READY_POLL_INTERVAL = 2 +INDEX_NAME = "vec" +INDEX_TYPES = ("embeddings", "aknn_v0") +SOURCE_FIELD = "vec_data" + + +def _httpx_host(host: str) -> str: + # macOS resolves localhost to ::1 first. The current antfly-zig listener is + # IPv4-only, so keep the user-facing flag but route httpx to IPv4 loopback. + return "127.0.0.1" if host == "localhost" else host + + +def _pack_dense_f32(values: list[float]) -> str: + return base64.b64encode(struct.pack(f"<{len(values)}f", *values)).decode("ascii") + + +def _make_client(base_url: str, timeout: float) -> httpx.Client: + raw_timeout = os.environ.get("ANTFLY_VDBBENCH_HTTP_TIMEOUT") + if raw_timeout: + try: + timeout = float(raw_timeout) + except ValueError: + log.warning("Ignoring invalid ANTFLY_VDBBENCH_HTTP_TIMEOUT=%r", raw_timeout) + return httpx.Client( + base_url=base_url, + timeout=timeout, + limits=httpx.Limits(max_keepalive_connections=8, max_connections=8), + ) + + +def _detect_api_root(host: str, port: int) -> str: + # Prefer the current antfly-zig public API root, falling back to the + # legacy /api/v1 root served by Go binaries. The content-type check + # matters: legacy binaries answer unknown /db/v1 paths with the dashboard + # SPA (200 text/html), not a 404. + for root in ("/db/v1", "/api/v1"): + try: + r = httpx.get(f"http://{host}:{port}{root}/tables", timeout=5) + except httpx.HTTPError: + continue + if r.status_code < 500 and "json" in r.headers.get("content-type", ""): + return root + return "/db/v1" + + +class Antfly(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DBCaseConfig, + collection_name: str = "vdbbench", + drop_old: bool = False, + with_scalar_labels: bool = False, + **kwargs, + ): + self.db_config = db_config + self.case_config = db_case_config + self.collection_name = collection_name + self.dim = dim + self.with_scalar_labels = with_scalar_labels + self._filter_query: dict[str, Any] | None = None + + # Antfly v0.1 used /api/v1; current antfly-zig serves the public DB API + # at /db/v1. Auto-detect unless ANTFLY_API_ROOT pins it explicitly. + api_root = os.environ.get("ANTFLY_API_ROOT", "").rstrip("/") + if not api_root: + api_root = _detect_api_root(_httpx_host(db_config["host"]), db_config["port"]) + log.info(f"Detected Antfly API root: {api_root}") + self._legacy_api = api_root == "/api/v1" + self._metadata_base_url = f"http://{_httpx_host(db_config['host'])}:{db_config['port']}{api_root}" + self._store_host = _httpx_host(db_config.get("store_host") or db_config["host"]) + self._store_port = db_config.get("store_port") + self._use_direct_store_search = bool(db_config.get("use_direct_store_search")) + self._pack_query_vectors = bool(db_config.get("pack_query_vectors")) + self._write_sync_level = os.environ.get("ANTFLY_VDBBENCH_SYNC_LEVEL", "write") + self._direct_shard_id: str | None = None + self._bench_status_last_log = 0.0 + num_shards = db_config.get("num_shards", 1) + + if self._use_direct_store_search and not self._store_port: + raise ValueError("Antfly direct store search requires store_port to be configured") + + client = _make_client(self._metadata_base_url, 60) + try: + if drop_old: + r = client.delete(f"/tables/{self.collection_name}") + log.info(f"Drop table response: {r.status_code}") + + table = self._get_table_status_or_none(client) + if table is None: + r = client.post(f"/tables/{self.collection_name}", json={"num_shards": num_shards}) + log.info(f"Create table response: {r.status_code}") + r.raise_for_status() + else: + log.info("Reusing existing table: %s", self.collection_name) + + self._wait_for_shard_ready(client) + # Wait for the write path before touching indexes: legacy (Go) + # binaries can permanently orphan a shard if an index-add lands + # while the shard is still initializing. + self._wait_for_write_ready(client) + + self._ensure_external_index(client, dim) + # Do not wait for an empty external index to finish rebuilding here. + # antfly-zig keeps an empty external dense index in backfill state + # until writes arrive, and optimize(data_size=...) performs the + # real post-load readiness wait below. + self._refresh_direct_search_routing(client) + finally: + client.close() + + def _ensure_external_index(self, client: httpx.Client, dim: int) -> None: + """Create the external embeddings index, verifying shard registration. + + Tries each index type, with and without an explicit source field, to + handle both old binaries (require field) and new source (reject field + with external). + + On the legacy API a 2xx from the create call is not proof of success: + either variant can be accepted by the metadata layer and then rejected + by shard-level registration, in which case the reconciler retries the + bad config forever and every vector write fails with "index not + found". The two known legacy behaviours are opposites -- some builds + require the field ("field or template must be specified"), v0.1.3 + rejects it ("external embeddings index config cannot specify field or + template") -- so ordering alone cannot be right. Instead, after each + 2xx poll the index status until every shard reports a non-null + registration; delete and move on if it stays broken. + """ + if self._get_index_status(client) is not None: + log.info("Reusing existing embeddings index: %s", INDEX_NAME) + return + index_def = { + "name": INDEX_NAME, + "dimension": dim, + "external": True, + **self.case_config.index_param(), + } + field_variants = ({}, {"field": SOURCE_FIELD}) + index_error = None + for index_type in INDEX_TYPES: + for extra in field_variants: + r = client.post( + f"/tables/{self.collection_name}/indexes/{INDEX_NAME}", + json={"type": index_type, **index_def, **extra}, + ) + log.info(f"Add embeddings index response ({index_type}, field={'field' in extra}): {r.status_code}") + if not r.is_success: + index_error = r + continue + if not self._legacy_api or self._wait_for_index_registration(client): + index_error = None + break + log.warning( + f"Index ({index_type}, field={'field' in extra}) was accepted by " + "the metadata layer but never registered on the shard; " + "deleting it and trying the next variant" + ) + client.delete(f"/tables/{self.collection_name}/indexes/{INDEX_NAME}") + index_error = r + if index_error is None: + break + if index_error is not None: + index_error.raise_for_status() + + def _wait_for_shard_ready(self, client: httpx.Client): + deadline = time.monotonic() + TABLE_READY_TIMEOUT + while time.monotonic() < deadline: + try: + table = self._get_table_status_or_none(client) + if table is not None: + log.info("Shard metadata is ready") + return + except Exception as exc: + log.debug("Shard readiness probe failed", exc_info=exc) + time.sleep(TABLE_READY_POLL_INTERVAL) + log.warning(f"Shard readiness timeout after {TABLE_READY_TIMEOUT}s, proceeding anyway") + + def _wait_for_write_ready(self, client: httpx.Client): + # Table metadata appearing does not mean shards accept writes yet: + # legacy (Go) binaries return 500 "shard is still initializing" for a + # few seconds after table creation. Probe with a throwaway document + # (no embedding, so it never lands in the vector index) until a batch + # write succeeds. The probe doc is intentionally left in place: deleting + # it would leave a tombstone, and a table that has ever deleted a doc + # loses the "all docs visible" fast path. Every dense query would then + # materialize the complete live-doc set as a positive filter. + probe_key = "key:__circus_write_probe__" + deadline = time.monotonic() + TABLE_READY_TIMEOUT + last_error = None + while time.monotonic() < deadline: + try: + r = client.post( + f"/tables/{self.collection_name}/batch", + json={"inserts": {probe_key: {"id": -1}}, "sync_level": "write"}, + ) + if r.is_success: + log.info("Write path is ready") + return + last_error = f"{r.status_code}: {r.text[:120]}" + except Exception as exc: + last_error = str(exc) + time.sleep(TABLE_READY_POLL_INTERVAL) + log.warning(f"Write readiness timeout after {TABLE_READY_TIMEOUT}s ({last_error}), proceeding anyway") + + def _get_index_status(self, client: httpx.Client) -> dict | None: + r = client.get(f"/tables/{self.collection_name}/indexes/{INDEX_NAME}") + if r.status_code == 404: + return None + r.raise_for_status() + return r.json() + + def _wait_for_index_registration(self, client: httpx.Client) -> bool: + """True once every shard reports a non-null registration for the index. + + Legacy binaries accept external-index configs at the metadata layer + and then fail to register them on the shard; while that is unresolved + the status endpoint reports null shard entries. Give the reconciler a + bounded window to register (or visibly fail) before the caller deletes + the index and tries the next variant. + """ + deadline = time.monotonic() + 8 * TABLE_READY_POLL_INTERVAL + while time.monotonic() < deadline: + status = self._get_index_status(client) + shards = (status or {}).get("shard_status") or {} + if shards and all(isinstance(entry, dict) for entry in shards.values()): + return True + time.sleep(TABLE_READY_POLL_INTERVAL) + return False + + def _get_table_status(self, client: httpx.Client) -> dict: + r = client.get(f"/tables/{self.collection_name}") + r.raise_for_status() + return r.json() + + def _get_table_status_or_none(self, client: httpx.Client) -> dict | None: + r = client.get(f"/tables/{self.collection_name}") + if r.status_code == 404: + return None + r.raise_for_status() + return r.json() + + def _bench_status_enabled(self) -> bool: + return os.environ.get("ANTFLY_BENCH_STATUS") == "1" + + def _bench_status_interval(self) -> float: + raw = os.environ.get("ANTFLY_BENCH_STATUS_INTERVAL", "30") + try: + return max(float(raw), 1.0) + except ValueError: + return 30.0 + + def _maybe_log_bench_status( + self, + client: httpx.Client, + phase: str, + *, + force: bool = False, + ) -> None: + if not self._bench_status_enabled(): + return + now = time.monotonic() + if not force and now - self._bench_status_last_log < self._bench_status_interval(): + return + self._bench_status_last_log = now + + try: + table = self._get_table_status_or_none(client) + index = self._get_index_status(client) + log.info( + "antfly_bench_status %s", + json.dumps( + { + "phase": phase, + "table": self._compact_table_status(table), + "index": self._compact_index_status(index), + }, + sort_keys=True, + separators=(",", ":"), + ), + ) + except Exception as exc: + log.warning("Antfly bench status probe failed: %s", exc) + + @staticmethod + def _compact_table_status(table: dict | None) -> dict | None: + if table is None: + return None + shards = table.get("shards") or {} + storage = table.get("storage_status") or {} + return { + "name": table.get("name"), + "shard_count": len(shards), + "empty": storage.get("empty"), + "lsm": storage.get("lsm"), + } + + @staticmethod + def _compact_index_status(index: dict | None) -> dict | None: + if index is None: + return None + status = index.get("status") or {} + async_indexing = status.get("async_indexing") or {} + dense_catch_up = async_indexing.get("dense_catch_up") or {} + hbc_cache = status.get("hbc_cache") or {} + return { + "type": (index.get("config") or {}).get("type"), + "rebuilding": status.get("rebuilding"), + "total_indexed": status.get("total_indexed"), + "doc_count": status.get("doc_count"), + "total_nodes": status.get("total_nodes"), + "query_visible_doc_count": status.get("query_visible_doc_count"), + "published_doc_count": status.get("published_doc_count"), + "backfill_state": status.get("backfill_state"), + "backfill_progress": status.get("backfill_progress"), + "catch_up_active": status.get("catch_up_active"), + "catch_up_phase": status.get("catch_up_phase"), + "catch_up_applied_sequence": status.get("catch_up_applied_sequence"), + "catch_up_target_sequence": status.get("catch_up_target_sequence"), + "dense_publish_pending": status.get("dense_publish_pending"), + "dense_catch_up": { + "phase": dense_catch_up.get("phase"), + "current_sequence": dense_catch_up.get("current_sequence"), + "current_target_sequence": dense_catch_up.get("current_target_sequence"), + "finish_calls": dense_catch_up.get("finish_calls"), + "finish_ns": dense_catch_up.get("finish_ns"), + "finalize_ns": dense_catch_up.get("finalize_ns"), + "maintenance_steps": dense_catch_up.get("maintenance_steps"), + "maintenance_ns": dense_catch_up.get("maintenance_ns"), + "manifest_writes": dense_catch_up.get("manifest_writes"), + "write_pressure_compactions": dense_catch_up.get("write_pressure_compactions"), + "write_pressure_ns": dense_catch_up.get("write_pressure_ns"), + }, + "hbc_cache_total_bytes": hbc_cache.get("total_bytes"), + } + + def _refresh_direct_search_routing(self, client: httpx.Client): + if not self._use_direct_store_search: + return + table = self._get_table_status(client) + shards = table.get("shards") or {} + if len(shards) != 1: + msg = f"Antfly direct store search currently requires exactly one shard; found {len(shards)} shards" + raise ValueError(msg) + self._direct_shard_id = next(iter(shards)) + + def _index_status_is_ready( + self, + payload: dict | None, + status: dict | None, + expected_total: int | None = None, + ) -> bool: + if payload is None: + return False + if status is None: + return expected_total == 0 + + rebuilding = bool(status.get("rebuilding")) + wal_backlog = int(status.get("wal_backlog", 0) or 0) + total_indexed = int(status.get("total_indexed", 0) or 0) + has_error = bool(status.get("error")) + + if has_error or rebuilding or wal_backlog > 0: + return False + return expected_total is None or total_indexed >= expected_total + + def _wait_for_index_ready(self, client: httpx.Client, expected_total: int | None = None): + deadline = time.monotonic() + INDEX_READY_TIMEOUT + last_status = None + + while time.monotonic() < deadline: + try: + payload = self._get_index_status(client) + status = payload.get("status") if payload else None + last_status = status + self._maybe_log_bench_status(client, "optimize_wait") + if self._index_status_is_ready(payload, status, expected_total): + log.info( + "Embeddings index is ready: %s", + self._compact_index_status(payload), + ) + return + except Exception as e: + last_status = {"error": str(e)} + time.sleep(INDEX_READY_POLL_INTERVAL) + + log.warning( + "Embeddings index readiness timeout after %ss, expected_total=%s, last_status=%s", + INDEX_READY_TIMEOUT, + expected_total, + last_status, + ) + + @contextmanager + def init(self): + self.client = _make_client(self._metadata_base_url, 120) + self.store_client = None + try: + if self._use_direct_store_search: + self.store_client = _make_client(self._store_base_url, 120) + yield + finally: + self.client.close() + self.client = None + if self.store_client is not None: + self.store_client.close() + self.store_client = None + + @property + def _store_base_url(self) -> str: + if self._store_port is None: + raise ValueError("Antfly store_base_url requested without store_port configured") + return f"http://{self._store_host}:{self._store_port}" + + def need_normalize_cosine(self) -> bool: + # Antfly computes cosine norms natively. Returning True would make the + # shared runner normalize every vector before the adapter sees it. + return False + + @staticmethod + def _pack_vector(vector: list[float]) -> str: + raw = struct.pack(f"<{len(vector)}f", *vector) + return base64.b64encode(raw).decode("ascii") + + def _serialize_query_vector(self, vector: list[float]) -> list[float] | str: + if self._pack_query_vectors or os.environ.get("ANTFLY_PACK_VECTORS") == "1": + return self._pack_vector(vector) + return vector + + def _serialize_insert_vector(self, vector: list[float]) -> str | list[float]: + # Legacy 0.1.0 binaries reject the packed base64 format on writes + # ("embedding ... must be an array (dense) or object (sparse), got + # string"); plain JSON arrays are accepted by every version. + if self._legacy_api: + return vector + return self._pack_vector(vector) + + def _metadata_query_body(self, query: list[float], k: int) -> dict[str, Any]: + body = { + "embeddings": {"vec": self._serialize_query_vector(query)}, + "limit": k, + "fields": [], + **self.case_config.search_param(), + } + if getattr(self, "_filter_query", None) is not None: + body["filter_query"] = self._filter_query + return body + + def _store_query_body(self, query: list[float], k: int) -> dict[str, Any]: + search_params = self.case_config.search_param() + vector_paging_options: dict[str, Any] = {"limit": k} + if "search_effort" in search_params: + vector_paging_options["search_effort"] = search_params["search_effort"] + return { + "star": True, + "limit": k, + "vector_searches": {INDEX_NAME: self._serialize_query_vector(query)}, + "vector_paging_options": vector_paging_options, + } + + def _parse_metadata_hits(self, data: dict) -> list[int]: + resp = data.get("responses", [{}])[0] + hits_obj = resp.get("hits") or {} + hits = hits_obj.get("hits") or [] + results = [] + for hit in hits: + doc_key = hit.get("_id", "") + try: + results.append(int(doc_key.split(":", 1)[1])) + except (IndexError, ValueError): + log.warning(f"Could not parse id from _id: {doc_key}") + return results + + def _parse_store_hits(self, data: dict) -> list[int]: + vec_result = (data.get("search_result") or {}).get(INDEX_NAME) or {} + hits = vec_result.get("hits") or [] + results = [] + for hit in hits: + fields = hit.get("fields") or {} + if "id" in fields: + results.append(int(fields["id"])) + continue + doc_key = hit.get("id", "") + try: + results.append(int(doc_key.split(":", 1)[1])) + except (IndexError, ValueError): + log.warning(f"Could not parse id from direct-store hit id: {doc_key}") + return results + + def ready_to_search(self) -> bool: + if getattr(self, "client", None) is not None: + payload = self._get_index_status(self.client) + return self._index_status_is_ready(payload, payload.get("status") if payload else None) + with _make_client(self._metadata_base_url, 120) as client: + payload = self._get_index_status(client) + return self._index_status_is_ready(payload, payload.get("status") if payload else None) + + def optimize(self, data_size: int | None = None): + if getattr(self, "client", None) is not None: + self._maybe_log_bench_status(self.client, "optimize_start", force=True) + self._wait_for_index_ready(self.client, expected_total=data_size) + self._maybe_log_bench_status(self.client, "optimize_end", force=True) + return + with _make_client(self._metadata_base_url, 120) as client: + self._maybe_log_bench_status(client, "optimize_start", force=True) + self._wait_for_index_ready(client, expected_total=data_size) + self._maybe_log_bench_status(client, "optimize_end", force=True) + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self._filter_query = None + elif filters.type == FilterOp.NumGE: + self._filter_query = { + "numeric_range": { + "field": filters.int_field, + "min": filters.int_value, + "inclusive_min": True, + } + } + elif filters.type == FilterOp.StrEqual: + self._filter_query = {"term": {filters.label_field: filters.label_value}} + else: + msg = f"Unsupported Antfly filter: {filters}" + raise ValueError(msg) + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs: Any, + ) -> tuple[int, Exception]: + total = len(embeddings) + if self.with_scalar_labels and labels_data is None: + return 0, ValueError("Antfly label-filter load requires labels_data") + try: + for start in range(0, total, BATCH_CHUNK_SIZE): + end = min(start + BATCH_CHUNK_SIZE, total) + inserts = {} + for i in range(start, end): + key = f"key:{metadata[i]}" + serialized_embedding = self._serialize_insert_vector(embeddings[i]) + inserts[key] = { + "id": metadata[i], + "metadata": metadata[i], + SOURCE_FIELD: str(metadata[i]), + "_embeddings": {"vec": serialized_embedding}, + } + if self.with_scalar_labels: + inserts[key]["labels"] = labels_data[i] + payload = {"inserts": inserts, "sync_level": self._write_sync_level} + r = self.client.post(f"/tables/{self.collection_name}/batch", json=payload) + r.raise_for_status() + self._maybe_log_bench_status(self.client, "insert") + except Exception as e: + log.warning(f"Antfly insert error: {e}") + return 0, e + return total, None + + def search_embedding( + self, + query: list[float], + k: int = 100, + payload_profile: PayloadProfile = PayloadProfile.IDS_ONLY, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + if payload_profile != PayloadProfile.IDS_ONLY: + msg = f"Antfly VDBBench adapter only supports payload_profile={PayloadProfile.IDS_ONLY.value}" + raise NotImplementedError(msg) + if self._use_direct_store_search and self._filter_query is not None: + raise ValueError("Antfly filtered ANN requires the public metadata query API") + + if self._use_direct_store_search: + if self._direct_shard_id is None: + self._refresh_direct_search_routing(self.client) + r = self.store_client.post( + "/search", + headers={"X-Raft-Shard-Id": self._direct_shard_id}, + json=self._store_query_body(query, k), + ) + r.raise_for_status() + return self._parse_store_hits(r.json()) + r = self.client.post( + f"/tables/{self.collection_name}/query", + json=self._metadata_query_body(query, k), + ) + r.raise_for_status() + return self._parse_metadata_hits(r.json()) diff --git a/vectordb_bench/backend/clients/antfly/cli.py b/vectordb_bench/backend/clients/antfly/cli.py new file mode 100644 index 000000000..25029c30a --- /dev/null +++ b/vectordb_bench/backend/clients/antfly/cli.py @@ -0,0 +1,107 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + get_custom_case_config, + run, +) +from ...cases import CaseType +from .. import DB +from ..api import MetricType + + +class AntflyTypedDict(TypedDict): + host: Annotated[ + str, click.option("--host", type=str, help="Antfly metadata API host", default="localhost", show_default=True) + ] + port: Annotated[ + int, click.option("--port", type=int, help="Antfly metadata API port", default=8080, show_default=True) + ] + store_host: Annotated[str, click.option("--store-host", type=str, help="Antfly store API host", default=None)] + store_port: Annotated[ + int, click.option("--store-port", type=int, help="Antfly store API port for direct search", default=None) + ] + username: Annotated[str, click.option("--username", type=str, help="Antfly username", default=None)] + password: Annotated[str, click.option("--password", type=str, help="Antfly password", default=None)] + num_shards: Annotated[ + int, click.option("--num-shards", type=int, help="Number of shards", default=1, show_default=True) + ] + use_direct_store_search: Annotated[ + bool, + click.option( + "--use-direct-store-search/--no-direct-store-search", + help="Query Antfly through the store /search API instead of the metadata table query API", + default=False, + show_default=True, + ), + ] + pack_query_vectors: Annotated[ + bool, + click.option( + "--pack-query-vectors/--no-pack-query-vectors", + help="Send dense vectors in Antfly's packed base64 float32 wire format for inserts and queries", + default=True, + show_default=True, + ), + ] + search_effort: Annotated[ + float | None, + click.option( + "--search-effort", + type=float, + help="Search effort 0.0-1.0 (higher=better recall, slower)", + default=None, + show_default=True, + ), + ] + metric_type: Annotated[ + str | None, + click.option( + "--metric-type", + type=click.Choice([metric.value for metric in MetricType]), + help="Distance metric override. Defaults to the selected VDBBench case dataset metric.", + default=None, + ), + ] + + +class AntflyAKNNTypedDict(CommonTypedDict, AntflyTypedDict): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(AntflyAKNNTypedDict) +def AntflyAKNN(**parameters: Unpack[AntflyAKNNTypedDict]): + from .config import AntflyConfig, AntflyIndexConfig + + metric_type = ( + MetricType(parameters["metric_type"]) + if parameters["metric_type"] + else CaseType[parameters["case_type"]].case_cls(get_custom_case_config(parameters)).dataset.data.metric_type + ) + + run( + db=DB.Antfly, + db_config=AntflyConfig( + db_label=parameters["db_label"], + host=parameters["host"], + port=parameters["port"], + store_host=parameters["store_host"], + store_port=parameters["store_port"], + username=SecretStr(parameters["username"]) if parameters["username"] else None, + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=parameters["num_shards"], + use_direct_store_search=parameters["use_direct_store_search"], + pack_query_vectors=parameters["pack_query_vectors"], + ), + db_case_config=AntflyIndexConfig( + num_shards=parameters["num_shards"], + search_effort=parameters["search_effort"], + metric_type=metric_type, + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/antfly/config.py b/vectordb_bench/backend/clients/antfly/config.py new file mode 100644 index 000000000..aae2cf035 --- /dev/null +++ b/vectordb_bench/backend/clients/antfly/config.py @@ -0,0 +1,49 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType + + +class AntflyConfig(DBConfig): + host: str = "localhost" + port: int = 8080 + store_host: str | None = None + store_port: int | None = None + username: SecretStr | None = None + password: SecretStr | None = None + num_shards: int = 1 + use_direct_store_search: bool = False + pack_query_vectors: bool = True + + def to_dict(self) -> dict: + return { + "host": self.host, + "port": self.port, + "store_host": self.store_host, + "store_port": self.store_port, + "username": self.username.get_secret_value() if self.username else None, + "password": self.password.get_secret_value() if self.password else None, + "num_shards": self.num_shards, + "use_direct_store_search": self.use_direct_store_search, + "pack_query_vectors": self.pack_query_vectors, + } + + +class AntflyIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + num_shards: int = 1 + search_effort: float | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.COSINE: + return "cosine" + if self.metric_type in (MetricType.IP, MetricType.DP): + return "inner_product" + return "l2_squared" + + def index_param(self) -> dict: + return {"distance_metric": self.parse_metric()} + + def search_param(self) -> dict: + if self.search_effort is not None: + return {"search_effort": self.search_effort} + return {} diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index 6942f4fc9..909c47966 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -3,6 +3,8 @@ import chromadb +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import VectorDB from .config import ChromaIndexConfig @@ -17,6 +19,11 @@ class ChromaClient(VectorDB): To change to running in process, modify the HttpClient() in __init__() and init(). """ + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + def __init__( self, dim: int, @@ -35,13 +42,14 @@ def __init__( if drop_old: try: - client.reset() - except Exception: - drop_old = False log.info(f"Chroma client drop_old collection: {self.collection_name}") + client.delete_collection(self.collection_name) + except Exception as e: + log.info(f"Chroma client collection was not dropped: {self.collection_name}, error: {e!s}") self.client = None self.collection = None + self._where_filter: dict | None = None @contextmanager def init(self): @@ -85,10 +93,17 @@ def search_embedding( self, query: list[float], k: int = 100, filters: dict | None = None, timeout: int | None = None ) -> list[int]: assert self.client is not None, "Please call self.init() before" - if filters: - results = self.collection.query( - query_embeddings=[query], n_results=k, where={"id": {"$gt": filters.get("id")}} - ) + if self._where_filter is not None: + results = self.collection.query(query_embeddings=[query], n_results=k, where=self._where_filter) else: results = self.collection.query(query_embeddings=[query], n_results=k) return [int(idx) for idx in results["ids"][0]] + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self._where_filter = None + elif filters.type == FilterOp.NumGE: + self._where_filter = {"index": {"$gte": filters.int_value}} + else: + msg = f"Unsupported Chroma filter: {filters}" + raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/chroma/cli.py b/vectordb_bench/backend/clients/chroma/cli.py index 64a2f972f..3445660a8 100644 --- a/vectordb_bench/backend/clients/chroma/cli.py +++ b/vectordb_bench/backend/clients/chroma/cli.py @@ -50,6 +50,7 @@ def Chroma(**parameters: Unpack[ChromaTypeDict]): run( db=DBTYPE, db_config=ChromaConfig( + db_label=parameters["db_label"], user=parameters["user"], password=SecretStr(parameters["password"]) if parameters["password"] else None, host=SecretStr(parameters["host"]), diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index c930dc057..92951a0dc 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -217,18 +217,19 @@ def insert_documents( def prepare_filter(self, filters: Filter): self.routing_key = None + is_fts = getattr(self, "_is_fts", False) if filters.type == FilterOp.NonFilter: - self.filter = None if self._is_fts else [] + self.filter = None if is_fts else [] elif filters.type == FilterOp.NumGE: - if self._is_fts: + if is_fts: if getattr(filters, "int_field", None) != self.filter_id_col_name: msg = f"ElasticCloud FTS filters only support int_field='{self.filter_id_col_name}'" raise ValueError(msg) self.filter = {"range": {self.filter_id_col_name: {"gte": filters.int_value}}} else: - self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} + self.filter = {"range": {self.id_col_name: {"gte": filters.int_value}}} elif filters.type == FilterOp.StrEqual: - if self._is_fts: + if is_fts: msg = f"Not support Filter for ElasticCloud FTS - {filters}" raise ValueError(msg) self.filter = {"term": {self.label_col_name: filters.label_value}} diff --git a/vectordb_bench/backend/clients/qdrant_local/cli.py b/vectordb_bench/backend/clients/qdrant_local/cli.py index 7995b99b3..ee3309dc2 100644 --- a/vectordb_bench/backend/clients/qdrant_local/cli.py +++ b/vectordb_bench/backend/clients/qdrant_local/cli.py @@ -49,7 +49,10 @@ def QdrantLocal(**parameters: Unpack[QdrantLocalTypedDict]): run( db=DBTYPE, - db_config=QdrantLocalConfig(url=SecretStr(parameters["url"])), + db_config=QdrantLocalConfig( + db_label=parameters["db_label"], + url=SecretStr(parameters["url"]), + ), db_case_config=QdrantLocalIndexConfig( on_disk=parameters["on_disk"], m=parameters["m"], diff --git a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py index 15c790c61..e8e253ac2 100644 --- a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py +++ b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py @@ -19,6 +19,9 @@ VectorParams, ) +from vectordb_bench.backend.filter import Filter as BenchFilter +from vectordb_bench.backend.filter import FilterOp + from ..api import VectorDB from .config import QdrantLocalIndexConfig @@ -40,6 +43,11 @@ def qdrant_collection_exists(client: QdrantClient, collection_name: str) -> bool class QdrantLocal(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + def __init__( self, dim: int, @@ -60,6 +68,7 @@ def __init__( self._primary_field = "pk" self._vector_field = "vector" + self._query_filter: Filter | None = None client = QdrantClient(**self.db_config) @@ -209,25 +218,29 @@ def search_embedding( """ assert self.client is not None - f = None - if filters: - f = Filter( - must=[ - FieldCondition( - key=self._primary_field, - range=Range( - gt=filters.get("id"), - ), - ), - ], - ) res = self.client.query_points( collection_name=self.collection_name, query=query, limit=k, - query_filter=f, + query_filter=self._query_filter, search_params=SearchParams(**self.search_parameter), timeout=timeout, ).points return [result.id for result in res] + + def prepare_filter(self, filters: BenchFilter): + if filters.type == FilterOp.NonFilter: + self._query_filter = None + elif filters.type == FilterOp.NumGE: + self._query_filter = Filter( + must=[ + FieldCondition( + key=self._primary_field, + range=Range(gte=filters.int_value), + ) + ] + ) + else: + msg = f"Unsupported QdrantLocal filter: {filters}" + raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index d6111c8da..453a22296 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -7,12 +7,20 @@ import weaviate from weaviate.exceptions import WeaviateBaseError +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import DBCaseConfig, VectorDB log = logging.getLogger(__name__) class WeaviateCloud(VectorDB): + thread_safe: bool = False + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + ] + def __init__( self, dim: int, @@ -23,6 +31,7 @@ def __init__( **kwargs, ): """Initialize wrapper around the weaviate vector database.""" + self.name = "WeaviateCloud" db_config.update( { "auth_client_secret": weaviate.AuthApiKey( @@ -37,6 +46,7 @@ def __init__( self._scalar_field = "key" self._vector_field = "vector" self._index_name = "vector_idx" + self._where_filter: dict | None = None # If local setup is used, we if db_config["no_auth"]: @@ -145,16 +155,24 @@ def search_embedding( .with_near_vector({"vector": query}) .with_limit(k) ) - if filters: - where_filter = { - "path": "key", - "operator": "GreaterThanEqual", - "valueInt": filters.get("id"), - } - query_obj = query_obj.with_where(where_filter) + if self._where_filter is not None: + query_obj = query_obj.with_where(self._where_filter) # Perform the search. res = query_obj.do() # Organize results. return [result[self._scalar_field] for result in res["data"]["Get"][self.collection_name]] + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self._where_filter = None + elif filters.type == FilterOp.NumGE: + self._where_filter = { + "path": [self._scalar_field], + "operator": "GreaterThanEqual", + "valueInt": filters.int_value, + } + else: + msg = f"Unsupported Weaviate filter: {filters}" + raise ValueError(msg) diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index e0cb98652..261f3ebff 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,6 +1,7 @@ from ..backend.clients.adbpg.cli import AdbpgNova from ..backend.clients.alisql.cli import AliSQLHNSW from ..backend.clients.alloydb.cli import AlloyDBScaNN +from ..backend.clients.antfly.cli import AntflyAKNN from ..backend.clients.aws_opensearch.cli import AWSOpenSearch from ..backend.clients.chroma.cli import Chroma from ..backend.clients.clickhouse.cli import Clickhouse @@ -113,6 +114,7 @@ cli.add_command(PolarDBHNSWSQ) cli.add_command(SeekDBHNSW) cli.add_command(VolcMySQLHNSW) +cli.add_command(AntflyAKNN) if __name__ == "__main__":