From 41d6e44f96f4cb0cfcf8e93cb2b84c10135f0085 Mon Sep 17 00:00:00 2001 From: gouyt13clear Date: Thu, 12 Nov 2026 18:07:26 +0800 Subject: [PATCH] test: stop requiring every graph point after nondeterministic builds HNSW-RaBitQ, SymphonyQG and hnswlib-based IVF routing do not guarantee that every point is reachable. Tests that required every point after a multi-threaded or unseeded build failed now and then in CI. Tests that check exact results now build with a fixed seed and one thread: the large-dimension round trip, the SymphonyQG fixture, rebuild and portable cases, and the HNSW add, resize and remove cases whose searches compare graphs. Tests about parallel construction keep their threads and check the self-recall of every point instead: the example scripts, and IVF HNSW routing over 1,024 centroids. Tests whose graphs keep every reverse link stay exact. With M equal to the point count, the base layer holds 2M >= N - 1 links and never prunes one, so every point stays reachable under any thread interleaving. Their comments now say so. --- tests/python/test_examples.py | 10 +++++++++- tests/python/test_hnsw.py | 23 ++++++++++++++--------- tests/python/test_ivf_initializers.py | 5 ++++- tests/python/test_large_dimensions.py | 5 +++-- tests/python/test_portable_indexes.py | 12 ++++++++++-- tests/python/test_symqg.py | 9 ++++++--- tests/unit/rabitqlib/index/hnsw_test.cpp | 5 +++-- 7 files changed, 49 insertions(+), 20 deletions(-) diff --git a/tests/python/test_examples.py b/tests/python/test_examples.py index 35c0c436..a66655ac 100644 --- a/tests/python/test_examples.py +++ b/tests/python/test_examples.py @@ -200,7 +200,15 @@ def check_saved_index_and_query(tmp_path, kind, metric, data): search_args = {"nprobe": 2} if kind == "ivf" else {"ef": len(data)} ids, distances = index.search(queries, k=5, num_threads=2, **search_args) assert index.metric == metric - np.testing.assert_array_equal(ids[:, 0], np.arange(len(queries))) + if kind == "ivf": + np.testing.assert_array_equal(ids[:, 0], np.arange(len(queries))) + else: + # The examples build graphs with two threads (SymphonyQG also without a fixed + # seed), and graph indexes do not guarantee that every point is reachable: + # check the self-recall of every point instead. + self_ids, _ = index.search(data, k=1, num_threads=2, **search_args) + found = np.count_nonzero(self_ids[:, 0] == np.arange(len(data))) + assert found >= 0.95 * len(data) assert np.isfinite(distances).all() restored_path = tmp_path / "restored.index" index.save(str(restored_path)) diff --git a/tests/python/test_hnsw.py b/tests/python/test_hnsw.py index bf612c6d..c478fcb5 100644 --- a/tests/python/test_hnsw.py +++ b/tests/python/test_hnsw.py @@ -86,13 +86,14 @@ def test_seed_reproduces_serial_build_and_search(tmp_path): def test_parallel_build_preserves_self_retrieval(base_data, clusters): - # Keep every neighbor so insertion-order-dependent pruning cannot isolate a - # point. This checks parallel construction, not sparse-graph recall. + # M equals the point count, so the base layer holds 2M >= N - 1 links and never + # prunes a reverse link: every point stays reachable under any thread + # interleaving. This checks parallel construction, not sparse-graph recall. idx = HnswIndex(DIM, N_VECTORS, M=N_VECTORS, ef_construction=N_VECTORS, nbits=4) centroids, cluster_ids = clusters idx.build(base_data, centroids, cluster_ids, num_threads=8) - ids, distances = idx.search(base_data[:10], k=1, ef=N_VECTORS) - np.testing.assert_array_equal(ids[:, 0], np.arange(10)) + ids, distances = idx.search(base_data, k=1, ef=N_VECTORS) + np.testing.assert_array_equal(ids[:, 0], np.arange(N_VECTORS)) assert np.isfinite(distances).all() @@ -251,7 +252,10 @@ def test_add_returns_dense_labels_and_preserves_search( _, cluster_ids = clusters assert partial_hnsw.num_points == 400 before, _ = partial_hnsw.search(base_data[:10], k=1, ef=N_VECTORS) - ids = partial_hnsw.add(base_data[400:], cluster_ids=cluster_ids[400:]) + # One thread, as the build used, so the grown graph and `after` are reproducible. + ids = partial_hnsw.add( + base_data[400:], cluster_ids=cluster_ids[400:], num_threads=1 + ) np.testing.assert_array_equal(ids, np.arange(400, N_VECTORS)) assert partial_hnsw.num_points == N_VECTORS # A sparse graph can isolate the odd point, as it can after build, so this @@ -265,8 +269,8 @@ def test_add_returns_dense_labels_and_preserves_search( def test_add_into_a_dense_graph_retrieves_every_point(base_data, clusters): - # Keeping every neighbor removes the pruning that isolates points, so the - # added points must all be found. + # M equals the point count, so the base layer holds 2M >= N - 1 links and never + # prunes a reverse link: even a parallel add keeps every point reachable. centroids, cluster_ids = clusters idx = HnswIndex(DIM, N_VECTORS, M=N_VECTORS, ef_construction=N_VECTORS, nbits=4) idx.build(base_data[:400], centroids, cluster_ids[:400]) @@ -331,7 +335,7 @@ def test_resize_grows_capacity_and_allows_more(base_data, clusters): assert idx.max_elements == N_VECTORS assert idx.num_points == 400 - idx.add(base_data[400:], cluster_ids=cluster_ids[400:]) + idx.add(base_data[400:], cluster_ids=cluster_ids[400:], num_threads=1) assert idx.num_points == N_VECTORS ids, _ = idx.search(base_data[:10], k=1, ef=N_VECTORS) np.testing.assert_array_equal(ids[:, 0], np.arange(10)) @@ -457,7 +461,8 @@ def check(index): plain = HnswIndex.load(str(plain_path)) for index in (idx, loaded, plain): index.resize(count + len(new)) - index.add(new) + # One thread: graphs grown in parallel differ, and the searches compare them. + index.add(new, num_threads=1) assert index.num_stored_points == count + len(new) assert index.num_points == count + len(new) - (0 if index is plain else 4) diff --git a/tests/python/test_ivf_initializers.py b/tests/python/test_ivf_initializers.py index 57c08f9e..93393aba 100644 --- a/tests/python/test_ivf_initializers.py +++ b/tests/python/test_ivf_initializers.py @@ -17,10 +17,13 @@ def test_hnsw_parallel_build_routes_and_reloads(tmp_path, metric, threads): data /= np.linalg.norm(data, axis=1, keepdims=True) index = IvfIndex(dim, count, count, 4, metric, initializer="hnsw") index.build(data, data, np.arange(count, dtype=np.uint32), num_threads=threads) + # A parallel build may leave a centroid unreachable in the routing graph, so + # check the aggregate self-recall of every point (one point per cluster). + every, _ = index.search(data, 1, 8, num_threads=threads) + assert np.count_nonzero(every[:, 0] == np.arange(count)) >= 0.99 * count labels = np.array([0, 17, 511, 1023], dtype=np.uint32) queries = data[labels] expected = index.search(queries, 1, 8, num_threads=threads) - np.testing.assert_array_equal(expected[0][:, 0], labels) assert np.isfinite(expected[1]).all() path = tmp_path / "parallel-build.index" index.save(str(path)) diff --git a/tests/python/test_large_dimensions.py b/tests/python/test_large_dimensions.py index 29765daf..708680b2 100644 --- a/tests/python/test_large_dimensions.py +++ b/tests/python/test_large_dimensions.py @@ -28,12 +28,13 @@ def test_large_dimension_round_trip(tmp_path, dim, metric, kind): search_args = {"ef": count} else: index = SymqgIndex( - dim, max_degree=32, ef_construction=count, nbits=4, metric=metric + dim, max_degree=32, ef_construction=count, nbits=4, metric=metric, seed=42 ) search_args = {"ef": count} + # A seeded single-thread build is reproducible, so every self-query must succeed. if kind == "qg": - index.add(data, num_threads=2) + index.add(data, num_threads=1) else: index.set_centroids(data.mean(axis=0, keepdims=True)) index.add(data, cluster_ids=np.zeros(count, np.uint32), num_threads=1) diff --git a/tests/python/test_portable_indexes.py b/tests/python/test_portable_indexes.py index cf9e9db0..35476f1e 100644 --- a/tests/python/test_portable_indexes.py +++ b/tests/python/test_portable_indexes.py @@ -25,10 +25,18 @@ def check_index_round_trip(tmp_path, metric, dim, kind, bits): ) search_args = {"ef": 96} else: - index = SymqgIndex(dim, max_degree=32, quantization_bits=bits, metric=metric) + # Seeded and single-threaded, so every self-query below must succeed (#216). + index = SymqgIndex( + dim, + max_degree=32, + nbits=bits or 32, + ef_construction=96, + metric=metric, + seed=42, + ) search_args = {"ef": 96} if kind == "qg": - index.build(data, ef_construction=96) + index.add(data, num_threads=1) else: index.build( data, data.mean(axis=0, keepdims=True), np.zeros(len(data), np.uint32) diff --git a/tests/python/test_symqg.py b/tests/python/test_symqg.py index 220acd7b..4d2be889 100644 --- a/tests/python/test_symqg.py +++ b/tests/python/test_symqg.py @@ -21,8 +21,10 @@ @pytest.fixture(scope="module") def built_symqg(base_data): - idx = SymqgIndex(DIM, max_degree=_MAX_DEGREE) - idx.build(base_data, ef_construction=_EF_BUILD) + # A seeded single-thread build is reproducible, so the self-retrieval test can + # require every probe (#216). + idx = SymqgIndex(DIM, max_degree=_MAX_DEGREE, ef_construction=_EF_BUILD, seed=42) + idx.add(base_data, num_threads=1) return idx @@ -234,7 +236,8 @@ def test_failed_rebuild_preserves_index( @pytest.mark.parametrize("bits", [0, 4, 8]) def test_successful_rebuild_replaces_index(base_data, tmp_path, bits): - index = SymqgIndex(DIM, max_degree=_MAX_DEGREE, quantization_bits=bits) + # Seeded, and build uses one thread, so every self-query below must succeed. + index = SymqgIndex(DIM, max_degree=_MAX_DEGREE, quantization_bits=bits, seed=42) index.build(base_data[:40], ef_construction=_EF_BUILD) replacement = base_data[40:88] + 10 index.build(replacement, ef_construction=_EF_BUILD) diff --git a/tests/unit/rabitqlib/index/hnsw_test.cpp b/tests/unit/rabitqlib/index/hnsw_test.cpp index 8835a2c4..1d778ff1 100644 --- a/tests/unit/rabitqlib/index/hnsw_test.cpp +++ b/tests/unit/rabitqlib/index/hnsw_test.cpp @@ -716,8 +716,9 @@ TEST(HnswResizeTest, SurvivesSaveAndLoad) { } } -// Labels must not depend on the order threads happen to finish in, and every -// added point must still be reachable. +// Labels must not depend on the order threads happen to finish in. M equals the point +// count, so the base layer holds 2M >= N - 1 links and never prunes a reverse link: +// every point stays reachable under any thread interleaving, and must be found. TEST(HnswAddTest, ParallelAddKeepsLabelsAndTheGraph) { constexpr size_t kBuilt = 256; constexpr size_t kAdded = 256;