Skip to content

Commit 3a7ec14

Browse files
meiravgrimeiravgri
authored
[0.8] [MOD-11237] Block Size Boundary Oscillation Benchmark (#768) (#770)
* [MOD-11237] Block Size Boundary Oscillation Benchmark (#768) * imp bemchmar fix counters * fix (cherry picked from commit fd08b55) * fix memory
1 parent c94c37a commit 3a7ec14

File tree

4 files changed

+112
-7
lines changed

4 files changed

+112
-7
lines changed

‎tests/benchmark/bm_common.h‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,9 @@ void BM_VecSimCommon<index_type_t>::Memory_FLAT(benchmark::State &st, unsigned s
6262
for (auto _ : st) {
6363
// Do nothing...
6464
}
65-
st.counters["memory"] = (double)VecSimIndex_StatsInfo(index).memory;
65+
st.counters["memory"] =
66+
benchmark::Counter((double)VecSimIndex_StatsInfo(index).memory,
67+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
6668
}
6769
template <typename index_type_t>
6870
void BM_VecSimCommon<index_type_t>::Memory_HNSW(benchmark::State &st, unsigned short index_offset) {
@@ -72,7 +74,9 @@ void BM_VecSimCommon<index_type_t>::Memory_HNSW(benchmark::State &st, unsigned s
7274
for (auto _ : st) {
7375
// Do nothing...
7476
}
75-
st.counters["memory"] = (double)VecSimIndex_StatsInfo(index).memory;
77+
st.counters["memory"] =
78+
benchmark::Counter((double)VecSimIndex_StatsInfo(index).memory,
79+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
7680
}
7781
template <typename index_type_t>
7882
void BM_VecSimCommon<index_type_t>::Memory_Tiered(benchmark::State &st,
@@ -82,7 +86,9 @@ void BM_VecSimCommon<index_type_t>::Memory_Tiered(benchmark::State &st,
8286
for (auto _ : st) {
8387
// Do nothing...
8488
}
85-
st.counters["memory"] = (double)VecSimIndex_StatsInfo(index).memory;
89+
st.counters["memory"] =
90+
benchmark::Counter((double)VecSimIndex_StatsInfo(index).memory,
91+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
8692
}
8793

8894
// TopK search BM

‎tests/benchmark/bm_vecsim_basics.h‎

Lines changed: 89 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ class BM_VecSimBasics : public BM_VecSimCommon<index_type_t> {
3232
static void Range_BF(benchmark::State &st);
3333
static void Range_HNSW(benchmark::State &st);
3434

35+
// Reproduces allocation/deallocation oscillation issue at block size boundaries.
36+
// Sets up index at blockSize+1 capacity, then repeatedly deletes and re-adds the same vector,
37+
// triggering constant grow-shrink cycles.
38+
// This behavior was fixed by PR #753 with a conservative resize strategy that only
39+
// shrinks containers when there are 2+ free blocks, preventing oscillation cycles.
40+
// Expected: High allocation overhead before fix, stable performance after fix.
41+
static void UpdateAtBlockSize(benchmark::State &st);
42+
3543
private:
3644
// Vectors of vector to store deleted labels' data.
3745
using LabelData = std::vector<std::vector<data_t>>;
@@ -66,7 +74,9 @@ void BM_VecSimBasics<index_type_t>::AddLabel(benchmark::State &st) {
6674
// For tiered index, wait for all threads to finish indexing
6775
BM_VecSimGeneral::mock_thread_pool.thread_pool_wait();
6876

69-
st.counters["memory_per_vector"] = (double)memory_delta / (double)added_vec_count;
77+
st.counters["memory_per_vector"] =
78+
benchmark::Counter((double)memory_delta / (double)added_vec_count,
79+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
7080
st.counters["vectors_per_label"] = vec_per_label;
7181

7282
assert(VecSimIndex_IndexSize(index) == N_VECTORS + added_vec_count);
@@ -114,7 +124,9 @@ void BM_VecSimBasics<index_type_t>::AddLabel_AsyncIngest(benchmark::State &st) {
114124
}
115125

116126
size_t memory_delta = index->getAllocationSize() - memory_before;
117-
st.counters["memory_per_vector"] = (double)memory_delta / (double)added_vec_count;
127+
st.counters["memory_per_vector"] =
128+
benchmark::Counter((double)memory_delta / (double)added_vec_count,
129+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
118130
st.counters["vectors_per_label"] = vec_per_label;
119131
st.counters["num_threads"] = BM_VecSimGeneral::mock_thread_pool.thread_pool_size;
120132

@@ -164,7 +176,9 @@ void BM_VecSimBasics<index_type_t>::DeleteLabel(algo_t *index, benchmark::State
164176
if (VecSimIndex_BasicInfo(index).algo == VecSimAlgo_TIERED) {
165177
dynamic_cast<TieredHNSWIndex<data_t, dist_t> *>(index)->executeReadySwapJobs();
166178
}
167-
st.counters["memory_per_vector"] = memory_delta / (double)removed_vectors_count;
179+
st.counters["memory_per_vector"] =
180+
benchmark::Counter((double)memory_delta / (double)removed_vectors_count,
181+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
168182

169183
// Restore index state.
170184
// For each label in removed_labels_data
@@ -214,7 +228,10 @@ void BM_VecSimBasics<index_type_t>::DeleteLabel_AsyncRepair(benchmark::State &st
214228
// Avg. memory delta per vector equals the total memory delta divided by the number
215229
// of deleted vectors.
216230
double memory_delta = tiered_index->getAllocationSize() - memory_before;
217-
st.counters["memory_per_vector"] = memory_delta / (double)removed_vectors_count;
231+
232+
st.counters["memory_per_vector"] =
233+
benchmark::Counter((double)memory_delta / (double)removed_vectors_count,
234+
benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024);
218235
st.counters["num_threads"] = (double)BM_VecSimGeneral::mock_thread_pool.thread_pool_size;
219236
st.counters["num_zombies"] = tiered_index->idToSwapJob.size();
220237

@@ -286,6 +303,69 @@ void BM_VecSimBasics<index_type_t>::Range_HNSW(benchmark::State &st) {
286303
st.counters["Recall"] = (float)total_res / total_res_bf;
287304
}
288305

306+
template <typename index_type_t>
307+
void BM_VecSimBasics<index_type_t>::UpdateAtBlockSize(benchmark::State &st) {
308+
auto index = INDICES[st.range(0)];
309+
size_t initial_index_size = VecSimIndex_IndexSize(index);
310+
// Calculate vectors needed to reach next block boundary
311+
size_t vecs_to_blocksize =
312+
BM_VecSimGeneral::block_size - (initial_index_size % BM_VecSimGeneral::block_size);
313+
assert(vecs_to_blocksize < BM_VecSimGeneral::block_size);
314+
labelType initial_label_count = index->indexLabelCount();
315+
labelType curr_label = initial_label_count;
316+
317+
// Set up index at blockSize+1 to trigger oscillation issue
318+
// Make sure we have enough queries to add a new label.
319+
assert(N_QUERIES > BM_VecSimGeneral::block_size);
320+
size_t overhead = 1;
321+
size_t added_vec_count = vecs_to_blocksize + overhead;
322+
for (size_t i = 0; i < added_vec_count; ++i) {
323+
VecSimIndex_AddVector(index, QUERIES[added_vec_count % N_QUERIES].data(), curr_label++);
324+
}
325+
// For tiered index, wait for all threads to finish indexing
326+
BM_VecSimGeneral::mock_thread_pool.thread_pool_wait();
327+
assert(VecSimIndex_IndexSize(index) % BM_VecSimGeneral::block_size == overhead);
328+
assert(VecSimIndex_IndexSize(index) == N_VECTORS + added_vec_count);
329+
330+
std::cout << "Added " << added_vec_count << " vectors to reach block size boundary."
331+
<< std::endl;
332+
std::cout << "Index size is now " << VecSimIndex_IndexSize(index) << std::endl;
333+
std::cout << "Last label is " << curr_label - 1 << std::endl;
334+
335+
// Benchmark loop: repeatedly delete/add same vector to trigger grow-shrink cycles
336+
labelType label_to_update = curr_label - 1;
337+
size_t index_cap = index->indexCapacity();
338+
for (auto _ : st) {
339+
// Remove the vector directly from hnsw
340+
size_t ret = VecSimIndex_DeleteVector(
341+
INDICES[st.range(0) == VecSimAlgo_TIERED ? VecSimAlgo_HNSWLIB : st.range(0)],
342+
label_to_update);
343+
assert(ret == 1);
344+
assert(index->indexCapacity() == index_cap - BM_VecSimGeneral::block_size);
345+
// Capacity should shrink by one block after deletion
346+
ret = VecSimIndex_AddVector(index, QUERIES[(added_vec_count - 1) % N_QUERIES].data(),
347+
label_to_update);
348+
assert(ret == 1);
349+
BM_VecSimGeneral::mock_thread_pool.thread_pool_wait();
350+
assert(VecSimIndex_IndexSize(
351+
INDICES[st.range(0) == VecSimAlgo_TIERED ? VecSimAlgo_HNSWLIB : st.range(0)]) ==
352+
N_VECTORS + added_vec_count);
353+
// Capacity should grow back to original size after addition
354+
assert(index->indexCapacity() == index_cap);
355+
}
356+
assert(VecSimIndex_IndexSize(index) == N_VECTORS + added_vec_count);
357+
358+
// Clean-up all the new vectors to restore the index size to its original value.
359+
360+
size_t new_label_count = index->indexLabelCount();
361+
for (size_t label = initial_label_count; label < new_label_count; label++) {
362+
// If index is tiered HNSW, remove directly from the underline HNSW.
363+
VecSimIndex_DeleteVector(
364+
INDICES[st.range(0) == VecSimAlgo_TIERED ? VecSimAlgo_HNSWLIB : st.range(0)], label);
365+
}
366+
assert(VecSimIndex_IndexSize(index) == N_VECTORS);
367+
}
368+
289369
#define UNIT_AND_ITERATIONS Unit(benchmark::kMillisecond)->Iterations(BM_VecSimGeneral::block_size)
290370

291371
// The actual radius will be the given arg divided by 100, since arg must be an integer.
@@ -331,3 +411,8 @@ void BM_VecSimBasics<index_type_t>::Range_HNSW(benchmark::State &st) {
331411
}
332412
#define REGISTER_DeleteLabel(BM_FUNC) \
333413
BENCHMARK_REGISTER_F(BM_VecSimBasics, BM_FUNC)->UNIT_AND_ITERATIONS
414+
415+
#define REGISTER_UpdateAtBlockSize(BM_FUNC, VecSimAlgo) \
416+
BENCHMARK_REGISTER_F(BM_VecSimBasics, BM_FUNC) \
417+
->UNIT_AND_ITERATIONS->Arg(VecSimAlgo) \
418+
->ArgName(#VecSimAlgo)

‎tests/benchmark/run_files/bm_basics_multi_fp32.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,10 @@ DEFINE_DELETE_LABEL(BM_FUNC_NAME(DeleteLabel, Tiered), fp32_index_t, TieredHNSWI
3333
VecSimAlgo_TIERED)
3434
#include "benchmark/bm_initialization/bm_basics_initialize_fp32.h"
3535

36+
// Test oscillations at block size boundaries.
37+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, UpdateAtBlockSize_Multi, fp32_index_t)
38+
(benchmark::State &st) { UpdateAtBlockSize(st); }
39+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Multi, VecSimAlgo_BF);
40+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Multi, VecSimAlgo_HNSWLIB);
41+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Multi, VecSimAlgo_TIERED);
3642
BENCHMARK_MAIN();

‎tests/benchmark/run_files/bm_basics_single_fp32.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,13 @@ DEFINE_DELETE_LABEL(BM_FUNC_NAME(DeleteLabel, HNSW), fp32_index_t, HNSWIndex_Sin
3131
VecSimAlgo_HNSWLIB)
3232
DEFINE_DELETE_LABEL(BM_FUNC_NAME(DeleteLabel, Tiered), fp32_index_t, TieredHNSWIndex, float, float,
3333
VecSimAlgo_TIERED)
34+
3435
#include "benchmark/bm_initialization/bm_basics_initialize_fp32.h"
36+
37+
// Test oscillations at block size boundaries.
38+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, UpdateAtBlockSize_Single, fp32_index_t)
39+
(benchmark::State &st) { UpdateAtBlockSize(st); }
40+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Single, VecSimAlgo_BF);
41+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Single, VecSimAlgo_HNSWLIB);
42+
REGISTER_UpdateAtBlockSize(UpdateAtBlockSize_Single, VecSimAlgo_TIERED);
3543
BENCHMARK_MAIN();

0 commit comments

Comments
 (0)