Skip to content

Commit c2ba07a

Browse files
lerman25lerman25
committed
Merge remote-tracking branch 'origin/main' into omerL-save_load_test_multi
2 parents 8f429a6 + f6d9ca6 commit c2ba07a

File tree

11 files changed

+289
-23
lines changed

11 files changed

+289
-23
lines changed

‎src/VecSim/algorithms/hnsw/hnsw_tiered.h

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1157,11 +1157,6 @@ VecSimDebugInfoIterator *TieredHNSWIndex<DataType, DistType>::debugInfoIterator(
11571157
// Get the base tiered fields.
11581158
auto *infoIterator = VecSimTieredIndex<DataType, DistType>::debugInfoIterator();
11591159

1160-
infoIterator->addInfoField(VecSim_InfoField{
1161-
.fieldName = VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING,
1162-
.fieldType = INFOFIELD_INT64,
1163-
.fieldValue = {FieldValue{.integerValue = info.tieredInfo.backgroundIndexing}}});
1164-
11651160
// Tiered HNSW specific param.
11661161
infoIterator->addInfoField(VecSim_InfoField{
11671162
.fieldName = VecSimCommonStrings::TIERED_HNSW_SWAP_JOBS_THRESHOLD_STRING,

‎src/VecSim/algorithms/svs/svs_tiered.h

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -888,11 +888,6 @@ class TieredSVSIndex : public VecSimTieredIndex<DataType, float> {
888888
auto *infoIterator = Base::debugInfoIterator();
889889
VecSimIndexDebugInfo info = this->debugInfo();
890890

891-
infoIterator->addInfoField(VecSim_InfoField{
892-
.fieldName = VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING,
893-
.fieldType = INFOFIELD_INT64,
894-
.fieldValue = {FieldValue{.integerValue = info.tieredInfo.backgroundIndexing}}});
895-
896891
infoIterator->addInfoField(VecSim_InfoField{
897892
.fieldName = VecSimCommonStrings::TIERED_SVS_TRAINING_THRESHOLD_STRING,
898893
.fieldType = INFOFIELD_UINT64,

‎src/VecSim/vec_sim_tiered_index.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,11 @@ VecSimDebugInfoIterator *VecSimTieredIndex<DataType, DistType>::debugInfoIterato
404404
.fieldType = INFOFIELD_UINT64,
405405
.fieldValue = {FieldValue{.uintegerValue = info.tieredInfo.management_layer_memory}}});
406406

407+
infoIterator->addInfoField(VecSim_InfoField{
408+
.fieldName = VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING,
409+
.fieldType = INFOFIELD_INT64,
410+
.fieldValue = {FieldValue{.integerValue = info.tieredInfo.backgroundIndexing}}});
411+
407412
infoIterator->addInfoField(
408413
VecSim_InfoField{.fieldName = VecSimCommonStrings::TIERED_BUFFER_LIMIT_STRING,
409414
.fieldType = INFOFIELD_UINT64,

‎tests/unit/test_bruteforce.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,26 @@ TYPED_TEST(BruteForceTest, test_dynamic_bf_info_iterator) {
609609
VecSimIndex_Free(index);
610610
}
611611

612+
TYPED_TEST(BruteForceTest, debugInfoIteratorFieldOrder) {
613+
namespace expected_output = test_utils::test_debug_info_iterator_order;
614+
615+
size_t d = 4;
616+
BFParams params = {.dim = d, .metric = VecSimMetric_L2, .blockSize = 1};
617+
VecSimIndex *index = this->CreateNewIndex(params);
618+
619+
// Add a vector to ensure the index is not empty
620+
GenerateAndAddVector<TEST_DATA_T>(index, d, 1, 1);
621+
622+
VecSimDebugInfoIterator *infoIterator = VecSimIndex_DebugInfoIterator(index);
623+
624+
// Test the field order using the common function
625+
expected_output::testDebugInfoIteratorFieldOrder(infoIterator,
626+
expected_output::getFlatFields());
627+
628+
VecSimDebugInfoIterator_Free(infoIterator);
629+
VecSimIndex_Free(index);
630+
}
631+
612632
TYPED_TEST(BruteForceTest, brute_force_vector_search_test_ip) {
613633
size_t dim = 4;
614634
size_t n = 100;

‎tests/unit/test_common.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -726,7 +726,7 @@ TEST(CommonAPITest, SearchDifferentScores) {
726726
};
727727
auto mock_thread_pool = tieredIndexMock();
728728
auto *tiered_index = dynamic_cast<TieredHNSWIndex<float, float> *>(
729-
test_utils::CreateNewTieredHNSWIndex(params, mock_thread_pool));
729+
test_utils::CreateNewTieredVecSimIndex(params, mock_thread_pool));
730730
ASSERT_NE(tiered_index, nullptr);
731731

732732
auto hnsw_index = tiered_index->getHNSWIndex();
@@ -848,7 +848,8 @@ TEST_P(CommonTypeMetricTieredTests, TestDataSizeTieredHNSW) {
848848
VecSimMetric metric = std::get<1>(GetParam());
849849

850850
HNSWParams hnsw_params = {.type = type, .dim = 4, .metric = metric};
851-
VecSimIndex *index = test_utils::CreateNewTieredHNSWIndex(hnsw_params, this->mock_thread_pool);
851+
VecSimIndex *index =
852+
test_utils::CreateNewTieredVecSimIndex(hnsw_params, this->mock_thread_pool);
852853

853854
auto verify_data_size = [&](const auto &tiered_index) {
854855
auto hnsw_index = tiered_index->getHNSWIndex();

‎tests/unit/test_hnsw.cpp

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -540,6 +540,27 @@ TYPED_TEST(HNSWTest, test_dynamic_hnsw_info_iterator) {
540540
this->CastToHNSW(index)->curElementCount = actual_element_count;
541541
VecSimIndex_Free(index);
542542
}
543+
544+
TYPED_TEST(HNSWTest, debugInfoIteratorFieldOrder) {
545+
namespace expected_output = test_utils::test_debug_info_iterator_order;
546+
547+
size_t d = 4;
548+
HNSWParams params = {.dim = d, .metric = VecSimMetric_L2};
549+
VecSimIndex *index = this->CreateNewIndex(params);
550+
551+
// Add a vector to ensure the index is not empty
552+
GenerateAndAddVector<TEST_DATA_T>(index, d, 1, 1);
553+
554+
VecSimDebugInfoIterator *infoIterator = VecSimIndex_DebugInfoIterator(index);
555+
556+
// Test the field order using the common function
557+
expected_output::testDebugInfoIteratorFieldOrder(infoIterator,
558+
expected_output::getHNSWFields());
559+
560+
VecSimDebugInfoIterator_Free(infoIterator);
561+
VecSimIndex_Free(index);
562+
}
563+
543564
TYPED_TEST(HNSWTest, test_query_runtime_params_default_build_args) {
544565
size_t n = 100;
545566
size_t d = 4;

‎tests/unit/test_hnsw_tiered.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ class HNSWTieredIndexTest : public ::testing::Test {
2929
using dist_t = typename index_type_t::dist_t;
3030

3131
protected:
32+
VecSimWriteMode original_mode;
33+
void SetUp() { original_mode = VecSimIndexInterface::asyncWriteMode; }
34+
35+
void TearDown() { VecSimIndexInterface::asyncWriteMode = original_mode; }
36+
3237
HNSWIndex<data_t, dist_t> *CastToHNSW(VecSimIndex *index) {
3338
auto tiered_index = reinterpret_cast<TieredHNSWIndex<data_t, dist_t> *>(index);
3439
return tiered_index->getHNSWIndex();
@@ -2860,6 +2865,28 @@ TYPED_TEST(HNSWTieredIndexTest, testInfoIterator) {
28602865
VecSimDebugInfoIterator_Free(infoIterator);
28612866
}
28622867

2868+
TYPED_TEST(HNSWTieredIndexTest, debugInfoIteratorFieldOrder) {
2869+
2870+
namespace expected_output = test_utils::test_debug_info_iterator_order;
2871+
2872+
// Create TieredHNSW index instance with a mock queue.
2873+
size_t dim = 4;
2874+
HNSWParams hnsw_params = {.type = TypeParam::get_index_type(),
2875+
.dim = dim,
2876+
.metric = VecSimMetric_L2,
2877+
.multi = TypeParam::isMulti()};
2878+
auto mock_thread_pool = tieredIndexMock();
2879+
auto index = test_utils::CreateNewTieredVecSimIndex(hnsw_params, mock_thread_pool);
2880+
GenerateAndAddVector(index, dim, 1, 1);
2881+
VecSimDebugInfoIterator *infoIterator = VecSimIndex_DebugInfoIterator(index);
2882+
2883+
// Test the field order using the common function
2884+
expected_output::testDebugInfoIteratorFieldOrder(infoIterator,
2885+
expected_output::getTieredHNSWFields());
2886+
2887+
VecSimDebugInfoIterator_Free(infoIterator);
2888+
}
2889+
28632890
TYPED_TEST(HNSWTieredIndexTest, writeInPlaceMode) {
28642891
// Create TieredHNSW index instance with a mock queue.
28652892
size_t dim = 4;

‎tests/unit/test_svs.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,6 +1156,26 @@ TYPED_TEST(SVSTest, test_dynamic_svs_info_iterator) {
11561156
}
11571157
}
11581158

1159+
TYPED_TEST(SVSTest, debugInfoIteratorFieldOrder) {
1160+
namespace expected_output = test_utils::test_debug_info_iterator_order;
1161+
1162+
size_t d = 4;
1163+
SVSParams params = {.dim = d, .metric = VecSimMetric_L2, .blockSize = 1};
1164+
VecSimIndex *index = this->CreateNewIndex(params);
1165+
ASSERT_INDEX(index);
1166+
1167+
// Add a vector to ensure the index is not empty
1168+
GenerateAndAddVector<TEST_DATA_T>(index, d, 1, 1);
1169+
1170+
VecSimDebugInfoIterator *infoIterator = VecSimIndex_DebugInfoIterator(index);
1171+
1172+
// Test the field order using the common function
1173+
expected_output::testDebugInfoIteratorFieldOrder(infoIterator, expected_output::getSVSFields());
1174+
1175+
VecSimDebugInfoIterator_Free(infoIterator);
1176+
VecSimIndex_Free(index);
1177+
}
1178+
11591179
TYPED_TEST(SVSTest, svs_vector_search_test_ip) {
11601180
const size_t dim = 4;
11611181
const size_t n = 10;

‎tests/unit/test_svs_tiered.cpp

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2489,6 +2489,30 @@ TYPED_TEST(SVSTieredIndexTest, testInfoIterator) {
24892489
VecSimDebugInfoIterator_Free(infoIterator);
24902490
}
24912491

2492+
TYPED_TEST(SVSTieredIndexTest, debugInfoIteratorFieldOrder) {
2493+
namespace expected_output = test_utils::test_debug_info_iterator_order;
2494+
2495+
// Create TieredSVS index instance with a mock queue.
2496+
size_t dim = 4;
2497+
SVSParams params = {.type = TypeParam::get_index_type(),
2498+
.dim = dim,
2499+
.metric = VecSimMetric_L2,
2500+
.multi = TypeParam::isMulti()};
2501+
auto mock_thread_pool = tieredIndexMock();
2502+
2503+
auto *index = test_utils::CreateNewTieredVecSimIndex(params, mock_thread_pool);
2504+
ASSERT_INDEX(index);
2505+
2506+
GenerateAndAddVector(index, dim, 1, 1);
2507+
VecSimDebugInfoIterator *infoIterator = VecSimIndex_DebugInfoIterator(index);
2508+
2509+
// Test the field order using the common function
2510+
expected_output::testDebugInfoIteratorFieldOrder(infoIterator,
2511+
expected_output::getTieredSVSFields());
2512+
2513+
VecSimDebugInfoIterator_Free(infoIterator);
2514+
}
2515+
24922516
TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) {
24932517
// Create TieredSVS index instance with a mock queue.
24942518
size_t dim = 4;

‎tests/unit/unit_test_utils.cpp

Lines changed: 131 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -679,14 +679,136 @@ TieredIndexParams CreateTieredParams(VecSimParams &primary_params,
679679
return tiered_params;
680680
}
681681

682-
VecSimIndex *CreateNewTieredHNSWIndex(const HNSWParams &hnsw_params,
683-
tieredIndexMock &mock_thread_pool) {
684-
VecSimParams primary_params = CreateParams(hnsw_params);
685-
auto tiered_params = CreateTieredParams(primary_params, mock_thread_pool);
686-
VecSimParams params = CreateParams(tiered_params);
687-
VecSimIndex *index = VecSimIndex_New(&params);
688-
mock_thread_pool.ctx->index_strong_ref.reset(index);
689-
690-
return index;
682+
namespace test_debug_info_iterator_order {
683+
std::vector<std::string> getCommonFields() {
684+
return {
685+
VecSimCommonStrings::TYPE_STRING, // 1. TYPE
686+
VecSimCommonStrings::DIMENSION_STRING, // 2. DIMENSION
687+
VecSimCommonStrings::METRIC_STRING, // 3. METRIC
688+
VecSimCommonStrings::IS_MULTI_STRING, // 4. IS_MULTI
689+
VecSimCommonStrings::INDEX_SIZE_STRING, // 5. INDEX_SIZE
690+
VecSimCommonStrings::INDEX_LABEL_COUNT_STRING, // 6. INDEX_LABEL_COUNT
691+
VecSimCommonStrings::MEMORY_STRING, // 7. MEMORY
692+
VecSimCommonStrings::SEARCH_MODE_STRING // 8. SEARCH_MODE
693+
};
691694
}
695+
696+
std::vector<std::string> getFlatFields() {
697+
std::vector<std::string> fields;
698+
fields.push_back(VecSimCommonStrings::ALGORITHM_STRING); // ALGORITHM
699+
auto commonFields = getCommonFields();
700+
fields.insert(fields.end(), commonFields.begin(), commonFields.end());
701+
fields.push_back(VecSimCommonStrings::BLOCK_SIZE_STRING); // BLOCK_SIZE
702+
return fields;
703+
}
704+
705+
// Imitates HNSWIndex<DataType, DistType>::debugInfoIterator()
706+
std::vector<std::string> getHNSWFields() {
707+
std::vector<std::string> fields;
708+
fields.push_back(VecSimCommonStrings::ALGORITHM_STRING); // ALGORITHM
709+
auto commonFields = getCommonFields();
710+
fields.insert(fields.end(), commonFields.begin(), commonFields.end());
711+
// Then HNSW-specific fields:
712+
fields.push_back(VecSimCommonStrings::BLOCK_SIZE_STRING);
713+
fields.push_back(VecSimCommonStrings::HNSW_M_STRING);
714+
fields.push_back(VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING);
715+
fields.push_back(VecSimCommonStrings::HNSW_EF_RUNTIME_STRING);
716+
fields.push_back(VecSimCommonStrings::HNSW_MAX_LEVEL);
717+
fields.push_back(VecSimCommonStrings::HNSW_ENTRYPOINT);
718+
fields.push_back(VecSimCommonStrings::EPSILON_STRING);
719+
fields.push_back(VecSimCommonStrings::NUM_MARKED_DELETED);
720+
return fields;
721+
}
722+
723+
// Imitates SVSIndex<DataType, DistType>::debugInfoIterator()
724+
std::vector<std::string> getSVSFields() {
725+
std::vector<std::string> fields;
726+
fields.push_back(VecSimCommonStrings::ALGORITHM_STRING); // ALGORITHM
727+
auto commonFields = getCommonFields();
728+
fields.insert(fields.end(), commonFields.begin(), commonFields.end());
729+
// Then SVS-specific fields:
730+
fields.push_back(VecSimCommonStrings::BLOCK_SIZE_STRING);
731+
fields.push_back(VecSimCommonStrings::SVS_QUANT_BITS_STRING);
732+
fields.push_back(VecSimCommonStrings::SVS_ALPHA_STRING);
733+
fields.push_back(VecSimCommonStrings::SVS_GRAPH_MAX_DEGREE_STRING);
734+
fields.push_back(VecSimCommonStrings::SVS_CONSTRUCTION_WS_STRING);
735+
fields.push_back(VecSimCommonStrings::SVS_MAX_CANDIDATE_POOL_SIZE_STRING);
736+
fields.push_back(VecSimCommonStrings::SVS_PRUNE_TO_STRING);
737+
fields.push_back(VecSimCommonStrings::SVS_USE_SEARCH_HISTORY_STRING);
738+
fields.push_back(VecSimCommonStrings::SVS_NUM_THREADS_STRING);
739+
fields.push_back(VecSimCommonStrings::NUM_MARKED_DELETED);
740+
fields.push_back(VecSimCommonStrings::SVS_SEARCH_WS_STRING);
741+
fields.push_back(VecSimCommonStrings::SVS_SEARCH_BC_STRING);
742+
fields.push_back(VecSimCommonStrings::SVS_LEANVEC_DIM_STRING);
743+
fields.push_back(VecSimCommonStrings::EPSILON_STRING);
744+
return fields;
745+
}
746+
747+
// Imitates VecSimTieredIndex<DataType, DistType>::debugInfoIterator()
748+
std::vector<std::string> getTieredCommonFields() {
749+
std::vector<std::string> fields;
750+
fields.push_back(VecSimCommonStrings::ALGORITHM_STRING); // ALGORITHM (set to "TIERED")
751+
auto commonFields = getCommonFields();
752+
fields.insert(fields.end(), commonFields.begin(),
753+
commonFields.end()); // backendIndex->addCommonInfoToIterator()
754+
// Then tiered-specific fields:
755+
fields.push_back(VecSimCommonStrings::TIERED_MANAGEMENT_MEMORY_STRING);
756+
fields.push_back(VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING);
757+
fields.push_back(VecSimCommonStrings::TIERED_BUFFER_LIMIT_STRING);
758+
fields.push_back(VecSimCommonStrings::FRONTEND_INDEX_STRING);
759+
fields.push_back(VecSimCommonStrings::BACKEND_INDEX_STRING);
760+
return fields;
761+
}
762+
763+
// Imitates TieredSVSIndex<DataType, DistType>::debugInfoIterator()
764+
std::vector<std::string> getTieredSVSFields() {
765+
auto fields = getTieredCommonFields();
766+
// Add SVS tiered-specific fields:
767+
fields.push_back(
768+
VecSimCommonStrings::TIERED_SVS_TRAINING_THRESHOLD_STRING); // 15.
769+
// TIERED_SVS_TRAINING_THRESHOLD
770+
fields.push_back(
771+
VecSimCommonStrings::TIERED_SVS_UPDATE_THRESHOLD_STRING); // 16. TIERED_SVS_UPDATE_THRESHOLD
772+
fields.push_back(
773+
VecSimCommonStrings::
774+
TIERED_SVS_THREADS_RESERVE_TIMEOUT_STRING); // 17. TIERED_SVS_THREADS_RESERVE_TIMEOUT
775+
return fields;
776+
}
777+
778+
// Imitates TieredHNSWIndex<DataType, DistType>::debugInfoIterator()
779+
std::vector<std::string> getTieredHNSWFields() {
780+
auto fields = getTieredCommonFields();
781+
// Add HNSW tiered-specific field:
782+
fields.push_back(
783+
VecSimCommonStrings::
784+
TIERED_HNSW_SWAP_JOBS_THRESHOLD_STRING); // 15. TIERED_HNSW_SWAP_JOBS_THRESHOLD
785+
return fields;
786+
}
787+
788+
void testDebugInfoIteratorFieldOrder(VecSimDebugInfoIterator *infoIterator,
789+
const std::vector<std::string> &expectedFieldOrder) {
790+
// Verify the total number of fields matches expected
791+
ASSERT_EQ(VecSimDebugInfoIterator_NumberOfFields(infoIterator), expectedFieldOrder.size());
792+
793+
// Iterate through the fields and verify the order
794+
size_t fieldIndex = 0;
795+
while (VecSimDebugInfoIterator_HasNextField(infoIterator)) {
796+
VecSim_InfoField *infoField = VecSimDebugInfoIterator_NextField(infoIterator);
797+
ASSERT_LT(fieldIndex, expectedFieldOrder.size())
798+
<< "More fields than expected. Field index: " << fieldIndex;
799+
800+
ASSERT_STREQ(infoField->fieldName, expectedFieldOrder[fieldIndex].c_str())
801+
<< "Field order mismatch at index " << fieldIndex
802+
<< ". Expected: " << expectedFieldOrder[fieldIndex]
803+
<< ", Got: " << infoField->fieldName;
804+
805+
fieldIndex++;
806+
}
807+
808+
// Verify we processed all expected fields
809+
ASSERT_EQ(fieldIndex, expectedFieldOrder.size())
810+
<< "Fewer fields than expected. Processed: " << fieldIndex
811+
<< ", Expected: " << expectedFieldOrder.size();
812+
}
813+
} // namespace test_debug_info_iterator_order
692814
} // namespace test_utils

0 commit comments

Comments
 (0)