[MOD-9685] Introduce SVS Basic Benchmarks

.github/workflows/benchmark.yml

@@ -43,6 +43,7 @@ on:
           - bm-updated-fp32-single
           - bm-svs-train-fp32
           - bm-svs-train-fp16
+          - bm-basics-svs-fp32-single
           - bm-spaces
         description: 'Benchmarks set to run'
         default: benchmarks-all

src/VecSim/index_factories/tiered_factory.h

@@ -13,6 +13,7 @@
 #include "VecSim/memory/vecsim_malloc.h"
 #include "VecSim/vec_sim_index.h"
 #include "VecSim/algorithms/hnsw/hnsw_tiered.h"
+#include "VecSim/algorithms/svs/svs_tiered.h"
 #include "VecSim/algorithms/brute_force/brute_force.h"
 #include "VecSim/index_factories/factory_utils.h"
 
@@ -58,8 +59,35 @@ VecSimIndex *NewIndex(const TieredIndexParams *params, HNSWIndex<DataType, DistT
 // verification of the backend index algorithm. To be removed once a proper verification is
 // introduced.
 namespace TieredSVSFactory {
-BFParams NewBFParams(const TieredIndexParams *params);
+
+#if HAVE_SVS
+template <typename DataType>
+inline VecSimIndex *NewIndex(const TieredIndexParams *params,
+                             VecSimIndexAbstract<DataType, float> *svs_index) {
+    // Initialize brute force index.
+    BFParams bf_params = {.type = svs_index->getType(),
+                          .dim = svs_index->getDim(),
+                          .metric = svs_index->getMetric(),
+                          .multi = svs_index->isMultiValue(),
+                          .blockSize = svs_index->getBlockSize()};
+
+    AbstractIndexInitParams abstractInitParams =
+        VecSimFactory::NewAbstractInitParams(&bf_params, params->primaryIndexParams->logCtx, false);
+    assert(svs_index->getInputBlobSize() == abstractInitParams.storedDataSize);
+    assert(svs_index->getStoredDataSize() == abstractInitParams.storedDataSize);
+    auto frontendIndex = static_cast<BruteForceIndex<DataType, float> *>(
+        BruteForceFactory::NewIndex(&bf_params, abstractInitParams, false));
+
+    // Create new tiered svs index
+    std::shared_ptr<VecSimAllocator> management_layer_allocator =
+        VecSimAllocator::newVecsimAllocator();
+
+    return new (management_layer_allocator)
+        TieredSVSIndex<DataType>(svs_index, frontendIndex, *params, management_layer_allocator);
 }
 #endif
+BFParams NewBFParams(const TieredIndexParams *params);
+} // namespace TieredSVSFactory
+#endif
 
 }; // namespace TieredFactory

tests/benchmark/benchmarks.sh

@@ -11,6 +11,8 @@ if [ -z "$BM_TYPE"  ] || [ "$BM_TYPE" = "benchmarks-all" ]; then
     echo updated_index_single_fp32
     echo svs_training_fp32
     echo svs_training_fp16
+    echo basics_svs_single_fp32
+    echo basics_svs_single_fp32_LVQ8
     echo spaces_fp32
     echo spaces_fp64
     echo spaces_bf16
@@ -87,10 +89,14 @@ elif [ "$BM_TYPE" = "bm-batch-iter-uint8-multi" ] ; then
 elif [ "$BM_TYPE" = "bm-updated-fp32-single" ] ; then
     echo updated_index_single_fp32
 
+# SVS benchmarks
 elif [ "$BM_TYPE" = "bm-svs-train-fp32" ] ; then
     echo svs_training_fp32
 elif [ "$BM_TYPE" = "bm-svs-train-fp16" ] ; then
     echo svs_training_fp16
+elif [ "$BM_TYPE" = "bm-basics-svs-fp32-single" ] ; then
+    echo basics_svs_single_fp32
+    echo basics_svs_single_fp32_LVQ8
 
 # Spaces benchmarks
 elif [ "$BM_TYPE" = "bm-spaces" ] ; then

tests/benchmark/bm_files.sh

@@ -49,6 +49,9 @@ elif [ "$BM_TYPE" = "bm-svs-train-fp32" ] \
 then
     file_name="training"
     alg="svs"
+elif [ "$BM_TYPE" = "bm-basics-svs-fp32-single" ]; then
+    file_name="basic_fp32"
+    alg="svs"
 else
     echo "No files to download for BM_TYPE=$BM_TYPE"
     exit 0

tests/benchmark/bm_initialization/bm_basics_svs_initialize_fp32.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2006-Present, Redis Ltd.
+ * All rights reserved.
+ *
+ * Licensed under your choice of the Redis Source Available License 2.0
+ * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
+ * GNU Affero General Public License v3 (AGPLv3).
+ */
+
+#pragma once
+/**************************************
+  Define and register tests
+  NOTE: benchmarks' tests order can affect their results. Please add new benchmarks at the end of
+the file.
+***************************************/
+// deleteLabel one by one
+BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimSVS, BM_FUNC_NAME(BM_RunGC), DATA_TYPE_INDEX_T)
+(benchmark::State &st) { RunGC(st); }
+BENCHMARK_REGISTER_F(BM_VecSimSVS, BM_FUNC_NAME(BM_RunGC))
+    ->Unit(benchmark::kMillisecond)
+    ->Iterations(1)
+    ->Arg(50)
+    ->Arg(100)
+    ->Arg(250)
+    ->Arg(500)
+    ->ArgName("num_deletions");
+
+// AddLabel one by one
+BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimSVS, BM_FUNC_NAME(BM_AddLabelOneByOne), DATA_TYPE_INDEX_T)
+(benchmark::State &st) { AddLabel(st); }
+BENCHMARK_REGISTER_F(BM_VecSimSVS, BM_FUNC_NAME(BM_AddLabelOneByOne))
+    ->Unit(benchmark::kMillisecond)
+    ->Iterations(BM_VecSimGeneral::block_size);
+
+// Add vectors in batches via tiered index
+BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimSVS, BM_FUNC_NAME(BM_TriggerUpdateTiered), DATA_TYPE_INDEX_T)
+(benchmark::State &st) { TriggerUpdateTiered(st); }
+BENCHMARK_REGISTER_F(BM_VecSimSVS, BM_FUNC_NAME(BM_TriggerUpdateTiered))
+    ->Unit(benchmark::kMillisecond)
+    ->Iterations(1)
+    ->ArgsProduct({{static_cast<long int>(BM_VecSimGeneral::block_size), 5000,
+                    static_cast<long int>(10 * BM_VecSimGeneral::block_size)},
+                   {2, 4, 8}})
+    ->ArgNames({"update_threshold", "thread_count"})
+    ->MeasureProcessCPUTime();

tests/benchmark/bm_utils.h

@@ -29,12 +29,12 @@ CreateTieredSVSParams(VecSimParams &svs_params, tieredIndexMock &mock_thread_poo
 
 template <typename data_t>
 static void verifyNumThreads(TieredSVSIndex<data_t> *tiered_index, size_t expected_num_threads,
-                             size_t expected_capcity) {
+                             size_t expected_capcity, std::string msg = "") {
     ASSERT_EQ(tiered_index->GetSVSIndex()->getThreadPoolCapacity(), expected_capcity)
-        << "thread pool capacity mismatch";
+        << msg << ": thread pool capacity mismatch";
     size_t num_reserved_threads = tiered_index->GetSVSIndex()->getNumThreads();
     if (num_reserved_threads < expected_num_threads) {
-        std::cout << "WARNING: last reserved threads (" << num_reserved_threads
+        std::cout << msg << ": WARNING: last reserved threads (" << num_reserved_threads
                   << ") is less than expected (" << expected_num_threads << ")." << std::endl;
     }
 }