Cp 0.3.1 (#172)

* CP "HNSW- account for memory allocation of incoming edges set creation (#169)"

* CP Hnsw reclaim memory (#168)

* CP version h (#171)

* CP "HNSW delete improvements (#170)"

* version bump

Co-authored-by: DvirDukhan <[email protected]>

Author: alonre24

Makefile

@@ -61,6 +61,7 @@ ifeq ($(wildcard $(ROOT)/deps/readies/mk),)
 $(shell mkdir -p deps; cd deps; git clone https://github.com/RedisLabsModules/readies.git)
 endif
 include $(ROOT)/deps/readies/mk/main
+export ROOT
 
 #----------------------------------------------------------------------------------------------
 

src/VecSim/algorithms/brute_force/brute_force.cpp

@@ -39,11 +39,12 @@ BruteForceIndex::~BruteForceIndex() {
 /******************** Implementation **************/
 size_t BruteForceIndex::estimateInitialSize(const BFParams *params) {
     // Constant part (not effected by parameters).
-    size_t est = sizeof(BruteForceIndex);
-    est += sizeof(*allocator);
-    est += sizeof(*space);
+    size_t est = sizeof(VecSimAllocator) + sizeof(BruteForceIndex) + sizeof(size_t);
+    est += (params->metric == VecSimMetric_L2 ? sizeof(L2Space) : sizeof(InnerProductSpace)) +
+           sizeof(size_t);
     // Parameters related part.
-    est += params->initialCapacity * sizeof(decltype(idToVectorBlockMemberMapping)::value_type);
+    est += params->initialCapacity * sizeof(decltype(idToVectorBlockMemberMapping)::value_type) +
+           sizeof(size_t);
 
     return est;
 }

src/VecSim/algorithms/hnsw/hnsw_wrapper.cpp

@@ -32,35 +32,63 @@ HNSWIndex::HNSWIndex(const HNSWParams *params, std::shared_ptr<VecSimAllocator>
 
 /******************** Implementation **************/
 size_t HNSWIndex::estimateInitialSize(const HNSWParams *params) {
-    size_t est = sizeof(HNSWIndex);
-    est += sizeof(*space);
-    est += sizeof(*hnsw);
-    est += sizeof(VisitedNodesHandler);
+    size_t est = sizeof(VecSimAllocator) + sizeof(HNSWIndex) + sizeof(size_t);
+    est += (params->metric == VecSimMetric_L2 ? sizeof(L2Space) : sizeof(InnerProductSpace)) +
+           sizeof(size_t);
+    est += sizeof(*hnsw) + sizeof(size_t);
+    est += sizeof(VisitedNodesHandler) + sizeof(size_t);
     // used for synchronization only when parallel indexing / searching is enabled.
 #ifdef ENABLE_PARALLELIZATION
     est += sizeof(VisitedNodesHandlerPool);
 #endif
-    est += params->initialCapacity * sizeof(tag_t);
+    est += sizeof(tag_t) * params->initialCapacity + sizeof(size_t); // visited nodes
 
-    est += sizeof(void *) * params->initialCapacity; // link lists
-    est += sizeof(size_t) * params->initialCapacity; // element level
+    est += sizeof(void *) * params->initialCapacity + sizeof(size_t); // link lists (for levels > 0)
+    est += sizeof(size_t) * params->initialCapacity + sizeof(size_t); // element level
+    est += sizeof(size_t) * params->initialCapacity +
+           sizeof(size_t); // labels lookup hash table buckets
 
     size_t size_links_level0 =
         sizeof(linklistsizeint) + params->M * 2 * sizeof(tableint) + sizeof(void *);
-    size_t size_data_per_element =
+    size_t size_total_data_per_element =
         size_links_level0 + params->dim * sizeof(float) + sizeof(labeltype);
-    est += params->initialCapacity * size_data_per_element;
+    est += params->initialCapacity * size_total_data_per_element + sizeof(size_t);
 
     return est;
 }
 
 size_t HNSWIndex::estimateElementMemory(const HNSWParams *params) {
-    size_t size_links_level0 =
-        sizeof(linklistsizeint) + params->M * 2 * sizeof(tableint) + sizeof(void *);
-    size_t size_data_per_element =
-        size_links_level0 + params->dim * sizeof(float) + sizeof(labeltype);
+    size_t size_links_level0 = sizeof(linklistsizeint) + params->M * 2 * sizeof(tableint) +
+                               sizeof(void *) + sizeof(vecsim_stl::vector<tableint>);
+    size_t size_links_higher_level = sizeof(linklistsizeint) + params->M * sizeof(tableint) +
+                                     sizeof(void *) + sizeof(vecsim_stl::vector<tableint>);
+    // The Expectancy for the random variable which is the number of levels per element equals
+    // 1/ln(M). Since the max_level is rounded to the "floor" integer, the actual average number
+    // of levels is lower (intuitively, we "loose" a level every time the random generated number
+    // should have been rounded up to the larger integer). So, we "fix" the expectancy and take
+    // 1/2*ln(M) instead as an approximation.
+    size_t expected_size_links_higher_levels =
+        ceil((1 / (2 * log(params->M))) * (float)size_links_higher_level);
+
+    size_t size_total_data_per_element = size_links_level0 + expected_size_links_higher_levels +
+                                         params->dim * sizeof(float) + sizeof(labeltype);
 
-    return size_data_per_element + sizeof(tag_t) + sizeof(size_t) + sizeof(void *);
+    // For every new vector, a new node of size 24 is allocated in a bucket of the hash table.
+    size_t size_label_lookup_node =
+        24 + sizeof(size_t); // 24 + VecSimAllocator::allocation_header_size
+    // 1 entry in visited nodes + 1 entry in element levels + (approximately) 1 bucket in labels
+    // lookup hash map.
+    size_t size_meta_data =
+        sizeof(tag_t) + sizeof(size_t) + sizeof(size_t) + size_label_lookup_node;
+
+    /* Disclaimer: we are neglecting two additional factors that consume memory:
+     * 1. The overall bucket size in labels_lookup hash table is usually higher than the number of
+     * requested buckets (which is the index capacity), and it is auto selected according to the
+     * hashing policy and the max load factor.
+     * 2. The incoming edges that aren't bidirectional are stored in a dynamic array
+     * (vecsim_stl::vector) Those edges' memory *is omitted completely* from this estimation.
+     */
+    return size_meta_data + size_total_data_per_element;
 }
 
 int HNSWIndex::addVector(const void *vector_data, size_t id) {
@@ -82,7 +110,13 @@ int HNSWIndex::addVector(const void *vector_data, size_t id) {
     }
 }
 
-int HNSWIndex::deleteVector(size_t id) { return this->hnsw->removePoint(id); }
+int HNSWIndex::deleteVector(size_t id) {
+    bool res = this->hnsw->removePoint(id);
+    if (hnsw->getIndexSize() + this->blockSize <= this->hnsw->getIndexCapacity()) {
+        this->hnsw->resizeIndex(this->hnsw->getIndexCapacity() - this->blockSize);
+    }
+    return res;
+}
 
 VecSimResolveCode HNSWIndex::resolveParams(VecSimRawParam *rparams, int paramNum,
                                            VecSimQueryParams *qparams) {