Skip to content

Commit c09129a

Browse files
meiravgrimeiravgri
committed
Merge branch 'feature_HNSW_tiered_index' into meiravg_bufferlimit_bm
2 parents 5d8349a + f3f26ac commit c09129a

30 files changed

+742
-448
lines changed

‎Makefile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ endif
152152
#----------------------------------------------------------------------------------------------
153153

154154
pybind:
155-
$(SHOW)python3 -m poetry build
155+
$(SHOW)poetry build
156156
.PHONY: pybind
157157

158158
#----------------------------------------------------------------------------------------------
@@ -186,8 +186,8 @@ valgrind:
186186
#----------------------------------------------------------------------------------------------
187187

188188
flow_test:
189-
$(SHOW)$(MAKE) pybind
190-
$(SHOW)tox -e flowenv
189+
$(SHOW)poetry install
190+
$(SHOW)poetry run pytest tests/flow -v -s
191191

192192
.PHONY: flow_test
193193

‎pyproject.toml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
name = "VecSim"
33
version = "0.0.1"
44
description = "Python library around collection of vector similarity algorithms"
5-
build = "setup.py"
65

76
packages = [
87
{ include = 'src'}
@@ -28,8 +27,6 @@ python = "^3.8"
2827

2928
[tool.poetry.dev-dependencies]
3029
numpy = "^1.21"
31-
tox = "^3.25.0"
32-
tox-poetry = "^0.4.0"
3330
hnswlib = "^0.6.2"
3431
pytest = "^6.2.4"
3532
scipy = "^1.9.1"
@@ -39,3 +36,7 @@ h5py = "^3.7.0"
3936
[build-system]
4037
requires = ["poetry-core>=1.0.0", "setuptools"]
4138
build-backend = "poetry.core.masonry.api"
39+
40+
[tool.poetry.build]
41+
script = "setup.py"
42+
generate-setup-file = true

‎requirements.txt‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,2 @@
11
pip>=21.1
2-
poetry==1.1.15
3-
tox>=3.25.0
4-
tox-poetry>=0.4.1
2+
poetry==1.4.2

‎src/VecSim/algorithms/brute_force/brute_force.h‎

Lines changed: 2 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -345,15 +345,7 @@ VecSimIndexInfo BruteForceIndex<DataType, DistType>::info() const {
345345

346346
VecSimIndexInfo info;
347347
info.algo = VecSimAlgo_BF;
348-
info.bfInfo.dim = this->dim;
349-
info.bfInfo.type = this->vecType;
350-
info.bfInfo.metric = this->metric;
351-
info.bfInfo.indexSize = this->count;
352-
info.bfInfo.indexLabelCount = this->indexLabelCount();
353-
info.bfInfo.blockSize = this->blockSize;
354-
info.bfInfo.memory = this->getAllocationSize();
355-
info.bfInfo.isMulti = this->isMulti;
356-
info.bfInfo.last_mode = this->last_mode;
348+
info.commonInfo = this->getCommonInfo();
357349
return info;
358350
}
359351

@@ -368,44 +360,7 @@ VecSimInfoIterator *BruteForceIndex<DataType, DistType>::infoIterator() const {
368360
.fieldName = VecSimCommonStrings::ALGORITHM_STRING,
369361
.fieldType = INFOFIELD_STRING,
370362
.fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(info.algo)}}});
371-
infoIterator->addInfoField(VecSim_InfoField{
372-
.fieldName = VecSimCommonStrings::TYPE_STRING,
373-
.fieldType = INFOFIELD_STRING,
374-
.fieldValue = {FieldValue{.stringValue = VecSimType_ToString(info.bfInfo.type)}}});
375-
infoIterator->addInfoField(
376-
VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING,
377-
.fieldType = INFOFIELD_UINT64,
378-
.fieldValue = {FieldValue{.uintegerValue = info.bfInfo.dim}}});
379-
infoIterator->addInfoField(VecSim_InfoField{
380-
.fieldName = VecSimCommonStrings::METRIC_STRING,
381-
.fieldType = INFOFIELD_STRING,
382-
.fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(info.bfInfo.metric)}}});
383-
infoIterator->addInfoField(
384-
VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING,
385-
.fieldType = INFOFIELD_UINT64,
386-
.fieldValue = {FieldValue{.uintegerValue = info.bfInfo.isMulti}}});
387-
infoIterator->addInfoField(
388-
VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING,
389-
.fieldType = INFOFIELD_UINT64,
390-
.fieldValue = {FieldValue{.uintegerValue = info.bfInfo.indexSize}}});
391-
infoIterator->addInfoField(
392-
VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING,
393-
.fieldType = INFOFIELD_UINT64,
394-
.fieldValue = {FieldValue{.uintegerValue = info.bfInfo.indexLabelCount}}});
395-
infoIterator->addInfoField(
396-
VecSim_InfoField{.fieldName = VecSimCommonStrings::BLOCK_SIZE_STRING,
397-
.fieldType = INFOFIELD_UINT64,
398-
.fieldValue = {FieldValue{.uintegerValue = info.bfInfo.blockSize}}});
399-
infoIterator->addInfoField(
400-
VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING,
401-
.fieldType = INFOFIELD_UINT64,
402-
.fieldValue = {FieldValue{.uintegerValue = info.bfInfo.memory}}});
403-
infoIterator->addInfoField(
404-
VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING,
405-
.fieldType = INFOFIELD_STRING,
406-
.fieldValue = {FieldValue{
407-
.stringValue = VecSimSearchMode_ToString(info.bfInfo.last_mode)}}});
408-
363+
this->addCommonInfoToIterator(infoIterator, info.commonInfo);
409364
return infoIterator;
410365
}
411366

‎src/VecSim/algorithms/hnsw/hnsw.h‎

Lines changed: 16 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2150,96 +2150,69 @@ template <typename DataType, typename DistType>
21502150
VecSimIndexInfo HNSWIndex<DataType, DistType>::info() const {
21512151

21522152
VecSimIndexInfo info;
2153+
info.commonInfo = this->getCommonInfo();
2154+
21532155
info.algo = VecSimAlgo_HNSWLIB;
2154-
info.hnswInfo.dim = this->dim;
2155-
info.hnswInfo.type = this->vecType;
2156-
info.hnswInfo.isMulti = this->isMulti;
2157-
info.hnswInfo.metric = this->metric;
2158-
info.hnswInfo.blockSize = this->blockSize;
21592156
info.hnswInfo.M = this->getM();
21602157
info.hnswInfo.efConstruction = this->getEfConstruction();
21612158
info.hnswInfo.efRuntime = this->getEf();
21622159
info.hnswInfo.epsilon = this->epsilon_;
2163-
info.hnswInfo.indexSize = this->indexSize();
2164-
info.hnswInfo.indexLabelCount = this->indexLabelCount();
21652160
info.hnswInfo.max_level = this->getMaxLevel();
21662161
info.hnswInfo.entrypoint = this->getEntryPointLabel();
2167-
info.hnswInfo.memory = this->getAllocationSize();
2168-
info.hnswInfo.last_mode = this->last_mode;
21692162
info.hnswInfo.visitedNodesPoolSize = this->visited_nodes_handler_pool.getPoolSize();
2163+
info.hnswInfo.numberOfMarkedDeletedNodes = this->getNumMarkedDeleted();
21702164
return info;
21712165
}
21722166

21732167
template <typename DataType, typename DistType>
21742168
VecSimInfoIterator *HNSWIndex<DataType, DistType>::infoIterator() const {
21752169
VecSimIndexInfo info = this->info();
21762170
// For readability. Update this number when needed.
2177-
size_t numberOfInfoFields = 12;
2171+
size_t numberOfInfoFields = 13;
21782172
VecSimInfoIterator *infoIterator = new VecSimInfoIterator(numberOfInfoFields);
21792173

21802174
infoIterator->addInfoField(VecSim_InfoField{
21812175
.fieldName = VecSimCommonStrings::ALGORITHM_STRING,
21822176
.fieldType = INFOFIELD_STRING,
21832177
.fieldValue = {FieldValue{.stringValue = VecSimAlgo_ToString(info.algo)}}});
2184-
infoIterator->addInfoField(VecSim_InfoField{
2185-
.fieldName = VecSimCommonStrings::TYPE_STRING,
2186-
.fieldType = INFOFIELD_STRING,
2187-
.fieldValue = {FieldValue{.stringValue = VecSimType_ToString(info.hnswInfo.type)}}});
2188-
infoIterator->addInfoField(
2189-
VecSim_InfoField{.fieldName = VecSimCommonStrings::DIMENSION_STRING,
2190-
.fieldType = INFOFIELD_UINT64,
2191-
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.dim}}});
2192-
infoIterator->addInfoField(VecSim_InfoField{
2193-
.fieldName = VecSimCommonStrings::METRIC_STRING,
2194-
.fieldType = INFOFIELD_STRING,
2195-
.fieldValue = {FieldValue{.stringValue = VecSimMetric_ToString(info.hnswInfo.metric)}}});
21962178

2197-
infoIterator->addInfoField(
2198-
VecSim_InfoField{.fieldName = VecSimCommonStrings::IS_MULTI_STRING,
2199-
.fieldType = INFOFIELD_UINT64,
2200-
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.isMulti}}});
2201-
infoIterator->addInfoField(
2202-
VecSim_InfoField{.fieldName = VecSimCommonStrings::INDEX_SIZE_STRING,
2203-
.fieldType = INFOFIELD_UINT64,
2204-
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.indexSize}}});
2205-
infoIterator->addInfoField(VecSim_InfoField{
2206-
.fieldName = VecSimCommonStrings::INDEX_LABEL_COUNT_STRING,
2207-
.fieldType = INFOFIELD_UINT64,
2208-
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.indexLabelCount}}});
2179+
this->addCommonInfoToIterator(infoIterator, info.commonInfo);
2180+
22092181
infoIterator->addInfoField(
22102182
VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_M_STRING,
22112183
.fieldType = INFOFIELD_UINT64,
22122184
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.M}}});
2185+
22132186
infoIterator->addInfoField(VecSim_InfoField{
22142187
.fieldName = VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING,
22152188
.fieldType = INFOFIELD_UINT64,
22162189
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efConstruction}}});
2190+
22172191
infoIterator->addInfoField(
22182192
VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EF_RUNTIME_STRING,
22192193
.fieldType = INFOFIELD_UINT64,
22202194
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.efRuntime}}});
2195+
22212196
infoIterator->addInfoField(
22222197
VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_MAX_LEVEL,
22232198
.fieldType = INFOFIELD_UINT64,
22242199
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.max_level}}});
2200+
22252201
infoIterator->addInfoField(
22262202
VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_ENTRYPOINT,
22272203
.fieldType = INFOFIELD_UINT64,
22282204
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.entrypoint}}});
2229-
infoIterator->addInfoField(
2230-
VecSim_InfoField{.fieldName = VecSimCommonStrings::MEMORY_STRING,
2231-
.fieldType = INFOFIELD_UINT64,
2232-
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.memory}}});
2233-
infoIterator->addInfoField(
2234-
VecSim_InfoField{.fieldName = VecSimCommonStrings::SEARCH_MODE_STRING,
2235-
.fieldType = INFOFIELD_STRING,
2236-
.fieldValue = {FieldValue{
2237-
.stringValue = VecSimSearchMode_ToString(info.hnswInfo.last_mode)}}});
2205+
22382206
infoIterator->addInfoField(
22392207
VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EPSILON_STRING,
22402208
.fieldType = INFOFIELD_FLOAT64,
22412209
.fieldValue = {FieldValue{.floatingPointValue = info.hnswInfo.epsilon}}});
22422210

2211+
infoIterator->addInfoField(VecSim_InfoField{
2212+
.fieldName = VecSimCommonStrings::HNSW_NUM_MARKED_DELETED,
2213+
.fieldType = INFOFIELD_UINT64,
2214+
.fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.numberOfMarkedDeletedNodes}}});
2215+
22432216
return infoIterator;
22442217
}
22452218

‎src/VecSim/algorithms/hnsw/hnsw_tiered.h‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,8 @@ class TieredHNSWIndex : public VecSimTieredIndex<DataType, DistType> {
165165
public:
166166
TieredHNSWIndex(HNSWIndex<DataType, DistType> *hnsw_index,
167167
BruteForceIndex<DataType, DistType> *bf_index,
168-
const TieredIndexParams &tieredParams);
168+
const TieredIndexParams &tieredParams,
169+
std::shared_ptr<VecSimAllocator> allocator);
169170
virtual ~TieredHNSWIndex();
170171

171172
int addVector(const void *blob, labelType label, void *auxiliaryCtx = nullptr) override;
@@ -178,9 +179,6 @@ class TieredHNSWIndex : public VecSimTieredIndex<DataType, DistType> {
178179
// needed.
179180
void increaseCapacity() override {}
180181

181-
// TODO: Implement the actual methods instead of these temporary ones.
182-
VecSimIndexInfo info() const override { return this->backendIndex->info(); }
183-
VecSimInfoIterator *infoIterator() const override { return this->backendIndex->infoIterator(); }
184182
VecSimBatchIterator *newBatchIterator(const void *queryBlob,
185183
VecSimQueryParams *queryParams) const override {
186184
size_t blobSize = this->backendIndex->getDim() * sizeof(DataType);
@@ -527,8 +525,9 @@ void TieredHNSWIndex<DataType, DistType>::executeRepairJob(HNSWRepairJob *job) {
527525
template <typename DataType, typename DistType>
528526
TieredHNSWIndex<DataType, DistType>::TieredHNSWIndex(HNSWIndex<DataType, DistType> *hnsw_index,
529527
BruteForceIndex<DataType, DistType> *bf_index,
530-
const TieredIndexParams &tiered_index_params)
531-
: VecSimTieredIndex<DataType, DistType>(hnsw_index, bf_index, tiered_index_params),
528+
const TieredIndexParams &tiered_index_params,
529+
std::shared_ptr<VecSimAllocator> allocator)
530+
: VecSimTieredIndex<DataType, DistType>(hnsw_index, bf_index, tiered_index_params, allocator),
532531
labelToInsertJobs(this->allocator), idToRepairJobs(this->allocator),
533532
idToSwapJob(this->allocator) {
534533
// If the param for swapJobThreshold is 0 use the default value, if it exceeds the maximum

‎src/VecSim/algorithms/hnsw/hnsw_tiered_tests_friends.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_BatchIteratorSize1_Test)
2222
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_BatchIteratorReset_Test)
2323
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_BatchIteratorWithOverlaps_Test)
2424
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_parallelBatchIteratorSearch_Test)
25+
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_testInfo_Test)
26+
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_testInfoIterator_Test)
2527
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_writeInPlaceMode_Test)
2628
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_switchWriteModes_Test)
2729
INDEX_TEST_FRIEND_CLASS(HNSWTieredIndexTest_bufferLimit_Test)

‎src/VecSim/index_factories/tiered_factory.cpp‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) {
2727
.multi = params->primaryIndexParams->hnswParams.multi,
2828
.blockSize = params->primaryIndexParams->hnswParams.blockSize};
2929

30-
AbstractIndexInitParams abstractInitParams = {.allocator = hnsw_index->getAllocator(),
30+
std::shared_ptr<VecSimAllocator> flat_allocator = VecSimAllocator::newVecsimAllocator();
31+
AbstractIndexInitParams abstractInitParams = {.allocator = flat_allocator,
3132
.dim = bf_params.dim,
3233
.vecType = bf_params.type,
3334
.metric = bf_params.metric,
@@ -38,8 +39,11 @@ inline VecSimIndex *NewIndex(const TieredIndexParams *params) {
3839
BruteForceFactory::NewIndex(&bf_params, abstractInitParams));
3940

4041
// Create new tiered hnsw index
41-
return new (hnsw_index->getAllocator())
42-
TieredHNSWIndex<DataType, DistType>(hnsw_index, frontendIndex, *params);
42+
std::shared_ptr<VecSimAllocator> management_layer_allocator =
43+
VecSimAllocator::newVecsimAllocator();
44+
45+
return new (management_layer_allocator) TieredHNSWIndex<DataType, DistType>(
46+
hnsw_index, frontendIndex, *params, management_layer_allocator);
4347
}
4448

4549
inline size_t EstimateInitialSize(const TieredIndexParams *params, BFParams &bf_params_output) {
@@ -48,6 +52,10 @@ inline size_t EstimateInitialSize(const TieredIndexParams *params, BFParams &bf_
4852
// Add size estimation of VecSimTieredIndex sub indexes.
4953
size_t est = HNSWFactory::EstimateInitialSize(&hnsw_params);
5054

55+
// Management layer allocator overhead.
56+
size_t allocations_overhead = VecSimAllocator::getAllocationOverheadSize();
57+
est += sizeof(VecSimAllocator) + allocations_overhead;
58+
5159
// Size of the TieredHNSWIndex struct.
5260
if (hnsw_params.type == VecSimType_FLOAT32) {
5361
est += sizeof(TieredHNSWIndex<float, float>);
@@ -87,6 +95,7 @@ VecSimIndex *NewIndex(const TieredIndexParams *params) {
8795
size_t EstimateInitialSize(const TieredIndexParams *params) {
8896

8997
size_t est = 0;
98+
9099
BFParams bf_params{};
91100
if (params->primaryIndexParams->algo == VecSimAlgo_HNSWLIB) {
92101
est += TieredHNSWFactory::EstimateInitialSize(params, bf_params);

‎src/VecSim/info_iterator.h‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,16 @@ typedef enum {
2121
INFOFIELD_STRING,
2222
INFOFIELD_INT64,
2323
INFOFIELD_UINT64,
24-
INFOFIELD_FLOAT64
24+
INFOFIELD_FLOAT64,
25+
INFOFIELD_ITERATOR
2526
} VecSim_InfoFieldType;
2627

2728
typedef union {
28-
double floatingPointValue; // Floating point value. 64 bits float.
29-
int64_t integerValue; // Integer value. Signed 64 bits integer.
30-
u_int64_t uintegerValue; // Unsigned value. Unsigned 64 buts integer.
31-
const char *stringValue; // String value.
29+
double floatingPointValue; // Floating point value. 64 bits float.
30+
int64_t integerValue; // Integer value. Signed 64 bits integer.
31+
u_int64_t uintegerValue; // Unsigned value. Unsigned 64 buts integer.
32+
const char *stringValue; // String value.
33+
VecSimInfoIterator *iteratorValue; // Iterator value.
3234
} FieldValue;
3335

3436
/**

‎src/VecSim/info_iterator_struct.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,12 @@ struct VecSimInfoIterator {
2727

2828
inline size_t numberOfFields() { return array_len(this->fields); }
2929

30-
virtual ~VecSimInfoIterator() { array_free(this->fields); }
30+
virtual ~VecSimInfoIterator() {
31+
for (size_t i = 0; i < array_len(this->fields); i++) {
32+
if (this->fields[i].fieldType == INFOFIELD_ITERATOR) {
33+
delete this->fields[i].fieldValue.iteratorValue;
34+
}
35+
}
36+
array_free(this->fields);
37+
}
3138
};

0 commit comments

Comments
 (0)