fix vector store dump bug

Author: sewenew

Chinese.md

@@ -463,13 +463,14 @@ LLM.CREATE-VECTOR-STORE key [--NX] [--XX] [--TYPE vector-store-type] [--LLM llm-
 目前我们只支持HNSW类型，这也是默认类型。当然，你也可以显式的通过`--TYPE hnsw`指定。它的参数如下：
 
 ```JSON
-{"max_elements": 100000, "m": 16, "ef_construction": 200, "dim": 0}
+{"max_elements": 100000, "m": 16, "ef_construction": 200}
 ```
 
 所有的参数都是键值对。必选参数使用*required*标识，可选参数给出了默认值。
 
 - *max_elements*: vector store能存储的最大数量。
-- *dim*: vector store存储的embedding的维度。如果使用默认值，0，那么使用第一条插入到vector store的embedding的维度作为vector store的维度。
+
+**NOTE**: vector store使用第一条插入到vector store的embedding的维度作为vector store的维度。
 
 #### 返回值
 

README.md

@@ -465,13 +465,14 @@ LLM.CREATE-VECTOR-STORE key [--NX] [--XX] [--TYPE vector-store-type] [--LLM llm-
 Currently, we only support vector store of HNSW type, and this is also the default one. Of course, you can specify `--TYPE hnsw` explicitly. The parameters are as follows:
 
 ```JSON
-{"max_elements": 100000, "m": 16, "ef_construction": 200, "dim": 0}
+{"max_elements": 100000, "m": 16, "ef_construction": 200}
 ```
 
 All parameters are key-value pairs. The required ones are set as *required*. The optional ones are set with default values. If parameter is not specified, the default value is used.
 
 - *max_elements*: Max number of items that can be stored in the vector store.
-- *dim*: Set the dimension of the vector saved in vector store. If it's 0, i.e. the default value, the dimension of the first inserted vector is used as the dimension of the vector store.
+
+**NOTE**: The dimension of the first inserted vector is used as the dimension of the vector store.
 
 #### Return
 

src/sw/redis-llm/redis_llm.cpp

@@ -64,7 +64,7 @@ void rewrite_vector_store(RedisModuleIO *aof, RedisModuleString *key, VectorStor
 
 void rdb_save_vector_store(RedisModuleIO *rdb, VectorStore &store);
 
-void rdb_load_vector_store(RedisModuleIO *rdb, VectorStore &store);
+void rdb_load_vector_store(RedisModuleIO *rdb, VectorStore &store, std::size_t dim);
 
 void* rdb_load_vector_store(RedisModuleIO *rdb);
 
@@ -502,6 +502,7 @@ void rdb_save_vector_store(RedisModuleIO *rdb, VectorStore &store) {
     rdb_save_config(rdb, store.conf());
     rdb_save_string(rdb, store.llm().to_string());
     rdb_save_number(rdb, store.id_idx());
+    rdb_save_number(rdb, store.dim());
 
     const auto &data_store = store.data_store();
     rdb_save_number(rdb, data_store.size());
@@ -524,8 +525,7 @@ void rdb_save_app(RedisModuleIO *rdb, void *value) {
     rdb_save_config(rdb, app->conf());
 }
 
-void rdb_load_vector_store(RedisModuleIO *rdb, VectorStore &store) {
-    auto dim = store.dim();
+void rdb_load_vector_store(RedisModuleIO *rdb, VectorStore &store, std::size_t dim) {
     auto size = rdb_load_number(rdb);
     for (auto idx = 0UL; idx < size; ++idx) {
         auto id = rdb_load_number(rdb);
@@ -563,11 +563,12 @@ void* rdb_load_vector_store(RedisModuleIO *rdb) {
     auto info_str = rdb_load_string(rdb);
     LlmInfo llm_info(to_sv(info_str));
     auto id_idx = rdb_load_number(rdb);
+    auto dim = rdb_load_number(rdb);
 
     auto store = llm.create_vector_store(type, conf, llm_info);
     store->set_id_idx(id_idx);
 
-    rdb_load_vector_store(rdb, *store);
+    rdb_load_vector_store(rdb, *store, dim);
 
     return store.get();
 }

src/sw/redis-llm/vector_store.cpp

@@ -31,6 +31,8 @@ uint64_t VectorStore::add(uint64_t id, const std::string_view &data, const Vecto
         // Use the first item's dimension as the dimension of the vector store.
         _dim = embedding.size();
 
+        assert(_dim > 0);
+
         _lazily_init(_dim);
     }
 
@@ -100,20 +102,6 @@ uint64_t VectorStore::_auto_gen_id() {
     return ++_id_idx;
 }
 
-std::size_t VectorStore::_dimension() const {
-    auto iter = _conf.find("dim");
-    if (iter == _conf.end()) {
-        return 0;
-    }
-
-    const auto &dim = iter.value();
-    if (!dim.is_number_unsigned()) {
-        throw Error("invalid dim");
-    }
-
-    return dim.get<std::size_t>();
-}
-
 VectorStoreFactory::VectorStoreFactory() {
     _register("hnsw", std::make_unique<VectorStoreCreatorTpl<Hnsw>>());
 }

src/sw/redis-llm/vector_store.h

@@ -32,7 +32,7 @@ namespace sw::redis::llm {
 class VectorStore : public Object {
 public:
     VectorStore(const std::string &type, const nlohmann::json &conf, const LlmInfo &llm) :
-        _type(type), _conf(conf), _dim(_dimension()), _llm(llm) {}
+        _type(type), _conf(conf), _dim(0), _llm(llm) {}
 
     virtual ~VectorStore() = default;
 
@@ -92,8 +92,6 @@ class VectorStore : public Object {
 
     uint64_t _auto_gen_id();
 
-    std::size_t _dimension() const;
-
     std::string _type;
 
     nlohmann::json _conf;