perf(vector): Improve hnsw by sharding vectors

posting/list.go

@@ -711,6 +711,14 @@ type ListOptions struct {
 	First     int
 }
 
+func NewVectorPosting(uid uint64, vec *[]byte) *pb.Posting {
+	return &pb.Posting{
+		Value:   *vec,
+		ValType: pb.Posting_BINARY,
+		Op:      Set,
+	}
+}
+
 // NewPosting takes the given edge and returns its equivalent representation as a posting.
 func NewPosting(t *pb.DirectedEdge) *pb.Posting {
 	var op uint32

posting/mvcc.go

@@ -297,6 +297,8 @@ func (txn *Txn) CommitToDisk(writer *TxnWriter, commitTs uint64) error {
 			for ; idx < len(keys); idx++ {
 				key := keys[idx]
 				data := cache.deltas[key]
+				//pk, _ := x.Parse([]byte(key))
+				//fmt.Println(pk, data)
 				if len(data) == 0 {
 					continue
 				}
@@ -634,6 +636,7 @@ func ReadPostingList(key []byte, it *badger.Iterator) (*List, error) {
 	// lists ended up being rolled-up multiple times. This issue was caught by the
 	// uid-set Jepsen test.
 	pk, err := x.Parse(key)
+	//fmt.Println("READING ", pk)
 	if err != nil {
 		return nil, errors.Wrapf(err, "while reading posting list with key [%v]", key)
 	}
@@ -712,6 +715,7 @@ func ReadPostingList(key []byte, it *badger.Iterator) (*List, error) {
 					l.mutationMap = newMutableLayer()
 				}
 				l.mutationMap.insertCommittedPostings(pl)
+				//fmt.Println("HERE", pk, pl, l.mutationMap)
 				return nil
 			})
 			if err != nil {

schema/parse.go

@@ -306,7 +306,7 @@ func parseTokenOrVectorIndexSpec(
 	tokenizer, has := tok.GetTokenizer(tokenOrFactoryName)
 	if !has {
 		return tokenOrFactoryName, nil, false,
-			next.Errorf("Invalid tokenizer %s", next.Val)
+			next.Errorf("Invalid tokenizer 1 %s", next.Val)
 	}
 	tokenizerType, ok := types.TypeForName(tokenizer.Type())
 	x.AssertTrue(ok) // Type is validated during tokenizer loading.

schema/schema.go

@@ -473,6 +473,11 @@ func (s *state) PredicatesToDelete(pred string) []string {
 			preds = append(preds, pred+hnsw.VecEntry)
 			preds = append(preds, pred+hnsw.VecKeyword)
 			preds = append(preds, pred+hnsw.VecDead)
+			for i := range 1000 {
+				preds = append(preds, fmt.Sprintf("%s%s_%d", pred, hnsw.VecEntry, i))
+				preds = append(preds, fmt.Sprintf("%s%s_%d", pred, hnsw.VecKeyword, i))
+				preds = append(preds, fmt.Sprintf("%s%s_%d", pred, hnsw.VecDead, i))
+			}
 		}
 	}
 	return preds