fix(Tokenizer): bpe tokenizing

marcpinet · marcpinet · commit 5871657a81cb · 2024-12-05T21:03:56.000+01:00
diff --git a/neuralnetlib/preprocessing.py b/neuralnetlib/preprocessing.py
@@ -470,10 +470,10 @@ def learn_bpe(self, texts: list[str], cache: bool = True) -> dict:
                     word = word.lower()
                 chars = list(word)
                 vocab[tuple(chars)] += 1
-                
+
                 word_pairs = self.get_pairs(chars)
                 for pair in word_pairs:
-                    pairs[pair] += vocab[tuple(chars)]
+                    pairs[pair] += 1
         
         merges = {}
         for i in range(self.bpe_merges):
