feat: add Transformer model and layer architecture (wip)

Author: marcpinet

examples/classification-regression/sentiment_analysis.ipynb

@@ -68,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-11-14T19:16:46.274852700Z",
@@ -240,7 +240,7 @@
     },
     {
      "data": {
-      "text/plain": ""
+      "text/plain": []
      },
      "execution_count": 6,
      "metadata": {},

examples/generation/transformer-text-generation/transformer-for-translation.ipynb

@@ -33,19 +33,21 @@ def from_config(config: dict) -> 'LossFunction':
     @staticmethod
     def from_name(name: str) -> "LossFunction":
         name = name.lower().replace("_", "")
-        if name == "mse":
+        if name == "mse" or name == "meansquarederror":
             return MeanSquaredError()
-        elif name == "bce":
+        elif name == "bce" or name == "binarycrossentropy":
             return BinaryCrossentropy()
-        elif name == "cce":
+        elif name == "cce" or name == "categorycrossentropy":
             return CategoricalCrossentropy()
-        elif name == "scce":
+        elif name == "scce" or name == "sparsecategoricalcrossentropy":
             return SparseCategoricalCrossentropy()
-        elif name == "mae":
+        elif name == "mae" or name == "meanabsoluteerror":
             return MeanAbsoluteError()
-        elif name == "kld":
+        elif name == "kld" or name == "kullbackleiblerdivergence":
             return KullbackLeiblerDivergence()
-        elif name.startswith("huber"):
+        elif name == "sequencecrossentropy" or name == "sce":
+            return SequenceCrossEntropy()
+        elif name.startswith("huber") and len(name.split("_")) == 2:
             delta = float(name.split("_")[-1])
             return HuberLoss(delta)
         else:
@@ -162,4 +164,36 @@ def derivative(self, mu: np.ndarray, log_var: np.ndarray) -> tuple:
         return d_mu, d_log_var
 
     def __str__(self):
-        return "KullbackLeiblerDivergence"
+        return "KullbackLeiblerDivergence"
+    
+
+class SequenceCrossEntropy(LossFunction):
+    def __init__(self):
+        super().__init__()
+    
+    def __call__(self, y_true, y_pred):
+        batch_size, seq_len = y_true.shape
+        y_pred = np.clip(y_pred, 1e-10, 1.0)
+        
+        y_true_one_hot = np.zeros_like(y_pred)
+        for i in range(batch_size):
+            for j in range(seq_len):
+                if y_true[i,j] < y_pred.shape[-1]:
+                    y_true_one_hot[i,j,y_true[i,j]] = 1
+        
+        loss = -np.sum(y_true_one_hot * np.log(y_pred))
+        loss = loss / (batch_size * seq_len)
+        return loss
+    
+    def derivative(self, y_true, y_pred):
+        batch_size, seq_len = y_true.shape
+        y_pred = np.clip(y_pred, 1e-10, 1.0)
+        
+        grad = np.zeros_like(y_pred)
+        for i in range(batch_size):
+            for j in range(seq_len):
+                if y_true[i,j] < y_pred.shape[-1]:
+                    grad[i,j,y_true[i,j]] = -1.0 / (y_pred[i,j,y_true[i,j]])
+        
+        grad = grad / (batch_size * seq_len)
+        return grad

examples/generation/vae-image_generation/autoencoder_vae_example.ipynb renamed to examples/generation/vae-image-generation/autoencoder_vae_example.ipynb

@@ -23,6 +23,8 @@ def __init__(self, name: str):
     def _get_function_by_name(self, name: str):
         if name in ['accuracy', 'accuracy_score', 'accuracy-score', 'acc']:
             return accuracy_score
+        elif name in ['sparse_categorical_accuracy', 'sparse-categorical-accuracy', 'sparse_acc']:
+            return sparse_categorical_accuracy_score
         elif name in ['f1', 'f1_score', 'f1-score']:
             return f1_score
         elif name in ['recall', 'recall_score', 'recall-score', 'sensitivity', 'rec']:
@@ -64,6 +66,24 @@ def accuracy_score(y_pred: np.ndarray, y_true: np.ndarray, threshold: float = 0.
     return np.mean(np.argmax(y_pred, axis=1) == np.argmax(y_true, axis=1))
 
 
+def sparse_categorical_accuracy_score(y_pred: np.ndarray, y_true: np.ndarray, threshold: float = 0.5) -> float:
+    y_pred = np.asarray(y_pred)
+    y_true = np.asarray(y_true)
+    
+    if y_pred.ndim == 1:
+        y_pred = y_pred.reshape(-1, 1)
+    
+    if y_true.ndim > 1:
+        if y_true.shape[1] == 1:
+            y_true = y_true.ravel()
+        else:
+            raise ValueError("y_true should be a 1D array of shape (n_samples,) containing integer class indices")
+    
+    predicted_classes = np.argmax(y_pred, axis=1)
+    
+    return np.mean(predicted_classes == y_true)
+
+
 def precision_score(y_pred: np.ndarray, y_true: np.ndarray, threshold: float = 0.5) -> float:
     y_pred, y_true = _reshape_inputs(y_pred, y_true)
     if y_pred.shape[1] == 1: