add ReLU to MLP predictor and add tests for it (#1031)

This PR adds ReLU to Multilayer perceptron predictor.
The ReLU implementation is the same as for Neural Network predictor. Tests for MLP regressor and classifier with ReLU activation function are passing.

Author: ekloberdanz

pymoose/pymoose/predictors/fixtures/MPL_classfier_1hidden_layers_10classes_relu.onnx

@@ -11,6 +11,7 @@
 class Activation(Enum):
     IDENTITY = 1
     SIGMOID = 2
+    RELU = 3
 
 
 class MLPPredictor(aes_predictor.AesPredictor, metaclass=abc.ABCMeta):
@@ -72,6 +73,8 @@ def from_onnx(cls, model_proto):
         )
         if activation_str == "Sigmoid":
             activation = Activation.SIGMOID
+        elif activation_str == "Relu":
+            activation = Activation.RELU
         else:
             activation = Activation.IDENTITY
 
@@ -95,13 +98,14 @@ def apply_layer(self, input, num_hidden_layers, i, fixedpoint_dtype):
     def activation_fn(self, z):
         if self.activation == Activation.SIGMOID:
             activation_output = edsl.sigmoid(z)
-        # There is a bug in edsl.shape
-        #  Relu code:
-        #     y_1_shape = edsl.slice(edsl.shape(x), begin=0, end=1)
-        #     ones = edsl.ones(y_1_shape, dtype=edsl.float64)
-        #     ones = edsl.cast(ones, dtype=fixedpoint_dtype)
-        #     zeros = edsl.sub(ones, ones)
-        #     activation_output = edsl.maximum([zeros, y_1])
+        elif self.activation == Activation.RELU:
+            z_shape = edsl.shape(z)
+            with self.bob:
+                ones_1 = edsl.ones(z_shape, dtype=predictor_utils.DEFAULT_FLOAT_DTYPE)
+                ones_2 = edsl.ones(z_shape, dtype=predictor_utils.DEFAULT_FLOAT_DTYPE)
+                zeros = edsl.sub(ones_1, ones_2)
+                zeros = edsl.cast(zeros, dtype=predictor_utils.DEFAULT_FIXED_DTYPE)
+            activation_output = edsl.maximum([zeros, z])
         elif self.activation == Activation.IDENTITY:
             activation_output = z
         else:

pymoose/pymoose/predictors/fixtures/MPL_regressor_1hidden_layers_1target_relu.onnx

@@ -25,6 +25,7 @@
         "MLP_regressor_2hidden_layers_2target_identity",
         [[550.57188666, 431.31144628], [57.07484159, 128.76424831]],
     ),
+    ("MPL_regressor_1hidden_layers_1target_relu", [32.55903733, 16.92872095]),
 ]
 _SK_CLASSIFIER_MODELS = [
     ("MLP_classifier_1hidden_layers_2classes_logistic", [0.60823407, 0.39176593]),
@@ -60,6 +61,21 @@
             0.01489627,
         ],
     ),
+    (
+        "MPL_classfier_1hidden_layers_10classes_relu",
+        [
+            0.18669855,
+            0.01198506,
+            0.02292363,
+            0.04383366,
+            0.4636751,
+            0.08422828,
+            0.0207348,
+            0.07604881,
+            0.0196595,
+            0.07021261,
+        ],
+    ),
 ]