Added product implementations for scaling pipes, added minmax and gaussian scaling based on objects

Author: mandar2812

build.sbt

@@ -12,7 +12,7 @@ packageDescription := "DynaML is a scala library/repl for implementing and worki
   "which can be extended easily to implement advanced models for small and large scale applications.\n\n"+
   "But the library can also be used as an educational/research tool for data analysis."
 
-val mainVersion = "v1.4-beta.11"
+val mainVersion = "v1.4-beta.12"
 
 val dataDirectory = settingKey[File]("The directory holding the data files for running example scripts")
 

dynaml-core/src/main/scala-2.11/io/github/mandar2812/dynaml/DynaMLPipe.scala

@@ -7,6 +7,7 @@ import io.github.mandar2812.dynaml.models.ParameterizedLearner
 import io.github.mandar2812.dynaml.models.gp.AbstractGPRegressionModel
 import io.github.mandar2812.dynaml.optimization.{CoupledSimulatedAnnealing, GPMLOptimizer, GloballyOptWithGrad, GridSearch}
 import io.github.mandar2812.dynaml.pipes.{DataPipe, ReversibleScaler, Scaler, StreamDataPipe}
+import io.github.mandar2812.dynaml.utils.{GaussianScaler, MinMaxScaler}
 import org.apache.log4j.Logger
 
 /**
@@ -284,6 +285,62 @@ object DynaMLPipe {
         trainTest._2.map(normalizationFunc)), (mean, stdDev))
     })
 
+  /**
+    * Perform gaussian normalization on a data stream which
+    * is a [[Tuple2]] of the form.
+    *
+    * (Stream(training data), Stream(test data))
+    * */
+  val gaussianScalingTrainTest =
+    DataPipe((trainTest: (Stream[(DenseVector[Double], DenseVector[Double])],
+      Stream[(DenseVector[Double], DenseVector[Double])])) => {
+
+      val (num_features, num_targets) = (trainTest._1.head._1.length, trainTest._1.head._2.length)
+
+      val (mean, variance) = utils.getStats(trainTest._1.map(tup =>
+        DenseVector(tup._1.toArray ++ tup._2.toArray)).toList)
+
+      val stdDev: DenseVector[Double] = variance.map(v =>
+        math.sqrt(v/(trainTest._1.length.toDouble - 1.0)))
+
+
+      val featuresScaler = new GaussianScaler(mean(0 until num_features), stdDev(0 until num_features))
+
+      val targetsScaler = new GaussianScaler(
+        mean(num_features until num_features + num_targets - 1),
+        stdDev(num_features until num_features + num_targets - 1))
+
+      val scaler = featuresScaler * targetsScaler
+
+      (scaler(trainTest._1), scaler(trainTest._2), scaler)
+    })
+
+  /**
+    * Perform [0,1] scaling on a data stream which
+    * is a [[Tuple2]] of the form.
+    *
+    * (Stream(training data), Stream(test data))
+    * */
+  val minMaxScalingTrainTest =
+    DataPipe((trainTest: (Stream[(DenseVector[Double], DenseVector[Double])],
+      Stream[(DenseVector[Double], DenseVector[Double])])) => {
+
+      val (num_features, num_targets) = (trainTest._1.head._1.length, trainTest._1.head._2.length)
+
+      val (min, max) = utils.getMinMax(trainTest._1.map(tup =>
+        DenseVector(tup._1.toArray ++ tup._2.toArray)).toList)
+
+      val featuresScaler = new GaussianScaler(min(0 until num_features), max(0 until num_features))
+
+      val targetsScaler = new MinMaxScaler(
+        min(num_features until num_features + num_targets - 1),
+        max(num_features until num_features + num_targets - 1))
+
+      val scaler = featuresScaler * targetsScaler
+
+      (scaler(trainTest._1), scaler(trainTest._2), scaler)
+    })
+
   /**
     * Extract a subset of the data into a [[Tuple2]] which
     * can be used as a training, test combo for model learning and evaluation.

dynaml-core/src/main/scala-2.11/io/github/mandar2812/dynaml/utils/package.scala

@@ -75,18 +75,43 @@ package object utils {
                     i: Int):
     (DenseVector[Double], DenseVector[Double]) = d match {
       case Nil => (m, s)
-      case x :: rest => {
+      case x :: rest =>
         getStatsRec(rest, m + (x - m)/i.toDouble,
           s + ((x - m) :* (x - (m + (x - m)/i.toDouble))),
           i - 1)
-      }
+
     }
 
     getStatsRec(data.tail, data.head,
       DenseVector.zeros[Double](data.head.length),
       data.length)
   }
 
+
+  def getMinMax(data: List[DenseVector[Double]]):
+  (DenseVector[Double], DenseVector[Double]) = {
+    @tailrec
+    def getMinMaxRec(d: List[DenseVector[Double]],
+                     m: DenseVector[Double],
+                     s: DenseVector[Double],
+                     i: Int):
+    (DenseVector[Double], DenseVector[Double]) = d match {
+      case Nil => (m, s)
+      case x :: rest =>
+        getMinMaxRec(rest,
+          DenseVector((x.toArray zip m.toArray).map(c => math.min(c._1, c._2))),
+          DenseVector((x.toArray zip s.toArray).map(c => math.max(c._1, c._2))),
+          i - 1)
+
+    }
+
+    getMinMaxRec(
+      data.tail,
+      data.head,
+      data.head,
+      data.length)
+  }
+
   /*
   * Calculate the value of the hermite polynomials
   * tail recursively. This is needed to calculate

dynaml-pipes/src/main/scala-2.11/io/github/mandar2812/dynaml/pipes/Scaler.scala

@@ -9,6 +9,14 @@ package io.github.mandar2812.dynaml.pipes
 trait Scaler[S] extends DataPipe[S, S]{
   def apply[T[S] <: Traversable[S]](data: T[S]) =
     data.map(run _).asInstanceOf[T[S]]
+
+  def *[T](that: Scaler[T]) = {
+    val firstRun = this.run _
+    new Scaler[(S,T)] {
+      override def run(data: (S, T)): (S, T) = (firstRun(data._1), that(data._2))
+    }
+  }
+
 }
 
 object Scaler {
@@ -30,4 +38,18 @@ trait ReversibleScaler[S] extends Scaler[S] {
     *
     * */
   val i: Scaler[S]
+
+  def *[T](that: ReversibleScaler[T]) = {
+
+    val firstInv = this.i
+
+    val firstRun = this.run _
+
+    new ReversibleScaler[(S, T)] {
+
+      val i: Scaler[(S,T)] = firstInv * that.i
+
+      override def run(data: (S, T)): (S, T) = (firstRun(data._1), that(data._2))
+    }
+  }
 }