From 40e469ff0b451103465fbcdd6897d5359aab865f Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Wed, 28 Mar 2018 11:25:58 +0200 Subject: [PATCH 01/16] handle errors and prepare for the implementation of hidden multi-layers Added "Rest parameters" for the future implementation of hidden multi-layers, where you will pass arguments like this: new NeuralNetwork(3,6,8,6,4) new NeuralNetwork(input, hidden_1, hidden_2, hidden_3..., output) Added "handle errors" in constructor method. --- lib/nn.js | 50 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index 0260280..f864d7b 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -19,22 +19,36 @@ let tanh = new ActivationFunction( class NeuralNetwork { - // TODO: document what a, b, c are - constructor(a, b, c) { - if (a instanceof NeuralNetwork) { - this.input_nodes = a.input_nodes; - this.hidden_nodes = a.hidden_nodes; - this.output_nodes = a.output_nodes; - - this.weights_ih = a.weights_ih.copy(); - this.weights_ho = a.weights_ho.copy(); - - this.bias_h = a.bias_h.copy(); - this.bias_o = a.bias_o.copy(); - } else { - this.input_nodes = a; - this.hidden_nodes = b; - this.output_nodes = c; + /** + * Construct method. + * + * The user can enter 3 parameters (integer) to create a new NeuralNetwork, + * where: the first parameter represents the number of inputs, the second + * parameter represents the number of hidden nodes and the third parameter + * represents the number of outputs of the network. + * The user can copy an instance of NeuralNetwork by passing the same as an + * argument to the constructor. + * + * @param {NeuralNetwork|integer} args (Rest parameters) + */ + constructor(...args) { + if (args.length === 1 && args[0] instanceof NeuralNetwork) { + this.input_nodes = args[0].input_nodes; + this.hidden_nodes = args[0].hidden_nodes; + this.output_nodes = args[0].output_nodes; + + this.weights_ih = args[0].weights_ih.copy(); + this.weights_ho = args[0].weights_ho.copy(); + + this.bias_h = args[0].bias_h.copy(); + this.bias_o = args[0].bias_o.copy(); + } else if(args.length === 3) { + if(!args.every( v => Number.isInteger(v) )) { + throw new Error('All arguments must be integer.'); + } + this.input_nodes = args[0]; + this.hidden_nodes = args[1]; + this.output_nodes = args[2]; this.weights_ih = new Matrix(this.hidden_nodes, this.input_nodes); this.weights_ho = new Matrix(this.output_nodes, this.hidden_nodes); @@ -45,13 +59,13 @@ class NeuralNetwork { this.bias_o = new Matrix(this.output_nodes, 1); this.bias_h.randomize(); this.bias_o.randomize(); + } else { + throw new Error('Invalid arguments. Read the documentation!'); } // TODO: copy these as well this.setLearningRate(); this.setActivationFunction(); - - } predict(input_array) { From 29c7a7e8f17917d22f1e5079e2ecc438281ce252 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Fri, 30 Mar 2018 20:32:27 +0200 Subject: [PATCH 02/16] Refactory constructor method for multi-hidden layers --- lib/nn.js | 101 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 29 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index f864d7b..87cc975 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -20,7 +20,7 @@ let tanh = new ActivationFunction( class NeuralNetwork { /** - * Construct method. + * Constructor method. * * The user can enter 3 parameters (integer) to create a new NeuralNetwork, * where: the first parameter represents the number of inputs, the second @@ -29,37 +29,83 @@ class NeuralNetwork { * The user can copy an instance of NeuralNetwork by passing the same as an * argument to the constructor. * - * @param {NeuralNetwork|integer} args (Rest parameters) + * @param {NeuralNetwork|...Integer} args (Rest parameters) */ constructor(...args) { + this.hiddenLayers = []; + this.weightLayers = []; + if (args.length === 1 && args[0] instanceof NeuralNetwork) { - this.input_nodes = args[0].input_nodes; - this.hidden_nodes = args[0].hidden_nodes; - this.output_nodes = args[0].output_nodes; - this.weights_ih = args[0].weights_ih.copy(); - this.weights_ho = args[0].weights_ho.copy(); + let nn = args[0]; + + // INPUT LAYER + this.inputLayer = { + nodes: nn.inputLayer.nodes + }; - this.bias_h = args[0].bias_h.copy(); - this.bias_o = args[0].bias_o.copy(); - } else if(args.length === 3) { + // HIDDEN LAYERS + for(let hidden of nn.hiddenLayers) { + this.hiddenLayers.push({ + nodes: hidden.nodes, + bias: hidden.bias.copy() + }); + } + + // OUTPUT LAYER + this.outputLayer = { + nodes: nn.outputLayer.nodes, + bias: nn.outputLayer.bias.copy() + }; + + // WEIGHTS CONNECTIONS + for(let weight of nn.weightLayers) { + this.weightLayers.push( + weight.copy() + ); + } + + } else if(args.length >= 3) { + + // CHECK ARGUMENTS if(!args.every( v => Number.isInteger(v) )) { throw new Error('All arguments must be integer.'); } - this.input_nodes = args[0]; - this.hidden_nodes = args[1]; - this.output_nodes = args[2]; - - this.weights_ih = new Matrix(this.hidden_nodes, this.input_nodes); - this.weights_ho = new Matrix(this.output_nodes, this.hidden_nodes); - this.weights_ih.randomize(); - this.weights_ho.randomize(); - - this.bias_h = new Matrix(this.hidden_nodes, 1); - this.bias_o = new Matrix(this.output_nodes, 1); - this.bias_h.randomize(); - this.bias_o.randomize(); + + // INPUT LAYER + this.inputLayer = { + nodes: args[0] + }; + + let lastIndex = args.length-1; + + // HIDDEN LAYERS + for(let i = 1; i < lastIndex; i++) { + let nodes = args[i]; + this.hiddenLayers.push({ + nodes: nodes, + bias: new Matrix(nodes, 1).randomize() + }); + } + + // OUTPUT LAYER + this.outputLayer = { + nodes: args[lastIndex], + bias: new Matrix(args[lastIndex], 1).randomize() + }; + + // WEIGHTS CONNECTIONS + let connections = lastIndex; + for(let i = 0; i < connections; i++) { + let primaryNodes = args[i], + secondaryNodes = args[i+1]; + this.weightLayers.push( + new Matrix(secondaryNodes, primaryNodes).randomize() + ); + } + } else { + throw new Error('Invalid arguments. Read the documentation!'); } @@ -86,12 +132,12 @@ class NeuralNetwork { return output.toArray(); } - setLearningRate(learning_rate = 0.1) { - this.learning_rate = learning_rate; + setLearningRate(learningRate = 0.1) { + this.learningRate = learningRate; } setActivationFunction(func = sigmoid) { - this.activation_function = func; + this.activationFunction = func; } train(input_array, target_array) { @@ -188,7 +234,4 @@ class NeuralNetwork { this.bias_h.map(mutate); this.bias_o.map(mutate); } - - - } From aa6fefc3bdf36321baaf4985986040ee3d6073ec Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Fri, 30 Mar 2018 21:51:44 +0200 Subject: [PATCH 03/16] Refactory train method for multi-hidden layers =) --- lib/nn.js | 103 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 18 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index 87cc975..1b48200 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -22,16 +22,17 @@ class NeuralNetwork { /** * Constructor method. * - * The user can enter 3 parameters (integer) to create a new NeuralNetwork, + * The user can enter with parameters (integer) to create a new NeuralNetwork, * where: the first parameter represents the number of inputs, the second - * parameter represents the number of hidden nodes and the third parameter - * represents the number of outputs of the network. + * (or more) parameter represents the number of hidden nodes and the last + * parameter represents the number of outputs of the network. * The user can copy an instance of NeuralNetwork by passing the same as an * argument to the constructor. * * @param {NeuralNetwork|...Integer} args (Rest parameters) */ constructor(...args) { + this.hiddenLayers = []; this.weightLayers = []; @@ -140,33 +141,97 @@ class NeuralNetwork { this.activationFunction = func; } - train(input_array, target_array) { + train(inputArray, targetArray) { + + if(inputArray.length !== this.inputLayer.nodes || + targetArray.length !== this.outputLayer.nodes) { + + throw new Error('Array size error.'); + } + // Generating the Hidden Outputs - let inputs = Matrix.fromArray(input_array); - let hidden = Matrix.multiply(this.weights_ih, inputs); - hidden.add(this.bias_h); - // activation function! - hidden.map(this.activation_function.func); + this.inputLayer.matrix = Matrix.fromArray(inputArray); + + let lastHiddenIndex; + let lastWeightIndex; + + for(let i = 0; i < this.hiddenLayers.length; i++) { + this.hiddenLayers[i].matrix = Matrix.multiply( + this.weightLayers[i], + i == 0 ? this.inputLayer.matrix : this.hiddenLayers[i-1].matrix + ) + .add(this.hiddenLayers[i].bias) + // activation function! + .map(this.activationFunction.func); + + lastHiddenIndex = i; + lastWeightIndex = i + 1; + } // Generating the output's output! - let outputs = Matrix.multiply(this.weights_ho, hidden); - outputs.add(this.bias_o); - outputs.map(this.activation_function.func); + this.outputLayer.matrix = Matrix.multiply( + this.weightLayers[lastWeightIndex], + this.hiddenLayers[lastHiddenIndex].matrix + ) + .add(this.outputLayer.bias) + .map(this.activationFunction.func); // Convert array to matrix object - let targets = Matrix.fromArray(target_array); + let targets = Matrix.fromArray(targetArray); // Calculate the error // ERROR = TARGETS - OUTPUTS - let output_errors = Matrix.subtract(targets, outputs); + let outputErrors = Matrix.subtract(targets, this.outputLayer.matrix); - // let gradient = outputs * (1 - outputs); + + // OUTPUT LAYER -> HIDDEN LAYERS // Calculate gradient - let gradients = Matrix.map(outputs, this.activation_function.dfunc); - gradients.multiply(output_errors); - gradients.multiply(this.learning_rate); + this.outputLayer.gradients = Matrix.map( + this.outputLayer.matrix, + this.activationFunction.dfunc + ) + .multiply(outputErrors) + .multiply(this.learningRate); + + this.weightLayers[lastWeightIndex].add( + Matrix.multiply( + this.outputLayer.gradients, + Matrix.transpose( + this.hiddenLayers[lastHiddenIndex] + ) + ) + ); + this.outputLayer.bias.add(this.outputLayer.gradients); + + // HIDDEN LAYERS -> INPUT LAYER + for(let i = lastHiddenIndex; i >= 0; i--) { + + let hiddenErrors = Matrix.multiply( + Matrix.transpose(this.weightLayers[i+1]), + outputErrors + ); + + this.hiddenLayers[i].gradients = Matrix.map( + this.hiddenLayers[i].matrix, + this.activationFunction.dfunc + ) + .multiply(hiddenErrors) + .multiply(this.learningRate); + + this.weightLayers[i].add( + Matrix.multiply( + this.hiddenLayers[i].gradients, + Matrix.transpose( + i == 0 ? this.inputLayer.matrix : this.hiddenLayers[i-1] + ) + ) + ); + this.hiddenLayers[i].bias.add(this.hiddenLayers[i].gradients); + } + + /* // Calculate deltas let hidden_T = Matrix.transpose(hidden); let weight_ho_deltas = Matrix.multiply(gradients, hidden_T); @@ -175,6 +240,7 @@ class NeuralNetwork { this.weights_ho.add(weight_ho_deltas); // Adjust the bias by its deltas (which is just the gradients) this.bias_o.add(gradients); + // Calculate the hidden layer errors let who_t = Matrix.transpose(this.weights_ho); @@ -196,6 +262,7 @@ class NeuralNetwork { // outputs.print(); // targets.print(); // error.print(); + */ } serialize() { From b7d0a2172412db7c891fbf8034c0fd788d9cedab Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Fri, 30 Mar 2018 22:43:46 +0200 Subject: [PATCH 04/16] Fix train method, add _walk method and refactory predict method --- lib/nn.js | 127 +++++++++++++++++++----------------------------------- 1 file changed, 44 insertions(+), 83 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index 1b48200..bf16e93 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -115,60 +115,28 @@ class NeuralNetwork { this.setActivationFunction(); } - predict(input_array) { - - // Generating the Hidden Outputs - let inputs = Matrix.fromArray(input_array); - let hidden = Matrix.multiply(this.weights_ih, inputs); - hidden.add(this.bias_h); - // activation function! - hidden.map(this.activation_function.func); - - // Generating the output's output! - let output = Matrix.multiply(this.weights_ho, hidden); - output.add(this.bias_o); - output.map(this.activation_function.func); - - // Sending back to the caller! - return output.toArray(); - } - - setLearningRate(learningRate = 0.1) { - this.learningRate = learningRate; - } - - setActivationFunction(func = sigmoid) { - this.activationFunction = func; - } - - train(inputArray, targetArray) { + _walk(inputArray) { - if(inputArray.length !== this.inputLayer.nodes || - targetArray.length !== this.outputLayer.nodes) { - - throw new Error('Array size error.'); + if(inputArray.length !== this.inputLayer.nodes) { + throw new Error('ERROR: Input array size.'); } - // Generating the Hidden Outputs + // Convert input to array this.inputLayer.matrix = Matrix.fromArray(inputArray); - let lastHiddenIndex; - let lastWeightIndex; - + // Walk for(let i = 0; i < this.hiddenLayers.length; i++) { this.hiddenLayers[i].matrix = Matrix.multiply( this.weightLayers[i], i == 0 ? this.inputLayer.matrix : this.hiddenLayers[i-1].matrix ) .add(this.hiddenLayers[i].bias) - // activation function! .map(this.activationFunction.func); - - lastHiddenIndex = i; - lastWeightIndex = i + 1; } - // Generating the output's output! + let lastHiddenIndex = this.hiddenLayers.length - 1; + let lastWeightIndex = this.weightLayers.length - 1; + this.outputLayer.matrix = Matrix.multiply( this.weightLayers[lastWeightIndex], this.hiddenLayers[lastHiddenIndex].matrix @@ -176,13 +144,41 @@ class NeuralNetwork { .add(this.outputLayer.bias) .map(this.activationFunction.func); + return this.outputLayer.matrix; + } + + predict(inputArray) { + return this._walk(inputArray).toArray(); + } + + setLearningRate(learningRate = 0.1) { + this.learningRate = learningRate; + } + + setActivationFunction(func = sigmoid) { + this.activationFunction = func; + } + + train(inputArray, targetArray) { + + this._walk(inputArray); + + if(targetArray.length !== this.outputLayer.nodes) { + throw new Error('ERROR: Target array size.'); + } + + let lastHiddenIndex = this.hiddenLayers.length - 1; + let lastWeightIndex = this.weightLayers.length - 1; + // Convert array to matrix object let targets = Matrix.fromArray(targetArray); // Calculate the error // ERROR = TARGETS - OUTPUTS - let outputErrors = Matrix.subtract(targets, this.outputLayer.matrix); - + this.outputLayer.errors = Matrix.subtract( + targets, + this.outputLayer.matrix + ); // OUTPUT LAYER -> HIDDEN LAYERS // Calculate gradient @@ -190,14 +186,14 @@ class NeuralNetwork { this.outputLayer.matrix, this.activationFunction.dfunc ) - .multiply(outputErrors) + .multiply(this.outputLayer.errors) .multiply(this.learningRate); this.weightLayers[lastWeightIndex].add( Matrix.multiply( this.outputLayer.gradients, Matrix.transpose( - this.hiddenLayers[lastHiddenIndex] + this.hiddenLayers[lastHiddenIndex].matrix ) ) ); @@ -206,63 +202,28 @@ class NeuralNetwork { // HIDDEN LAYERS -> INPUT LAYER for(let i = lastHiddenIndex; i >= 0; i--) { - let hiddenErrors = Matrix.multiply( + this.hiddenLayers[i].errors = Matrix.multiply( Matrix.transpose(this.weightLayers[i+1]), - outputErrors + i == lastHiddenIndex ? this.outputLayer.errors : this.hiddenLayers[i+1].errors ); this.hiddenLayers[i].gradients = Matrix.map( this.hiddenLayers[i].matrix, this.activationFunction.dfunc ) - .multiply(hiddenErrors) + .multiply(this.hiddenLayers[i].errors) .multiply(this.learningRate); this.weightLayers[i].add( Matrix.multiply( this.hiddenLayers[i].gradients, Matrix.transpose( - i == 0 ? this.inputLayer.matrix : this.hiddenLayers[i-1] + i == 0 ? this.inputLayer.matrix : this.hiddenLayers[i-1].matrix ) ) ); this.hiddenLayers[i].bias.add(this.hiddenLayers[i].gradients); } - - - - /* - // Calculate deltas - let hidden_T = Matrix.transpose(hidden); - let weight_ho_deltas = Matrix.multiply(gradients, hidden_T); - - // Adjust the weights by deltas - this.weights_ho.add(weight_ho_deltas); - // Adjust the bias by its deltas (which is just the gradients) - this.bias_o.add(gradients); - - - // Calculate the hidden layer errors - let who_t = Matrix.transpose(this.weights_ho); - let hidden_errors = Matrix.multiply(who_t, output_errors); - - // Calculate hidden gradient - let hidden_gradient = Matrix.map(hidden, this.activation_function.dfunc); - hidden_gradient.multiply(hidden_errors); - hidden_gradient.multiply(this.learning_rate); - - // Calcuate input->hidden deltas - let inputs_T = Matrix.transpose(inputs); - let weight_ih_deltas = Matrix.multiply(hidden_gradient, inputs_T); - - this.weights_ih.add(weight_ih_deltas); - // Adjust the bias by its deltas (which is just the gradients) - this.bias_h.add(hidden_gradient); - - // outputs.print(); - // targets.print(); - // error.print(); - */ } serialize() { From 697d9e8204fcdf0f7fce3e3a5c5cf4544c0e9480 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Fri, 30 Mar 2018 22:59:02 +0200 Subject: [PATCH 05/16] Change to multi-hidden layers --- examples/doodle_classification/sketch.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/doodle_classification/sketch.js b/examples/doodle_classification/sketch.js index 57ec727..7082b18 100644 --- a/examples/doodle_classification/sketch.js +++ b/examples/doodle_classification/sketch.js @@ -31,8 +31,8 @@ function setup() { prepareData(rainbows, rainbowsData, RAINBOW); prepareData(trains, trainsData, TRAIN); - // Making the neural network - nn = new NeuralNetwork(784, 64, 3); + // Making the neural network (multi-hidden layers) + nn = new NeuralNetwork(784, 256, 64, 3); // Randomizing the data let training = []; From 1327a088b36b8dd7f4c5a78d8da7eafea5d84c2f Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Fri, 30 Mar 2018 23:01:05 +0200 Subject: [PATCH 06/16] Add TODO list after multi-hidden implementation --- lib/nn.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/nn.js b/lib/nn.js index bf16e93..2be325a 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -230,6 +230,7 @@ class NeuralNetwork { return JSON.stringify(this); } + // TODO: convert to multi-hidden layers static deserialize(data) { if (typeof data == 'string') { data = JSON.parse(data); @@ -243,12 +244,12 @@ class NeuralNetwork { return nn; } - // Adding function for neuro-evolution copy() { return new NeuralNetwork(this); } + // TODO: convert to multi-hidden layers mutate(rate) { function mutate(val) { if (Math.random() < rate) { From 3341a367c8f4e36bb8197c80dbfb95a35c6edaf4 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sat, 31 Mar 2018 00:05:30 +0200 Subject: [PATCH 07/16] Refactory static method deserialize to multi-hidden layers --- lib/nn.js | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index 2be325a..f4aac60 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -230,18 +230,32 @@ class NeuralNetwork { return JSON.stringify(this); } - // TODO: convert to multi-hidden layers static deserialize(data) { if (typeof data == 'string') { data = JSON.parse(data); } - let nn = new NeuralNetwork(data.input_nodes, data.hidden_nodes, data.output_nodes); - nn.weights_ih = Matrix.deserialize(data.weights_ih); - nn.weights_ho = Matrix.deserialize(data.weights_ho); - nn.bias_h = Matrix.deserialize(data.bias_h); - nn.bias_o = Matrix.deserialize(data.bias_o); - nn.learning_rate = data.learning_rate; - return nn; + + let args = [null]; + args.push(data.inputLayer.nodes); + for(let hidden of data.hiddenLayers) { + args.push(hidden.nodes); + } + args.push(data.outputLayer.nodes); + + let neuralNetwork = new (Function.prototype.bind.apply(this, args)); + + neuralNetwork.learningRate = data.learningRate; + + for(let i = 0; i < data.hiddenLayers.length; i++) { + neuralNetwork.hiddenLayers[i].bias = Matrix.deserialize(data.hiddenLayers[i].bias); + } + neuralNetwork.outputLayer.bias = Matrix.deserialize(data.outputLayer.bias); + + for(let i = 0; i < data.weightLayers.length; i++) { + neuralNetwork.weightLayers[i] = Matrix.deserialize(data.weightLayers[i]); + } + + return neuralNetwork; } // Adding function for neuro-evolution From 3e5ab2c4d0bc736d95c739145682ae553fa158d4 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sat, 31 Mar 2018 00:14:21 +0200 Subject: [PATCH 08/16] Refactory mutate method to multi-hidden layers --- lib/nn.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index f4aac60..ecf72dc 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -263,7 +263,6 @@ class NeuralNetwork { return new NeuralNetwork(this); } - // TODO: convert to multi-hidden layers mutate(rate) { function mutate(val) { if (Math.random() < rate) { @@ -272,9 +271,12 @@ class NeuralNetwork { return val; } } - this.weights_ih.map(mutate); - this.weights_ho.map(mutate); - this.bias_h.map(mutate); - this.bias_o.map(mutate); + for(let weight of this.weightLayers) { + weight.map(mutate); + } + for(let hidden of this.hiddenLayers) { + hidden.bias.map(mutate); + } + this.outputLayer.bias.map(mutate); } } From 5a6ab6857a42ddeb7c4563a77ad8c0ef8edba30c Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sun, 15 Apr 2018 11:59:04 +0200 Subject: [PATCH 09/16] Update semantics --- lib/nn.js | 363 +++++++++++++++++++++++++++++------------------------- 1 file changed, 195 insertions(+), 168 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index ecf72dc..79ed14c 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -29,207 +29,256 @@ class NeuralNetwork { * The user can copy an instance of NeuralNetwork by passing the same as an * argument to the constructor. * - * @param {NeuralNetwork|...Integer} args (Rest parameters) + * @param {NeuralNetwork|Array|...Integer} args (Rest parameters) */ constructor(...args) { - this.hiddenLayers = []; - this.weightLayers = []; + if(args.length === 1 && args[0] instanceof NeuralNetwork) { - if (args.length === 1 && args[0] instanceof NeuralNetwork) { + let layers = []; - let nn = args[0]; + for(let layer of args[0].layers) { + layers.push(layer.nodes); + } - // INPUT LAYER - this.inputLayer = { - nodes: nn.inputLayer.nodes - }; + this._build(layers, args[0].learningRate); - // HIDDEN LAYERS - for(let hidden of nn.hiddenLayers) { - this.hiddenLayers.push({ - nodes: hidden.nodes, - bias: hidden.bias.copy() - }); + for(let i = 0; i < this.connections.length; i++) { + this.connections[i].weights = args[0].connections[i].weights.copy(); + this.connections[i].bias = args[0].connections[i].bias.copy(); } - // OUTPUT LAYER - this.outputLayer = { - nodes: nn.outputLayer.nodes, - bias: nn.outputLayer.bias.copy() - }; + } else if(args.length === 1 && Array.isArray(args[0])) { - // WEIGHTS CONNECTIONS - for(let weight of nn.weightLayers) { - this.weightLayers.push( - weight.copy() - ); - } + this._build(args[0]); - } else if(args.length >= 3) { + } else if(args.length === 3 && Number.isInteger(args[0]) && + Array.isArray(args[1]) && Number.isInteger(args[2])) { - // CHECK ARGUMENTS - if(!args.every( v => Number.isInteger(v) )) { - throw new Error('All arguments must be integer.'); + let layers = []; + + layers.push(args[0]); + + for(let arg of args[1]) { + layers.push(arg); } - // INPUT LAYER - this.inputLayer = { - nodes: args[0] - }; + layers.push(args[2]); - let lastIndex = args.length-1; + this._build(layers); - // HIDDEN LAYERS - for(let i = 1; i < lastIndex; i++) { - let nodes = args[i]; - this.hiddenLayers.push({ - nodes: nodes, - bias: new Matrix(nodes, 1).randomize() - }); - } + } else if(args.length >= 3) { - // OUTPUT LAYER - this.outputLayer = { - nodes: args[lastIndex], - bias: new Matrix(args[lastIndex], 1).randomize() - }; - - // WEIGHTS CONNECTIONS - let connections = lastIndex; - for(let i = 0; i < connections; i++) { - let primaryNodes = args[i], - secondaryNodes = args[i+1]; - this.weightLayers.push( - new Matrix(secondaryNodes, primaryNodes).randomize() - ); - } + this._build(args); } else { throw new Error('Invalid arguments. Read the documentation!'); } - - // TODO: copy these as well - this.setLearningRate(); - this.setActivationFunction(); } - _walk(inputArray) { + _build(layers, learningRate = 0.1) { - if(inputArray.length !== this.inputLayer.nodes) { - throw new Error('ERROR: Input array size.'); + if(!Array.isArray(layers)) { + throw new Error('Must be array of nodes.'); + } else if(!layers.every( value => Number.isInteger(value) )) { + throw new Error('All arguments must be integer.'); } - // Convert input to array - this.inputLayer.matrix = Matrix.fromArray(inputArray); - - // Walk - for(let i = 0; i < this.hiddenLayers.length; i++) { - this.hiddenLayers[i].matrix = Matrix.multiply( - this.weightLayers[i], - i == 0 ? this.inputLayer.matrix : this.hiddenLayers[i-1].matrix - ) - .add(this.hiddenLayers[i].bias) - .map(this.activationFunction.func); - } + this.learningRate = learningRate; + this.activationFunction = sigmoid; - let lastHiddenIndex = this.hiddenLayers.length - 1; - let lastWeightIndex = this.weightLayers.length - 1; + this.layers = []; + this.connections = []; - this.outputLayer.matrix = Matrix.multiply( - this.weightLayers[lastWeightIndex], - this.hiddenLayers[lastHiddenIndex].matrix - ) - .add(this.outputLayer.bias) - .map(this.activationFunction.func); + for(let nodes of layers) { + this.layers.push({ + nodes: nodes + }); + } - return this.outputLayer.matrix; + for(let i = 0; i < this.layers.length - 1; i++) { + let primaryNodes = this.layers[i].nodes, + secondaryNodes = this.layers[i+1].nodes; + this.connections.push({ + weights: new Matrix(secondaryNodes, primaryNodes).randomize(), + bias: new Matrix(secondaryNodes, 1).randomize() + }); + } } - predict(inputArray) { - return this._walk(inputArray).toArray(); - } + _walk(inputs) { - setLearningRate(learningRate = 0.1) { - this.learningRate = learningRate; - } + this.inputs = inputs; + + for(let i = 0; i < this.connections.length; i++) { + this.layers[i+1].results = Matrix.multiply( + this.connections[i].weights, + i === 0 ? this.inputs : this.layers[i].results + ) + .add(this.connections[i].bias) + .map(this.activationFunction.func); + } - setActivationFunction(func = sigmoid) { - this.activationFunction = func; + return this.outputs; } - train(inputArray, targetArray) { + train(inputs, targets) { - this._walk(inputArray); + this._walk(inputs); - if(targetArray.length !== this.outputLayer.nodes) { + // TODO: Handle errors + if(targets.length !== this.outputsNodes) { throw new Error('ERROR: Target array size.'); } - let lastHiddenIndex = this.hiddenLayers.length - 1; - let lastWeightIndex = this.weightLayers.length - 1; - - // Convert array to matrix object - let targets = Matrix.fromArray(targetArray); - - // Calculate the error - // ERROR = TARGETS - OUTPUTS - this.outputLayer.errors = Matrix.subtract( - targets, - this.outputLayer.matrix - ); - - // OUTPUT LAYER -> HIDDEN LAYERS - // Calculate gradient - this.outputLayer.gradients = Matrix.map( - this.outputLayer.matrix, - this.activationFunction.dfunc - ) - .multiply(this.outputLayer.errors) - .multiply(this.learningRate); - - this.weightLayers[lastWeightIndex].add( - Matrix.multiply( - this.outputLayer.gradients, - Matrix.transpose( - this.hiddenLayers[lastHiddenIndex].matrix - ) - ) - ); - this.outputLayer.bias.add(this.outputLayer.gradients); + for(let i = this._outputsIndex; i > 0; i--) { - // HIDDEN LAYERS -> INPUT LAYER - for(let i = lastHiddenIndex; i >= 0; i--) { + if(i === this._outputsIndex) { - this.hiddenLayers[i].errors = Matrix.multiply( - Matrix.transpose(this.weightLayers[i+1]), - i == lastHiddenIndex ? this.outputLayer.errors : this.hiddenLayers[i+1].errors - ); + this.layers[i].errors = Matrix.subtract( + Matrix.fromArray(targets), + this.outputs + ); + + } else { - this.hiddenLayers[i].gradients = Matrix.map( - this.hiddenLayers[i].matrix, + this.layers[i].errors = Matrix.multiply( + Matrix.transpose( + this.connections[i].weights + ), + this.layers[i+1].errors + ); + } + + let gradients = Matrix.map( + this.layers[i].results, this.activationFunction.dfunc ) - .multiply(this.hiddenLayers[i].errors) + .multiply(this.layers[i].errors) .multiply(this.learningRate); - this.weightLayers[i].add( - Matrix.multiply( - this.hiddenLayers[i].gradients, - Matrix.transpose( - i == 0 ? this.inputLayer.matrix : this.hiddenLayers[i-1].matrix - ) - ) + let deltas = Matrix.multiply( + gradients, + Matrix.transpose(i === 1 ? this.inputs : this.layers[i-1].results) ); - this.hiddenLayers[i].bias.add(this.hiddenLayers[i].gradients); + + this.connections[i-1].weights.add(deltas); + this.connections[i-1].bias.add(gradients); } } + predict(inputs) { + return this._walk(inputs).toArray(); + } + + copy() { + return new NeuralNetwork(this); + } + serialize() { - return JSON.stringify(this); + let nn = this.copy(); + delete nn._activationFunction; + + for(let layer of nn.layers) { + delete layer.matrix; + delete layer.results; + delete layer.errors; + } + + return JSON.stringify(nn); } + mutate(rate) { + + if(Number(rate) !== rate || (0 < rate && rate < 1)) { + throw new Error('Mutate rate must be a number between 0 and 1.'); + } + + let mutate = value => { + if(Math.random() < rate) { + return Math.random() * 1000 - 1; + } else { + return value; + } + }; + + for(let connection of this.connections) { + connection.weights.map(mutate); + connection.bias.map(mutate); + } + } + + // SETTERS + set inputs(inputs) { + + if(!Array.isArray(inputs)) { + throw new Error('Inputs must be array.'); + } else if(inputs.length !== this.inputsNodes) { + throw new Error('Inputs size.'); + } else if(!inputs.every( val => true )) { + throw new Error('Inputs value must be a number.'); + } + + this.layers[this._inputsIndex].matrix = Matrix.fromArray(inputs); + } + + set learningRate(rate) { + + if(Number(rate) !== rate) { + throw new Error('Learning rate must be a number'); + } + + this._learningRate = rate; + } + + set activationFunction(func) { + + if(!func instanceof ActivationFunction) { + throw new Error('Activation function must be a instance of ActivationFunction.'); + } + + this._activationFunction = func; + } + + // GETTERS + get _inputsIndex() { + return 0; + } + + get _outputsIndex() { + return this.layers.length - 1; + } + + get inputsNodes() { + return this.layers[this._inputsIndex].nodes; + } + + get inputs() { + return this.layers[this._inputsIndex].matrix; + } + + get outputsNodes() { + return this.layers[this._outputsIndex].nodes; + } + + get outputs() { + return this.layers[this._outputsIndex].results; + } + + get learningRate() { + return this._learningRate; + } + + get activationFunction() { + return this._activationFunction; + } + + + + + + static deserialize(data) { if (typeof data == 'string') { data = JSON.parse(data); @@ -257,26 +306,4 @@ class NeuralNetwork { return neuralNetwork; } - - // Adding function for neuro-evolution - copy() { - return new NeuralNetwork(this); - } - - mutate(rate) { - function mutate(val) { - if (Math.random() < rate) { - return Math.random() * 1000 - 1; - } else { - return val; - } - } - for(let weight of this.weightLayers) { - weight.map(mutate); - } - for(let hidden of this.hiddenLayers) { - hidden.bias.map(mutate); - } - this.outputLayer.bias.map(mutate); - } } From ed1cf8facb64910f05ebbdee1aae6f5f7c3df510 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sun, 15 Apr 2018 12:18:41 +0200 Subject: [PATCH 10/16] add TODO --- lib/nn.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index 79ed14c..1d306c8 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -277,8 +277,7 @@ class NeuralNetwork { - - + // TODO: refactory deserialized method static deserialize(data) { if (typeof data == 'string') { data = JSON.parse(data); From d66ef8c0f82a7f49c43c0bf81d133df591fe4f32 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sun, 15 Apr 2018 12:35:23 +0200 Subject: [PATCH 11/16] add comment and TODO --- lib/nn.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/nn.js b/lib/nn.js index 1d306c8..81ebd69 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -132,6 +132,7 @@ class NeuralNetwork { throw new Error('ERROR: Target array size.'); } + // Backpropagation for(let i = this._outputsIndex; i > 0; i--) { if(i === this._outputsIndex) { @@ -216,7 +217,7 @@ class NeuralNetwork { throw new Error('Inputs must be array.'); } else if(inputs.length !== this.inputsNodes) { throw new Error('Inputs size.'); - } else if(!inputs.every( val => true )) { + } else if(!inputs.every( val => true )) { // TODO: Check if is float number throw new Error('Inputs value must be a number.'); } From 61049e6fac31b888b4be9598e026ec6ee4cbdd5a Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sun, 15 Apr 2018 13:04:27 +0200 Subject: [PATCH 12/16] Refactored deserialize function --- lib/nn.js | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index 81ebd69..4a8b821 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -77,6 +77,7 @@ class NeuralNetwork { } } + // PRIVATE _build(layers, learningRate = 0.1) { if(!Array.isArray(layers)) { @@ -123,6 +124,7 @@ class NeuralNetwork { return this.outputs; } + // PUBLIC train(inputs, targets) { this._walk(inputs); @@ -275,35 +277,27 @@ class NeuralNetwork { return this._activationFunction; } - - - - // TODO: refactory deserialized method + // STATIC static deserialize(data) { if (typeof data == 'string') { data = JSON.parse(data); } let args = [null]; - args.push(data.inputLayer.nodes); - for(let hidden of data.hiddenLayers) { - args.push(hidden.nodes); - } - args.push(data.outputLayer.nodes); - let neuralNetwork = new (Function.prototype.bind.apply(this, args)); + for(let layer of data.layers) { + args.push(layer.nodes); + } - neuralNetwork.learningRate = data.learningRate; + let nn = new (Function.prototype.bind.apply(this, args)); - for(let i = 0; i < data.hiddenLayers.length; i++) { - neuralNetwork.hiddenLayers[i].bias = Matrix.deserialize(data.hiddenLayers[i].bias); - } - neuralNetwork.outputLayer.bias = Matrix.deserialize(data.outputLayer.bias); + nn.learningRate = data._learningRate; - for(let i = 0; i < data.weightLayers.length; i++) { - neuralNetwork.weightLayers[i] = Matrix.deserialize(data.weightLayers[i]); + for(let i = 0; i < nn.connections.length; i++) { + nn.connections[i].weights = Matrix.deserialize(data.connections[i].weights); + nn.connections[i].bias = Matrix.deserialize(data.connections[i].bias); } - return neuralNetwork; + return nn; } } From b4b6efea535239e41f82574f8eafae1c4b9ed626 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sun, 15 Apr 2018 21:12:13 +0200 Subject: [PATCH 13/16] Bug fix in mutate function @Versatilus's help on pull request #107 https://github.com/CodingTrain/Toy-Neural-Network-JS/pull/107/commits/5a6ab6857a42ddeb7c4563a77ad8c0ef8edba30c#diff-8db8691495148458de348350d97cd073R194 --- lib/nn.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/nn.js b/lib/nn.js index 4a8b821..a1b8c73 100644 --- a/lib/nn.js +++ b/lib/nn.js @@ -194,8 +194,8 @@ class NeuralNetwork { mutate(rate) { - if(Number(rate) !== rate || (0 < rate && rate < 1)) { - throw new Error('Mutate rate must be a number between 0 and 1.'); + if(Number(rate) !== rate || !(0 < rate && rate <= 1)) { + throw new Error('Mutate rate must be a number > 0 and <= 1.'); } let mutate = value => { From c54a8f59dd13506d48cff9c3b2e56175c921b8b1 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sun, 15 Apr 2018 21:39:11 +0200 Subject: [PATCH 14/16] Update README file. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a7a35d7..cb29bb6 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Neural Network JavaScript library for Coding Train tutorials Here are some demos running directly in the browser: * [XOR problem](https://codingtrain.github.io/Toy-Neural-Network-JS/examples/xor/) * [Handwritten digit recognition](https://codingtrain.github.io/Toy-Neural-Network-JS/examples/mnist/) +* [Multiple hidden layers](https://maksuel.github.io/Toy-Neural-Network-JS/examples/doodle_classification/) ## To-Do List @@ -20,7 +21,7 @@ Here are some demos running directly in the browser: * only use testing data * [ ] Support for saving / restoring network (see [#50](https://github.com/CodingTrain/Toy-Neural-Network-JS/pull/50)) * [ ] Support for different activation functions (see [#45](https://github.com/CodingTrain/Toy-Neural-Network-JS/pull/45), [#62](https://github.com/CodingTrain/Toy-Neural-Network-JS/pull/62)) -* [ ] Support for multiple hidden layers (see [#61](https://github.com/CodingTrain/Toy-Neural-Network-JS/pull/61)) +* [x] Support for multiple hidden layers (see [#107](https://github.com/CodingTrain/Toy-Neural-Network-JS/pull/107)) * [ ] Support for neuro-evolution * [ ] play flappy bird (many players at once). * [ ] play pong (many game simulations at once) From a9ca77f4957e0b1ca0fd62a7eb0b3078f275e7c2 Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Mon, 16 Apr 2018 10:10:26 +0200 Subject: [PATCH 15/16] Introduction Visualization of NeuralNetwork --- examples/doodle_classification/index.html | 1 + examples/doodle_classification/sketch.js | 4 ++++ .../doodle_classification/visualization.js | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 examples/doodle_classification/visualization.js diff --git a/examples/doodle_classification/index.html b/examples/doodle_classification/index.html index 1419153..863920d 100644 --- a/examples/doodle_classification/index.html +++ b/examples/doodle_classification/index.html @@ -12,6 +12,7 @@ + diff --git a/examples/doodle_classification/sketch.js b/examples/doodle_classification/sketch.js index 7082b18..f8e87da 100644 --- a/examples/doodle_classification/sketch.js +++ b/examples/doodle_classification/sketch.js @@ -15,6 +15,8 @@ let rainbows = {}; let nn; +let visualization; + function preload() { catsData = loadBytes('data/cats1000.bin'); trainsData = loadBytes('data/trains1000.bin'); @@ -95,6 +97,8 @@ function setup() { // let percent = testAll(testing); // console.log("% Correct: " + percent); // } + + visualization = new Visualization(nn); } diff --git a/examples/doodle_classification/visualization.js b/examples/doodle_classification/visualization.js new file mode 100644 index 0000000..fb219da --- /dev/null +++ b/examples/doodle_classification/visualization.js @@ -0,0 +1,19 @@ +class Visualization { + constructor(nn) { + if(typeof p5 !== 'function') { + throw new Error('Need to include p5js'); + } else if(!nn instanceof NeuralNetwork) { + throw new Error('Need a instance of NeuralNetwork'); + } + + let layers = []; + + for(let layer of nn.layers) { + layers.push(layer.nodes); + } + + console.log('Visualization',layers); + + // this.graphics = createGraphics(); + } +} \ No newline at end of file From aeba18a745b94903662c545364f29d282324662e Mon Sep 17 00:00:00 2001 From: Maksuel Boni Date: Sun, 7 Oct 2018 01:49:00 +0200 Subject: [PATCH 16/16] clear --- .../doodle_classification/visualization.js | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/examples/doodle_classification/visualization.js b/examples/doodle_classification/visualization.js index fb219da..29c9832 100644 --- a/examples/doodle_classification/visualization.js +++ b/examples/doodle_classification/visualization.js @@ -1,19 +1,36 @@ class Visualization { - constructor(nn) { + + static _check(nn) { if(typeof p5 !== 'function') { throw new Error('Need to include p5js'); } else if(!nn instanceof NeuralNetwork) { throw new Error('Need a instance of NeuralNetwork'); } + } + static _getLayers(nn) { let layers = []; for(let layer of nn.layers) { layers.push(layer.nodes); } - console.log('Visualization',layers); + return layers; + } + + static graphics(nn) { + this._check(nn); + + let layers = this._getLayers(nn); + + let w = floor(layers.length * 20); + let h = floor(layers[0] * 20); + + let graphics = createGraphics(w,h); + + console.log(graphics.width, graphics.height); + + return graphics; - // this.graphics = createGraphics(); } } \ No newline at end of file