@@ -43,46 +43,46 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
4343 The Training algorithms used when training on <struct fann_train_data> with functions like
4444 <fann_train_on_data> or <fann_train_on_file>. The incremental training alters the weights
4545 after each time it is presented an input pattern, while batch only alters the weights once
46- after it has been presented to all the patterns.
46+ after it has been presented to all the patterns.
4747
4848 FANN_TRAIN_INCREMENTAL - Standard backpropagation algorithm, where the weights are
4949 updated after each training pattern. This means that the weights are updated many
50- times during a single epoch. For this reason some problems will train very fast with
51- this algorithm, while other more advanced problems will not train very well.
50+ times during a single epoch. For this reason some problems will train very fast
51+ with this algorithm, while other more advanced problems will not train very well.
5252 FANN_TRAIN_BATCH - Standard backpropagation algorithm, where the weights are updated after
5353 calculating the mean square error for the whole training set. This means that the
54- weights are only updated once during an epoch. For this reason some problems will train slower
55- with this algorithm. But since the mean square error is calculated more correctly than in
56- incremental training, some problems will reach better solutions with this algorithm.
57- FANN_TRAIN_RPROP - A more advanced batch training algorithm which achieves good results
58- for many problems. The RPROP training algorithm is adaptive, and does therefore not
59- use the learning_rate. Some other parameters can however be set to change the way
60- the RPROP algorithm works, but it is only recommended for users with insight in how the RPROP
61- training algorithm works. The RPROP training algorithm is described by
62- [Riedmiller and Braun, 1993], but the actual learning algorithm used here is the
63- iRPROP- training algorithm which is described by [Igel and Husken, 2000] which
64- is a variant of the standard RPROP training algorithm.
54+ weights are only updated once during an epoch. For this reason some problems will
55+ train slower with this algorithm. But since the mean square error is calculated
56+ more correctly than in incremental training, some problems will reach better
57+ solutions with this algorithm.
58+ FANN_TRAIN_RPROP - A more advanced batch training algorithm which achieves good
59+ results for many problems. The RPROP training algorithm is adaptive, and does
60+ therefore not use the learning_rate. Some other parameters can however be set to
61+ change the way the RPROP algorithm works, but it is only recommended for users with
62+ insight in how the RPROP training algorithm works. The RPROP training algorithm is
63+ described by [Riedmiller and Braun, 1993], but the actual Learning algorithm used
64+ here is the iRPROP- training algorithm which is described by [Igel and Husken,
65+ 2000] which is a variant of the standard RPROP training algorithm.
6566 FANN_TRAIN_QUICKPROP - A more advanced batch training algorithm which achieves good results
66- for many problems. The quickprop training algorithm uses the learning_rate parameter
67- along with other more advanced parameters, but it is only recommended to change
68- these advanced parameters, for users with insight in how the quickprop training algorithm works.
69- The quickprop training algorithm is described by [Fahlman, 1988].
70- FANN_TRAIN_SARPROP - A batch training algorithm which extends resilient
71- backpropagation (RPROP) with simulated annealing. SARPROP introduces
72- adaptive weight decay and controlled noise based on the training epoch
73- in order to improve convergence and reduce the risk of getting stuck
74- in local minima. The SARPROP training algorithm is described in
75- "The SARPROP Algorithm: A Simulated Annealing Enhancement to Resilient
67+ for many problems. The quickprop training algorithm uses the learning_rate
68+ parameter along with other more advanced parameters, but it is only recommended to
69+ change these advanced parameters, for users with insight in how the quickprop
70+ training algorithm works. The quickprop training algorithm is described by
71+ [Fahlman, 1988].
72+ FANN_TRAIN_SARPROP - A batch training algorithm which extends resilient backpropagation
73+ (RPROP) with simulated annealing. SARPROP introduces adaptive weight decay and
74+ controlled noise based on the training epoch in order to improve convergence and
75+ reduce the risk of getting stuck in local minima. The SARPROP training algorithm is
76+ described in "The SARPROP Algorithm: A Simulated Annealing Enhancement to Resilient
7677 Back Propagation".
7778 http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.47.8197&rep=rep1&type=pdf
78- FANN_TRAIN_ADAM - Adaptive Moment Estimation training algorithm which combines
79- momentum and RMSProp style updates by maintaining exponential moving
80- averages of both the gradients and the squared gradients, with bias
81- correction to compensate for initialization at zero. Adam uses the
82- learning_rate parameter together with additional optimizer parameters
83- (beta1, beta2, epsilon), and generally provides good performance across
84- a wide range of problems with minimal tuning. The Adam training algorithm
85- is described by [Kingma and Ba, 2015].
79+ FANN_TRAIN_ADAM - Adaptive Moment Estimation training algorithm which combines momentum and
80+ RMSProp style updates by maintaining exponential moving averages of both the
81+ gradients and the squared gradients, with bias correction to compensate for
82+ initialization at zero. Adam uses the learning_rate parameter together with
83+ additional optimizer parameters (beta1, beta2, epsilon), and generally provides
84+ good performance across a wide range of problems with minimal tuning. The Adam
85+ training algorithm is described by [Kingma and Ba, 2015].
8686
8787 See also:
8888 <fann_set_training_algorithm>, <fann_get_training_algorithm>
@@ -108,8 +108,8 @@ enum fann_train_enum {
108108 <fann_train_enum>
109109*/
110110static char const * const FANN_TRAIN_NAMES [] = {"FANN_TRAIN_INCREMENTAL" , "FANN_TRAIN_BATCH" ,
111- "FANN_TRAIN_RPROP" , "FANN_TRAIN_QUICKPROP" ,
112- "FANN_TRAIN_SARPROP" , "FANN_TRAIN_ADAM" };
111+ "FANN_TRAIN_RPROP" , "FANN_TRAIN_QUICKPROP" ,
112+ "FANN_TRAIN_SARPROP" , "FANN_TRAIN_ADAM" };
113113
114114/* Enums: fann_activationfunc_enum
115115
@@ -771,19 +771,19 @@ struct fann {
771771 /* Adam optimizer parameters */
772772 /* First moment vector (mean of gradients) for Adam optimizer */
773773 fann_type * adam_m ;
774-
774+
775775 /* Second moment vector (variance of gradients) for Adam optimizer */
776776 fann_type * adam_v ;
777-
777+
778778 /* Exponential decay rate for the first moment estimates (default 0.9) */
779779 float adam_beta1 ;
780-
780+
781781 /* Exponential decay rate for the second moment estimates (default 0.999) */
782782 float adam_beta2 ;
783-
783+
784784 /* Small constant for numerical stability (default 1e-8) */
785785 float adam_epsilon ;
786-
786+
787787 /* Current timestep for Adam optimizer */
788788 unsigned int adam_timestep ;
789789
0 commit comments