Remove unnecessary parameters for RecordGradient.

Author: Oceania2018

src/TensorFlowNET.Core/Contexts/Context.cs

@@ -18,6 +18,7 @@ limitations under the License.
 using System.Diagnostics;
 using System.Linq;
 using Tensorflow.Eager;
+using static Tensorflow.Binding;
 
 namespace Tensorflow.Contexts
 {
@@ -114,6 +115,36 @@ public T RunInAutoMode<T>(Func<T> graphAction, Func<T> eagerAction, params Tenso
             }
         }
 
+        [DebuggerStepThrough]
+        public Tensors RunInAutoMode2(Func<Tensors> graphAction, 
+            Func<Tensors> eagerAction, 
+            Action<Operation> recordGradient,
+            Tensors tensors)
+        {
+            var shouldRunInEager = executing_eagerly()
+                && tensors.Count(x => x.IsEagerTensor) == tensors.Length;
+
+            if (shouldRunInEager)
+                return eagerAction();
+            else
+            {
+                if (executing_eagerly())
+                {
+                    graph_mode();
+                    var result = graphAction();
+                    restore_mode();
+                    return result;
+                }
+                else
+                {
+                    var result = graphAction();
+                    if (tf.Runner.MustRecordGradient())
+                        recordGradient(result[0].op);
+                    return result;
+                }
+            }
+        }
+
         public void Dispose()
             => Handle.Dispose();
     }

src/TensorFlowNET.Core/Eager/EagerRunner.RecordGradient.cs

@@ -11,7 +11,8 @@ public partial class EagerRunner
         public bool RecordGradient(string op_name,
             Tensor[] inputs,
             object[] attrs,
-            Tensor[] results)
+            Tensor[] results,
+            Func<BackwardFunction> getBackwardFunction = null)
         {
             var input_ids = MakeTensorIDList(inputs);
             var input_dtypes = MakeTensorDtypeList(inputs);
@@ -77,13 +78,20 @@ public bool RecordGradient(string op_name,
             else
                 op_inputs = inputs;
 
-            TapeSetRecordOperation(op_name, inputs, results, input_ids, input_dtypes,
-                () => GetGradientFunction(op_name, inputs, attrs, results));
-
+            TapeSetRecordOperation(op_name, inputs, results,
+                getBackwardFunction ?? GetBackwradFunction(op_name, inputs, attrs, results));
 
             return true;
         }
 
+        Func<BackwardFunction> GetBackwradFunction(string op_name,
+                     Tensor[] op_inputs,
+                     object[] attrs,
+                     Tensor[] op_outputs)
+        {
+            return () => GetGradientFunction(op_name, op_inputs, attrs, op_outputs);
+        }
+
         BackwardFunction GetGradientFunction(string op_name,
                      Tensor[] op_inputs,
                      object[] attrs,

src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordBackprop.cs

@@ -10,8 +10,6 @@ public partial class EagerRunner
         void TapeSetRecordBackprop(string op_type,
             Tensor[] input_tensors,
             TapeTensor[] output_tensors,
-            long[] input_ids,
-            TF_DataType[] input_dtypes,
             Func<BackwardFunction> backward_function_getter)
         {
             if (!CouldBackprop())
@@ -22,7 +20,6 @@ void TapeSetRecordBackprop(string op_type,
             foreach (var tape in tf.GetTapeSet())
             {
                 tape.RecordOperation(op_type, input_tensors, output_tensors,
-                    input_ids, input_dtypes,
                     backward_function_getter);
             }
         }

src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordForwardprop.cs

@@ -9,8 +9,6 @@ public partial class EagerRunner
         bool TapeSetRecordForwardprop(string op_type,
             Tensor[] input_tensors,
             TapeTensor[] output_tensors,
-            long[] input_ids,
-            TF_DataType[] input_dtypes,
             Func<BackwardFunction> backward_function_getter)
         {
             if (!CouldForwardprop())

src/TensorFlowNET.Core/Eager/EagerRunner.TapeSetRecordOperation.cs

@@ -7,11 +7,9 @@ namespace Tensorflow.Eager
 {
     public partial class EagerRunner
     {
-        bool TapeSetRecordOperation(string op_type,
+        public bool TapeSetRecordOperation(string op_type,
             Tensor[] input_tensors,
             Tensor[] output_tensors,
-            long[] input_ids,
-            TF_DataType[] input_dtypes,
             Func<BackwardFunction> backward_function_getter)
         {
             var output_info = new List<TapeTensor>();
@@ -20,11 +18,11 @@ bool TapeSetRecordOperation(string op_type,
                 return false;
 
             if (!TapeSetRecordForwardprop(op_type, input_tensors, output_info.ToArray(),
-                    input_ids, input_dtypes, backward_function_getter))
+                    backward_function_getter))
                 return false;
 
             TapeSetRecordBackprop(op_type, input_tensors, output_info.ToArray(),
-                input_ids, input_dtypes, backward_function_getter);
+                backward_function_getter);
 
             return true;
         }

src/TensorFlowNET.Core/Eager/IEagerRunner.cs

@@ -1,6 +1,7 @@
 using System;
 using Tensorflow.Contexts;
 using Tensorflow.Gradients;
+using static Tensorflow.tensorflow;
 
 namespace Tensorflow.Eager
 {
@@ -37,7 +38,8 @@ Tensor[] TFE_TapeGradient(ITape tape,
         bool RecordGradient(string op_name,
             Tensor[] inputs,
             object[] attrs,
-            Tensor[] results);
+            Tensor[] results,
+            Func<BackwardFunction> getBackwardFunction = null);
 
         bool MustRecordGradient();
 

src/TensorFlowNET.Core/Gradients/Tape.cs

@@ -54,10 +54,7 @@ public bool ShouldRecord(long[] tensor_ids, TF_DataType[] dtypes)
                 if (tensor_tape_.find(tensor_ids[i]))
                 {
                     if (IsDtypeTrainable(dtypes[i]))
-                    {
-                        tf.Logger.Debug($"tape.h->ShouldRecord: should_record = true, tensor_tape_.size()={tensor_tape_.Count}, tensor_ids[{i}]={tensor_ids[i]}");
                         return true;
-                    }
                 }
             }
             return false;

src/TensorFlowNET.Core/Gradients/ops.gradient_function_mapping.cs

@@ -19,6 +19,7 @@ limitations under the License.
 using System.Linq;
 using System.Reflection;
 using Tensorflow.Gradients;
+using static Tensorflow.Binding;
 
 namespace Tensorflow
 {
@@ -47,11 +48,17 @@ public static void RegisterFromAssembly()
                     {
                         RegisterGradientFunction(m.GetCustomAttribute<RegisterGradient>().Name,
                             (oper, out_grads) =>
-                                 g.InvokeMember(m.Name,
-                                    BindingFlags.InvokeMethod,
-                                    null,
-                                    null,
-                                    args: new object[] { oper, out_grads }) as Tensor[]
+                            {
+                                tf.Logger.Debug($"Caculate Gradient: {m.Name}");
+                                var results = g.InvokeMember(m.Name,
+                                   BindingFlags.InvokeMethod,
+                                   null,
+                                   null,
+                                   args: new object[] { oper, out_grads }) as Tensor[];
+                                foreach (var result in results.Where(x => x != null))
+                                    tf.Logger.Debug($"{result.TensorShape}");
+                                return results;
+                            }
                         );
                     }
 

src/TensorFlowNET.Core/Operations/gen_math_ops.cs

@@ -358,7 +358,7 @@ public static Tensor sigmoid_grad(Tensor y, Tensor dy, string name = "SigmoidGra
             }
 
             var op = tf.OpDefLib._apply_op_helper("SigmoidGrad", name: name, args: new { y, dy });
-
+            
             return op.output;
         }
 

src/TensorFlowNET.Keras/Engine/Functional.cs

@@ -335,9 +335,10 @@ Tensors run_internal_graph(Tensors inputs, bool training = false, Tensors mask =
 
                     var layer_inputs = node.MapArguments(tensor_dict);
 
-                    tf.Logger.Debug($"{node.Layer}: {node.Layer.Name}");
+                    tf.Logger.Debug($"{depth}: {node.Layer}: {node.Layer.Name}");
                     var outputs = node.Layer.Apply(layer_inputs, is_training: training);
-
+                    foreach (var output in outputs.Where(x => x != null))
+                        tf.Logger.Debug($"{output.TensorShape}");
                     // Update tensor_dict for next input
                     foreach (var (x_id, y) in zip(node.FlatOutputIds, outputs))
                         tensor_dict[x_id] = new Queue<Tensor>(Enumerable.Range(0, tensor_usage_count[x_id]).Select(x => y));