remove duplicate metric vs step log for train loop (#4173)

* remove duplicate metric vs step log

* remove duplicate metric vs step log

* remove duplicate metric vs step log

* fix ddp index issue

Author: williamFalcon

pytorch_lightning/trainer/training_loop.py

@@ -477,13 +477,18 @@ def _track_gradient_norm(self):
     def log_training_step_metrics(self, opt_closure_result, batch_callback_metrics, batch_log_metrics):
         # track callback metrics
         callback_metrics = opt_closure_result.training_step_output.callback_metrics
-        batch_callback_metrics.append(callback_metrics)
 
         # decide which metrics to log (results vs dict return)
         using_results_obj = isinstance(opt_closure_result.training_step_output, Result)
         if using_results_obj:
-            metrics_to_log = opt_closure_result.training_step_output.batch_log_metrics
-            step_pbar_metrics = opt_closure_result.training_step_output.batch_pbar_metrics
+            metrics_to_log = opt_closure_result.training_step_output.get_batch_log_metrics(
+                include_forked_originals=False
+            )
+            step_pbar_metrics = opt_closure_result.training_step_output.get_batch_pbar_metrics(
+                include_forked_originals=False
+            )
+            forked_metrics = opt_closure_result.training_step_output.get_forked_metrics()
+            callback_metrics.update(forked_metrics)
         else:
             metrics_to_log = opt_closure_result.training_step_output.log_metrics
             step_pbar_metrics = opt_closure_result.training_step_output.pbar_on_batch_end
@@ -496,6 +501,8 @@ def log_training_step_metrics(self, opt_closure_result, batch_callback_metrics,
             self.trainer.logger_connector.add_progress_bar_metrics(step_pbar_metrics)
             self.trainer.logger_connector.callback_metrics.update(step_pbar_metrics)
 
+        batch_callback_metrics.append(callback_metrics)
+
     def process_hiddens(self, opt_closure_result):
         hiddens = opt_closure_result.hiddens
         if isinstance(opt_closure_result.training_step_output, Result):

tests/trainer/logging/test_eval_loop_logging_1_0.py

@@ -64,7 +64,6 @@ def backward(self, loss, optimizer, optimizer_idx):
 
     # make sure all the metrics are available for callbacks
     expected_logged_metrics = {
-        'a',
         'a2',
         'a_step',
         'a_epoch',
@@ -137,7 +136,6 @@ def backward(self, loss, optimizer, optimizer_idx):
     expected_logged_metrics = {
         'epoch',
         'a',
-        'b',
         'b_step',
         'b_epoch',
         'c',

tests/trainer/logging/test_train_loop_logging_1_0.py

@@ -19,7 +19,7 @@
 import torch
 import pytest
 
-from pytorch_lightning import Trainer
+from pytorch_lightning import Trainer, callbacks
 from tests.base.deterministic_model import DeterministicModel
 from torch.utils.data import Dataset
 
@@ -80,6 +80,7 @@ def backward(self, loss, optimizer, optimizer_idx):
         max_epochs=2,
         log_every_n_steps=1,
         weights_summary=None,
+        checkpoint_callback=callbacks.ModelCheckpoint(monitor='l_se')
     )
     trainer.fit(model)
 
@@ -95,7 +96,6 @@ def backward(self, loss, optimizer, optimizer_idx):
         'default',
         'l_e',
         'l_s',
-        'l_se',
         'l_se_step',
         'l_se_epoch',
     }
@@ -105,7 +105,6 @@ def backward(self, loss, optimizer, optimizer_idx):
     expected_pbar_metrics = {
         'p_e',
         'p_s',
-        'p_se',
         'p_se_step',
         'p_se_epoch',
     }
@@ -116,6 +115,7 @@ def backward(self, loss, optimizer, optimizer_idx):
     expected_callback_metrics = set()
     expected_callback_metrics = expected_callback_metrics.union(logged_metrics)
     expected_callback_metrics = expected_callback_metrics.union(pbar_metrics)
+    expected_callback_metrics.update({'p_se', 'l_se'})
     expected_callback_metrics.remove('epoch')
     assert callback_metrics == expected_callback_metrics
 
@@ -163,7 +163,7 @@ def backward(self, loss, optimizer, optimizer_idx):
 
     # make sure all the metrics are available for callbacks
     logged_metrics = set(trainer.logged_metrics.keys())
-    expected_logged_metrics = {'epoch', 'a', 'a_step', 'a_epoch', 'b', 'b1', 'a1', 'a2'}
+    expected_logged_metrics = {'epoch', 'a_step', 'a_epoch', 'b', 'b1', 'a1', 'a2'}
     assert logged_metrics == expected_logged_metrics
 
     pbar_metrics = set(trainer.progress_bar_metrics.keys())
@@ -178,6 +178,7 @@ def backward(self, loss, optimizer, optimizer_idx):
     expected_callback_metrics = expected_callback_metrics.union(logged_metrics)
     expected_callback_metrics = expected_callback_metrics.union(pbar_metrics)
     expected_callback_metrics.remove('epoch')
+    expected_callback_metrics.add('a')
     assert callback_metrics == expected_callback_metrics
 
 
@@ -226,23 +227,24 @@ def training_epoch_end(self, outputs):
     # make sure all the metrics are available for callbacks
     logged_metrics = set(trainer.logged_metrics.keys())
     expected_logged_metrics = {
-        'a', 'a_step', 'a_epoch',
-        'b', 'b_step', 'b_epoch',
+        'a_step', 'a_epoch',
+        'b_step', 'b_epoch',
         'c',
         'd/e/f',
         'epoch'
     }
     assert logged_metrics == expected_logged_metrics
 
     pbar_metrics = set(trainer.progress_bar_metrics.keys())
-    expected_pbar_metrics = {'b', 'c', 'b_epoch', 'b_step'}
+    expected_pbar_metrics = {'c', 'b_epoch', 'b_step'}
     assert pbar_metrics == expected_pbar_metrics
 
     callback_metrics = set(trainer.callback_metrics.keys())
     callback_metrics.remove('debug_epoch')
     expected_callback_metrics = set()
     expected_callback_metrics = expected_callback_metrics.union(logged_metrics)
     expected_callback_metrics = expected_callback_metrics.union(pbar_metrics)
+    expected_callback_metrics.update({'a', 'b'})
     expected_callback_metrics.remove('epoch')
     assert callback_metrics == expected_callback_metrics
 
@@ -355,7 +357,7 @@ def train_dataloader(self):
     trainer.fit(model)
 
     generated = set(trainer.logged_metrics.keys())
-    expected = {'a', 'a_step', 'a_epoch', 'epoch'}
+    expected = {'a_step', 'a_epoch', 'epoch'}
     assert generated == expected
 
 

tests/trainer/optimization/test_manual_optimization.py

@@ -213,7 +213,7 @@ def configure_optimizers(self):
     num_manual_backward_calls = 3
     assert trainer.dev_debugger.count_events('backward_call') == limit_train_batches * num_manual_backward_calls
 
-    expected = {'a', 'a_step', 'a_epoch', 'epoch'}
+    expected = {'a_step', 'a_epoch', 'epoch'}
     logged = set(trainer.logged_metrics.keys())
     assert expected == logged