Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

remove duplicate metric vs step log for train loop #4173

Merged
merged 4 commits into from
Oct 15, 2020
Merged

remove duplicate metric vs step log for train loop #4173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d2aedd1
remove duplicate metric vs step log
williamFalcon Oct 15, 2020
e8d8417
remove duplicate metric vs step log
williamFalcon Oct 15, 2020
7bf9e4d
remove duplicate metric vs step log
williamFalcon Oct 15, 2020
c502e39
fix ddp index issue
williamFalcon Oct 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 10 additions & 3 deletions pytorch_lightning/trainer/training_loop.py
View file
Open in desktop
Expand Up @@ -477,13 +477,18 @@ def _track_gradient_norm(self):
def log_training_step_metrics(self, opt_closure_result, batch_callback_metrics, batch_log_metrics):
# track callback metrics
callback_metrics = opt_closure_result.training_step_output.callback_metrics
batch_callback_metrics.append(callback_metrics)

# decide which metrics to log (results vs dict return)
using_results_obj = isinstance(opt_closure_result.training_step_output, Result)
if using_results_obj:
metrics_to_log = opt_closure_result.training_step_output.batch_log_metrics
step_pbar_metrics = opt_closure_result.training_step_output.batch_pbar_metrics
metrics_to_log = opt_closure_result.training_step_output.get_batch_log_metrics(
include_forked_originals=False
)
step_pbar_metrics = opt_closure_result.training_step_output.get_batch_pbar_metrics(
include_forked_originals=False
)
forked_metrics = opt_closure_result.training_step_output.get_forked_metrics()
callback_metrics.update(forked_metrics)
else:
metrics_to_log = opt_closure_result.training_step_output.log_metrics
step_pbar_metrics = opt_closure_result.training_step_output.pbar_on_batch_end
Expand All @@ -496,6 +501,8 @@ def log_training_step_metrics(self, opt_closure_result, batch_callback_metrics,
self.trainer.logger_connector.add_progress_bar_metrics(step_pbar_metrics)
self.trainer.logger_connector.callback_metrics.update(step_pbar_metrics)

batch_callback_metrics.append(callback_metrics)

def process_hiddens(self, opt_closure_result):
hiddens = opt_closure_result.hiddens
if isinstance(opt_closure_result.training_step_output, Result):
Expand Down
2 changes: 0 additions & 2 deletions tests/trainer/logging/test_eval_loop_logging_1_0.py
View file
Open in desktop
Expand Up @@ -64,7 +64,6 @@ def backward(self, loss, optimizer, optimizer_idx):

# make sure all the metrics are available for callbacks
expected_logged_metrics = {
'a',
'a2',
'a_step',
'a_epoch',
Expand Down Expand Up @@ -137,7 +136,6 @@ def backward(self, loss, optimizer, optimizer_idx):
expected_logged_metrics = {
'epoch',
'a',
'b',
'b_step',
'b_epoch',
'c',
Expand Down
18 changes: 10 additions & 8 deletions tests/trainer/logging/test_train_loop_logging_1_0.py
View file
Open in desktop
Expand Up @@ -19,7 +19,7 @@
import torch
import pytest

from pytorch_lightning import Trainer
from pytorch_lightning import Trainer, callbacks
from tests.base.deterministic_model import DeterministicModel
from torch.utils.data import Dataset

Expand Down Expand Up @@ -80,6 +80,7 @@ def backward(self, loss, optimizer, optimizer_idx):
max_epochs=2,
log_every_n_steps=1,
weights_summary=None,
checkpoint_callback=callbacks.ModelCheckpoint(monitor='l_se')
)
trainer.fit(model)

Expand All @@ -95,7 +96,6 @@ def backward(self, loss, optimizer, optimizer_idx):
'default',
'l_e',
'l_s',
'l_se',
'l_se_step',
'l_se_epoch',
}
Expand All @@ -105,7 +105,6 @@ def backward(self, loss, optimizer, optimizer_idx):
expected_pbar_metrics = {
'p_e',
'p_s',
'p_se',
'p_se_step',
'p_se_epoch',
}
Expand All @@ -116,6 +115,7 @@ def backward(self, loss, optimizer, optimizer_idx):
expected_callback_metrics = set()
expected_callback_metrics = expected_callback_metrics.union(logged_metrics)
expected_callback_metrics = expected_callback_metrics.union(pbar_metrics)
expected_callback_metrics.update({'p_se', 'l_se'})
expected_callback_metrics.remove('epoch')
assert callback_metrics == expected_callback_metrics

Expand Down Expand Up @@ -163,7 +163,7 @@ def backward(self, loss, optimizer, optimizer_idx):

# make sure all the metrics are available for callbacks
logged_metrics = set(trainer.logged_metrics.keys())
expected_logged_metrics = {'epoch', 'a', 'a_step', 'a_epoch', 'b', 'b1', 'a1', 'a2'}
expected_logged_metrics = {'epoch', 'a_step', 'a_epoch', 'b', 'b1', 'a1', 'a2'}
assert logged_metrics == expected_logged_metrics

pbar_metrics = set(trainer.progress_bar_metrics.keys())
Expand All @@ -178,6 +178,7 @@ def backward(self, loss, optimizer, optimizer_idx):
expected_callback_metrics = expected_callback_metrics.union(logged_metrics)
expected_callback_metrics = expected_callback_metrics.union(pbar_metrics)
expected_callback_metrics.remove('epoch')
expected_callback_metrics.add('a')
assert callback_metrics == expected_callback_metrics


Expand Down Expand Up @@ -226,23 +227,24 @@ def training_epoch_end(self, outputs):
# make sure all the metrics are available for callbacks
logged_metrics = set(trainer.logged_metrics.keys())
expected_logged_metrics = {
'a', 'a_step', 'a_epoch',
'b', 'b_step', 'b_epoch',
'a_step', 'a_epoch',
'b_step', 'b_epoch',
'c',
'd/e/f',
'epoch'
}
assert logged_metrics == expected_logged_metrics

pbar_metrics = set(trainer.progress_bar_metrics.keys())
expected_pbar_metrics = {'b', 'c', 'b_epoch', 'b_step'}
expected_pbar_metrics = {'c', 'b_epoch', 'b_step'}
assert pbar_metrics == expected_pbar_metrics

callback_metrics = set(trainer.callback_metrics.keys())
callback_metrics.remove('debug_epoch')
expected_callback_metrics = set()
expected_callback_metrics = expected_callback_metrics.union(logged_metrics)
expected_callback_metrics = expected_callback_metrics.union(pbar_metrics)
expected_callback_metrics.update({'a', 'b'})
expected_callback_metrics.remove('epoch')
assert callback_metrics == expected_callback_metrics

Expand Down Expand Up @@ -355,7 +357,7 @@ def train_dataloader(self):
trainer.fit(model)

generated = set(trainer.logged_metrics.keys())
expected = {'a', 'a_step', 'a_epoch', 'epoch'}
expected = {'a_step', 'a_epoch', 'epoch'}
assert generated == expected


Expand Down
2 changes: 1 addition & 1 deletion tests/trainer/optimization/test_manual_optimization.py
View file
Open in desktop
Expand Up @@ -213,7 +213,7 @@ def configure_optimizers(self):
num_manual_backward_calls = 3
assert trainer.dev_debugger.count_events('backward_call') == limit_train_batches * num_manual_backward_calls

expected = {'a', 'a_step', 'a_epoch', 'epoch'}
expected = {'a_step', 'a_epoch', 'epoch'}
logged = set(trainer.logged_metrics.keys())
assert expected == logged

Expand Down