mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-12 12:40:20 +08:00
Progress bar callback (#1450)
* squash and rebase sanity check hooks sanity check callback hook finish moved core progress bar functionality into callback wip remove duplicate merge clean up imports docs sanity check progress bar main sanity move callback calls init progrss bar callback configuration and docs changelog rate decorator pass process_position disable on rank > 0 position index is_enabled remove decorator refactor init tqdm bars callback method ordering cannot reset when disabled sequence -> list default values fix has no attr _time() move on_val_end to proper place fix the pickle issue update warning properties check for None remove old comment switch order pull out non-tqdm functionality into base class documentation for the base class docs fix refresh rate issue in validation restrict type hint of trainer arg more docs update trainer docs rst docs fix lines too long fix test add missing type hints fix typo move docstring to __init__ solves doctest failures remove doctest :(( can't fix the pickle error fix example simplify by saving trainer reference fix docs errors move docstring initial value multiple val checks per epoch simpler handling of inf dataset sizes update inf docs renamed training_tqdm_dict rename get_tqdm_dict rename occurences of tqdm update changelog fix doctest fix formatting errors added callback tests progress bar on off test more tests for progress bar weird test fix? add ignored property disable default progress bar in LR finder change enable/disable behavior trying doctest in CI again undo doctest pickle error undo doctest pickle error :(( remove progress_bar_callback Trainer arg and fix tests restore progress bar after auto lr find update docs fix rebase fix wrong negation * fix fast dev run total * more thorough testing * remove old args * fix merge * fix merge * separate tests * type hint total batches * reduce if Co-Authored-By: Jirka Borovec <Borda@users.noreply.github.com> * is_disabled Co-Authored-By: Jirka Borovec <Borda@users.noreply.github.com> * is_enabled Co-Authored-By: Jirka Borovec <Borda@users.noreply.github.com> * rename enabled/disabled * move deprecated api * remove duplicated test from merge * fix rename is_disabled * newline * test also testprogress for fast dev run Co-authored-by: J. Borovec <jirka.borovec@seznam.cz> Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>
This commit is contained in:
Adrian Wälchli
co-authored by
Jirka Borovec
J. Borovec
Jirka Borovec
J. Borovec
parent
fe2b6666e0
commit
3e8f2d99a9
@@ -123,14 +123,12 @@ In this second case, the options you pass to trainer will be used when running
|
||||
|
||||
"""
|
||||
|
||||
import sys
|
||||
from abc import ABC, abstractmethod
|
||||
from pprint import pprint
|
||||
from typing import Callable
|
||||
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from pytorch_lightning.core.lightning import LightningModule
|
||||
from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel, LightningDataParallel
|
||||
@@ -157,9 +155,6 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
|
||||
# this is just a summary on variables used in this abstract class,
|
||||
# the proper values/initialisation should be done in child class
|
||||
test_progress_bar: ...
|
||||
val_progress_bar: ...
|
||||
main_progress_bar: ...
|
||||
on_gpu: bool
|
||||
use_ddp: bool
|
||||
use_dp: bool
|
||||
@@ -171,9 +166,8 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
num_test_batches: int
|
||||
num_val_batches: int
|
||||
fast_dev_run: ...
|
||||
process_position: ...
|
||||
process_output: ...
|
||||
training_tqdm_dict: ...
|
||||
progress_bar_dict: ...
|
||||
proc_rank: int
|
||||
current_epoch: int
|
||||
callback_metrics: ...
|
||||
@@ -181,9 +175,12 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
val_dataloaders: DataLoader
|
||||
use_tpu: bool
|
||||
reload_dataloaders_every_epoch: ...
|
||||
progress_bar_refresh_rate: ...
|
||||
|
||||
# Callback system
|
||||
on_validation_batch_start: Callable
|
||||
on_validation_batch_end: Callable
|
||||
on_test_batch_start: Callable
|
||||
on_test_batch_end: Callable
|
||||
on_validation_start: Callable
|
||||
on_validation_end: Callable
|
||||
on_test_start: Callable
|
||||
@@ -210,7 +207,7 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
"""Warning: this is just empty shell for code implemented in other class."""
|
||||
|
||||
@abstractmethod
|
||||
def add_tqdm_metrics(self, *args):
|
||||
def add_progress_bar_metrics(self, *args):
|
||||
"""Warning: this is just empty shell for code implemented in other class."""
|
||||
|
||||
@abstractmethod
|
||||
@@ -265,6 +262,12 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
if batch_idx >= max_batches:
|
||||
break
|
||||
|
||||
# callbacks
|
||||
if test_mode:
|
||||
self.on_test_batch_start()
|
||||
else:
|
||||
self.on_validation_batch_start()
|
||||
|
||||
# -----------------
|
||||
# RUN EVALUATION STEP
|
||||
# -----------------
|
||||
@@ -280,22 +283,17 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
model_ref = self.get_model()
|
||||
with self.profiler.profile('test_step_end'):
|
||||
output = model_ref.test_step_end(output)
|
||||
self.on_test_batch_end()
|
||||
else:
|
||||
if self.is_overriden('validation_step_end'):
|
||||
model_ref = self.get_model()
|
||||
with self.profiler.profile('validation_step_end'):
|
||||
output = model_ref.validation_step_end(output)
|
||||
self.on_validation_batch_end()
|
||||
|
||||
# track outputs for collation
|
||||
dl_outputs.append(output)
|
||||
|
||||
# batch done
|
||||
if self.progress_bar_refresh_rate >= 1 and batch_idx % self.progress_bar_refresh_rate == 0:
|
||||
if test_mode:
|
||||
self.test_progress_bar.update(self.progress_bar_refresh_rate)
|
||||
else:
|
||||
self.val_progress_bar.update(self.progress_bar_refresh_rate)
|
||||
self.main_progress_bar.update(self.progress_bar_refresh_rate)
|
||||
outputs.append(dl_outputs)
|
||||
|
||||
eval_results = {}
|
||||
@@ -343,12 +341,6 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
"You called `.test()` without defining model's `.test_step()`."
|
||||
" Please define and try again")
|
||||
|
||||
# Validation/Test begin callbacks
|
||||
if test_mode:
|
||||
self.on_test_start()
|
||||
else:
|
||||
self.on_validation_start()
|
||||
|
||||
# hook
|
||||
model = self.get_model()
|
||||
model.on_pre_performance_check()
|
||||
@@ -372,21 +364,18 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
if self.fast_dev_run:
|
||||
max_batches = 1
|
||||
|
||||
# init validation or test progress bar
|
||||
# main progress bar will already be closed when testing so initial position is free
|
||||
position = 2 * self.process_position + (not test_mode)
|
||||
desc = 'Testing' if test_mode else 'Validating'
|
||||
total = max_batches if max_batches != float('inf') else None
|
||||
pbar = tqdm(desc=desc, total=total, leave=test_mode, position=position,
|
||||
disable=not self.progress_bar_refresh_rate, dynamic_ncols=True, file=sys.stdout)
|
||||
setattr(self, f'{"test" if test_mode else "val"}_progress_bar', pbar)
|
||||
# Validation/Test begin callbacks
|
||||
if test_mode:
|
||||
self.on_test_start()
|
||||
else:
|
||||
self.on_validation_start()
|
||||
|
||||
# run evaluation
|
||||
eval_results = self._evaluate(self.model, dataloaders, max_batches, test_mode)
|
||||
_, prog_bar_metrics, log_metrics, callback_metrics, _ = self.process_output(eval_results)
|
||||
|
||||
# add metrics to prog bar
|
||||
self.add_tqdm_metrics(prog_bar_metrics)
|
||||
self.add_progress_bar_metrics(prog_bar_metrics)
|
||||
|
||||
# log results of test
|
||||
if test_mode and self.proc_rank == 0:
|
||||
@@ -404,16 +393,6 @@ class TrainerEvaluationLoopMixin(ABC):
|
||||
# hook
|
||||
model.on_post_performance_check()
|
||||
|
||||
# add model specific metrics
|
||||
if not test_mode:
|
||||
self.main_progress_bar.set_postfix(**self.training_tqdm_dict)
|
||||
|
||||
# close progress bar
|
||||
if test_mode:
|
||||
self.test_progress_bar.close()
|
||||
else:
|
||||
self.val_progress_bar.close()
|
||||
|
||||
# eventual dataset reloading
|
||||
if test_mode:
|
||||
if self.reload_dataloaders_every_epoch:
|
||||
|
||||
Reference in New Issue
Block a user