add rank warning (#1428)

* add rank warning

* changelog

* use rank_zero_warn

* user trainer_init

* replace warnings

* fix test

* flake8

* docs

* changelog

* bug lol
This commit is contained in:
Jirka Borovec
2020-04-09 14:05:46 -04:00
committed by GitHub
parent b4eb3884cf
commit 17f58d2e11
41 changed files with 213 additions and 187 deletions
@@ -6,12 +6,11 @@ Stop training when a monitored quantity has stopped improving.
"""
import warnings
import numpy as np
from pytorch_lightning import _logger as log
from pytorch_lightning.callbacks.base import Callback
from pytorch_lightning.utilities import rank_zero_warn
class EarlyStopping(Callback):
@@ -80,7 +79,7 @@ class EarlyStopping(Callback):
if self.strict:
raise RuntimeError(error_msg)
if self.verbose > 0:
warnings.warn(error_msg, RuntimeWarning)
rank_zero_warn(error_msg, RuntimeWarning)
return False
@@ -113,6 +112,6 @@ class EarlyStopping(Callback):
def on_train_end(self, trainer, pl_module):
if self.stopped_epoch > 0 and self.verbose > 0:
warnings.warn('Displayed epoch numbers by `EarlyStopping` start from "1" until v0.6.x,'
' but will start from "0" in v0.8.0.', DeprecationWarning)
rank_zero_warn('Displayed epoch numbers by `EarlyStopping` start from "1" until v0.6.x,'
' but will start from "0" in v0.8.0.', DeprecationWarning)
log.info(f'Epoch {self.stopped_epoch + 1:05d}: early stopping')
@@ -6,9 +6,8 @@ Change gradient accumulation factor according to scheduling.
"""
import warnings
from pytorch_lightning.callbacks.base import Callback
from pytorch_lightning.utilities import rank_zero_warn
class GradientAccumulationScheduler(Callback):
@@ -46,8 +45,8 @@ class GradientAccumulationScheduler(Callback):
raise TypeError("All epoches and accumulation factor must be integers")
minimal_epoch = min(scheduling.keys())
warnings.warn('Epochs indexing of `scheduling` starts from "1" until v0.6.x,'
' but will start from "0" in v0.8.0.', DeprecationWarning)
rank_zero_warn('Epochs indexing of `scheduling` starts from "1" until v0.6.x,'
' but will start from "0" in v0.8.0.', DeprecationWarning)
if minimal_epoch < 1:
msg = f"Epochs indexing from 1, epoch {minimal_epoch} cannot be interpreted correct"
raise IndexError(msg)
@@ -7,14 +7,13 @@ Automatically save model checkpoints during training.
"""
import os
import shutil
import warnings
import re
import numpy as np
from pytorch_lightning.callbacks.base import Callback
from pytorch_lightning import _logger as log
from pytorch_lightning.callbacks.base import Callback
from pytorch_lightning.utilities import rank_zero_warn
class ModelCheckpoint(Callback):
@@ -83,7 +82,7 @@ class ModelCheckpoint(Callback):
mode: str = 'auto', period: int = 1, prefix: str = ''):
super().__init__()
if save_top_k > 0 and os.path.isdir(filepath) and len(os.listdir(filepath)) > 0:
warnings.warn(
rank_zero_warn(
f"Checkpoint directory {filepath} exists and is not empty with save_top_k != 0."
"All files in this directory will be deleted when a checkpoint is saved!"
)
@@ -115,9 +114,7 @@ class ModelCheckpoint(Callback):
}
if mode not in mode_dict:
warnings.warn(
f'ModelCheckpoint mode {mode} is unknown, '
'fallback to auto mode.', RuntimeWarning)
rank_zero_warn(f'ModelCheckpoint mode {mode} is unknown, fallback to auto mode.', RuntimeWarning)
mode = 'auto'
self.monitor_op, self.kth_value, self.mode = mode_dict[mode]
@@ -206,7 +203,7 @@ class ModelCheckpoint(Callback):
current = metrics.get(self.monitor)
if current is None:
warnings.warn(f'Can save best model only with {self.monitor} available, skipping.', RuntimeWarning)
rank_zero_warn(f'Can save best model only with {self.monitor} available, skipping.', RuntimeWarning)
elif self.check_monitor_top_k(current):
self._do_check_save(filepath, current, epoch)
elif self.verbose > 0: