mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-12 12:40:20 +08:00
proper checkpoint implementation (#1043)
* enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * enabled early stopping/checkpooiunt even without val step * name formatting * version * testing * add test * fix test * Update model_checkpoint.py * doctests * pylint * tests * debug * debug * enabled early stopping/checkpooiunt even without val step * fix MNIST download (#1044) * fix MNIST download * simple * name formatting * version * testing * add test * fix test * doctests * tests * debug * debug * rebased 1041 * rebased 1041 * tests * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 * rebased 1041 Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>
This commit is contained in:
William Falcon
co-authored by
Jirka Borovec
Jirka Borovec
parent
165b9fb3f3
commit
bcb45d906d
@@ -1,5 +1,8 @@
|
||||
import os
|
||||
|
||||
import tests.models.utils as tutils
|
||||
from pytorch_lightning import Trainer, LightningModule
|
||||
from pytorch_lightning.callbacks import ModelCheckpoint
|
||||
from tests.models import (
|
||||
TestModelBase,
|
||||
LightTrainDataloader,
|
||||
|
||||
@@ -27,6 +27,28 @@ from pytorch_lightning.trainer.logging import TrainerLoggingMixin
|
||||
from pytorch_lightning.utilities.debugging import MisconfigurationException
|
||||
|
||||
|
||||
def test_hparams_save_load(tmpdir):
|
||||
model = DictHparamsModel({'in_features': 28 * 28, 'out_features': 10})
|
||||
|
||||
# logger file to get meta
|
||||
trainer_options = dict(
|
||||
default_save_path=tmpdir,
|
||||
max_epochs=2,
|
||||
)
|
||||
|
||||
# fit model
|
||||
trainer = Trainer(**trainer_options)
|
||||
result = trainer.fit(model)
|
||||
|
||||
assert result == 1
|
||||
|
||||
# try to load the model now
|
||||
pretrained_model = tutils.load_model_from_checkpoint(
|
||||
trainer.checkpoint_callback.dirpath,
|
||||
module_class=DictHparamsModel
|
||||
)
|
||||
|
||||
|
||||
def test_no_val_module(tmpdir):
|
||||
"""Tests use case where trainer saves the model, and user loads it from tags independently."""
|
||||
tutils.reset_seed()
|
||||
@@ -126,7 +148,8 @@ def test_gradient_accumulation_scheduling(tmpdir):
|
||||
assert Trainer(accumulate_grad_batches={1: 2.5, 3: 5})
|
||||
|
||||
# test optimizer call freq matches scheduler
|
||||
def _optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, second_order_closure=None):
|
||||
def _optimizer_step(self, epoch, batch_idx, optimizer,
|
||||
optimizer_idx, second_order_closure=None):
|
||||
# only test the first 12 batches in epoch
|
||||
if batch_idx < 12:
|
||||
if epoch == 0:
|
||||
@@ -255,11 +278,11 @@ def test_model_checkpoint_options(tmp_path):
|
||||
assert len(file_lists) == len(losses), "Should save all models when save_top_k=-1"
|
||||
|
||||
# verify correct naming
|
||||
for fname in {'_epoch=4_val_loss=2.50.ckpt',
|
||||
'_epoch=3_val_loss=5.00.ckpt',
|
||||
'_epoch=2_val_loss=2.80.ckpt',
|
||||
'_epoch=1_val_loss=9.00.ckpt',
|
||||
'_epoch=0_val_loss=10.00.ckpt'}:
|
||||
for fname in {'epoch=4.ckpt',
|
||||
'epoch=3.ckpt',
|
||||
'epoch=2.ckpt',
|
||||
'epoch=1.ckpt',
|
||||
'epoch=0.ckpt'}:
|
||||
assert fname in file_lists
|
||||
|
||||
save_dir = tmp_path / "2"
|
||||
@@ -286,7 +309,7 @@ def test_model_checkpoint_options(tmp_path):
|
||||
|
||||
# -----------------
|
||||
# CASE K=1 (2.5, epoch 4)
|
||||
checkpoint_callback = ModelCheckpoint(save_dir, save_top_k=1, verbose=1, prefix='test_prefix')
|
||||
checkpoint_callback = ModelCheckpoint(save_dir, save_top_k=1, verbose=1, prefix='test_prefix_')
|
||||
checkpoint_callback.save_function = mock_save_function
|
||||
trainer = Trainer()
|
||||
|
||||
@@ -299,7 +322,7 @@ def test_model_checkpoint_options(tmp_path):
|
||||
file_lists = set(os.listdir(save_dir))
|
||||
|
||||
assert len(file_lists) == 1, "Should save 1 model when save_top_k=1"
|
||||
assert 'test_prefix_epoch=4_val_loss=2.50.ckpt' in file_lists
|
||||
assert 'test_prefix_epoch=4.ckpt' in file_lists
|
||||
|
||||
save_dir = tmp_path / "4"
|
||||
save_dir.mkdir()
|
||||
@@ -322,8 +345,8 @@ def test_model_checkpoint_options(tmp_path):
|
||||
file_lists = set(os.listdir(save_dir))
|
||||
|
||||
assert len(file_lists) == 3, 'Should save 2 model when save_top_k=2'
|
||||
for fname in {'_epoch=4_val_loss=2.50.ckpt',
|
||||
'_epoch=2_val_loss=2.80.ckpt',
|
||||
for fname in {'epoch=4.ckpt',
|
||||
'epoch=2.ckpt',
|
||||
'other_file.ckpt'}:
|
||||
assert fname in file_lists
|
||||
|
||||
@@ -368,9 +391,9 @@ def test_model_checkpoint_options(tmp_path):
|
||||
file_lists = set(os.listdir(save_dir))
|
||||
|
||||
assert len(file_lists) == 3, 'Should save 3 models when save_top_k=3'
|
||||
for fname in {'_epoch=0_val_loss=2.80.ckpt',
|
||||
'_epoch=0_val_loss=2.50.ckpt',
|
||||
'_epoch=0_val_loss=5.00.ckpt'}:
|
||||
for fname in {'epoch=0.ckpt',
|
||||
'epoch=0.ckpt',
|
||||
'epoch=0.ckpt'}:
|
||||
assert fname in file_lists
|
||||
|
||||
|
||||
@@ -620,25 +643,3 @@ def test_default_args(tmpdir):
|
||||
|
||||
assert isinstance(trainer, Trainer)
|
||||
assert trainer.max_epochs == 5
|
||||
|
||||
|
||||
def test_hparams_save_load(tmpdir):
|
||||
model = DictHparamsModel({'in_features': 28 * 28, 'out_features': 10})
|
||||
|
||||
# logger file to get meta
|
||||
trainer_options = dict(
|
||||
default_save_path=tmpdir,
|
||||
max_epochs=2,
|
||||
)
|
||||
|
||||
# fit model
|
||||
trainer = Trainer(**trainer_options)
|
||||
result = trainer.fit(model)
|
||||
|
||||
assert result == 1
|
||||
|
||||
# try to load the model now
|
||||
pretrained_model = tutils.load_model_from_checkpoint(
|
||||
trainer.checkpoint_callback.dirpath,
|
||||
module_class=DictHparamsModel
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user