diff --git a/docs/Trainer/Checkpointing.md b/docs/Trainer/Checkpointing.md index b88d3d4a..5d3b257b 100644 --- a/docs/Trainer/Checkpointing.md +++ b/docs/Trainer/Checkpointing.md @@ -2,13 +2,20 @@ Lightning can automate saving and loading checkpoints. --- ### Model saving -To enable checkpointing, define the checkpoint callback and give it to the trainer. +Checkpointing is enabled by default to the current working directory. +To change the checkpoint path pass in : +```python +Trainer(default_save_path='/your/path/to/save/checkpoints') +``` + +To modify the behavior of checkpointing pass in your own callback. ``` {.python} from pytorch_lightning.callbacks import ModelCheckpoint +# DEFAULTS used by the Trainer checkpoint_callback = ModelCheckpoint( - filepath='/path/to/store/weights/', + filepath=os.getcwd(), save_best_only=True, verbose=True, monitor='val_loss', diff --git a/docs/Trainer/Logging.md b/docs/Trainer/Logging.md index 8c3cdc9e..5fc6cc4f 100644 --- a/docs/Trainer/Logging.md +++ b/docs/Trainer/Logging.md @@ -1,16 +1,33 @@ Lighting offers options for logging information about model, gpu usage, etc, via several different logging frameworks. It also offers printing options for training monitoring. +--- +### default_save_path +Lightning sets a default TestTubeLogger and CheckpointCallback for you which log to +```os.getcwd()``` by default. To modify the logging path you can set: +```python +Trainer(default_save_path='/your/path/to/save/checkpoints') +``` + +If you need more custom behavior (different paths for both, different metrics, etc...) +from the logger and the checkpointCallback, pass in your own instances as explained below. + --- ### Setting up logging -Initialize your logger, which should inherit from `LightningBaseLogger`, and pass -it to `Trainer`. +The trainer inits a default logger for you (TestTubeLogger). All logs will +go to the current working directory under a folder named ```os.getcwd()/lightning_logs``. + +If you want to modify the default logging behavior even more, pass in a logger +(which should inherit from `LightningBaseLogger`). + ```{.python} my_logger = MyLightningLogger(...) trainer = Trainer(logger=my_logger) ``` +The path in this logger will overwrite default_save_path. + Lightning supports several common experiment tracking frameworks out of the box --- diff --git a/docs/Trainer/Training Loop.md b/docs/Trainer/Training Loop.md index 8d95bc28..b8ccfc85 100644 --- a/docs/Trainer/Training Loop.md +++ b/docs/Trainer/Training Loop.md @@ -21,17 +21,18 @@ trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000) --- #### Early stopping -To enable early-stopping, define the callback and give it to the trainer. +The trainer already sets up default early stopping for you. +To modify this behavior, pass in your own EarlyStopping callback. ``` {.python} from pytorch_lightning.callbacks import EarlyStopping -# DEFAULTS +# DEFAULTS used by Trainer early_stop_callback = EarlyStopping( monitor='val_loss', min_delta=0.00, - patience=0, + patience=3, verbose=False, - mode='auto' + mode='min' ) trainer = Trainer(early_stop_callback=early_stop_callback) diff --git a/docs/examples/Examples.md b/docs/examples/Examples.md index 11541407..c3d42dc2 100644 --- a/docs/examples/Examples.md +++ b/docs/examples/Examples.md @@ -47,57 +47,17 @@ def main(hparams, cluster, results_dict): :param hparams: :return: """ - # init experiment - log_dir = os.path.dirname(os.path.realpath(__file__)) - exp = Experiment( - name='test_tube_exp', - debug=True, - save_dir=log_dir, - version=0, - autosave=False, - description='test demo' - ) - - # set the hparams for the experiment - exp.argparse(hparams) - exp.save() - # build model model = MyLightningModule(hparams) - # callbacks - early_stop = EarlyStopping( - monitor=hparams.early_stop_metric, - patience=hparams.early_stop_patience, - verbose=True, - mode=hparams.early_stop_mode - ) - - model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) - checkpoint = ModelCheckpoint( - filepath=model_save_path, - save_function=None, - save_best_only=True, - verbose=True, - monitor=hparams.model_save_monitor_value, - mode=hparams.model_save_monitor_mode - ) - # configure trainer - trainer = Trainer( - experiment=exp, - cluster=cluster, - checkpoint_callback=checkpoint, - early_stop_callback=early_stop, - ) + trainer = Trainer() # train model trainer.fit(model) ``` - - The __main__ function will start training on your **main** function. If you use the HyperParameterOptimizer in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple argument parser you get the default arguments in the argument parser. diff --git a/docs/index.md b/docs/index.md index 7edc06ed..74703afc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,7 +47,7 @@ if use_bert: else: model = CoolerNotBERT() -trainer = Trainer(gpus=[0, 1, 2, 3], use_amp=True) +trainer = Trainer(gpus=4, use_amp=True) trainer.fit(model) ``` diff --git a/examples/new_project_templates/multi_node_examples/multi_node_cluster_auto_slurm.py b/examples/new_project_templates/multi_node_examples/multi_node_cluster_auto_slurm.py index 9367e9d0..90b75922 100644 --- a/examples/new_project_templates/multi_node_examples/multi_node_cluster_auto_slurm.py +++ b/examples/new_project_templates/multi_node_examples/multi_node_cluster_auto_slurm.py @@ -55,32 +55,11 @@ def main(hparams, cluster): exp.argparse(hparams) exp.save() - # ------------------------ - # 3 DEFINE CALLBACKS - # ------------------------ - model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) - early_stop = EarlyStopping( - monitor='val_acc', - patience=3, - verbose=True, - mode='max' - ) - - checkpoint = ModelCheckpoint( - filepath=model_save_path, - save_best_only=True, - verbose=True, - monitor='val_loss', - mode='min' - ) - # ------------------------ # 4 INIT TRAINER # ------------------------ trainer = Trainer( experiment=exp, - checkpoint_callback=checkpoint, - early_stop_callback=early_stop, gpus=hparams.per_experiment_nb_gpus, nb_gpu_nodes=hyperparams.nb_gpu_nodes, distributed_backend=hyperparams.distributed_backend diff --git a/examples/new_project_templates/single_cpu_template.py b/examples/new_project_templates/single_cpu_template.py index 9e1c71d3..5b7bf1a4 100644 --- a/examples/new_project_templates/single_cpu_template.py +++ b/examples/new_project_templates/single_cpu_template.py @@ -42,35 +42,12 @@ def main(hparams): exp.save() # ------------------------ - # 3 DEFINE CALLBACKS + # 3 INIT TRAINER # ------------------------ - model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) - early_stop = EarlyStopping( - monitor='val_acc', - patience=3, - verbose=True, - mode='max' - ) - - checkpoint = ModelCheckpoint( - filepath=model_save_path, - save_best_only=True, - verbose=True, - monitor='val_loss', - mode='min' - ) + trainer = Trainer(experiment=exp) # ------------------------ - # 4 INIT TRAINER - # ------------------------ - trainer = Trainer( - experiment=exp, - checkpoint_callback=checkpoint, - early_stop_callback=early_stop, - ) - - # ------------------------ - # 5 START TRAINING + # 4 START TRAINING # ------------------------ trainer.fit(model) diff --git a/examples/new_project_templates/single_gpu_node_16bit_template.py b/examples/new_project_templates/single_gpu_node_16bit_template.py index faee8ddc..d0b28d1c 100644 --- a/examples/new_project_templates/single_gpu_node_16bit_template.py +++ b/examples/new_project_templates/single_gpu_node_16bit_template.py @@ -45,37 +45,16 @@ def main(hparams): exp.save() # ------------------------ - # 3 DEFINE CALLBACKS - # ------------------------ - model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) - early_stop = EarlyStopping( - monitor='val_acc', - patience=3, - verbose=True, - mode='max' - ) - - checkpoint = ModelCheckpoint( - filepath=model_save_path, - save_best_only=True, - verbose=True, - monitor='val_loss', - mode='min' - ) - - # ------------------------ - # 4 INIT TRAINER + # 3 INIT TRAINER # ------------------------ trainer = Trainer( experiment=exp, - checkpoint_callback=checkpoint, - early_stop_callback=early_stop, gpus=hparams.gpus, use_amp=True ) # ------------------------ - # 5 START TRAINING + # 4 START TRAINING # ------------------------ trainer.fit(model) diff --git a/examples/new_project_templates/single_gpu_node_ddp_template.py b/examples/new_project_templates/single_gpu_node_ddp_template.py index b5966a0e..e6553cfe 100644 --- a/examples/new_project_templates/single_gpu_node_ddp_template.py +++ b/examples/new_project_templates/single_gpu_node_ddp_template.py @@ -45,37 +45,16 @@ def main(hparams): exp.save() # ------------------------ - # 3 DEFINE CALLBACKS - # ------------------------ - model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) - early_stop = EarlyStopping( - monitor='val_acc', - patience=3, - verbose=True, - mode='max' - ) - - checkpoint = ModelCheckpoint( - filepath=model_save_path, - save_best_only=True, - verbose=True, - monitor='val_loss', - mode='min' - ) - - # ------------------------ - # 4 INIT TRAINER + # 3 INIT TRAINER # ------------------------ trainer = Trainer( experiment=exp, - checkpoint_callback=checkpoint, - early_stop_callback=early_stop, gpus=hparams.gpus, distributed_backend=hparams.dist_backend ) # ------------------------ - # 5 START TRAINING + # 4 START TRAINING # ------------------------ trainer.fit(model) diff --git a/examples/new_project_templates/single_gpu_node_dp_template.py b/examples/new_project_templates/single_gpu_node_dp_template.py index 3c285178..c6941009 100644 --- a/examples/new_project_templates/single_gpu_node_dp_template.py +++ b/examples/new_project_templates/single_gpu_node_dp_template.py @@ -30,9 +30,8 @@ def main(hparams): print('model built') # ------------------------ - # 2 INIT TEST TUBE EXP + # 2 INIT Logger # ------------------------ - # init experiment exp = Experiment( name=hyperparams.experiment_name, @@ -45,37 +44,16 @@ def main(hparams): exp.save() # ------------------------ - # 3 DEFINE CALLBACKS - # ------------------------ - model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) - early_stop = EarlyStopping( - monitor='val_acc', - patience=3, - verbose=True, - mode='max' - ) - - checkpoint = ModelCheckpoint( - filepath=model_save_path, - save_best_only=True, - verbose=True, - monitor='val_loss', - mode='min' - ) - - # ------------------------ - # 4 INIT TRAINER + # 3 INIT TRAINER # ------------------------ trainer = Trainer( experiment=exp, - checkpoint_callback=checkpoint, - early_stop_callback=early_stop, gpus=hparams.gpus, distributed_backend=hparams.dist_backend, ) # ------------------------ - # 5 START TRAINING + # 4 START TRAINING # ------------------------ trainer.fit(model) diff --git a/examples/new_project_templates/trainer_cpu_template.py b/examples/new_project_templates/trainer_cpu_template.py index 0123c0f4..f3611615 100644 --- a/examples/new_project_templates/trainer_cpu_template.py +++ b/examples/new_project_templates/trainer_cpu_template.py @@ -31,29 +31,8 @@ def main(hparams): # build model model = LightningTemplateModel(hparams) - # callbacks - early_stop = EarlyStopping( - monitor='val_acc', - patience=3, - mode='min', - verbose=True, - ) - - model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) - checkpoint = ModelCheckpoint( - filepath=model_save_path, - save_best_only=True, - verbose=True, - monitor='val_acc', - mode='min' - ) - # configure trainer - trainer = Trainer( - experiment=exp, - checkpoint_callback=checkpoint, - early_stop_callback=early_stop, - ) + trainer = Trainer(experiment=exp) # train model trainer.fit(model) diff --git a/pytorch_lightning/logging/base.py b/pytorch_lightning/logging/base.py index 0545eafd..351b945e 100644 --- a/pytorch_lightning/logging/base.py +++ b/pytorch_lightning/logging/base.py @@ -15,7 +15,7 @@ def rank_zero_only(fn): return wrapped_fn -class LightningLoggerBase: +class LightningLoggerBase(object): """Base class for experiment loggers""" def __init__(self): diff --git a/pytorch_lightning/logging/test_tube_logger.py b/pytorch_lightning/logging/test_tube_logger.py index c20cc1d3..ffffe2cd 100644 --- a/pytorch_lightning/logging/test_tube_logger.py +++ b/pytorch_lightning/logging/test_tube_logger.py @@ -7,6 +7,8 @@ from test_tube import Experiment class TestTubeLogger(LightningLoggerBase): + __test__ = False + def __init__( self, save_dir, name="default", debug=False, version=None, create_git_tag=False ): diff --git a/pytorch_lightning/root_module/memory.py b/pytorch_lightning/root_module/memory.py index 818e9643..b4fe2977 100644 --- a/pytorch_lightning/root_module/memory.py +++ b/pytorch_lightning/root_module/memory.py @@ -35,7 +35,7 @@ class ModelSummary(object): out_sizes = [] input_ = self.model.example_input_array - if self.model.use_ddp or self.model.use_dp: + if self.model.use_ddp or self.model.use_dp or self.model.single_gpu: input_ = input_.cuda(0) if self.model.trainer.use_amp: diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 179cc089..6eae341b 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -16,10 +16,12 @@ from torch.optim.optimizer import Optimizer from pytorch_lightning.root_module.root_module import LightningModule from pytorch_lightning.root_module.memory import get_gpu_memory_map +from pytorch_lightning.logging import TestTubeLogger from pytorch_lightning.trainer.trainer_io import TrainerIO from pytorch_lightning.pt_overrides.override_data_parallel import ( LightningDistributedDataParallel, LightningDataParallel) -from pytorch_lightning.callbacks import GradientAccumulationScheduler +from pytorch_lightning.callbacks import GradientAccumulationScheduler, \ + ModelCheckpoint, EarlyStopping from pytorch_lightning.utilities.debugging import MisconfigurationException import pdb from pytorch_lightning.trainer import ignored_warnings @@ -57,8 +59,9 @@ class Trainer(TrainerIO): def __init__(self, logger=None, - early_stop_callback=None, checkpoint_callback=None, + early_stop_callback=None, + default_save_path=None, gradient_clip_val=0, process_position=0, nb_gpu_nodes=1, @@ -88,8 +91,9 @@ class Trainer(TrainerIO): """ :param logger: Logger for experiment tracking - :param early_stop_callback: Callback for early stopping :param checkpoint_callback: Callback for checkpointing + :param early_stop_callback: Callback for early stopping + :param default_save_path: Default path for logs+weights if no logger/ckpt_callback passed :param gradient_clip_val: int. 0 means don't clip. :param process_position: shown in the tqdm bar :param nb_gpu_nodes: number of GPU nodes @@ -133,6 +137,11 @@ class Trainer(TrainerIO): self.nb_sanity_val_steps = nb_sanity_val_steps self.print_nan_grads = print_nan_grads + # set default save path if user didn't provide one + self.default_save_path = default_save_path + if self.default_save_path is None: + self.default_save_path = os.getcwd() + # training bookeeping self.total_batch_nb = 0 self.running_loss = [] @@ -156,13 +165,39 @@ class Trainer(TrainerIO): self.total_batches = 0 # configure early stop callback + # creates a default one if none passed in self.early_stop_callback = early_stop_callback - - # configure weights save path - self.__configure_weights_path(checkpoint_callback, weights_save_path) + if self.early_stop_callback is None: + self.early_stop = EarlyStopping( + monitor='val_loss', + patience=3, + verbose=True, + mode='min' + ) # configure logger self.logger = logger + if self.logger is None: + self.logger = TestTubeLogger( + save_dir=self.default_save_path, + name='lightning_logs' + ) + + # configure checkpoint callback + self.checkpoint_callback = checkpoint_callback + if self.checkpoint_callback is None: + if isinstance(logger, TestTubeLogger): + ckpt_path = '{}/{}/{}'.format(self.default_save_path, self.logger.name, + self.logger.version) + else: + ckpt_path = self.default_save_path + + self.checkpoint_callback = ModelCheckpoint( + filepath=ckpt_path + ) + + # configure weights save path + self.__configure_weights_path(checkpoint_callback, weights_save_path) # accumulated grads self.__configure_accumulated_gradients(accumulate_grad_batches) @@ -214,8 +249,6 @@ class Trainer(TrainerIO): """ self.weights_save_path = weights_save_path - # configure checkpoint callback - self.checkpoint_callback = checkpoint_callback if self.checkpoint_callback is not None: self.checkpoint_callback.save_function = self.save_checkpoint @@ -224,7 +257,7 @@ class Trainer(TrainerIO): # if weights_save_path is still none here, set to current workingdir if self.weights_save_path is None: - self.weights_save_path = os.getcwd() + self.weights_save_path = self.default_save_path def __init_amp(self, use_amp): self.use_amp = use_amp and APEX_AVAILABLE @@ -900,6 +933,7 @@ class Trainer(TrainerIO): # set local properties on the model ref_model.on_gpu = self.on_gpu + ref_model.single_gpu = self.single_gpu ref_model.use_dp = self.use_dp ref_model.use_ddp = self.use_ddp ref_model.use_ddp2 = self.use_ddp2 diff --git a/tests/test_logging.py b/tests/test_logging.py index 742ef7a7..54f76e1d 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -22,6 +22,7 @@ def test_testtube_logger(): trainer_options = dict( max_nb_epochs=1, + train_percent_check=0.01, logger=logger ) @@ -46,6 +47,7 @@ def test_testtube_pickle(): trainer_options = dict( max_nb_epochs=1, + train_percent_check=0.01, logger=logger ) @@ -74,6 +76,7 @@ def test_mlflow_logger(): trainer_options = dict( max_nb_epochs=1, + train_percent_check=0.01, logger=logger ) diff --git a/tests/test_models.py b/tests/test_models.py index a54dfd07..d6075474 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -39,6 +39,30 @@ np.random.seed(SEED) # ------------------------------------------------------------------------ # TESTS # ------------------------------------------------------------------------ +def test_default_logger_callbacks_cpu_model(): + """ + Test each of the trainer options + :return: + """ + + trainer_options = dict( + max_nb_epochs=1, + gradient_clip_val=1.0, + overfit_pct=0.20, + print_nan_grads=True, + show_progress_bar=False, + train_percent_check=0.01, + val_percent_check=0.01 + ) + + model, hparams = get_model() + run_model_test_no_loggers(trainer_options, model, hparams, on_gpu=False) + + # test freeze on cpu + model.freeze() + model.unfreeze() + + def test_multi_gpu_model_ddp2(): """ Make sure DDP2 works @@ -1336,6 +1360,32 @@ def test_multiple_test_dataloader(): # ------------------------------------------------------------------------ # UTILS # ------------------------------------------------------------------------ +def run_model_test_no_loggers(trainer_options, model, hparams, on_gpu=True): + save_dir = init_save_dir() + + trainer_options['default_save_path'] = save_dir + + # fit model + trainer = Trainer(**trainer_options) + result = trainer.fit(model) + + # correct result and ok accuracy + assert result == 1, 'amp + ddp model failed to complete' + + # test model loading + pretrained_model = load_model(trainer.logger.experiment, save_dir) + + # test new model accuracy + [run_prediction(dataloader, pretrained_model) for dataloader in model.test_dataloader()] + + if trainer.use_ddp: + # on hpc this would work fine... but need to hack it for the purpose of the test + trainer.model = pretrained_model + trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers() + + clear_save_dir() + + def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True): save_dir = init_save_dir()