Fixes #292 (#303)

* early stopping callback is not default

* added a default logger

* added default checkpoint callback

* added default checkpoint/loggers

* added default checkpoint/loggers

* updated docs

* cleaned demos

* cleaned demos

* cleaned demos

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

* clean up docs around loggers

docs/Trainer/Checkpointing.md

@@ -2,13 +2,20 @@ Lightning can automate saving and loading checkpoints.
 
 ---
 ### Model saving
-To enable checkpointing, define the checkpoint callback and give it to the trainer.
+Checkpointing is enabled by default to the current working directory.
+To change the checkpoint path pass in :
+```python
+Trainer(default_save_path='/your/path/to/save/checkpoints')
+```
+
+To modify the behavior of checkpointing pass in your own callback.
 
 ``` {.python}
 from pytorch_lightning.callbacks import ModelCheckpoint
 
+# DEFAULTS used by the Trainer
 checkpoint_callback = ModelCheckpoint(
-    filepath='/path/to/store/weights/',
+    filepath=os.getcwd(),
     save_best_only=True,
     verbose=True,
     monitor='val_loss',

docs/Trainer/Logging.md

@@ -1,16 +1,33 @@
 Lighting offers options for logging information about model, gpu usage, etc, via several different logging frameworks. It also offers printing options for training monitoring.
 
+---   
+### default_save_path   
+Lightning sets a default TestTubeLogger and CheckpointCallback for you which log to
+```os.getcwd()``` by default. To modify the logging path you can set:
+```python
+Trainer(default_save_path='/your/path/to/save/checkpoints')
+```
+ 
+If you need more custom behavior (different paths for both, different metrics, etc...)
+from the logger and the checkpointCallback, pass in your own instances as explained below.
+
 
 ---
 ### Setting up logging
 
-Initialize your logger, which should inherit from `LightningBaseLogger`, and pass
-it to `Trainer`.
+The trainer inits a default logger for you (TestTubeLogger). All logs will
+go to the current working directory under a folder named ```os.getcwd()/lightning_logs``. 
+
+If you want to modify the default logging behavior even more, pass in a logger
+(which should inherit from `LightningBaseLogger`).   
+
 ```{.python}
 my_logger = MyLightningLogger(...)
 trainer = Trainer(logger=my_logger)
 ```
 
+The path in this logger will overwrite default_save_path.
+
 Lightning supports several common experiment tracking frameworks out of the box
 
 ---

docs/Trainer/Training Loop.md

@@ -21,17 +21,18 @@ trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)
 
 ---
 #### Early stopping
-To enable early-stopping, define the callback and give it to the trainer.
+The trainer already sets up default early stopping for you. 
+To modify this behavior, pass in your own EarlyStopping callback.
 ``` {.python}
 from pytorch_lightning.callbacks import EarlyStopping
 
-# DEFAULTS
+# DEFAULTS used by Trainer
 early_stop_callback = EarlyStopping(
     monitor='val_loss',
     min_delta=0.00,
-    patience=0,
+    patience=3,
     verbose=False,
-    mode='auto'
+    mode='min'
 )
 
 trainer = Trainer(early_stop_callback=early_stop_callback)

docs/examples/Examples.md

@@ -47,57 +47,17 @@ def main(hparams, cluster, results_dict):
     :param hparams:
     :return:
     """
-    # init experiment
-    log_dir = os.path.dirname(os.path.realpath(__file__))
-    exp = Experiment(
-        name='test_tube_exp',
-        debug=True,
-        save_dir=log_dir,
-        version=0,
-        autosave=False,
-        description='test demo'
-    )
-    
-    # set the hparams for the experiment
-    exp.argparse(hparams)
-    exp.save()
-
     # build model
     model = MyLightningModule(hparams)
 
-    # callbacks
-    early_stop = EarlyStopping(
-        monitor=hparams.early_stop_metric,
-        patience=hparams.early_stop_patience,
-        verbose=True,
-        mode=hparams.early_stop_mode
-    )
-
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_function=None,
-        save_best_only=True,
-        verbose=True,
-        monitor=hparams.model_save_monitor_value,
-        mode=hparams.model_save_monitor_mode
-    )
-
     # configure trainer
-    trainer = Trainer(
-        experiment=exp,
-        cluster=cluster,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
+    trainer = Trainer()
 
     # train model
     trainer.fit(model)
 ```
 
 
-
-
 The __main__ function will start training on your **main** function. If you use the HyperParameterOptimizer
 in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple
 argument parser you get the default arguments in the argument parser.

docs/index.md

@@ -47,7 +47,7 @@ if use_bert:
 else:
     model = CoolerNotBERT()
     
-trainer = Trainer(gpus=[0, 1, 2, 3], use_amp=True)
+trainer = Trainer(gpus=4, use_amp=True)
 trainer.fit(model)
 ```
 

examples/new_project_templates/multi_node_examples/multi_node_cluster_auto_slurm.py

@@ -55,32 +55,11 @@ def main(hparams, cluster):
     exp.argparse(hparams)
     exp.save()
 
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
     # ------------------------
     # 4 INIT TRAINER
     # ------------------------
     trainer = Trainer(
         experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
         gpus=hparams.per_experiment_nb_gpus,
         nb_gpu_nodes=hyperparams.nb_gpu_nodes,
         distributed_backend=hyperparams.distributed_backend

examples/new_project_templates/single_cpu_template.py

@@ -42,35 +42,12 @@ def main(hparams):
     exp.save()
 
     # ------------------------
-    # 3 DEFINE CALLBACKS
+    # 3 INIT TRAINER
     # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
+    trainer = Trainer(experiment=exp)
 
     # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
-
-    # ------------------------
-    # 5 START TRAINING
+    # 4 START TRAINING
     # ------------------------
     trainer.fit(model)
 

examples/new_project_templates/single_gpu_node_16bit_template.py

@@ -45,37 +45,16 @@ def main(hparams):
     exp.save()
 
     # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
+    # 3 INIT TRAINER
     # ------------------------
     trainer = Trainer(
         experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
         gpus=hparams.gpus,
         use_amp=True
     )
 
     # ------------------------
-    # 5 START TRAINING
+    # 4 START TRAINING
     # ------------------------
     trainer.fit(model)
 

examples/new_project_templates/single_gpu_node_ddp_template.py

@@ -45,37 +45,16 @@ def main(hparams):
     exp.save()
 
     # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
+    # 3 INIT TRAINER
     # ------------------------
     trainer = Trainer(
         experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
         gpus=hparams.gpus,
         distributed_backend=hparams.dist_backend
     )
 
     # ------------------------
-    # 5 START TRAINING
+    # 4 START TRAINING
     # ------------------------
     trainer.fit(model)
 

examples/new_project_templates/single_gpu_node_dp_template.py

@@ -30,9 +30,8 @@ def main(hparams):
     print('model built')
 
     # ------------------------
-    # 2 INIT TEST TUBE EXP
+    # 2 INIT Logger
     # ------------------------
-
     # init experiment
     exp = Experiment(
         name=hyperparams.experiment_name,
@@ -45,37 +44,16 @@ def main(hparams):
     exp.save()
 
     # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
+    # 3 INIT TRAINER
     # ------------------------
     trainer = Trainer(
         experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
         gpus=hparams.gpus,
         distributed_backend=hparams.dist_backend,
     )
 
     # ------------------------
-    # 5 START TRAINING
+    # 4 START TRAINING
     # ------------------------
     trainer.fit(model)
 

examples/new_project_templates/trainer_cpu_template.py

@@ -31,29 +31,8 @@ def main(hparams):
     # build model
     model = LightningTemplateModel(hparams)
 
-    # callbacks
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        mode='min',
-        verbose=True,
-    )
-
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_acc',
-        mode='min'
-    )
-
     # configure trainer
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
+    trainer = Trainer(experiment=exp)
 
     # train model
     trainer.fit(model)

pytorch_lightning/logging/base.py

@@ -15,7 +15,7 @@ def rank_zero_only(fn):
     return wrapped_fn
 
 
-class LightningLoggerBase:
+class LightningLoggerBase(object):
     """Base class for experiment loggers"""
 
     def __init__(self):

pytorch_lightning/logging/test_tube_logger.py

@@ -7,6 +7,8 @@ from test_tube import Experiment
 
 
 class TestTubeLogger(LightningLoggerBase):
+    __test__ = False
+
     def __init__(
         self, save_dir, name="default", debug=False, version=None, create_git_tag=False
     ):

pytorch_lightning/root_module/memory.py

@@ -35,7 +35,7 @@ class ModelSummary(object):
         out_sizes = []
         input_ = self.model.example_input_array
 
-        if self.model.use_ddp or self.model.use_dp:
+        if self.model.use_ddp or self.model.use_dp or self.model.single_gpu:
             input_ = input_.cuda(0)
 
         if self.model.trainer.use_amp:

pytorch_lightning/trainer/trainer.py

@@ -16,10 +16,12 @@ from torch.optim.optimizer import Optimizer
 
 from pytorch_lightning.root_module.root_module import LightningModule
 from pytorch_lightning.root_module.memory import get_gpu_memory_map
+from pytorch_lightning.logging import TestTubeLogger
 from pytorch_lightning.trainer.trainer_io import TrainerIO
 from pytorch_lightning.pt_overrides.override_data_parallel import (
     LightningDistributedDataParallel, LightningDataParallel)
-from pytorch_lightning.callbacks import GradientAccumulationScheduler
+from pytorch_lightning.callbacks import GradientAccumulationScheduler, \
+    ModelCheckpoint, EarlyStopping
 from pytorch_lightning.utilities.debugging import MisconfigurationException
 import pdb
 from pytorch_lightning.trainer import ignored_warnings
@@ -57,8 +59,9 @@ class Trainer(TrainerIO):
 
     def __init__(self,
                  logger=None,
-                 early_stop_callback=None,
                  checkpoint_callback=None,
+                 early_stop_callback=None,
+                 default_save_path=None,
                  gradient_clip_val=0,
                  process_position=0,
                  nb_gpu_nodes=1,
@@ -88,8 +91,9 @@ class Trainer(TrainerIO):
         """
 
         :param logger: Logger for experiment tracking
-        :param early_stop_callback: Callback for early stopping
         :param checkpoint_callback: Callback for checkpointing
+        :param early_stop_callback: Callback for early stopping
+        :param default_save_path: Default path for logs+weights if no logger/ckpt_callback passed
         :param gradient_clip_val: int. 0 means don't clip.
         :param process_position: shown in the tqdm bar
         :param nb_gpu_nodes: number of GPU nodes
@@ -133,6 +137,11 @@ class Trainer(TrainerIO):
         self.nb_sanity_val_steps = nb_sanity_val_steps
         self.print_nan_grads = print_nan_grads
 
+        # set default save path if user didn't provide one
+        self.default_save_path = default_save_path
+        if self.default_save_path is None:
+            self.default_save_path = os.getcwd()
+
         # training bookeeping
         self.total_batch_nb = 0
         self.running_loss = []
@@ -156,13 +165,39 @@ class Trainer(TrainerIO):
         self.total_batches = 0
 
         # configure early stop callback
+        # creates a default one if none passed in
         self.early_stop_callback = early_stop_callback
-
-        # configure weights save path
-        self.__configure_weights_path(checkpoint_callback, weights_save_path)
+        if self.early_stop_callback is None:
+            self.early_stop = EarlyStopping(
+                monitor='val_loss',
+                patience=3,
+                verbose=True,
+                mode='min'
+            )
 
         # configure logger
         self.logger = logger
+        if self.logger is None:
+            self.logger = TestTubeLogger(
+                save_dir=self.default_save_path,
+                name='lightning_logs'
+            )
+
+        # configure checkpoint callback
+        self.checkpoint_callback = checkpoint_callback
+        if self.checkpoint_callback is None:
+            if isinstance(logger, TestTubeLogger):
+                ckpt_path = '{}/{}/{}'.format(self.default_save_path, self.logger.name,
+                                              self.logger.version)
+            else:
+                ckpt_path = self.default_save_path
+
+            self.checkpoint_callback = ModelCheckpoint(
+                filepath=ckpt_path
+            )
+
+        # configure weights save path
+        self.__configure_weights_path(checkpoint_callback, weights_save_path)
 
         # accumulated grads
         self.__configure_accumulated_gradients(accumulate_grad_batches)
@@ -214,8 +249,6 @@ class Trainer(TrainerIO):
         """
         self.weights_save_path = weights_save_path
 
-        # configure checkpoint callback
-        self.checkpoint_callback = checkpoint_callback
         if self.checkpoint_callback is not None:
             self.checkpoint_callback.save_function = self.save_checkpoint
 
@@ -224,7 +257,7 @@ class Trainer(TrainerIO):
 
         # if weights_save_path is still none here, set to current workingdir
         if self.weights_save_path is None:
-            self.weights_save_path = os.getcwd()
+            self.weights_save_path = self.default_save_path
 
     def __init_amp(self, use_amp):
         self.use_amp = use_amp and APEX_AVAILABLE
@@ -900,6 +933,7 @@ class Trainer(TrainerIO):
 
         # set local properties on the model
         ref_model.on_gpu = self.on_gpu
+        ref_model.single_gpu = self.single_gpu
         ref_model.use_dp = self.use_dp
         ref_model.use_ddp = self.use_ddp
         ref_model.use_ddp2 = self.use_ddp2

tests/test_logging.py

@@ -22,6 +22,7 @@ def test_testtube_logger():
 
     trainer_options = dict(
         max_nb_epochs=1,
+        train_percent_check=0.01,
         logger=logger
     )
 
@@ -46,6 +47,7 @@ def test_testtube_pickle():
 
     trainer_options = dict(
         max_nb_epochs=1,
+        train_percent_check=0.01,
         logger=logger
     )
 
@@ -74,6 +76,7 @@ def test_mlflow_logger():
 
     trainer_options = dict(
         max_nb_epochs=1,
+        train_percent_check=0.01,
         logger=logger
     )
 

tests/test_models.py

@@ -39,6 +39,30 @@ np.random.seed(SEED)
 # ------------------------------------------------------------------------
 # TESTS
 # ------------------------------------------------------------------------
+def test_default_logger_callbacks_cpu_model():
+    """
+    Test each of the trainer options
+    :return:
+    """
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        gradient_clip_val=1.0,
+        overfit_pct=0.20,
+        print_nan_grads=True,
+        show_progress_bar=False,
+        train_percent_check=0.01,
+        val_percent_check=0.01
+    )
+
+    model, hparams = get_model()
+    run_model_test_no_loggers(trainer_options, model, hparams, on_gpu=False)
+
+    # test freeze on cpu
+    model.freeze()
+    model.unfreeze()
+
+
 def test_multi_gpu_model_ddp2():
     """
     Make sure DDP2 works
@@ -1336,6 +1360,32 @@ def test_multiple_test_dataloader():
 # ------------------------------------------------------------------------
 # UTILS
 # ------------------------------------------------------------------------
+def run_model_test_no_loggers(trainer_options, model, hparams, on_gpu=True):
+    save_dir = init_save_dir()
+
+    trainer_options['default_save_path'] = save_dir
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    # correct result and ok accuracy
+    assert result == 1, 'amp + ddp model failed to complete'
+
+    # test model loading
+    pretrained_model = load_model(trainer.logger.experiment, save_dir)
+
+    # test new model accuracy
+    [run_prediction(dataloader, pretrained_model) for dataloader in model.test_dataloader()]
+
+    if trainer.use_ddp:
+        # on hpc this would work fine... but need to hack it for the purpose of the test
+        trainer.model = pretrained_model
+        trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers()
+
+    clear_save_dir()
+
+
 def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True):
     save_dir = init_save_dir()