diff --git a/pytorch_lightning/trainer/__init__.py b/pytorch_lightning/trainer/__init__.py index 39cced3a..92521ac0 100644 --- a/pytorch_lightning/trainer/__init__.py +++ b/pytorch_lightning/trainer/__init__.py @@ -5,15 +5,14 @@ the Trainer automates everything else. .. figure:: /_images/lightning_module/pt_trainer.png :alt: Convert from PyTorch to Lightning -This abstraction achieves the folowing: +This abstraction achieves the following: -1. You maintain control over all aspects via PyTorch -code without an added abstraction. + 1. You maintain control over all aspects via PyTorch code without an added abstraction. -2. The trainer uses best practices embedded by contributors and users -from top AI labs such as Facebook AI Research, NYU, MIT, Stanford, etc... + 2. The trainer uses best practices embedded by contributors and users + from top AI labs such as Facebook AI Research, NYU, MIT, Stanford, etc... -3. The trainer allows overriding any key part that you don't want automated. + 3. The trainer allows overriding any key part that you don't want automated. ----------- @@ -102,11 +101,13 @@ accumulate_grad_batches ^^^^^^^^^^^^^^^^^^^^^^^ Accumulates grads every k batches or as set up in the dict. -Example:: +.. code-block:: python # default used by the Trainer (no accumulation) trainer = Trainer(accumulate_grad_batches=1) +Example:: + # accumulate every 4 batches (effective batch size is batch*4) trainer = Trainer(accumulate_grad_batches=4) @@ -118,7 +119,7 @@ amp_level The optimization level to use (O1, O2, etc...) for 16-bit GPU precision (using NVIDIA apex under the hood). -Check nvidia docs for level (https://nvidia.github.io/apex/amp.html#opt-levels) +Check `NVIDIA apex docs `_ for level Example:: @@ -137,10 +138,17 @@ The speedup comes from allowing the cudnn auto-tuner to find the best algorithm for the hardware `[see discussion here] `_. +Example:: + + # default used by the Trainer + trainer = Trainer(benchmark=False) + callbacks ^^^^^^^^^ -callbacks: Add a list of callbacks. +Add a list of user defined callbacks. + +.. note:: Only user defined callbacks (ie: Not EarlyStopping or ModelCheckpoint) .. code-block:: python @@ -196,7 +204,10 @@ Example:: default_save_path ^^^^^^^^^^^^^^^^^ -Default path for logs and weights when no logger/ckpt_callback passed +Default path for logs and weights when no logger +or :class:`pytorch_lightning.callbacks.ModelCheckpoint` callback passed. +On certain clusters you might want to separate where logs and checkpoints +are stored. If you don't then use this method for convenience. Example:: @@ -207,27 +218,25 @@ distributed_backend ^^^^^^^^^^^^^^^^^^^ The distributed backend to use. -- ('dp') is DataParallel (split batch among GPUs of same machine) -- ('ddp') is DistributedDataParallel (each gpu on each node trains, and syncs grads) -- ('ddp2') dp on node, ddp across nodes +- (```dp```) is DataParallel (split batch among GPUs of same machine) +- (```ddp```) is DistributedDataParallel (each gpu on each node trains, and syncs grads) +- (```ddp2```) dp on node, ddp across nodes. Useful for things like increasing + the number of negative samples -Example:: +.. code-block:: python # default used by the Trainer trainer = Trainer(distributed_backend=None) - # dp = DataParallel (split a batch onto k gpus on same machine). +Example:: + + # dp = DataParallel trainer = Trainer(gpus=2, distributed_backend='dp') # ddp = DistributedDataParallel - # Each gpu trains by itself on a subset of the data. - # Gradients sync across all gpus and all machines. trainer = Trainer(gpus=2, num_nodes=2, distributed_backend='ddp') # ddp2 = DistributedDataParallel + dp - # behaves like dp on every node - # syncs gradients across nodes like ddp - # useful for things like increasing the number of negative samples trainer = Trainer(gpus=2, num_nodes=2, distributed_backend='ddp2') early_stop_callback @@ -236,13 +245,14 @@ early_stop_callback Callback for early stopping. early_stop_callback (:class:`pytorch_lightning.callbacks.EarlyStopping`) -- If set to ``True``, then a default callback monitoring ``'val_loss'`` is created. -- Will raise an error if ``'val_loss'`` is not found. -- If set to ``False``, then early stopping will be disabled. -- If set to ``None``, then the default callback monitoring ``'val_loss'`` is created. -- If ``'val_loss'`` is not found will work as if early stopping is disabled. +- ``True``: A default callback monitoring ``'val_loss'`` is created. + Will raise an error if ``'val_loss'`` is not found. +- ``False``: Early stopping will be disabled. +- ``None``: The default callback monitoring ``'val_loss'`` is created. - Default: ``None``. +.. note:: If ``'val_loss'`` is not found will work as if early stopping is disabled. + .. code-block:: python trainer = Trainer(early_stop_callback=early_stop_callback) @@ -314,7 +324,10 @@ Example:: trainer = Trainer(gpus='-1') # equivalent # combine with num_nodes to train on multiple GPUs across nodes - trainer = Trainer(gpus=2, num_nodes=4) # uses 8 gpus in total + # uses 8 gpus in total + trainer = Trainer(gpus=2, num_nodes=4) + +.. note:: See the `multi-gpu computing guide `_ gradient_clip_val ^^^^^^^^^^^^^^^^^ @@ -357,7 +370,7 @@ Example:: log_save_interval ^^^^^^^^^^^^^^^^^ -Writes logs to disk this often +Writes logs to disk this often. Example:: @@ -367,7 +380,7 @@ Example:: logger ^^^^^^ -Logger (or iterable collection of loggers) for experiment tracking. +`Logger `_ (or iterable collection of loggers) for experiment tracking. .. code-block:: python @@ -414,9 +427,14 @@ min_nb_epochs: max_steps ^^^^^^^^^ -Stop training after this number of steps. Disabled by default (None). +Stop training after this number of steps Training will stop if max_steps or max_epochs have reached (earliest). +.. code-block:: python + + # Default (disabled) + trainer = Trainer(max_steps=None) + Example:: # Stop after 100 steps @@ -425,9 +443,14 @@ Example:: min_steps ^^^^^^^^^ -Force training for at least these number of steps. Disabled by default (None). +Force training for at least these number of steps. Trainer will train model for at least min_steps or min_epochs (latest). +.. code-block:: python + + # Default (disabled) + trainer = Trainer(min_steps=None) + Example:: # Run at least for 100 steps (disable min_epochs) @@ -479,8 +502,9 @@ A single TPU v2 or v3 has 8 cores. A TPU pod has up to 2048 cores. A slice of a POD means you get as many cores as you request. -You MUST use DistributedDataSampler with your dataloader for this -to work. Your effective batch size is batch_size * total tpu cores. +Your effective batch size is batch_size * total tpu cores. + +.. note:: No need to add a DistributedDataSampler, Lightning automatically does it for you. This parameter can be either 1 or 8. @@ -518,7 +542,8 @@ Example:: overfit_pct ^^^^^^^^^^^ -uses this much data of all datasets. +Uses this much data of all datasets. +Useful for quickly debugging or trying to overfit on purpose Example:: @@ -559,7 +584,8 @@ Example:: process_position ^^^^^^^^^^^^^^^^ -orders the tqdm bar when running multiple models on same machine. +Orders the tqdm bar. Useful when running multiple trainers +on the same node. Example:: @@ -570,6 +596,8 @@ profiler ^^^^^^^^ To profile individual steps during training and assist in identifying bottlenecks. +See the `profiler documentation `_. for more details. + Example:: from pytorch_lightning.profiler import Profiler, AdvancedProfiler @@ -590,12 +618,19 @@ Example:: progress_bar_refresh_rate ^^^^^^^^^^^^^^^^^^^^^^^^^ -How often to refresh progress bar (in steps) -Default is 50. Useful for notebooks with slow refresh rate. +How often to refresh progress bar (in steps). +Faster refresh rates (lower number), in notebooks is known to crash them +because of their screen refresh rates. 50 is optimal for those cases. + +Example:: + + # default used by the Trainer + trainer = Trainer(progress_bar_refresh_rate=50) + reload_dataloaders_every_epoch ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Set to True to reload dataloaders every epoch +Set to True to reload dataloaders every epoch. .. code-block:: python @@ -657,7 +692,6 @@ test_percent_check ^^^^^^^^^^^^^^^^^^ How much of test dataset to check. -Useful when debugging or testing something that happens at the end of an epoch. Example:: @@ -667,18 +701,22 @@ Example:: # run through only 25% of the test set each epoch trainer = Trainer(test_percent_check=0.25) -val_check_interval: +val_check_interval +^^^^^^^^^^^^^^^^^^ + How often within one training epoch to check the validation set. Can specify as float or int. - use (float) to check within a training epoch - use (int) to check every n steps (batches) -Example:: +.. code-block:: python # default used by the Trainer trainer = Trainer(val_check_interval=1.0) +Example:: + # check validation set 4 times during a training epoch trainer = Trainer(val_check_interval=0.25) @@ -693,11 +731,13 @@ track_grad_norm - no tracking (-1) - Otherwise tracks that norm (2 for 2-norm) -Example:: +.. code-block:: python # default used by the Trainer trainer = Trainer(track_grad_norm=-1) +Example:: + # track the 2-norm trainer = Trainer(track_grad_norm=2) @@ -707,6 +747,11 @@ train_percent_check How much of training dataset to check. Useful when debugging or testing something that happens at the end of an epoch. +.. code-block::python + + # default used by the Trainer + trainer = Trainer(train_percent_check=1.0) + Example:: # default used by the Trainer @@ -719,12 +764,17 @@ truncated_bptt_steps ^^^^^^^^^^^^^^^^^^^^ Truncated back prop breaks performs backprop every k steps of -a much longer sequence If this is enabled, your batches will automatically get truncated -and the trainer will apply Truncated Backprop to it. Make sure your batches have a sequence -dimension. (`Williams et al. "An efficient gradient-based algorithm for on-line training of +a much longer sequence. + +If this is enabled, your batches will automatically get truncated +and the trainer will apply Truncated Backprop to it. + +(`Williams et al. "An efficient gradient-based algorithm for on-line training of recurrent network trajectories." `_) +.. note:: Make sure your batches have a sequence dimension. + Example:: # default used by the Trainer (ie: disabled) @@ -736,11 +786,38 @@ Example:: Lightning takes care to split your batch along the time-dimension. -.. note:: If you need to modify how the batch is split, - override :meth:`pytorch_lightning.core.LightningModule.tbptt_split_batch`. +.. code-block:: python + + # we use the second as the time dimension + # (batch, time, ...) + sub_batch = batch[0, 0:t, ...] + +Using this feature requires updating your LightningModule's +:meth:`pytorch_lightning.core.LightningModule.training_step` to include a `hiddens` arg +with the hidden + +.. code-block:: python + + # Truncated back-propagation through time + def training_step(self, batch, batch_idx, hiddens): + # hiddens are the hiddens from the previous truncated backprop step + out, hiddens = self.lstm(data, hiddens) + + return { + "loss": ..., + "hiddens": hiddens # remember to detach() this + } + +To modify how the batch is split, +override :meth:`pytorch_lightning.core.LightningModule.tbptt_split_batch`: + +.. code-block:: python + + class LitMNIST(pl.LightningModule): + def tbptt_split_batch(self, batch, split_size): + # do your own splitting on the batch + return splits -.. note:: Using this feature requires updating your LightningModule's - :meth:`pytorch_lightning.core.LightningModule.training_step` to include a `hiddens` arg. val_percent_check ^^^^^^^^^^^^^^^^^ @@ -758,13 +835,15 @@ Example:: weights_save_path ^^^^^^^^^^^^^^^^^ -Where to save weights if specified. +Directory of where to save weights if specified. -Example:: +.. code-block:: python # default used by the Trainer trainer = Trainer(weights_save_path=os.getcwd()) +Example:: + # save to your custom path trainer = Trainer(weights_save_path='my/path') diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index a868b049..6b8d484e 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -55,31 +55,6 @@ class TrainerDataLoadingMixin(ABC): if not 0. <= value <= 1.: raise ValueError(msg) - def call_prepare_data(self, model): - """ - Let model download the data on proc==0 only - :param model: - """ - # download data on DDP+ - if self.use_ddp or self.use_ddp2: - if self.proc_rank == 0: - model.prepare_data() - - # all processes wait until data download has happened - dist.barrier() - - # data download/load on TPU - elif self.use_tpu and XLA_AVAILABLE: - if self.tpu_local_core_rank == 0: - model.prepare_data() - - # all processes wait until data download has happened - torch_xla.core.xla_model.rendezvous("pl.TrainerDataLoadingMixin.get_dataloaders") - - else: - # regular download - model.prepare_data() - def auto_add_sampler(self, dataloader, train): if self.use_ddp or self.use_ddp2 or self.use_tpu: dl_args = { diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 212edfdc..3d9bde72 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -550,7 +550,12 @@ class Trainer(TrainerIOMixin, model.logger = self.logger # set up the passed in dataloaders (if needed) - self.__set_fit_dataloaders(model, train_dataloader, val_dataloaders, test_dataloaders) + self.__attach_dataloaders(model, train_dataloader, val_dataloaders, test_dataloaders) + + # download the data and do whatever transforms we need + # do before any spawn calls so that the model can assign properties + # only on proc 0 because no spawn has happened yet + model.prepare_data() # route to appropriate start method # when using multi-node or DDP within a node start each module in a separate process @@ -627,7 +632,7 @@ class Trainer(TrainerIOMixin, default_port = random.randint(10000, 19000) os.environ['MASTER_PORT'] = str(default_port) - def __set_fit_dataloaders(self, model, train_dataloader, val_dataloaders, test_dataloaders): + def __attach_dataloaders(self, model, train_dataloader, val_dataloaders, test_dataloaders): # when dataloader is passed via fit, patch the train_dataloader # functions to overwrite with these implementations if train_dataloader is not None: @@ -761,9 +766,6 @@ class Trainer(TrainerIOMixin, # restore training and model before hpc call self.restore_weights(model) - # download the data and do whatever transforms we need - self.call_prepare_data(ref_model) - # when testing requested only run test and return if self.testing: # only load test dataloader for testing