diff --git a/README.md b/README.md index eac326f8..825db916 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ To use lightning do 2 things: def forward(self, x): return torch.relu(self.l1(x.view(x.size(0), -1))) - def training_step(self, batch, batch_nb): + def training_step(self, batch, batch_idx): # REQUIRED x, y = batch y_hat = self.forward(x) @@ -104,7 +104,7 @@ To use lightning do 2 things: tensorboard_logs = {'train_loss': loss} return {'loss': loss, 'log': tensorboard_logs} - def validation_step(self, batch, batch_nb): + def validation_step(self, batch, batch_idx): # OPTIONAL x, y = batch y_hat = self.forward(x) @@ -154,16 +154,16 @@ use something other than tensorboard). Here are more advanced examples ```python # train on cpu using only 10% of the data (for demo purposes) -trainer = Trainer(max_nb_epochs=1, train_percent_check=0.1) +trainer = Trainer(max_num_epochs=1, train_percent_check=0.1) # train on 4 gpus (lightning chooses GPUs for you) -# trainer = Trainer(max_nb_epochs=1, gpus=4, distributed_backend='ddp') +# trainer = Trainer(max_num_epochs=1, gpus=4, distributed_backend='ddp') # train on 4 gpus (you choose GPUs) -# trainer = Trainer(max_nb_epochs=1, gpus=[0, 1, 3, 7], distributed_backend='ddp') +# trainer = Trainer(max_num_epochs=1, gpus=[0, 1, 3, 7], distributed_backend='ddp') # train on 32 gpus across 4 nodes (make sure to submit appropriate SLURM job) -# trainer = Trainer(max_nb_epochs=1, gpus=8, nb_gpu_nodes=4, distributed_backend='ddp') +# trainer = Trainer(max_num_epochs=1, gpus=8, num_gpu_nodes=4, distributed_backend='ddp') # train (1 epoch only here for demo) trainer.fit(model) @@ -187,10 +187,10 @@ You define the blue parts using the LightningModule interface: ```python # what to do in the training loop -def training_step(self, batch, batch_nb): +def training_step(self, batch, batch_idx): # what to do in the validation loop -def validation_step(self, batch, batch_nb): +def validation_step(self, batch, batch_idx): # how to aggregate validation_step outputs def validation_end(self, outputs): @@ -205,7 +205,7 @@ def test_dataloader(): ```python # define what happens for training here -def training_step(self, batch, batch_nb): +def training_step(self, batch, batch_idx): x, y = batch # define your own forward and loss calculation @@ -232,7 +232,7 @@ def training_step(self, batch, batch_nb): ```python # define what happens for validation here -def validation_step(self, batch, batch_nb): +def validation_step(self, batch, batch_idx): x, y = batch # or as basic as a CNN classification diff --git a/docs/source/intro.md b/docs/source/intro.md index c45584ea..4155f660 100644 --- a/docs/source/intro.md +++ b/docs/source/intro.md @@ -16,7 +16,7 @@ class BERT(pl.LightningModule): elif model_name == 'my_cool_version': self.net = MyCoolVersion() - def training_step(self, batch, batch_nb): + def training_step(self, batch, batch_idx): if self.task == 'standard_bert': # do standard bert training with self.net... # return loss @@ -35,7 +35,7 @@ class CoolerNotBERT(pl.LightningModule): def __init__(self): self.net = ... - def training_step(self, batch, batch_nb): + def training_step(self, batch, batch_idx): # do some other cool task # return loss ``` diff --git a/pl_examples/domain_templates/gan.py b/pl_examples/domain_templates/gan.py index a8468c4d..f86628a1 100644 --- a/pl_examples/domain_templates/gan.py +++ b/pl_examples/domain_templates/gan.py @@ -90,12 +90,12 @@ class GAN(pl.LightningModule): def adversarial_loss(self, y_hat, y): return F.binary_cross_entropy(y_hat, y) - def training_step(self, batch, batch_nb, optimizer_i): + def training_step(self, batch, batch_idx, optimizer_idx): imgs, _ = batch self.last_imgs = imgs # train generator - if optimizer_i == 0: + if optimizer_idx == 0: # sample noise z = torch.randn(imgs.shape[0], self.hparams.latent_dim) @@ -125,7 +125,7 @@ class GAN(pl.LightningModule): return output # train discriminator - if optimizer_i == 1: + if optimizer_idx == 1: # Measure discriminator's ability to classify real from generated samples # how well can it label as real? diff --git a/pl_examples/full_examples/imagenet/imagenet_example.py b/pl_examples/full_examples/imagenet/imagenet_example.py index 3dbe4a60..e20979df 100644 --- a/pl_examples/full_examples/imagenet/imagenet_example.py +++ b/pl_examples/full_examples/imagenet/imagenet_example.py @@ -234,7 +234,7 @@ def main(hparams): trainer = pl.Trainer( default_save_path=hparams.save_path, gpus=hparams.gpus, - max_nb_epochs=hparams.epochs, + max_num_epochs=hparams.epochs, distributed_backend=hparams.distributed_backend, use_amp=hparams.use_16bit ) diff --git a/pl_examples/multi_node_examples/multi_node_ddp2_demo.py b/pl_examples/multi_node_examples/multi_node_ddp2_demo.py index 0a4a45db..0fcf423c 100644 --- a/pl_examples/multi_node_examples/multi_node_ddp2_demo.py +++ b/pl_examples/multi_node_examples/multi_node_ddp2_demo.py @@ -31,7 +31,7 @@ def main(hparams): # ------------------------ trainer = Trainer( gpus=2, - nb_gpu_nodes=2, + num_nodes=2, distributed_backend='ddp2' ) diff --git a/pl_examples/multi_node_examples/multi_node_ddp_demo.py b/pl_examples/multi_node_examples/multi_node_ddp_demo.py index 46877d39..bb8dcf01 100644 --- a/pl_examples/multi_node_examples/multi_node_ddp_demo.py +++ b/pl_examples/multi_node_examples/multi_node_ddp_demo.py @@ -31,7 +31,7 @@ def main(hparams): # ------------------------ trainer = Trainer( gpus=2, - nb_gpu_nodes=2, + num_nodes=2, distributed_backend='ddp' ) diff --git a/pytorch_lightning/core/__init__.py b/pytorch_lightning/core/__init__.py index 33452107..0bb2323f 100644 --- a/pytorch_lightning/core/__init__.py +++ b/pytorch_lightning/core/__init__.py @@ -34,13 +34,13 @@ Minimal example def forward(self, x): return torch.relu(self.l1(x.view(x.size(0), -1))) - def training_step(self, batch, batch_nb): + def training_step(self, batch, batch_idx): # REQUIRED x, y = batch y_hat = self.forward(x) return {'loss': F.cross_entropy(y_hat, y)} - def validation_step(self, batch, batch_nb): + def validation_step(self, batch, batch_idx): # OPTIONAL x, y = batch y_hat = self.forward(x) @@ -51,7 +51,7 @@ Minimal example avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() return {'avg_val_loss': avg_loss} - def test_step(self, batch, batch_nb): + def test_step(self, batch, batch_idx): # OPTIONAL x, y = batch y_hat = self.forward(x) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 95a98e51..390c4341 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -109,7 +109,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): """return loss, dict with metrics for tqdm :param batch: The output of your dataloader. A tensor, tuple or list - :param int batch_nb: Integer displaying which batch this is + :param int batch_idx: Integer displaying which batch this is :return: dict with loss key and optional log, progress keys if implementing training_step, return whatever you need in that step: - loss -> tensor scalar [REQUIRED] @@ -124,7 +124,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python - def training_step(self, batch, batch_nb): + def training_step(self, batch, batch_idx): x, y, z = batch # implement your own @@ -150,7 +150,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # Multiple optimizers (ie: GANs) - def training_step(self, batch, batch_nb, optimizer_idx): + def training_step(self, batch, batch_idx, optimizer_idx): if optimizer_idx == 0: # do training_step with encoder if optimizer_idx == 1: @@ -163,7 +163,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # Truncated back-propagation through time - def training_step(self, batch, batch_nb, hiddens): + def training_step(self, batch, batch_idx, hiddens): # hiddens are the hiddens from the previous truncated backprop step You can also return a -1 instead of a dict to stop the current loop. This is useful @@ -192,9 +192,9 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # WITHOUT training_end - # if used in DP or DDP2, this batch is 1/nb_gpus large - def training_step(self, batch, batch_nb): - # batch is 1/nb_gpus big + # if used in DP or DDP2, this batch is 1/num_gpus large + def training_step(self, batch, batch_idx): + # batch is 1/num_gpus big x, y = batch out = self.forward(x) @@ -204,8 +204,8 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): # -------------- # with training_end to do softmax over the full batch - def training_step(self, batch, batch_nb): - # batch is 1/nb_gpus big + def training_step(self, batch, batch_idx): + # batch is 1/num_gpus big x, y = batch out = self.forward(x) @@ -225,7 +225,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # Multiple optimizers (ie: GANs) - def training_step(self, batch, batch_nb, optimizer_idx): + def training_step(self, batch, batch_idx, optimizer_idx): if optimizer_idx == 0: # do training_step with encoder if optimizer_idx == 1: @@ -237,7 +237,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # Truncated back-propagation through time - def training_step(self, batch, batch_nb, hiddens): + def training_step(self, batch, batch_idx, hiddens): # hiddens are the hiddens from the previous truncated backprop step You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to @@ -249,17 +249,17 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): """return whatever outputs will need to be aggregated in validation_end :param batch: The output of your dataloader. A tensor, tuple or list - :param int batch_nb: Integer displaying which batch this is + :param int batch_idx: Integer displaying which batch this is :param int dataloader_idx: Integer displaying which dataloader this is (only if multiple val datasets used) :return dict: Dict or OrderedDict - passed to the validation_end step .. code-block:: python # if you have one val dataloader: - def validation_step(self, batch, batch_nb) + def validation_step(self, batch, batch_idx) # if you have multiple val dataloaders: - def validation_step(self, batch, batch_nb, dataloader_idxdx) + def validation_step(self, batch, batch_idx, dataloader_idxdx) If you don't need to validate you don't need to implement this method. In this step you'd normally generate examples or calculate anything of interest such as accuracy. @@ -275,7 +275,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # CASE 1: A single validation dataset - def validation_step(self, batch, batch_nb): + def validation_step(self, batch, batch_idx): x, y = batch # implement your own @@ -307,7 +307,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # CASE 2: multiple validation datasets - def validation_step(self, batch, batch_nb, dataset_idx): + def validation_step(self, batch, batch_idx, dataset_idx): # dataset_idx tells you which dataset this is. The `dataset_idx` corresponds to the order of datasets returned in `val_dataloader`. @@ -318,17 +318,17 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): """return whatever outputs will need to be aggregated in test_end :param batch: The output of your dataloader. A tensor, tuple or list - :param int batch_nb: Integer displaying which batch this is + :param int batch_idx: Integer displaying which batch this is :param int dataloader_idx: Integer displaying which dataloader this is (only if multiple test datasets used) :return dict: Dict or OrderedDict with metrics to display in progress bar. All keys must be tensors. .. code-block:: python # if you have one test dataloader: - def test_step(self, batch, batch_nb) + def test_step(self, batch, batch_idx) # if you have multiple test dataloaders: - def test_step(self, batch, batch_nb, dataloader_idxdx) + def test_step(self, batch, batch_idx, dataloader_idxdx) **OPTIONAL** @@ -348,7 +348,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # CASE 1: A single test dataset - def test_step(self, batch, batch_nb): + def test_step(self, batch, batch_idx): x, y = batch # implement your own @@ -375,7 +375,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # CASE 2: multiple test datasets - def test_step(self, batch, batch_nb, dataset_idx): + def test_step(self, batch, batch_idx, dataset_idx): # dataset_idx tells you which dataset this is. @@ -694,13 +694,13 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): """ raise NotImplementedError - def optimizer_step(self, epoch_nb, batch_nb, optimizer, optimizer_i, second_order_closure=None): + def optimizer_step(self, epoch_idx, batch_idx, optimizer, optimizer_idx, second_order_closure=None): """Do something instead of the standard optimizer behavior - :param int epoch_nb: - :param int batch_nb: + :param int epoch_idx: + :param int batch_idx: :param optimizer: - :param optimizer_i: + :param optimizer_idx: :param second_order_closure: closure for second order methods :return: @@ -712,21 +712,21 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # DEFAULT - def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None): + def optimizer_step(self, current_epoch, batch_idx, optimizer, optimizer_idx, second_order_closure=None): optimizer.step() optimizer.zero_grad() # Alternating schedule for optimizer steps (ie: GANs) - def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None): + def optimizer_step(self, current_epoch, batch_idx, optimizer, optimizer_idx, second_order_closure=None): # update generator opt every 2 steps - if optimizer_i == 0: - if batch_nb % 2 == 0 : + if optimizer_idx == 0: + if batch_idx % 2 == 0 : optimizer.step() optimizer.zero_grad() # update discriminator opt every 4 steps - if optimizer_i == 1: - if batch_nb % 4 == 0 : + if optimizer_idx == 1: + if batch_idx % 4 == 0 : optimizer.step() optimizer.zero_grad() @@ -739,7 +739,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): .. code-block:: python # learning rate warm-up - def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None): + def optimizer_step(self, current_epoch, batch_idx, optimizer, optimizer_idx, second_order_closure=None): # warm up lr if self.trainer.global_step < 500: lr_scale = min(1., float(self.trainer.global_step + 1) / 500.) diff --git a/pytorch_lightning/core/memory.py b/pytorch_lightning/core/memory.py index 362688eb..b01451f8 100644 --- a/pytorch_lightning/core/memory.py +++ b/pytorch_lightning/core/memory.py @@ -174,20 +174,20 @@ def print_mem_stack(): # pragma: no cover def count_mem_items(): # pragma: no cover - nb_params = 0 - nb_tensors = 0 + num_params = 0 + num_tensors = 0 for obj in gc.get_objects(): try: if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)): obj_type = str(type(obj)) if 'parameter' in obj_type: - nb_params += 1 + num_params += 1 else: - nb_tensors += 1 + num_tensors += 1 except Exception: pass - return nb_params, nb_tensors + return num_params, num_tensors def get_memory_profile(mode): diff --git a/pytorch_lightning/logging/__init__.py b/pytorch_lightning/logging/__init__.py index 1482a710..23924a76 100644 --- a/pytorch_lightning/logging/__init__.py +++ b/pytorch_lightning/logging/__init__.py @@ -52,7 +52,7 @@ only the first process in DDP training logs data. pass @rank_zero_only - def log_metrics(self, metrics, step_num): + def log_metrics(self, metrics, step_idx): # metrics is a dictionary of metric names and values # your code to record metrics goes here pass diff --git a/pytorch_lightning/logging/base.py b/pytorch_lightning/logging/base.py index a72e072b..68f7c7b6 100644 --- a/pytorch_lightning/logging/base.py +++ b/pytorch_lightning/logging/base.py @@ -21,12 +21,11 @@ class LightningLoggerBase(object): def __init__(self): self._rank = 0 - def log_metrics(self, metrics, step_num): - """Record metrics + def log_metrics(self, metrics, step_idx): + """Record metrics. - :param metric: Dictionary with metric names as keys and measured - quanties as values - :param step_num: Step number at which the metrics should be recorded + :param float metric: Dictionary with metric names as keys and measured quanties as values + :param int|None step_idx: Step number at which the metrics should be recorded """ raise NotImplementedError() diff --git a/pytorch_lightning/logging/comet.py b/pytorch_lightning/logging/comet.py index 4ceaf508..3bfcf9b1 100644 --- a/pytorch_lightning/logging/comet.py +++ b/pytorch_lightning/logging/comet.py @@ -145,13 +145,13 @@ class CometLogger(LightningLoggerBase): self.experiment.log_parameters(vars(params)) @rank_zero_only - def log_metrics(self, metrics, step_num=None): + def log_metrics(self, metrics, step_idx=None): # Comet.ml expects metrics to be a dictionary of detached tensors on CPU for key, val in metrics.items(): if is_tensor(val): metrics[key] = val.cpu().detach() - self.experiment.log_metrics(metrics, step=step_num) + self.experiment.log_metrics(metrics, step=step_idx) @rank_zero_only def finalize(self, status): @@ -169,7 +169,7 @@ class CometLogger(LightningLoggerBase): def version(self): if self.project_name and self.rest_api_key: # Determines the number of experiments in this project, and returns the next integer as the version number - nb_exps = len(self.comet_api.get_experiments(self.workspace, self.project_name)) - return nb_exps + 1 + num_exps = len(self.comet_api.get_experiments(self.workspace, self.project_name)) + return num_exps + 1 else: return None diff --git a/pytorch_lightning/logging/mlflow.py b/pytorch_lightning/logging/mlflow.py index 6fae9f35..123b17db 100644 --- a/pytorch_lightning/logging/mlflow.py +++ b/pytorch_lightning/logging/mlflow.py @@ -68,7 +68,7 @@ class MLFlowLogger(LightningLoggerBase): self.experiment.log_param(self.run_id, k, v) @rank_zero_only - def log_metrics(self, metrics, step_num=None): + def log_metrics(self, metrics, step_idx=None): timestamp_ms = int(time() * 1000) for k, v in metrics.items(): if isinstance(v, str): @@ -76,7 +76,7 @@ class MLFlowLogger(LightningLoggerBase): f"Discarding metric with string value {k}={v}" ) continue - self.experiment.log_metric(self.run_id, k, v, timestamp_ms, step_num) + self.experiment.log_metric(self.run_id, k, v, timestamp_ms, step_idx) def save(self): pass diff --git a/pytorch_lightning/logging/test_tube.py b/pytorch_lightning/logging/test_tube.py index a3c11c62..089c6ebd 100644 --- a/pytorch_lightning/logging/test_tube.py +++ b/pytorch_lightning/logging/test_tube.py @@ -76,10 +76,10 @@ class TestTubeLogger(LightningLoggerBase): self.experiment.argparse(params) @rank_zero_only - def log_metrics(self, metrics, step_num=None): + def log_metrics(self, metrics, step_idx=None): # TODO: HACK figure out where this is being set to true self.experiment.debug = self.debug - self.experiment.log(metrics, global_step=step_num) + self.experiment.log(metrics, global_step=step_idx) @rank_zero_only def save(self): diff --git a/pytorch_lightning/testing/model_base.py b/pytorch_lightning/testing/model_base.py index b83b8a28..df9a3320 100644 --- a/pytorch_lightning/testing/model_base.py +++ b/pytorch_lightning/testing/model_base.py @@ -121,7 +121,7 @@ class LightningTestModelBase(LightningModule): loss_val = loss_val.unsqueeze(0) # alternate possible outputs to test - if self.trainer.batch_nb % 1 == 0: + if self.trainer.batch_idx % 1 == 0: output = OrderedDict({ 'loss': loss_val, 'progress_bar': {'some_val': loss_val * loss_val}, @@ -129,7 +129,7 @@ class LightningTestModelBase(LightningModule): }) return output - if self.trainer.batch_nb % 2 == 0: + if self.trainer.batch_idx % 2 == 0: return loss_val # --------------------- diff --git a/pytorch_lightning/trainer/data_loading_mixin.py b/pytorch_lightning/trainer/data_loading_mixin.py index 8533fdf4..205a7eac 100644 --- a/pytorch_lightning/trainer/data_loading_mixin.py +++ b/pytorch_lightning/trainer/data_loading_mixin.py @@ -35,10 +35,10 @@ class TrainerDataLoadingMixin(object): # determine number of training batches if EXIST_ITER_DATASET and isinstance(self.get_train_dataloader().dataset, IterableDataset): - self.nb_training_batches = float('inf') + self.num_training_batches = float('inf') else: - self.nb_training_batches = len(self.get_train_dataloader()) - self.nb_training_batches = int(self.nb_training_batches * self.train_percent_check) + self.num_training_batches = len(self.get_train_dataloader()) + self.num_training_batches = int(self.num_training_batches * self.train_percent_check) # determine when to check validation # if int passed in, val checks that often @@ -46,7 +46,7 @@ class TrainerDataLoadingMixin(object): if isinstance(self.val_check_interval, int): self.val_check_batch = self.val_check_interval else: - self.val_check_batch = int(self.nb_training_batches * self.val_check_interval) + self.val_check_batch = int(self.num_training_batches * self.val_check_interval) self.val_check_batch = max(1, self.val_check_batch) on_ddp = self.use_ddp or self.use_ddp2 @@ -82,9 +82,9 @@ class TrainerDataLoadingMixin(object): # determine number of validation batches # val datasets could be none, 1 or 2+ if self.get_val_dataloaders() is not None: - self.nb_val_batches = sum(len(dataloader) for dataloader in self.get_val_dataloaders()) - self.nb_val_batches = int(self.nb_val_batches * self.val_percent_check) - self.nb_val_batches = max(1, self.nb_val_batches) + self.num_val_batches = sum(len(dataloader) for dataloader in self.get_val_dataloaders()) + self.num_val_batches = int(self.num_val_batches * self.val_percent_check) + self.num_val_batches = max(1, self.num_val_batches) on_ddp = self.use_ddp or self.use_ddp2 if on_ddp and self.get_val_dataloaders() is not None: @@ -125,9 +125,9 @@ class TrainerDataLoadingMixin(object): # determine number of test batches if self.get_test_dataloaders() is not None: len_sum = sum(len(dataloader) for dataloader in self.get_test_dataloaders()) - self.nb_test_batches = len_sum - self.nb_test_batches = int(self.nb_test_batches * self.test_percent_check) - self.nb_test_batches = max(1, self.nb_test_batches) + self.num_test_batches = len_sum + self.num_test_batches = int(self.num_test_batches * self.test_percent_check) + self.num_test_batches = max(1, self.num_test_batches) on_ddp = self.use_ddp or self.use_ddp2 if on_ddp and self.get_test_dataloaders() is not None: diff --git a/pytorch_lightning/trainer/ddp_mixin.py b/pytorch_lightning/trainer/ddp_mixin.py index 67821ae9..b0e75f25 100644 --- a/pytorch_lightning/trainer/ddp_mixin.py +++ b/pytorch_lightning/trainer/ddp_mixin.py @@ -131,7 +131,8 @@ except ImportError: class TrainerDDPMixin(object): - def set_distributed_mode(self, distributed_backend, nb_gpu_nodes): + + def set_distributed_mode(self, distributed_backend, num_gpu_nodes): # skip for CPU if self.num_gpus == 0: return @@ -169,8 +170,8 @@ class TrainerDDPMixin(object): self.use_ddp2 = False # throw error to force user ddp or ddp2 choice - if nb_gpu_nodes > 1 and not (self.use_ddp2 or self.use_ddp): # pragma: no cover - w = 'DataParallel does not support nb_gpu_nodes > 1. ' \ + if num_gpu_nodes > 1 and not (self.use_ddp2 or self.use_ddp): # pragma: no cover + w = 'DataParallel does not support num_nodes > 1. ' \ 'Switching to DistributedDataParallel for you. ' \ 'To silence this warning set distributed_backend=ddp' \ 'or distributed_backend=ddp2' @@ -178,18 +179,18 @@ class TrainerDDPMixin(object): logging.info(f'gpu available: {torch.cuda.is_available()}, used: {self.on_gpu}') - def configure_slurm_ddp(self, nb_gpu_nodes): + def configure_slurm_ddp(self, num_gpu_nodes): self.is_slurm_managing_tasks = False # extract SLURM flag vars # whenever we have the correct number of tasks, we let slurm manage processes # otherwise we launch the required number of processes if self.use_ddp: - self.nb_requested_gpus = self.num_gpus * nb_gpu_nodes - self.nb_slurm_tasks = 0 + self.num_requested_gpus = self.num_gpus * num_gpu_nodes + self.num_slurm_tasks = 0 try: - self.nb_slurm_tasks = int(os.environ['SLURM_NTASKS']) - self.is_slurm_managing_tasks = self.nb_slurm_tasks == self.nb_requested_gpus + self.num_slurm_tasks = int(os.environ['SLURM_NTASKS']) + self.is_slurm_managing_tasks = self.num_slurm_tasks == self.num_requested_gpus # in interactive mode we don't manage tasks job_name = os.environ['SLURM_JOB_NAME'] @@ -226,10 +227,10 @@ class TrainerDDPMixin(object): logging.info(f'VISIBLE GPUS: {os.environ["CUDA_VISIBLE_DEVICES"]}') - def ddp_train(self, gpu_nb, model): + def ddp_train(self, gpu_idx, model): """ Entry point into a DP thread - :param gpu_nb: + :param gpu_idx: :param model: :param cluster_obj: :return: @@ -243,16 +244,16 @@ class TrainerDDPMixin(object): self.node_rank = 0 # show progressbar only on progress_rank 0 - self.show_progress_bar = self.show_progress_bar and self.node_rank == 0 and gpu_nb == 0 + self.show_progress_bar = self.show_progress_bar and self.node_rank == 0 and gpu_idx == 0 # determine which process we are and world size if self.use_ddp: - self.proc_rank = self.node_rank * self.num_gpus + gpu_nb - self.world_size = self.nb_gpu_nodes * self.num_gpus + self.proc_rank = self.node_rank * self.num_gpus + gpu_idx + self.world_size = self.num_gpu_nodes * self.num_gpus elif self.use_ddp2: self.proc_rank = self.node_rank - self.world_size = self.nb_gpu_nodes + self.world_size = self.num_gpu_nodes # let the exp know the rank to avoid overwriting logs if self.logger is not None: @@ -271,14 +272,14 @@ class TrainerDDPMixin(object): # MODEL # copy model to each gpu if self.distributed_backend == 'ddp': - torch.cuda.set_device(gpu_nb) - model.cuda(gpu_nb) + torch.cuda.set_device(gpu_idx) + model.cuda(gpu_idx) # set model properties before going into wrapper self.copy_trainer_model_properties(model) # override root GPU - self.root_gpu = gpu_nb + self.root_gpu = gpu_idx # AMP # run through amp wrapper before going to distributed DP @@ -289,9 +290,11 @@ class TrainerDDPMixin(object): # DDP2 uses all GPUs on the machine if self.distributed_backend == 'ddp': - device_ids = [gpu_nb] + device_ids = [gpu_idx] elif self.use_ddp2: device_ids = self.data_parallel_device_ids + else: + device_ids = None # allow user to configure ddp model = model.configure_ddp(model, device_ids) diff --git a/pytorch_lightning/trainer/dp_mixin.py b/pytorch_lightning/trainer/dp_mixin.py index aca6c488..497d165f 100644 --- a/pytorch_lightning/trainer/dp_mixin.py +++ b/pytorch_lightning/trainer/dp_mixin.py @@ -173,7 +173,7 @@ Multi-node training is easily done by specifying these flags. .. code-block:: python # train on 12*8 GPUs - trainer = Trainer(gpus=8, nb_gpu_nodes=12, distributed_backend='ddp') + trainer = Trainer(gpus=8, num_nodes=12, distributed_backend='ddp') You must configure your job submission script correctly for the trainer to work. diff --git a/pytorch_lightning/trainer/evaluation_loop_mixin.py b/pytorch_lightning/trainer/evaluation_loop_mixin.py index bc860d85..bb59d000 100644 --- a/pytorch_lightning/trainer/evaluation_loop_mixin.py +++ b/pytorch_lightning/trainer/evaluation_loop_mixin.py @@ -76,10 +76,10 @@ Lightning runs a few steps of validation in the beginning of training. .. code-block:: python # DEFAULT - trainer = Trainer(nb_sanity_val_steps=5) + trainer = Trainer(num_sanity_val_steps=5) -You can use `Trainer(nb_sanity_val_steps=0)` to skip the sanity check. +You can use `Trainer(num_sanity_val_steps=0)` to skip the sanity check. # Testing loop @@ -230,11 +230,11 @@ class TrainerEvaluationLoopMixin(object): # select dataloaders if test: dataloaders = self.get_test_dataloaders() - max_batches = self.nb_test_batches + max_batches = self.num_test_batches else: # val dataloaders = self.get_val_dataloaders() - max_batches = self.nb_val_batches + max_batches = self.num_val_batches # cap max batches to 1 when using fast_dev_run if self.fast_dev_run: diff --git a/pytorch_lightning/trainer/logging_mixin.py b/pytorch_lightning/trainer/logging_mixin.py index 0dae4da1..82376ce4 100644 --- a/pytorch_lightning/trainer/logging_mixin.py +++ b/pytorch_lightning/trainer/logging_mixin.py @@ -28,7 +28,7 @@ class TrainerLoggingMixin(object): # log actual metrics if self.proc_rank == 0 and self.logger is not None: - self.logger.log_metrics(scalar_metrics, step_num=self.global_step) + self.logger.log_metrics(scalar_metrics, step_idx=self.global_step) self.logger.save() def add_tqdm_metrics(self, metrics): @@ -68,8 +68,8 @@ class TrainerLoggingMixin(object): callback_metrics[k] = v if train and (self.use_dp or self.use_ddp2): - nb_gpus = self.num_gpus - callback_metrics = self.reduce_distributed_output(callback_metrics, nb_gpus) + num_gpus = self.num_gpus + callback_metrics = self.reduce_distributed_output(callback_metrics, num_gpus) for k, v in callback_metrics.items(): callback_metrics[k] = v.item() @@ -82,8 +82,8 @@ class TrainerLoggingMixin(object): # reduce progress metrics for tqdm when using dp if train and (self.use_dp or self.use_ddp2): - nb_gpus = self.num_gpus - progress_output = self.reduce_distributed_output(progress_output, nb_gpus) + num_gpus = self.num_gpus + progress_output = self.reduce_distributed_output(progress_output, num_gpus) progress_bar_metrics = progress_output except Exception: @@ -98,8 +98,8 @@ class TrainerLoggingMixin(object): # reduce progress metrics for tqdm when using dp if train and (self.use_dp or self.use_ddp2): - nb_gpus = self.num_gpus - log_output = self.reduce_distributed_output(log_output, nb_gpus) + num_gpus = self.num_gpus + log_output = self.reduce_distributed_output(log_output, num_gpus) log_metrics = log_output except Exception: @@ -142,8 +142,8 @@ class TrainerLoggingMixin(object): return loss, progress_bar_metrics, log_metrics, callback_metrics, hiddens - def reduce_distributed_output(self, output, nb_gpus): - if nb_gpus <= 1: + def reduce_distributed_output(self, output, num_gpus): + if num_gpus <= 1: return output # when using DP, we get one output per gpu @@ -154,14 +154,14 @@ class TrainerLoggingMixin(object): for k, v in output.items(): # recurse on nested dics if isinstance(output[k], dict): - output[k] = self.reduce_distributed_output(output[k], nb_gpus) + output[k] = self.reduce_distributed_output(output[k], num_gpus) # do nothing when there's a scalar elif isinstance(output[k], torch.Tensor) and output[k].dim() == 0: pass # reduce only metrics that have the same nb of gpus - elif output[k].size(0) == nb_gpus: + elif output[k].size(0) == num_gpus: reduced = torch.mean(output[k]) output[k] = reduced return output diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py index 5ee7ae06..0bff3d59 100644 --- a/pytorch_lightning/trainer/train_loop_mixin.py +++ b/pytorch_lightning/trainer/train_loop_mixin.py @@ -23,7 +23,7 @@ It can be useful to force training for a minimum number of epochs or limit to a .. code-block:: python # DEFAULT - trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000) + trainer = Trainer(min_num_epochs=1, max_num_epochs=1000) Early stopping -------------- @@ -123,7 +123,7 @@ When using PackedSequence, do 2 things: return x, y # In module - def training_step(self, batch, batch_nb): + def training_step(self, batch, batch_idx): x = rnn.pack_sequence(batch[0], enforce_sorted=False) y = rnn.pack_sequence(batch[1], enforce_sorted=False) @@ -165,46 +165,46 @@ class TrainerTrainLoopMixin(object): def train(self): # run all epochs - for epoch_nb in range(self.current_epoch, self.max_nb_epochs): + for epoch_idx in range(self.current_epoch, self.max_num_epochs): # set seed for distributed sampler (enables shuffling for each epoch) if self.use_ddp and hasattr(self.get_train_dataloader().sampler, 'set_epoch'): - self.get_train_dataloader().sampler.set_epoch(epoch_nb) + self.get_train_dataloader().sampler.set_epoch(epoch_idx) # get model model = self.get_model() # update training progress in trainer and model - model.current_epoch = epoch_nb - self.current_epoch = epoch_nb + model.current_epoch = epoch_idx + self.current_epoch = epoch_idx # val can be checked multiple times in epoch is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0 - val_checks_per_epoch = self.nb_training_batches // self.val_check_batch + val_checks_per_epoch = self.num_training_batches // self.val_check_batch val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0 # total batches includes multiple val checks - self.total_batches = (self.nb_training_batches + - self.nb_val_batches * val_checks_per_epoch) + self.total_batches = (self.num_training_batches + + self.num_val_batches * val_checks_per_epoch) self.batch_loss_value = 0 # accumulated grads if self.fast_dev_run: # limit the number of batches to 2 (1 train and 1 val) in fast_dev_run - nb_iterations = 2 + num_iterations = 2 elif self.is_iterable_train_dataloader: # for iterable train loader, the progress bar never ends - nb_iterations = None + num_iterations = None else: - nb_iterations = self.total_batches + num_iterations = self.total_batches # reset progress bar # .reset() doesn't work on disabled progress bar so we should check if not self.main_progress_bar.disable: - self.main_progress_bar.reset(nb_iterations) - desc = f'Epoch {epoch_nb + 1}' if not self.is_iterable_train_dataloader else '' + self.main_progress_bar.reset(num_iterations) + desc = f'Epoch {epoch_idx + 1}' if not self.is_iterable_train_dataloader else '' self.main_progress_bar.set_description(desc) # changing gradient according accumulation_scheduler - self.accumulation_scheduler.on_epoch_begin(epoch_nb, self) + self.accumulation_scheduler.on_epoch_begin(epoch_idx, self) # ----------------- # RUN TNG EPOCH @@ -225,9 +225,9 @@ class TrainerTrainLoopMixin(object): self.reduce_lr_on_plateau_scheduler.step(val_loss, epoch=self.current_epoch) # early stopping - met_min_epochs = epoch_nb > self.min_nb_epochs + met_min_epochs = epoch_idx > self.min_num_epochs if self.enable_early_stop and (met_min_epochs or self.fast_dev_run): - should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch_nb, + should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch_idx, logs=self.callback_metrics) # stop training stop = should_stop and met_min_epochs @@ -247,8 +247,8 @@ class TrainerTrainLoopMixin(object): model.on_epoch_start() # run epoch - for batch_nb, batch in enumerate(self.get_train_dataloader()): - self.batch_nb = batch_nb + for batch_idx, batch in enumerate(self.get_train_dataloader()): + self.batch_idx = batch_idx model = self.get_model() model.global_step = self.global_step @@ -256,7 +256,7 @@ class TrainerTrainLoopMixin(object): # --------------- # RUN TRAIN STEP # --------------- - output = self.run_training_batch(batch, batch_nb) + output = self.run_training_batch(batch, batch_idx) batch_result, grad_norm_dic, batch_step_metrics = output # when returning -1 from train_step, we end epoch early @@ -265,7 +265,7 @@ class TrainerTrainLoopMixin(object): # --------------- # RUN VAL STEP # --------------- - is_val_check_batch = (batch_nb + 1) % self.val_check_batch == 0 + is_val_check_batch = (batch_idx + 1) % self.val_check_batch == 0 can_check_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0 should_check_val = ((is_val_check_batch or early_stop_epoch) and can_check_epoch) @@ -274,19 +274,19 @@ class TrainerTrainLoopMixin(object): self.run_evaluation(test=self.testing) # when logs should be saved - should_save_log = (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch + should_save_log = (batch_idx + 1) % self.log_save_interval == 0 or early_stop_epoch if should_save_log or self.fast_dev_run: if self.proc_rank == 0 and self.logger is not None: self.logger.save() # when metrics should be logged - should_log_metrics = batch_nb % self.row_log_interval == 0 or early_stop_epoch + should_log_metrics = batch_idx % self.row_log_interval == 0 or early_stop_epoch if should_log_metrics or self.fast_dev_run: # logs user requested information to logger self.log_metrics(batch_step_metrics, grad_norm_dic) self.global_step += 1 - self.total_batch_nb += 1 + self.total_batch_idx += 1 # end epoch early # stop when the flag is changed or we've gone past the amount @@ -295,7 +295,7 @@ class TrainerTrainLoopMixin(object): break # stop epoch if we limited nb batches - met_batch_limit = batch_nb >= self.nb_training_batches + met_batch_limit = batch_idx >= self.num_training_batches if met_batch_limit: break @@ -304,7 +304,7 @@ class TrainerTrainLoopMixin(object): model = self.get_model() model.on_epoch_end() - def run_training_batch(self, batch, batch_nb): + def run_training_batch(self, batch, batch_idx): # track grad norms grad_norm_dic = {} @@ -331,8 +331,8 @@ class TrainerTrainLoopMixin(object): splits = model_ref.tbptt_split_batch(batch, self.truncated_bptt_steps) self.hiddens = None - for split_nb, split_batch in enumerate(splits): - self.split_nb = split_nb + for split_idx, split_batch in enumerate(splits): + self.split_idx = split_idx # call training_step once per optimizer for opt_idx, optimizer in enumerate(self.optimizers): @@ -341,7 +341,7 @@ class TrainerTrainLoopMixin(object): def optimizer_closure(): # forward pass output = self.training_forward( - split_batch, batch_nb, opt_idx, self.hiddens) + split_batch, batch_idx, opt_idx, self.hiddens) closure_loss = output[0] progress_bar_metrics = output[1] @@ -382,10 +382,10 @@ class TrainerTrainLoopMixin(object): self.batch_loss_value += loss.item() # gradient update with accumulated gradients - if (self.batch_nb + 1) % self.accumulate_grad_batches == 0: + if (self.batch_idx + 1) % self.accumulate_grad_batches == 0: # track gradient norms when requested - if batch_nb % self.row_log_interval == 0: + if batch_idx % self.row_log_interval == 0: if self.track_grad_norm > 0: model = self.get_model() grad_norm_dic = model.grad_norm( @@ -397,7 +397,7 @@ class TrainerTrainLoopMixin(object): # calls .step(), .zero_grad() # override function to modify this behavior model = self.get_model() - model.optimizer_step(self.current_epoch, batch_nb, + model.optimizer_step(self.current_epoch, batch_idx, optimizer, opt_idx, optimizer_closure) # calculate running loss for display @@ -422,18 +422,18 @@ class TrainerTrainLoopMixin(object): return 0, grad_norm_dic, all_log_metrics - def training_forward(self, batch, batch_nb, opt_idx, hiddens): + def training_forward(self, batch, batch_idx, opt_idx, hiddens): """ Handle forward for each training case (distributed, single gpu, etc...) :param batch: - :param batch_nb: + :param batch_idx: :return: """ # --------------- # FORWARD # --------------- # enable not needing to add opt_idx to training_step - args = [batch, batch_nb] + args = [batch, batch_idx] if len(self.optimizers) > 1: args.append(opt_idx) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index eb00cda4..649ec316 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -50,41 +50,47 @@ class Trainer(TrainerIOMixin, TrainerCallbackConfigMixin, TrainerModelHooksMixin): - def __init__(self, - logger=True, - checkpoint_callback=True, - early_stop_callback=True, - default_save_path=None, - gradient_clip_val=0, - gradient_clip=None, # backward compatible - process_position=0, - nb_gpu_nodes=1, - gpus=None, - log_gpu_memory=None, - show_progress_bar=True, - overfit_pct=0.0, - track_grad_norm=-1, - check_val_every_n_epoch=1, - fast_dev_run=False, - accumulate_grad_batches=1, - max_nb_epochs=1000, - min_nb_epochs=1, - train_percent_check=1.0, - val_percent_check=1.0, - test_percent_check=1.0, - val_check_interval=1.0, - log_save_interval=100, - row_log_interval=10, - add_row_log_interval=None, # backward compatible - distributed_backend=None, - use_amp=False, - print_nan_grads=False, - weights_summary='full', - weights_save_path=None, - amp_level='O1', - nb_sanity_val_steps=5, - truncated_bptt_steps=None, - resume_from_checkpoint=None): + def __init__( + self, + logger=True, + checkpoint_callback=True, + early_stop_callback=True, + default_save_path=None, + gradient_clip_val=0, + gradient_clip=None, # backward compatible, todo: remove in v0.8.0 + process_position=0, + nb_gpu_nodes=None, # backward compatible, todo: remove in v0.8.0 + num_nodes=1, + gpus=None, + log_gpu_memory=None, + show_progress_bar=True, + overfit_pct=0.0, + track_grad_norm=-1, + check_val_every_n_epoch=1, + fast_dev_run=False, + accumulate_grad_batches=1, + max_nb_epochs=None, # backward compatible, todo: remove in v0.8.0 + min_nb_epochs=None, # backward compatible, todo: remove in v0.8.0 + max_num_epochs=1000, + min_num_epochs=1, + train_percent_check=1.0, + val_percent_check=1.0, + test_percent_check=1.0, + val_check_interval=1.0, + log_save_interval=100, + row_log_interval=10, + add_row_log_interval=None, # backward compatible, todo: remove in v0.8.0 + distributed_backend=None, + use_amp=False, + print_nan_grads=False, + weights_summary='full', + weights_save_path=None, + amp_level='O1', + nb_sanity_val_steps=None, # backward compatible, todo: remove in v0.8.0 + num_sanity_val_steps=5, + truncated_bptt_steps=None, + resume_from_checkpoint=None, + ): """ :param logger: Logger for experiment tracking @@ -94,8 +100,8 @@ class Trainer(TrainerIOMixin, :param int gradient_clip_val: 0 means don't clip. :param int gradient_clip: 0 means don't clip. Deprecated. :param process_position: shown in the tqdm bar - :param int nb_gpu_nodes: number of GPU nodes - :param gpus: int. (ie: 2 gpus) OR list to specify which GPUs [0, 1] OR '0,1' + :param int num_nodes: number of GPU nodes + :param list|str|int gpus: int. (ie: 2 gpus) OR list to specify which GPUs [0, 1] OR '0,1' OR '-1' / -1 to use all available gpus :param str log_gpu_memory: None, 'min_max', 'all' :param bool show_progress_bar: If true shows tqdm bar @@ -104,8 +110,8 @@ class Trainer(TrainerIOMixin, :param int check_val_every_n_epoch: check val every n train epochs :param bool fast_dev_run: runs full iteration over everything to find bugs :param int accumulate_grad_batches: Accumulates grads every k batches - :param int max_nb_epochs: - :param int min_nb_epochs: + :param int max_num_epochs: + :param int min_num_epochs: :param int train_percent_check: How much of train set to check :param int val_percent_check: How much of val set to check :param int test_percent_check: How much of test set to check @@ -119,26 +125,55 @@ class Trainer(TrainerIOMixin, :param str weights_summary: Options: 'full', 'top', None to not print. :param bool weights_save_path: Where to save weights if on cluster :param str amp_level: Check nvidia docs for level - :param int nb_sanity_val_steps: How many val steps before a full train loop. + :param int num_sanity_val_steps: How many val steps before a full train loop. :param int truncated_bptt_steps: Enables multiple backward passes for each batch. + + .. warning:: Following arguments become deprecated and they will be removed in v0.8.0: + - `gradient_clip`, + - `nb_gpu_nodes`, + - `max_nb_epochs`, + - `min_nb_epochs`, + - `add_row_log_interval`, + - `nb_sanity_val_steps` + """ # Transfer params - self.nb_gpu_nodes = nb_gpu_nodes + if nb_gpu_nodes is not None: # Backward compatibility + warnings.warn("`nb_gpu_nodes` has renamed to `num_nodes` since v0.5.0" + " and will be removed in v0.8.0", DeprecationWarning) + if not num_nodes: # in case you did not set the proper value + num_nodes = nb_gpu_nodes + self.num_gpu_nodes = num_nodes self.log_gpu_memory = log_gpu_memory - if not (gradient_clip is None): - # Backward compatibility + if gradient_clip is not None: # Backward compatibility warnings.warn("`gradient_clip` has renamed to `gradient_clip_val` since v0.5.0" " and will be removed in v0.8.0", DeprecationWarning) - gradient_clip_val = gradient_clip + if not gradient_clip_val: # in case you did not set the proper value + gradient_clip_val = gradient_clip self.gradient_clip_val = gradient_clip_val self.check_val_every_n_epoch = check_val_every_n_epoch self.track_grad_norm = track_grad_norm self.on_gpu = True if (gpus and torch.cuda.is_available()) else False self.process_position = process_position self.weights_summary = weights_summary - self.max_nb_epochs = max_nb_epochs - self.min_nb_epochs = min_nb_epochs - self.nb_sanity_val_steps = nb_sanity_val_steps + if max_nb_epochs is not None: # Backward compatibility + warnings.warn("`max_nb_epochs` has renamed to `max_num_epochs` since v0.5.0" + " and will be removed in v0.8.0", DeprecationWarning) + if not max_num_epochs: # in case you did not set the proper value + max_num_epochs = max_nb_epochs + self.max_num_epochs = max_num_epochs + if min_nb_epochs is not None: # Backward compatibility + warnings.warn("`min_nb_epochs` has renamed to `min_num_epochs` since v0.5.0" + " and will be removed in v0.8.0", DeprecationWarning) + if not min_num_epochs: # in case you did not set the proper value + min_num_epochs = min_nb_epochs + self.min_num_epochs = min_num_epochs + if nb_sanity_val_steps is not None: # Backward compatibility + warnings.warn("`nb_sanity_val_steps` has renamed to `num_sanity_val_steps` since v0.5.0" + " and will be removed in v0.8.0", DeprecationWarning) + if not num_sanity_val_steps: # in case you did not set the proper value + num_sanity_val_steps = nb_sanity_val_steps + self.num_sanity_val_steps = num_sanity_val_steps self.print_nan_grads = print_nan_grads self.truncated_bptt_steps = truncated_bptt_steps self.resume_from_checkpoint = resume_from_checkpoint @@ -146,8 +181,8 @@ class Trainer(TrainerIOMixin, self.fast_dev_run = fast_dev_run if self.fast_dev_run: - self.nb_sanity_val_steps = 1 - self.max_nb_epochs = 1 + self.num_sanity_val_steps = 1 + self.max_num_epochs = 1 m = ''' Running in fast_dev_run mode: will run a full train, val loop using a single batch @@ -160,15 +195,15 @@ class Trainer(TrainerIOMixin, self.default_save_path = os.getcwd() # training bookeeping - self.total_batch_nb = 0 + self.total_batch_idx = 0 self.running_loss = [] self.avg_loss = 0 - self.batch_nb = 0 + self.batch_idx = 0 self.tqdm_metrics = {} self.callback_metrics = {} - self.nb_val_batches = 0 - self.nb_training_batches = 0 - self.nb_test_batches = 0 + self.num_val_batches = 0 + self.num_training_batches = 0 + self.num_test_batches = 0 self.get_train_dataloader = None self.get_test_dataloaders = None self.get_val_dataloaders = None @@ -207,13 +242,13 @@ class Trainer(TrainerIOMixin, self.use_dp = False self.single_gpu = False self.distributed_backend = distributed_backend - self.set_distributed_mode(distributed_backend, nb_gpu_nodes) + self.set_distributed_mode(distributed_backend, num_nodes) # init flags for SLURM+ddp to work self.proc_rank = 0 self.world_size = 1 self.node_rank = 0 - self.configure_slurm_ddp(nb_gpu_nodes) + self.configure_slurm_ddp(num_nodes) # nvidia setup self.set_nvidia_flags(self.is_slurm_managing_tasks, self.data_parallel_device_ids) @@ -248,7 +283,7 @@ class Trainer(TrainerIOMixin, try: job_id = os.environ['SLURM_JOB_ID'] job_id = int(job_id) - except Exception as e: + except Exception: job_id = None return job_id @@ -260,17 +295,17 @@ class Trainer(TrainerIOMixin, # if gpus = -1 then use all available devices # otherwise, split the string using commas if gpus is not None: - if type(gpus) is list: + if isinstance(gpus, list): gpus = gpus - elif type(gpus) is str: + elif isinstance(gpus, str): if gpus == '-1': gpus = list(range(0, torch.cuda.device_count())) else: gpus = [int(x.strip()) for x in gpus.split(',')] - elif type(gpus) is int: + elif isinstance(gpus, int): gpus = gpus else: - raise Exception('gpus has to be a string, int or list of ints') + raise ValueError('`gpus` has to be a string, int or list of ints') return gpus @@ -305,11 +340,11 @@ class Trainer(TrainerIOMixin, """ tqdm_dict = { 'loss': '{0:.3f}'.format(self.avg_loss), - 'batch_nb': '{}'.format(self.batch_nb), + 'batch_idx': '{}'.format(self.batch_idx), } if self.truncated_bptt_steps is not None: - tqdm_dict['split_nb'] = self.split_nb + tqdm_dict['split_idx'] = self.split_idx if self.logger is not None and self.logger.version is not None: tqdm_dict['v_nb'] = self.logger.version @@ -395,10 +430,9 @@ class Trainer(TrainerIOMixin, return schedulers, None def run_pretrain_routine(self, model): - """ - Sanity check a few things before starting actual training + """Sanity check a few things before starting actual training. + :param model: - :return: """ ref_model = model if self.data_parallel: @@ -455,16 +489,16 @@ class Trainer(TrainerIOMixin, # run tiny validation (if validation defined) # to make sure program won't crash during val ref_model.on_sanity_check_start() - if self.get_val_dataloaders() is not None and self.nb_sanity_val_steps > 0: + if self.get_val_dataloaders() is not None and self.num_sanity_val_steps > 0: # init progress bars for validation sanity check - pbar = tqdm.tqdm(desc='Validation sanity check', total=self.nb_sanity_val_steps, + pbar = tqdm.tqdm(desc='Validation sanity check', total=self.num_sanity_val_steps, leave=False, position=2 * self.process_position, disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch') self.main_progress_bar = pbar # dummy validation progress bar self.val_progress_bar = tqdm.tqdm(disable=True) - self.evaluate(model, self.get_val_dataloaders(), self.nb_sanity_val_steps, self.testing) + self.evaluate(model, self.get_val_dataloaders(), self.num_sanity_val_steps, self.testing) # close progress bars self.main_progress_bar.close() diff --git a/pytorch_lightning/utilities/arg_parse.py b/pytorch_lightning/utilities/arg_parse.py index afafc925..c126a83c 100644 --- a/pytorch_lightning/utilities/arg_parse.py +++ b/pytorch_lightning/utilities/arg_parse.py @@ -15,8 +15,8 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None parser.opt_list('--accumulate_grad_batches', default=1, type=int, tunable=False, help='accumulates gradients k times before applying update.' ' Simulates huge batch size') - parser.add_argument('--max_nb_epochs', default=200, type=int, help='cap epochs') - parser.add_argument('--min_nb_epochs', default=2, type=int, help='min epochs') + parser.add_argument('--max_num_epochs', default=200, type=int, help='cap epochs') + parser.add_argument('--min_num_epochs', default=2, type=int, help='min epochs') parser.add_argument('--train_percent_check', default=1.0, type=float, help='how much of training set to check') parser.add_argument('--val_percent_check', default=1.0, type=float, diff --git a/tests/debug.py b/tests/debug.py index 78b548f4..aa0614a9 100644 --- a/tests/debug.py +++ b/tests/debug.py @@ -24,12 +24,12 @@ class CoolModel(pl.LightningModule): def my_loss(self, y_hat, y): return F.cross_entropy(y_hat, y) - def training_step(self, batch, batch_nb): + def training_step(self, batch, batch_idx): x, y = batch y_hat = self.forward(x) return {'training_loss': self.my_loss(y_hat, y)} - def validation_step(self, batch, batch_nb): + def validation_step(self, batch, batch_idx): x, y = batch y_hat = self.forward(x) return {'val_loss': self.my_loss(y_hat, y)} diff --git a/tests/test_amp.py b/tests/test_amp.py index 3b1fec25..ea15a713 100644 --- a/tests/test_amp.py +++ b/tests/test_amp.py @@ -23,7 +23,7 @@ def test_amp_single_gpu(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=True, - max_nb_epochs=1, + max_num_epochs=1, gpus=1, distributed_backend='ddp', use_amp=True @@ -45,7 +45,7 @@ def test_no_amp_single_gpu(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=True, - max_nb_epochs=1, + max_num_epochs=1, gpus=1, distributed_backend='dp', use_amp=True @@ -69,7 +69,7 @@ def test_amp_gpu_ddp(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=True, - max_nb_epochs=1, + max_num_epochs=1, gpus=2, distributed_backend='ddp', use_amp=True @@ -94,7 +94,7 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir): trainer_options = dict( show_progress_bar=True, - max_nb_epochs=1, + max_num_epochs=1, gpus=[0], distributed_backend='ddp', use_amp=True @@ -153,7 +153,7 @@ def test_cpu_model_with_amp(tmpdir): default_save_path=tmpdir, show_progress_bar=False, logger=tutils.get_test_tube_logger(tmpdir), - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.4, use_amp=True @@ -175,7 +175,7 @@ def test_amp_gpu_dp(tmpdir): model, hparams = tutils.get_model() trainer_options = dict( default_save_path=tmpdir, - max_nb_epochs=1, + max_num_epochs=1, gpus='0, 1', # test init with gpu string distributed_backend='dp', use_amp=True diff --git a/tests/test_cpu_models.py b/tests/test_cpu_models.py index ddfa164a..7b2e717c 100644 --- a/tests/test_cpu_models.py +++ b/tests/test_cpu_models.py @@ -46,7 +46,7 @@ def test_lbfgs_cpu_model(tmpdir): trainer_options = dict( default_save_path=tmpdir, - max_nb_epochs=1, + max_num_epochs=1, print_nan_grads=True, show_progress_bar=False, weights_summary='top', @@ -64,7 +64,7 @@ def test_default_logger_callbacks_cpu_model(tmpdir): trainer_options = dict( default_save_path=tmpdir, - max_nb_epochs=1, + max_num_epochs=1, gradient_clip_val=1.0, overfit_pct=0.20, print_nan_grads=True, @@ -97,7 +97,7 @@ def test_running_test_after_fitting(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.2, test_percent_check=0.2, @@ -135,7 +135,7 @@ def test_running_test_without_val(tmpdir): trainer_options = dict( show_progress_bar=False, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.2, test_percent_check=0.2, @@ -209,7 +209,7 @@ def test_simple_cpu(tmpdir): # logger file to get meta trainer_options = dict( default_save_path=tmpdir, - max_nb_epochs=1, + max_num_epochs=1, val_percent_check=0.1, train_percent_check=0.1, ) @@ -230,7 +230,7 @@ def test_cpu_model(tmpdir): default_save_path=tmpdir, show_progress_bar=False, logger=tutils.get_test_tube_logger(tmpdir), - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.4 ) @@ -253,7 +253,7 @@ def test_all_features_cpu_model(tmpdir): show_progress_bar=False, logger=tutils.get_test_tube_logger(tmpdir), accumulate_grad_batches=2, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.4 ) @@ -314,7 +314,7 @@ def test_tbptt_cpu_model(tmpdir): trainer_options = dict( default_save_path=tmpdir, - max_nb_epochs=1, + max_num_epochs=1, truncated_bptt_steps=truncated_bptt_steps, val_percent_check=0, weights_summary=None, @@ -348,7 +348,7 @@ def test_single_gpu_model(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.1, val_percent_check=0.1, gpus=1 diff --git a/tests/test_gpu_models.py b/tests/test_gpu_models.py index 9f01b7ef..216f345a 100644 --- a/tests/test_gpu_models.py +++ b/tests/test_gpu_models.py @@ -33,7 +33,7 @@ def test_multi_gpu_model_ddp2(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=True, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.2, gpus=2, @@ -56,7 +56,7 @@ def test_multi_gpu_model_ddp(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.2, gpus=[0, 1], @@ -109,7 +109,7 @@ def test_cpu_slurm_save_load(tmpdir): version = logger.version trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, logger=logger, checkpoint_callback=ModelCheckpoint(tmpdir) ) @@ -143,7 +143,7 @@ def test_cpu_slurm_save_load(tmpdir): logger = tutils.get_test_tube_logger(tmpdir, False, version=version) trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, logger=logger, checkpoint_callback=ModelCheckpoint(tmpdir), ) @@ -177,7 +177,7 @@ def test_multi_gpu_none_backend(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.1, val_percent_check=0.1, gpus='-1' @@ -199,7 +199,7 @@ def test_multi_gpu_model_dp(tmpdir): default_save_path=tmpdir, show_progress_bar=False, distributed_backend='dp', - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.1, val_percent_check=0.1, gpus='-1' @@ -227,7 +227,7 @@ def test_ddp_sampler_error(tmpdir): trainer = Trainer( logger=logger, show_progress_bar=False, - max_nb_epochs=1, + max_num_epochs=1, gpus=[0, 1], distributed_backend='ddp', use_amp=True diff --git a/tests/test_logging.py b/tests/test_logging.py index fa0f8daf..37618a56 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -16,7 +16,7 @@ def test_testtube_logger(tmpdir): logger = tutils.get_test_tube_logger(tmpdir, False) trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.01, logger=logger ) @@ -39,7 +39,7 @@ def test_testtube_pickle(tmpdir): logger.save() trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.01, logger=logger ) @@ -67,7 +67,7 @@ def test_mlflow_logger(tmpdir): logger = MLFlowLogger("test", f"file://{mlflow_dir}") trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.01, logger=logger ) @@ -96,7 +96,7 @@ def test_mlflow_pickle(tmpdir): logger = MLFlowLogger("test", f"file://{mlflow_dir}") trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, logger=logger ) @@ -128,7 +128,7 @@ def test_comet_logger(tmpdir): ) trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.01, logger=logger ) @@ -162,7 +162,7 @@ def test_comet_pickle(tmpdir): ) trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, logger=logger ) @@ -185,7 +185,7 @@ def test_custom_logger(tmpdir): self.hparams_logged = params @rank_zero_only - def log_metrics(self, metrics, step_num): + def log_metrics(self, metrics, step_idx): self.metrics_logged = metrics @rank_zero_only @@ -206,7 +206,7 @@ def test_custom_logger(tmpdir): logger = CustomLogger() trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.01, logger=logger, default_save_path=tmpdir diff --git a/tests/test_restore_models.py b/tests/test_restore_models.py index c319bc31..5857741d 100644 --- a/tests/test_restore_models.py +++ b/tests/test_restore_models.py @@ -28,7 +28,7 @@ def test_running_test_pretrained_model_ddp(tmpdir): trainer_options = dict( show_progress_bar=False, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.2, checkpoint_callback=checkpoint, @@ -73,7 +73,7 @@ def test_running_test_pretrained_model(tmpdir): trainer_options = dict( show_progress_bar=False, - max_nb_epochs=4, + max_num_epochs=4, train_percent_check=0.4, val_percent_check=0.2, checkpoint_callback=checkpoint, @@ -106,7 +106,7 @@ def test_load_model_from_checkpoint(tmpdir): trainer_options = dict( show_progress_bar=False, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.2, checkpoint_callback=True, @@ -153,7 +153,7 @@ def test_running_test_pretrained_model_dp(tmpdir): trainer_options = dict( show_progress_bar=True, - max_nb_epochs=1, + max_num_epochs=1, train_percent_check=0.4, val_percent_check=0.2, checkpoint_callback=checkpoint, @@ -191,7 +191,7 @@ def test_dp_resume(tmpdir): trainer_options = dict( show_progress_bar=True, - max_nb_epochs=2, + max_num_epochs=2, gpus=2, distributed_backend='dp', ) @@ -230,7 +230,7 @@ def test_dp_resume(tmpdir): trainer_options['checkpoint_callback'] = ModelCheckpoint(tmpdir) trainer_options['train_percent_check'] = 0.2 trainer_options['val_percent_check'] = 0.2 - trainer_options['max_nb_epochs'] = 1 + trainer_options['max_num_epochs'] = 1 new_trainer = Trainer(**trainer_options) # set the epoch start hook so we can predict before the model does the full training @@ -269,7 +269,7 @@ def test_cpu_restore_training(tmpdir): logger = tutils.get_test_tube_logger(tmpdir, False, version=test_logger_version) trainer_options = dict( - max_nb_epochs=2, + max_num_epochs=2, val_check_interval=0.50, val_percent_check=0.2, train_percent_check=0.2, @@ -290,7 +290,7 @@ def test_cpu_restore_training(tmpdir): # we want to see if the weights come back correctly new_logger = tutils.get_test_tube_logger(tmpdir, False, version=test_logger_version) trainer_options = dict( - max_nb_epochs=2, + max_num_epochs=2, val_check_interval=0.50, val_percent_check=0.2, train_percent_check=0.2, @@ -329,7 +329,7 @@ def test_model_saving_loading(tmpdir): logger = tutils.get_test_tube_logger(tmpdir, False) trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, logger=logger, checkpoint_callback=ModelCheckpoint(tmpdir) ) diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 33324f16..3564c2a7 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -34,7 +34,7 @@ def test_no_val_module(tmpdir): logger = tutils.get_test_tube_logger(tmpdir, False) trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, logger=logger, checkpoint_callback=ModelCheckpoint(tmpdir) ) @@ -72,7 +72,7 @@ def test_no_val_end_module(tmpdir): logger = tutils.get_test_tube_logger(tmpdir, False) trainer_options = dict( - max_nb_epochs=1, + max_num_epochs=1, logger=logger, checkpoint_callback=ModelCheckpoint(tmpdir) ) @@ -114,40 +114,40 @@ def test_gradient_accumulation_scheduling(tmpdir): assert Trainer(accumulate_grad_batches={1: 2.5, 3: 5}) # test optimizer call freq matches scheduler - def optimizer_step(self, epoch_nb, batch_nb, optimizer, optimizer_i, second_order_closure=None): + def optimizer_step(self, epoch_idx, batch_idx, optimizer, optimizer_idx, second_order_closure=None): # only test the first 12 batches in epoch - if batch_nb < 12: - if epoch_nb == 0: + if batch_idx < 12: + if epoch_idx == 0: # reset counter when starting epoch - if batch_nb == 0: - self.prev_called_batch_nb = 0 + if batch_idx == 0: + self.prev_called_batch_idx = 0 # use this opportunity to test once assert self.trainer.accumulate_grad_batches == 1 - assert batch_nb == self.prev_called_batch_nb - self.prev_called_batch_nb += 1 + assert batch_idx == self.prev_called_batch_idx + self.prev_called_batch_idx += 1 - elif 1 <= epoch_nb <= 2: + elif 1 <= epoch_idx <= 2: # reset counter when starting epoch - if batch_nb == 1: - self.prev_called_batch_nb = 1 + if batch_idx == 1: + self.prev_called_batch_idx = 1 # use this opportunity to test once assert self.trainer.accumulate_grad_batches == 2 - assert batch_nb == self.prev_called_batch_nb - self.prev_called_batch_nb += 2 + assert batch_idx == self.prev_called_batch_idx + self.prev_called_batch_idx += 2 else: - if batch_nb == 3: - self.prev_called_batch_nb = 3 + if batch_idx == 3: + self.prev_called_batch_idx = 3 # use this opportunity to test once assert self.trainer.accumulate_grad_batches == 4 - assert batch_nb == self.prev_called_batch_nb - self.prev_called_batch_nb += 3 + assert batch_idx == self.prev_called_batch_idx + self.prev_called_batch_idx += 3 optimizer.step() @@ -161,12 +161,12 @@ def test_gradient_accumulation_scheduling(tmpdir): trainer = Trainer(accumulate_grad_batches=schedule, train_percent_check=0.1, val_percent_check=0.1, - max_nb_epochs=4, + max_num_epochs=4, default_save_path=tmpdir) # for the test trainer.optimizer_step = optimizer_step - model.prev_called_batch_nb = 0 + model.prev_called_batch_idx = 0 trainer.fit(model) @@ -198,10 +198,10 @@ def test_dp_output_reduce(): # test identity when we have a single gpu out = torch.rand(3, 1) - assert mixin.reduce_distributed_output(out, nb_gpus=1) is out + assert mixin.reduce_distributed_output(out, num_gpus=1) is out # average when we have multiples - assert mixin.reduce_distributed_output(out, nb_gpus=2) == out.mean() + assert mixin.reduce_distributed_output(out, num_gpus=2) == out.mean() # when we have a dict of vals out = { @@ -210,7 +210,7 @@ def test_dp_output_reduce(): 'c': out } } - reduced = mixin.reduce_distributed_output(out, nb_gpus=3) + reduced = mixin.reduce_distributed_output(out, num_gpus=3) assert reduced['a'] == out['a'] assert reduced['b']['c'] == out['b']['c'] @@ -354,7 +354,7 @@ def test_multiple_val_dataloader(tmpdir): # logger file to get meta trainer_options = dict( default_save_path=tmpdir, - max_nb_epochs=1, + max_num_epochs=1, val_percent_check=0.1, train_percent_check=1.0, ) @@ -391,7 +391,7 @@ def test_multiple_test_dataloader(tmpdir): # logger file to get meta trainer_options = dict( default_save_path=tmpdir, - max_nb_epochs=1, + max_num_epochs=1, val_percent_check=0.1, train_percent_check=0.1, )