diff --git a/pytorch_lightning/trainer/amp_mixin.py b/pytorch_lightning/trainer/amp_mixin.py index 42251bb3..481ea98f 100644 --- a/pytorch_lightning/trainer/amp_mixin.py +++ b/pytorch_lightning/trainer/amp_mixin.py @@ -1,3 +1,5 @@ +from abc import ABC + try: from apex import amp @@ -7,7 +9,7 @@ except ImportError: import logging -class TrainerAMPMixin(object): +class TrainerAMPMixin(ABC): def init_amp(self, use_amp): self.use_amp = use_amp and APEX_AVAILABLE diff --git a/pytorch_lightning/trainer/callback_config_mixin.py b/pytorch_lightning/trainer/callback_config_mixin.py index 58e6ec05..02db09c3 100644 --- a/pytorch_lightning/trainer/callback_config_mixin.py +++ b/pytorch_lightning/trainer/callback_config_mixin.py @@ -1,10 +1,19 @@ import os +from abc import ABC from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping from pytorch_lightning.logging import TestTubeLogger -class TrainerCallbackConfigMixin(object): +class TrainerCallbackConfigMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.default_save_path = None + self.save_checkpoint = None + self.slurm_job_id = None + def configure_checkpoint_callback(self): """ Weight path set in this priority: diff --git a/pytorch_lightning/trainer/data_loading_mixin.py b/pytorch_lightning/trainer/data_loading_mixin.py index 205a7eac..84413697 100644 --- a/pytorch_lightning/trainer/data_loading_mixin.py +++ b/pytorch_lightning/trainer/data_loading_mixin.py @@ -1,4 +1,5 @@ import warnings +from abc import ABC import torch.distributed as dist try: @@ -24,7 +25,17 @@ except ImportError: APEX_AVAILABLE = False -class TrainerDataLoadingMixin(object): +class TrainerDataLoadingMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.proc_rank = None + self.use_ddp = None + self.use_ddp2 = None + self.shown_warnings = None + self.val_check_interval = None + def init_train_dataloader(self, model): """ Dataloaders are provided by the model @@ -114,10 +125,9 @@ class TrainerDataLoadingMixin(object): break def init_test_dataloader(self, model): - """ - Dataloaders are provided by the model + """Dataloaders are provided by the model. + :param model: - :return: """ self.get_test_dataloaders = model.test_dataloader @@ -134,20 +144,22 @@ class TrainerDataLoadingMixin(object): for dataloader in self.get_test_dataloaders(): if not isinstance(dataloader.sampler, DistributedSampler): msg = """ - Your test_dataloader(s) don't use DistributedSampler. + Your `test_dataloader(s)` don't use DistributedSampler. You're using multiple gpus and multiple nodes without using a DistributedSampler to assign a subset of your data to each process. To silence this warning, pass a DistributedSampler to your DataLoader. - ie: this: - dataset = myDataset() - dataloader = Dataloader(dataset) + ie: this:: - becomes: - dataset = myDataset() - dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - dataloader = Dataloader(dataset, sampler=dist_sampler) + dataset = myDataset() + dataloader = Dataloader(dataset) + + becomes:: + + dataset = myDataset() + dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + dataloader = Dataloader(dataset, sampler=dist_sampler) If you want each process to load the full dataset, ignore this warning. """ @@ -181,8 +193,8 @@ class TrainerDataLoadingMixin(object): EXIST_ITER_DATASET and isinstance(self.get_train_dataloader().dataset, IterableDataset)) if self.is_iterable_train_dataloader and not isinstance(self.val_check_interval, int): m = ''' - When using an iterableDataset for train_dataloader, - Trainer(val_check_interval) must be an int. + When using an iterableDataset for `train_dataloader`, + `Trainer(val_check_interval)` must be an int. An int k specifies checking validation every k training batches ''' raise MisconfigurationException(m) diff --git a/pytorch_lightning/trainer/ddp_mixin.py b/pytorch_lightning/trainer/ddp_mixin.py index b0e75f25..665ec800 100644 --- a/pytorch_lightning/trainer/ddp_mixin.py +++ b/pytorch_lightning/trainer/ddp_mixin.py @@ -117,6 +117,7 @@ import os import re import logging import warnings +from abc import ABC, abstractmethod import torch @@ -130,7 +131,34 @@ except ImportError: APEX_AVAILABLE = False -class TrainerDDPMixin(object): +class TrainerDDPMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.num_gpus = None + self.on_gpu = None + self.num_gpu_nodes = None + self.logger = None + self.data_parallel_device_ids = None + self.distributed_backend = None + self.use_amp = None + self.amp_level = None + + @abstractmethod + def copy_trainer_model_properties(self, model): + # this is just empty shell for code from other class + pass + + @abstractmethod + def run_pretrain_routine(self, model): + # this is just empty shell for code from other class + pass + + @abstractmethod + def init_optimizers(self, optimizers): + # this is just empty shell for code from other class + pass def set_distributed_mode(self, distributed_backend, num_gpu_nodes): # skip for CPU diff --git a/pytorch_lightning/trainer/dp_mixin.py b/pytorch_lightning/trainer/dp_mixin.py index 497d165f..4c1f3cdc 100644 --- a/pytorch_lightning/trainer/dp_mixin.py +++ b/pytorch_lightning/trainer/dp_mixin.py @@ -302,6 +302,8 @@ Here lightning distributes parts of your module across available GPUs to optimiz """ +from abc import ABC, abstractmethod + import torch from pytorch_lightning.overrides.data_parallel import ( @@ -318,7 +320,31 @@ except ImportError: APEX_AVAILABLE = False -class TrainerDPMixin(object): +class TrainerDPMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.on_gpu = None + self.use_dp = None + self.use_ddp2 = None + self.use_ddp = None + self.use_amp = None + self.testing = None + self.single_gpu = None + self.root_gpu = None + self.amp_level = None + + @abstractmethod + def run_pretrain_routine(self, model): + # this is just empty shell for code from other class + pass + + @abstractmethod + def init_optimizers(self, optimizers): + # this is just empty shell for code from other class + pass + def copy_trainer_model_properties(self, model): if isinstance(model, LightningDataParallel): ref_model = model.module diff --git a/pytorch_lightning/trainer/evaluation_loop_mixin.py b/pytorch_lightning/trainer/evaluation_loop_mixin.py index bb59d000..4e507bdd 100644 --- a/pytorch_lightning/trainer/evaluation_loop_mixin.py +++ b/pytorch_lightning/trainer/evaluation_loop_mixin.py @@ -122,6 +122,7 @@ In this second case, the options you pass to trainer will be used when running """ +from abc import ABC, abstractmethod import torch import sys @@ -130,11 +131,67 @@ import tqdm from pytorch_lightning.utilities.debugging import MisconfigurationException -class TrainerEvaluationLoopMixin(object): +class TrainerEvaluationLoopMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.test_progress_bar = None + self.val_progress_bar = None + self.main_progress_bar = None + self.use_ddp = None + self.use_dp = None + self.use_ddp2 = None + self.single_gpu = None + self.data_parallel_device_ids = None + self.model = None + self.nb_test_batches = None + self.nb_val_batches = None + self.fast_dev_run = None + self.process_position = None + self.show_progress_bar = None + self.process_output = None + self.training_tqdm_dict = None + self.proc_rank = None + self.checkpoint_callback = None + self.current_epoch = None + self.callback_metrics = None + self.get_test_dataloaders = None + self.get_val_dataloaders = None + + @abstractmethod + def copy_trainer_model_properties(self, model): + # this is just empty shell for code from other class + pass + + @abstractmethod + def get_model(self): + # this is just empty shell for code from other class + pass + + @abstractmethod + def is_overriden(self, m): + # this is just empty shell for code from other class + pass + + @abstractmethod + def transfer_batch_to_gpu(self, batch, gpu): + # this is just empty shell for code from other class + pass + + @abstractmethod + def add_tqdm_metrics(self, metrics): + # this is just empty shell for code from other class + pass + + @abstractmethod + def log_metrics(self, metrics, grad_norm_dic): + # this is just empty shell for code from other class + pass def evaluate(self, model, dataloaders, max_batches, test=False): - """ - Run evaluation code + """Run evaluation code. + :param model: PT model :param dataloaders: list of PT dataloaders :param max_batches: Scalar @@ -304,7 +361,7 @@ class TrainerEvaluationLoopMixin(object): if self.single_gpu: # for single GPU put inputs on gpu manually root_gpu = 0 - if type(self.data_parallel_device_ids) is list: + if isinstance(self.data_parallel_device_ids, list): root_gpu = self.data_parallel_device_ids[0] batch = self.transfer_batch_to_gpu(batch, root_gpu) args[0] = batch diff --git a/pytorch_lightning/trainer/logging_mixin.py b/pytorch_lightning/trainer/logging_mixin.py index 82376ce4..cd23d217 100644 --- a/pytorch_lightning/trainer/logging_mixin.py +++ b/pytorch_lightning/trainer/logging_mixin.py @@ -1,16 +1,31 @@ +from abc import ABC + import torch from pytorch_lightning.core import memory -class TrainerLoggingMixin(object): +class TrainerLoggingMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.current_epoch = None + self.on_gpu = None + self.log_gpu_memory = None + self.logger = None + self.tqdm_metrics = None + self.global_step = None + self.proc_rank = None + self.use_dp = None + self.use_ddp2 = None + self.num_gpus = None def log_metrics(self, metrics, grad_norm_dic): - """ - Logs the metric dict passed in + """Logs the metric dict passed in. + :param metrics: :param grad_norm_dic: - :return: """ # added metrics by Lightning for convenience metrics['epoch'] = self.current_epoch @@ -52,8 +67,8 @@ class TrainerLoggingMixin(object): return new_metrics def process_output(self, output, train=False): - """ - Reduces output according to the training mode. + """Reduces output according to the training mode. + Separates loss from logging and tqdm metrics :param output: :return: diff --git a/pytorch_lightning/trainer/model_hooks_mixin.py b/pytorch_lightning/trainer/model_hooks_mixin.py index 442c5209..e5afc90a 100644 --- a/pytorch_lightning/trainer/model_hooks_mixin.py +++ b/pytorch_lightning/trainer/model_hooks_mixin.py @@ -1,8 +1,10 @@ import inspect +from abc import ABC, abstractmethod + from pytorch_lightning.core.lightning import LightningModule -class TrainerModelHooksMixin(object): +class TrainerModelHooksMixin(ABC): def is_function_implemented(self, f_name): model = self.get_model() @@ -21,3 +23,8 @@ class TrainerModelHooksMixin(object): model = self.get_model() f_op = getattr(model, f_name, None) return arg_name in inspect.signature(f_op).parameters + + @abstractmethod + def get_model(self): + # this is just empty shell for code from other class + pass diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py index 49cd3d70..b24ff868 100644 --- a/pytorch_lightning/trainer/train_loop_mixin.py +++ b/pytorch_lightning/trainer/train_loop_mixin.py @@ -150,6 +150,8 @@ When this flag is enabled each batch is split into sequences of size truncated_b """ import inspect +from abc import ABC, abstractmethod + import numpy as np from pytorch_lightning.utilities.debugging import MisconfigurationException @@ -162,7 +164,98 @@ except ImportError: APEX_AVAILABLE = False -class TrainerTrainLoopMixin(object): +class TrainerTrainLoopMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.max_nb_epochs = None + self.use_ddp = None + self.use_dp = None + self.use_ddp2 = None + self.single_gpu = None + self.data_parallel_device_ids = None + self.check_val_every_n_epoch = None + self.nb_training_batches = None + self.val_check_batch = None + self.nb_val_batches = None + self.fast_dev_run = None + self.is_iterable_train_dataloader = None + self.main_progress_bar = None + self.accumulation_scheduler = None + self.lr_schedulers = None + self.min_nb_epochs = None + self.enable_early_stop = None + self.early_stop_callback = None + self.callback_metrics = None + self.logger = None + self.global_step = None + self.testing = None + self.log_save_interval = None + self.proc_rank = None + self.row_log_interval = None + self.total_batch_nb = None + self.truncated_bptt_steps = None + self.optimizers = None + self.accumulate_grad_batches = None + self.use_amp = None + self.print_nan_grads = None + self.track_grad_norm = None + self.model = None + self.running_loss = None + self.training_tqdm_dict = None + self.get_train_dataloader = None + self.reduce_lr_on_plateau_scheduler = None + + @abstractmethod + def get_model(self): + # this is just empty shell for code from other class + pass + + @abstractmethod + def is_function_implemented(self, m): + # this is just empty shell for code from other class + pass + + @abstractmethod + def run_evaluation(self, test): + # this is just empty shell for code from other class + pass + + @abstractmethod + def transfer_batch_to_gpu(self, batch, gpu): + # this is just empty shell for code from other class + pass + + @abstractmethod + def clip_gradients(self): + # this is just empty shell for code from other class + pass + + @abstractmethod + def print_nan_gradients(self): + # this is just empty shell for code from other class + pass + + @abstractmethod + def is_overriden(self, m): + # this is just empty shell for code from other class + pass + + @abstractmethod + def add_tqdm_metrics(self, metrics): + # this is just empty shell for code from other class + pass + + @abstractmethod + def log_metrics(self, metrics, grad_norm_dic): + # this is just empty shell for code from other class + pass + + @abstractmethod + def process_output(self, output, train): + # this is just empty shell for code from other class + pass def train(self): # run all epochs @@ -456,7 +549,7 @@ class TrainerTrainLoopMixin(object): # single GPU forward elif self.single_gpu: gpu_id = 0 - if type(self.data_parallel_device_ids) is list: + if isinstance(self.data_parallel_device_ids, list): gpu_id = self.data_parallel_device_ids[0] batch = self.transfer_batch_to_gpu(batch.copy(), gpu_id) args[0] = batch diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 66753b23..319004a4 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -39,16 +39,17 @@ except ImportError: class Trainer(TrainerIOMixin, - TrainerDDPMixin, TrainerDPMixin, + TrainerDDPMixin, + TrainerLoggingMixin, + TrainerModelHooksMixin, + TrainerTrainingTricksMixin, TrainerDataLoadingMixin, TrainerAMPMixin, TrainerEvaluationLoopMixin, TrainerTrainLoopMixin, - TrainerLoggingMixin, - TrainerTrainingTricksMixin, TrainerCallbackConfigMixin, - TrainerModelHooksMixin): + ): def __init__( self, diff --git a/pytorch_lightning/trainer/trainer_io.py b/pytorch_lightning/trainer/trainer_io.py index 4bd32e81..7410a523 100644 --- a/pytorch_lightning/trainer/trainer_io.py +++ b/pytorch_lightning/trainer/trainer_io.py @@ -95,6 +95,7 @@ import signal import warnings from subprocess import call import logging +from abc import ABC import torch import torch.distributed as dist @@ -105,7 +106,24 @@ from pytorch_lightning.overrides.data_parallel import ( ) -class TrainerIOMixin(object): +class TrainerIOMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.model = None + self.on_gpu = None + self.root_gpu = None + self.resume_from_checkpoint = None + self.use_ddp = None + self.use_ddp2 = None + self.checkpoint_callback = None + self.proc_rank = None + self.weights_save_path = None + self.logger = None + self.early_stop_callback = None + self.lr_schedulers = None + self.optimizers = None def get_model(self): is_dp_module = isinstance(self.model, (LightningDistributedDataParallel, diff --git a/pytorch_lightning/trainer/training_tricks_mixin.py b/pytorch_lightning/trainer/training_tricks_mixin.py index 1cdef291..c9c31bb8 100644 --- a/pytorch_lightning/trainer/training_tricks_mixin.py +++ b/pytorch_lightning/trainer/training_tricks_mixin.py @@ -1,9 +1,21 @@ +from abc import ABC, abstractmethod + import torch import logging from pytorch_lightning.callbacks import GradientAccumulationScheduler -class TrainerTrainingTricksMixin(object): +class TrainerTrainingTricksMixin(ABC): + + def __init__(self): + # this is just a summary on variables used in this abstract class, + # the proper values/initialisation should be done in child class + self.gradient_clip_val = None + + @abstractmethod + def get_model(self): + # this is just empty shell for code from other class + pass def clip_gradients(self): if self.gradient_clip_val > 0: