Abstract Mixin classes (#572)

* make partial Trainer classes as abstract

* add empty attributes/methods

* flake8

* fix mixin order

* update abstact

* reorder

pytorch_lightning/trainer/amp_mixin.py

@@ -1,3 +1,5 @@
+from abc import ABC
+
 try:
     from apex import amp
 
@@ -7,7 +9,7 @@ except ImportError:
 import logging
 
 
-class TrainerAMPMixin(object):
+class TrainerAMPMixin(ABC):
 
     def init_amp(self, use_amp):
         self.use_amp = use_amp and APEX_AVAILABLE

pytorch_lightning/trainer/callback_config_mixin.py

@@ -1,10 +1,19 @@
 import os
+from abc import ABC
 
 from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping
 from pytorch_lightning.logging import TestTubeLogger
 
 
-class TrainerCallbackConfigMixin(object):
+class TrainerCallbackConfigMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.default_save_path = None
+        self.save_checkpoint = None
+        self.slurm_job_id = None
+
     def configure_checkpoint_callback(self):
         """
         Weight path set in this priority:

pytorch_lightning/trainer/data_loading_mixin.py

@@ -1,4 +1,5 @@
 import warnings
+from abc import ABC
 
 import torch.distributed as dist
 try:
@@ -24,7 +25,17 @@ except ImportError:
     APEX_AVAILABLE = False
 
 
-class TrainerDataLoadingMixin(object):
+class TrainerDataLoadingMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.proc_rank = None
+        self.use_ddp = None
+        self.use_ddp2 = None
+        self.shown_warnings = None
+        self.val_check_interval = None
+
     def init_train_dataloader(self, model):
         """
         Dataloaders are provided by the model
@@ -114,10 +125,9 @@ class TrainerDataLoadingMixin(object):
                     break
 
     def init_test_dataloader(self, model):
-        """
-        Dataloaders are provided by the model
+        """Dataloaders are provided by the model.
+
         :param model:
-        :return:
         """
 
         self.get_test_dataloaders = model.test_dataloader
@@ -134,20 +144,22 @@ class TrainerDataLoadingMixin(object):
             for dataloader in self.get_test_dataloaders():
                 if not isinstance(dataloader.sampler, DistributedSampler):
                     msg = """
-                    Your test_dataloader(s) don't use DistributedSampler.
+                    Your `test_dataloader(s)` don't use DistributedSampler.
 
                     You're using multiple gpus and multiple nodes without using a
                     DistributedSampler to assign a subset of your data to each process.
                     To silence this warning, pass a DistributedSampler to your DataLoader.
 
-                    ie: this:
-                    dataset = myDataset()
-                    dataloader = Dataloader(dataset)
+                    ie: this::
 
-                    becomes:
-                    dataset = myDataset()
-                    dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
-                    dataloader = Dataloader(dataset, sampler=dist_sampler)
+                        dataset = myDataset()
+                        dataloader = Dataloader(dataset)
+
+                    becomes::
+
+                        dataset = myDataset()
+                        dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
+                        dataloader = Dataloader(dataset, sampler=dist_sampler)
 
                     If you want each process to load the full dataset, ignore this warning.
                     """
@@ -181,8 +193,8 @@ class TrainerDataLoadingMixin(object):
             EXIST_ITER_DATASET and isinstance(self.get_train_dataloader().dataset, IterableDataset))
         if self.is_iterable_train_dataloader and not isinstance(self.val_check_interval, int):
             m = '''
-            When using an iterableDataset for train_dataloader,
-            Trainer(val_check_interval) must be an int.
+            When using an iterableDataset for `train_dataloader`,
+            `Trainer(val_check_interval)` must be an int.
             An int k specifies checking validation every k training batches
             '''
             raise MisconfigurationException(m)

pytorch_lightning/trainer/ddp_mixin.py

@@ -117,6 +117,7 @@ import os
 import re
 import logging
 import warnings
+from abc import ABC, abstractmethod
 
 import torch
 
@@ -130,7 +131,34 @@ except ImportError:
     APEX_AVAILABLE = False
 
 
-class TrainerDDPMixin(object):
+class TrainerDDPMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.num_gpus = None
+        self.on_gpu = None
+        self.num_gpu_nodes = None
+        self.logger = None
+        self.data_parallel_device_ids = None
+        self.distributed_backend = None
+        self.use_amp = None
+        self.amp_level = None
+
+    @abstractmethod
+    def copy_trainer_model_properties(self, model):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def run_pretrain_routine(self, model):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def init_optimizers(self, optimizers):
+        # this is just empty shell for code from other class
+        pass
 
     def set_distributed_mode(self, distributed_backend, num_gpu_nodes):
         # skip for CPU

pytorch_lightning/trainer/dp_mixin.py

@@ -302,6 +302,8 @@ Here lightning distributes parts of your module across available GPUs to optimiz
 
 """
 
+from abc import ABC, abstractmethod
+
 import torch
 
 from pytorch_lightning.overrides.data_parallel import (
@@ -318,7 +320,31 @@ except ImportError:
     APEX_AVAILABLE = False
 
 
-class TrainerDPMixin(object):
+class TrainerDPMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.on_gpu = None
+        self.use_dp = None
+        self.use_ddp2 = None
+        self.use_ddp = None
+        self.use_amp = None
+        self.testing = None
+        self.single_gpu = None
+        self.root_gpu = None
+        self.amp_level = None
+
+    @abstractmethod
+    def run_pretrain_routine(self, model):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def init_optimizers(self, optimizers):
+        # this is just empty shell for code from other class
+        pass
+
     def copy_trainer_model_properties(self, model):
         if isinstance(model, LightningDataParallel):
             ref_model = model.module

pytorch_lightning/trainer/evaluation_loop_mixin.py

@@ -122,6 +122,7 @@ In this second case, the options you pass to trainer will be used when running
 
 """
 
+from abc import ABC, abstractmethod
 
 import torch
 import sys
@@ -130,11 +131,67 @@ import tqdm
 from pytorch_lightning.utilities.debugging import MisconfigurationException
 
 
-class TrainerEvaluationLoopMixin(object):
+class TrainerEvaluationLoopMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.test_progress_bar = None
+        self.val_progress_bar = None
+        self.main_progress_bar = None
+        self.use_ddp = None
+        self.use_dp = None
+        self.use_ddp2 = None
+        self.single_gpu = None
+        self.data_parallel_device_ids = None
+        self.model = None
+        self.nb_test_batches = None
+        self.nb_val_batches = None
+        self.fast_dev_run = None
+        self.process_position = None
+        self.show_progress_bar = None
+        self.process_output = None
+        self.training_tqdm_dict = None
+        self.proc_rank = None
+        self.checkpoint_callback = None
+        self.current_epoch = None
+        self.callback_metrics = None
+        self.get_test_dataloaders = None
+        self.get_val_dataloaders = None
+
+    @abstractmethod
+    def copy_trainer_model_properties(self, model):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def get_model(self):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def is_overriden(self, m):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def transfer_batch_to_gpu(self, batch, gpu):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def add_tqdm_metrics(self, metrics):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def log_metrics(self, metrics, grad_norm_dic):
+        # this is just empty shell for code from other class
+        pass
 
     def evaluate(self, model, dataloaders, max_batches, test=False):
-        """
-        Run evaluation code
+        """Run evaluation code.
+
         :param model: PT model
         :param dataloaders: list of PT dataloaders
         :param max_batches: Scalar
@@ -304,7 +361,7 @@ class TrainerEvaluationLoopMixin(object):
         if self.single_gpu:
             # for single GPU put inputs on gpu manually
             root_gpu = 0
-            if type(self.data_parallel_device_ids) is list:
+            if isinstance(self.data_parallel_device_ids, list):
                 root_gpu = self.data_parallel_device_ids[0]
             batch = self.transfer_batch_to_gpu(batch, root_gpu)
             args[0] = batch

pytorch_lightning/trainer/logging_mixin.py

@@ -1,16 +1,31 @@
+from abc import ABC
+
 import torch
 
 from pytorch_lightning.core import memory
 
 
-class TrainerLoggingMixin(object):
+class TrainerLoggingMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.current_epoch = None
+        self.on_gpu = None
+        self.log_gpu_memory = None
+        self.logger = None
+        self.tqdm_metrics = None
+        self.global_step = None
+        self.proc_rank = None
+        self.use_dp = None
+        self.use_ddp2 = None
+        self.num_gpus = None
 
     def log_metrics(self, metrics, grad_norm_dic):
-        """
-        Logs the metric dict passed in
+        """Logs the metric dict passed in.
+
         :param metrics:
         :param grad_norm_dic:
-        :return:
         """
         # added metrics by Lightning for convenience
         metrics['epoch'] = self.current_epoch
@@ -52,8 +67,8 @@ class TrainerLoggingMixin(object):
         return new_metrics
 
     def process_output(self, output, train=False):
-        """
-        Reduces output according to the training mode.
+        """Reduces output according to the training mode.
+
         Separates loss from logging and tqdm metrics
         :param output:
         :return:

pytorch_lightning/trainer/model_hooks_mixin.py

@@ -1,8 +1,10 @@
 import inspect
+from abc import ABC, abstractmethod
+
 from pytorch_lightning.core.lightning import LightningModule
 
 
-class TrainerModelHooksMixin(object):
+class TrainerModelHooksMixin(ABC):
 
     def is_function_implemented(self, f_name):
         model = self.get_model()
@@ -21,3 +23,8 @@ class TrainerModelHooksMixin(object):
         model = self.get_model()
         f_op = getattr(model, f_name, None)
         return arg_name in inspect.signature(f_op).parameters
+
+    @abstractmethod
+    def get_model(self):
+        # this is just empty shell for code from other class
+        pass

pytorch_lightning/trainer/train_loop_mixin.py

@@ -150,6 +150,8 @@ When this flag is enabled each batch is split into sequences of size truncated_b
 """
 
 import inspect
+from abc import ABC, abstractmethod
+
 import numpy as np
 
 from pytorch_lightning.utilities.debugging import MisconfigurationException
@@ -162,7 +164,98 @@ except ImportError:
     APEX_AVAILABLE = False
 
 
-class TrainerTrainLoopMixin(object):
+class TrainerTrainLoopMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.max_nb_epochs = None
+        self.use_ddp = None
+        self.use_dp = None
+        self.use_ddp2 = None
+        self.single_gpu = None
+        self.data_parallel_device_ids = None
+        self.check_val_every_n_epoch = None
+        self.nb_training_batches = None
+        self.val_check_batch = None
+        self.nb_val_batches = None
+        self.fast_dev_run = None
+        self.is_iterable_train_dataloader = None
+        self.main_progress_bar = None
+        self.accumulation_scheduler = None
+        self.lr_schedulers = None
+        self.min_nb_epochs = None
+        self.enable_early_stop = None
+        self.early_stop_callback = None
+        self.callback_metrics = None
+        self.logger = None
+        self.global_step = None
+        self.testing = None
+        self.log_save_interval = None
+        self.proc_rank = None
+        self.row_log_interval = None
+        self.total_batch_nb = None
+        self.truncated_bptt_steps = None
+        self.optimizers = None
+        self.accumulate_grad_batches = None
+        self.use_amp = None
+        self.print_nan_grads = None
+        self.track_grad_norm = None
+        self.model = None
+        self.running_loss = None
+        self.training_tqdm_dict = None
+        self.get_train_dataloader = None
+        self.reduce_lr_on_plateau_scheduler = None
+
+    @abstractmethod
+    def get_model(self):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def is_function_implemented(self, m):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def run_evaluation(self, test):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def transfer_batch_to_gpu(self, batch, gpu):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def clip_gradients(self):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def print_nan_gradients(self):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def is_overriden(self, m):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def add_tqdm_metrics(self, metrics):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def log_metrics(self, metrics, grad_norm_dic):
+        # this is just empty shell for code from other class
+        pass
+
+    @abstractmethod
+    def process_output(self, output, train):
+        # this is just empty shell for code from other class
+        pass
 
     def train(self):
         # run all epochs
@@ -456,7 +549,7 @@ class TrainerTrainLoopMixin(object):
         # single GPU forward
         elif self.single_gpu:
             gpu_id = 0
-            if type(self.data_parallel_device_ids) is list:
+            if isinstance(self.data_parallel_device_ids, list):
                 gpu_id = self.data_parallel_device_ids[0]
             batch = self.transfer_batch_to_gpu(batch.copy(), gpu_id)
             args[0] = batch

pytorch_lightning/trainer/trainer.py

@@ -39,16 +39,17 @@ except ImportError:
 
 
 class Trainer(TrainerIOMixin,
-              TrainerDDPMixin,
               TrainerDPMixin,
+              TrainerDDPMixin,
+              TrainerLoggingMixin,
+              TrainerModelHooksMixin,
+              TrainerTrainingTricksMixin,
               TrainerDataLoadingMixin,
               TrainerAMPMixin,
               TrainerEvaluationLoopMixin,
               TrainerTrainLoopMixin,
-              TrainerLoggingMixin,
-              TrainerTrainingTricksMixin,
               TrainerCallbackConfigMixin,
-              TrainerModelHooksMixin):
+              ):
 
     def __init__(
             self,

pytorch_lightning/trainer/trainer_io.py

@@ -95,6 +95,7 @@ import signal
 import warnings
 from subprocess import call
 import logging
+from abc import ABC
 
 import torch
 import torch.distributed as dist
@@ -105,7 +106,24 @@ from pytorch_lightning.overrides.data_parallel import (
 )
 
 
-class TrainerIOMixin(object):
+class TrainerIOMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.model = None
+        self.on_gpu = None
+        self.root_gpu = None
+        self.resume_from_checkpoint = None
+        self.use_ddp = None
+        self.use_ddp2 = None
+        self.checkpoint_callback = None
+        self.proc_rank = None
+        self.weights_save_path = None
+        self.logger = None
+        self.early_stop_callback = None
+        self.lr_schedulers = None
+        self.optimizers = None
 
     def get_model(self):
         is_dp_module = isinstance(self.model, (LightningDistributedDataParallel,

pytorch_lightning/trainer/training_tricks_mixin.py

@@ -1,9 +1,21 @@
+from abc import ABC, abstractmethod
+
 import torch
 import logging
 from pytorch_lightning.callbacks import GradientAccumulationScheduler
 
 
-class TrainerTrainingTricksMixin(object):
+class TrainerTrainingTricksMixin(ABC):
+
+    def __init__(self):
+        # this is just a summary on variables used in this abstract class,
+        #  the proper values/initialisation should be done in child class
+        self.gradient_clip_val = None
+
+    @abstractmethod
+    def get_model(self):
+        # this is just empty shell for code from other class
+        pass
 
     def clip_gradients(self):
         if self.gradient_clip_val > 0: