Option to provide seed to random generators to ensure reproducibility (#1572)

* Option to provide seed to random generators to ensure reproducibility

I added small function in utilities which imports torch, numpy, python
random and sets seed for all of the libraries to ensure reproducibility
of results.

* Apply recommendations from core contributors on seeding

1. Moved the seeding code to another file
2. Make deterministic as a parameter for trainer class
3. Add assertions for seeding numpy
4. Added warnings
5. torch.manual_seed should be enough for seeding torch

* Revert "Apply recommendations from core contributors on seeding"

This reverts commit a213c8e6882eec8a9e7408b9418926d2db7c5461.

* Revert "Revert "Apply recommendations from core contributors on seeding""

This reverts commit 59b2da53c62878de7aab0aa3feb3115e105eea06.

* Change in test, for correct seeding

* Allow seed equal to 0

* Allow seed to be uint32.max

* Added deterministic to benchmarks

* Cuda manual seed as in benchmark seeding

* Seeding should be done before model initialization

* cuda manual_seed is not necessary

* Fixing seed test_cpu_lbfgs

On some seeds seems like lbfgs doesn't converge.
So I fixed the seed during testing.

* rebasing issue with old reproducibility.py

* Improved documentation and ability to seed before initializing Train
class

* Change in docs

* Removed seed from trainer, update for documentation

* Typo in the docs

* Added seed_everything to _all_

* Fixing old changes

* Model initialization should be earlier then Trainer

* Update pytorch_lightning/trainer/__init__.py

From Example to testcode

Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>

* Fixing according to the contributors suggestions

* Moving horovod deterministic to Trainer class

* deterministic flag affects horovod docs update

* Improved static typing

* Added deterministic to test runners of horovod

It is failing on some versions, not very predictable

* static seeds for horovod tests

* Change for reset_seed function in tests

* Seeding horovod using reset_seed from tutils

* Update pytorch_lightning/trainer/__init__.py

* chlog

* Update trainer.py

* change "testcode" to "Example" in trainer init documentation

* Update pytorch_lightning/trainer/seed.py, first line in comment

Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>

Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>
Co-authored-by: Jirka <jirka.borovec@seznam.cz>
Co-authored-by: William Falcon <waf2107@columbia.edu>

CHANGELOG.md

@@ -20,6 +20,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - Enable `NeptuneLogger` to work with `distributed_backend=ddp` ([#1753](https://github.com/PyTorchLightning/pytorch-lightning/pull/1753))
 
+- Added option to provide seed to random generators to ensure reproducibility ([#1572](https://github.com/PyTorchLightning/pytorch-lightning/pull/1572))
+
 ### Changed
 
 - Reduction when `batch_size < num_gpus` ([#1609](https://github.com/PyTorchLightning/pytorch-lightning/pull/1609))

benchmarks/test_rnn_parity.py

@@ -8,7 +8,7 @@ import torch.nn.functional as F
 from torch.utils.data import Dataset, DataLoader
 import tests.base.utils as tutils
 
-from pytorch_lightning import Trainer, LightningModule
+from pytorch_lightning import Trainer, LightningModule, seed_everything
 
 
 class AverageDataset(Dataset):
@@ -68,13 +68,6 @@ def test_pytorch_parity(tmpdir):
     tutils.assert_speed_parity(pl_times, pt_times, num_epochs)
 
 
-def set_seed(seed):
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed(seed)
-
-
 def vanilla_loop(MODEL, num_runs=10, num_epochs=10):
     """
     Returns an array with the last loss from each epoch for each run
@@ -83,12 +76,13 @@ def vanilla_loop(MODEL, num_runs=10, num_epochs=10):
     errors = []
     times = []
 
+    torch.backends.cudnn.deterministic = True
     for i in range(num_runs):
         time_start = time.perf_counter()
 
         # set seed
         seed = i
-        set_seed(seed)
+        seed_everything(seed)
 
         # init model parts
         model = MODEL()
@@ -134,10 +128,10 @@ def lightning_loop(MODEL, num_runs=10, num_epochs=10):
 
         # set seed
         seed = i
-        set_seed(seed)
+        seed_everything(seed)
+        model = MODEL()
 
         # init model parts
-        model = MODEL()
         trainer = Trainer(
             max_epochs=num_epochs,
             progress_bar_refresh_rate=0,
@@ -146,6 +140,7 @@ def lightning_loop(MODEL, num_runs=10, num_epochs=10):
             early_stop_callback=False,
             checkpoint_callback=False,
             distributed_backend='dp',
+            deterministic=True,
         )
         trainer.fit(model)
 

benchmarks/test_trainer_parity.py

@@ -10,7 +10,7 @@ from torch.utils.data import DataLoader
 from torchvision import transforms
 import tests.base.utils as tutils
 
-from pytorch_lightning import Trainer, LightningModule
+from pytorch_lightning import Trainer, LightningModule, seed_everything
 from tests.base.datasets import TrialMNIST
 
 
@@ -69,13 +69,6 @@ def test_pytorch_parity(tmpdir):
     tutils.assert_speed_parity(pl_times[1:], pt_times[1:], num_epochs)
 
 
-def _set_seed(seed):
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed(seed)
-
-
 def vanilla_loop(MODEL, num_runs=10, num_epochs=10):
     """
     Returns an array with the last loss from each epoch for each run
@@ -84,12 +77,13 @@ def vanilla_loop(MODEL, num_runs=10, num_epochs=10):
     errors = []
     times = []
 
+    torch.backends.cudnn.deterministic = True
     for i in range(num_runs):
         time_start = time.perf_counter()
 
         # set seed
         seed = i
-        _set_seed(seed)
+        seed_everything(seed)
 
         # init model parts
         model = MODEL()
@@ -135,17 +129,18 @@ def lightning_loop(MODEL, num_runs=10, num_epochs=10):
 
         # set seed
         seed = i
-        _set_seed(seed)
+        seed_everything(seed)
 
-        # init model parts
         model = MODEL()
+        # init model parts
         trainer = Trainer(
             max_epochs=num_epochs,
             progress_bar_refresh_rate=0,
             weights_summary=None,
             gpus=1,
             early_stop_callback=False,
-            checkpoint_callback=False
+            checkpoint_callback=False,
+            deterministic=True,
         )
         trainer.fit(model)
 

pytorch_lightning/__init__.py

@@ -52,6 +52,7 @@ if __LIGHTNING_SETUP__:
 else:
     from pytorch_lightning.core import LightningModule
     from pytorch_lightning.trainer import Trainer
+    from pytorch_lightning.trainer.seed import seed_everything
     from pytorch_lightning.callbacks import Callback
     from pytorch_lightning.core import data_loader
 
@@ -60,6 +61,7 @@ else:
         'LightningModule',
         'Callback',
         'data_loader'
+        'seed_everything'
     ]
 
     # necessary for regular bolts imports. Skip exception since bolts is not always installed

pytorch_lightning/trainer/__init__.py

@@ -101,6 +101,24 @@ Use it to do whatever!
     out = pretrained_model(x)
     api_write({'response': out}
 
+------------
+
+Reproducibility
+---------------
+
+To ensure full reproducibility from run to run you need to set seeds for pseudo-random generators,
+and set ``deterministic``` flag in ``Trainer``.
+
+.. code-block:: python
+
+    from pytorch-lightning import Trainer, seed_everything
+
+    seed_everything(42)
+    # sets seeds for numpy, torch, python.random and PYTHONHASHSEED.
+    model = Model()
+    trainer = Trainer(deterministic=True)
+
+
 -------
 
 Trainer flags
@@ -186,6 +204,21 @@ Example::
     # default used by the Trainer
     trainer = Trainer(benchmark=False)
 
+deterministic
+^^^^^^^^^^^^^
+
+If true enables cudnn.deterministic.
+Might make your system slower, but ensures reproducibility.
+Also sets ``$HOROVOD_FUSION_THRESHOLD=0``.
+
+For more info check `[pytorch docs]
+<https://pytorch.org/docs/stable/notes/randomness.html>`_.
+
+Example::
+
+    # default used by the Trainer
+    trainer = Trainer(deterministic=False)
+
 callbacks
 ^^^^^^^^^
 
@@ -980,5 +1013,6 @@ Trainer class
 """
 
 from pytorch_lightning.trainer.trainer import Trainer
+from pytorch_lightning.trainer.seed import seed_everything
 
-__all__ = ['Trainer']
+__all__ = ['Trainer', 'seed_everything']

pytorch_lightning/trainer/seed.py

@@ -0,0 +1,42 @@
+"""Helper functions to help with reproducibility of models. """
+
+import os
+from typing import Optional
+
+import numpy as np
+import random
+import torch
+
+from pytorch_lightning import _logger as log
+
+
+def seed_everything(seed: Optional[int] = None) -> int:
+    """Function that sets seed for pseudo-random number generators  in:
+        pytorch, numpy, python.random and sets PYTHONHASHSEED environment variable.
+    """
+    max_seed_value = np.iinfo(np.uint32).max
+    min_seed_value = np.iinfo(np.uint32).min
+
+    try:
+        seed = int(seed)
+    except (TypeError, ValueError):
+        seed = _select_seed_randomly(min_seed_value, max_seed_value)
+
+    if (seed > max_seed_value) or (seed < min_seed_value):
+        log.warning(
+            f"{seed} is not in bounds, \
+            numpy accepts from {min_seed_value} to {max_seed_value}"
+        )
+        seed = _select_seed_randomly(min_seed_value, max_seed_value)
+
+    os.environ["PYTHONHASHSEED"] = str(seed)
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    return seed
+
+
+def _select_seed_randomly(min_seed_value: int = 0, max_seed_value: int = 255) -> int:
+    seed = random.randint(min_seed_value, max_seed_value)
+    log.warning(f"No correct seed found, seed set to {seed}")
+    return seed

pytorch_lightning/trainer/trainer.py

@@ -14,6 +14,7 @@ from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, Callback
 from pytorch_lightning.core.lightning import LightningModule
 from pytorch_lightning.loggers import LightningLoggerBase
 from pytorch_lightning.profiler import SimpleProfiler, PassThroughProfiler, BaseProfiler
+from pytorch_lightning.trainer.seed import seed_everything
 from pytorch_lightning.trainer.auto_mix_precision import TrainerAMPMixin
 from pytorch_lightning.trainer.callback_config import TrainerCallbackConfigMixin
 from pytorch_lightning.trainer.callback_hook import TrainerCallbackHookMixin
@@ -32,8 +33,7 @@ from pytorch_lightning.trainer.training_loop import TrainerTrainLoopMixin
 from pytorch_lightning.trainer.training_tricks import TrainerTrainingTricksMixin
 from pytorch_lightning.trainer.lr_finder import TrainerLRFinderMixin
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
-from pytorch_lightning.utilities import rank_zero_warn
-from pytorch_lightning.utilities import parsing
+from pytorch_lightning.utilities import rank_zero_warn, parsing
 
 
 try:
@@ -126,10 +126,12 @@ class Trainer(
             resume_from_checkpoint: Optional[str] = None,
             profiler: Optional[BaseProfiler] = None,
             benchmark: bool = False,
+            deterministic: bool = False,
             reload_dataloaders_every_epoch: bool = False,
             auto_lr_find: Union[bool, str] = False,
             replace_sampler_ddp: bool = True,
             progress_bar_callback: Optional[Union[ProgressBarBase, bool]] = True,
+            terminate_on_nan: bool = False,
             auto_scale_batch_size: Optional[str] = None,
             amp_level: str = 'O1',  # backward compatible, todo: remove in v0.8.0
             default_save_path=None,  # backward compatible, todo: remove in v0.8.0
@@ -140,7 +142,6 @@ class Trainer(
             use_amp=None,  # backward compatible, todo: remove in v0.9.0
             show_progress_bar=None,  # backward compatible, todo: remove in v0.9.0
             nb_sanity_val_steps=None,  # backward compatible, todo: remove in v0.8.0
-            terminate_on_nan: bool = False,
             **kwargs
     ):
         r"""
@@ -293,6 +294,8 @@ class Trainer(
 
             benchmark: If true enables cudnn.benchmark.
 
+            deterministic: If true enables cudnn.deterministic
+
             terminate_on_nan: If set to True, will terminate training (by raising a `ValueError`) at the
                 end of each training batch, if any of the parameters or the loss are NaN or +/-inf.
 
@@ -303,6 +306,13 @@ class Trainer(
                 a power search or `binsearch` that estimates the batch size through a binary search.
         """
 
+        self.deterministic = deterministic
+        torch.backends.cudnn.deterministic = self.deterministic
+        if self.deterministic:
+            # fixing non-deterministic part of horovod
+            # https://github.com/PyTorchLightning/pytorch-lightning/pull/1572/files#r420279383
+            os.environ["HOROVOD_FUSION_THRESHOLD"] = str(0)
+
         # Init callbacks
         self.callbacks = callbacks or []
         self.on_init_start()

tests/base/utils.py

@@ -5,7 +5,7 @@ import numpy as np
 import torch
 
 # from pl_examples import LightningTemplateModel
-from pytorch_lightning import Trainer
+from pytorch_lightning import Trainer, seed_everything
 from pytorch_lightning.callbacks import ModelCheckpoint
 from pytorch_lightning.loggers import TensorBoardLogger
 from tests import TEMP_PATH, RANDOM_PORTS, RANDOM_SEEDS
@@ -188,8 +188,7 @@ def assert_ok_model_acc(trainer, key='test_acc', thr=0.5):
 
 def reset_seed():
     seed = RANDOM_SEEDS.pop()
-    torch.manual_seed(seed)
-    np.random.seed(seed)
+    seed_everything(seed)
 
 
 def set_random_master_port():

tests/models/test_horovod.py

@@ -39,6 +39,7 @@ def _nccl_available():
 
 def _run_horovod(trainer_options, on_gpu=False):
     """Execute the training script across multiple workers in parallel."""
+    tutils.reset_seed()
     cmdline = [
         'horovodrun',
         '-np', '2',
@@ -62,7 +63,8 @@ def test_horovod_cpu(tmpdir):
         max_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.2,
-        distributed_backend='horovod'
+        distributed_backend='horovod',
+        deterministic=True,
     )
     _run_horovod(trainer_options)
 
@@ -78,6 +80,7 @@ def test_horovod_cpu_implicit(tmpdir):
         max_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.2,
+        deterministic=True,
     )
     _run_horovod(trainer_options)
 
@@ -96,6 +99,7 @@ def test_horovod_multi_gpu(tmpdir):
         train_percent_check=0.4,
         val_percent_check=0.2,
         gpus=1,
+        deterministic=True,
         distributed_backend='horovod'
     )
     _run_horovod(trainer_options, on_gpu=True)
@@ -130,6 +134,7 @@ def test_horovod_transfer_batch_to_gpu(tmpdir):
         train_percent_check=0.4,
         val_percent_check=0.2,
         gpus=1,
+        deterministic=True,
         distributed_backend='horovod'
     )
     tutils.run_model_test_without_loggers(trainer_options, model)
@@ -147,6 +152,7 @@ def test_horovod_multi_optimizer(tmpdir):
         max_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.2,
+        deterministic=True,
         distributed_backend='horovod'
     )