diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83bfbef4..eb554aaa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
+## [0.7.3] - 2020-04-09
+
+### Added
+
+- Added `rank_zero_warn` for warning only in rank 0 ([#1428](https://github.com/PyTorchLightning/pytorch-lightning/pull/1428))
+
+### Fixed
+
+- Fixed default `DistributedSampler` for DDP training ([#1425](https://github.com/PyTorchLightning/pytorch-lightning/pull/1425))
+- Fixed workers warning not on windows ([#1430](https://github.com/PyTorchLightning/pytorch-lightning/pull/1430))
+
 ## [0.7.2] - 2020-04-07
 
 ### Added
diff --git a/pytorch_lightning/__init__.py b/pytorch_lightning/__init__.py
index 5355ee36..73f3d11f 100644
--- a/pytorch_lightning/__init__.py
+++ b/pytorch_lightning/__init__.py
@@ -1,6 +1,6 @@
 """Root package info."""
 
-__version__ = '0.7.2'
+__version__ = '0.7.3rc1'
 __author__ = 'William Falcon et al.'
 __author_email__ = 'waf2107@columbia.edu'
 __license__ = 'Apache-2.0'
diff --git a/pytorch_lightning/callbacks/early_stopping.py b/pytorch_lightning/callbacks/early_stopping.py
index f477cd72..70e90a70 100644
--- a/pytorch_lightning/callbacks/early_stopping.py
+++ b/pytorch_lightning/callbacks/early_stopping.py
@@ -6,12 +6,11 @@ Stop training when a monitored quantity has stopped improving.
 
 """
 
-import warnings
-
 import numpy as np
 
 from pytorch_lightning import _logger as log
 from pytorch_lightning.callbacks.base import Callback
+from pytorch_lightning.utilities import rank_zero_warn
 
 
 class EarlyStopping(Callback):
@@ -80,7 +79,7 @@ class EarlyStopping(Callback):
             if self.strict:
                 raise RuntimeError(error_msg)
             if self.verbose > 0:
-                warnings.warn(error_msg, RuntimeWarning)
+                rank_zero_warn(error_msg, RuntimeWarning)
 
             return False
 
@@ -113,6 +112,6 @@ class EarlyStopping(Callback):
 
     def on_train_end(self, trainer, pl_module):
         if self.stopped_epoch > 0 and self.verbose > 0:
-            warnings.warn('Displayed epoch numbers by `EarlyStopping` start from "1" until v0.6.x,'
-                          ' but will start from "0" in v0.8.0.', DeprecationWarning)
+            rank_zero_warn('Displayed epoch numbers by `EarlyStopping` start from "1" until v0.6.x,'
+                           ' but will start from "0" in v0.8.0.', DeprecationWarning)
             log.info(f'Epoch {self.stopped_epoch + 1:05d}: early stopping')
diff --git a/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py b/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py
index b0563f46..0d2fa63b 100644
--- a/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py
+++ b/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py
@@ -6,9 +6,8 @@ Change gradient accumulation factor according to scheduling.
 
 """
 
-import warnings
-
 from pytorch_lightning.callbacks.base import Callback
+from pytorch_lightning.utilities import rank_zero_warn
 
 
 class GradientAccumulationScheduler(Callback):
@@ -46,8 +45,8 @@ class GradientAccumulationScheduler(Callback):
                 raise TypeError("All epoches and accumulation factor must be integers")
 
         minimal_epoch = min(scheduling.keys())
-        warnings.warn('Epochs indexing of `scheduling` starts from "1" until v0.6.x,'
-                      ' but will start from "0" in v0.8.0.', DeprecationWarning)
+        rank_zero_warn('Epochs indexing of `scheduling` starts from "1" until v0.6.x,'
+                       ' but will start from "0" in v0.8.0.', DeprecationWarning)
         if minimal_epoch < 1:
             msg = f"Epochs indexing from 1, epoch {minimal_epoch} cannot be interpreted correct"
             raise IndexError(msg)
diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py
index 5a2fbb1c..14f420ee 100644
--- a/pytorch_lightning/callbacks/model_checkpoint.py
+++ b/pytorch_lightning/callbacks/model_checkpoint.py
@@ -7,14 +7,13 @@ Automatically save model checkpoints during training.
 """
 
 import os
-import shutil
-import warnings
 import re
 
 import numpy as np
 
-from pytorch_lightning.callbacks.base import Callback
 from pytorch_lightning import _logger as log
+from pytorch_lightning.callbacks.base import Callback
+from pytorch_lightning.utilities import rank_zero_warn
 
 
 class ModelCheckpoint(Callback):
@@ -83,7 +82,7 @@ class ModelCheckpoint(Callback):
                  mode: str = 'auto', period: int = 1, prefix: str = ''):
         super().__init__()
         if save_top_k > 0 and os.path.isdir(filepath) and len(os.listdir(filepath)) > 0:
-            warnings.warn(
+            rank_zero_warn(
                 f"Checkpoint directory {filepath} exists and is not empty with save_top_k != 0."
                 "All files in this directory will be deleted when a checkpoint is saved!"
             )
@@ -115,9 +114,7 @@ class ModelCheckpoint(Callback):
         }
 
         if mode not in mode_dict:
-            warnings.warn(
-                f'ModelCheckpoint mode {mode} is unknown, '
-                'fallback to auto mode.', RuntimeWarning)
+            rank_zero_warn(f'ModelCheckpoint mode {mode} is unknown, fallback to auto mode.', RuntimeWarning)
             mode = 'auto'
 
         self.monitor_op, self.kth_value, self.mode = mode_dict[mode]
@@ -206,7 +203,7 @@ class ModelCheckpoint(Callback):
             current = metrics.get(self.monitor)
 
             if current is None:
-                warnings.warn(f'Can save best model only with {self.monitor} available, skipping.', RuntimeWarning)
+                rank_zero_warn(f'Can save best model only with {self.monitor} available, skipping.', RuntimeWarning)
             elif self.check_monitor_top_k(current):
                 self._do_check_save(filepath, current, epoch)
             elif self.verbose > 0:
diff --git a/pytorch_lightning/core/decorators.py b/pytorch_lightning/core/decorators.py
index 3cd22c26..3979a4fc 100644
--- a/pytorch_lightning/core/decorators.py
+++ b/pytorch_lightning/core/decorators.py
@@ -1,4 +1,4 @@
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
 
 def data_loader(fn):
@@ -7,7 +7,7 @@ def data_loader(fn):
     Warnings:
         This decorator deprecated in v0.7.0 and it will be removed v0.9.0.
     """
-    warnings.warn('`data_loader` decorator deprecated in v0.7.0. Will be removed v0.9.0', DeprecationWarning)
+    rank_zero_warn('`data_loader` decorator deprecated in v0.7.0. Will be removed v0.9.0', DeprecationWarning)
 
     def inner_fx(self):
         return fn(self)
diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py
index ee0b2463..a34a5a93 100644
--- a/pytorch_lightning/core/lightning.py
+++ b/pytorch_lightning/core/lightning.py
@@ -1,7 +1,6 @@
 import collections
 import inspect
 import os
-import warnings
 from abc import ABC, abstractmethod
 from argparse import Namespace
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union, Sequence
@@ -20,6 +19,7 @@ from pytorch_lightning.core.memory import ModelSummary
 from pytorch_lightning.core.saving import ModelIO, load_hparams_from_tags_csv
 from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
+from pytorch_lightning.utilities import rank_zero_warn
 
 try:
     import torch_xla.core.xla_model as xm
@@ -225,7 +225,7 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks):
             The loss value shown in the progress bar is smoothed (averaged) over the last values,
             so it differs from the actual loss returned in train/validation step.
         """
-        warnings.warn('`training_step` must be implemented to be used with the Lightning Trainer')
+        rank_zero_warn('`training_step` must be implemented to be used with the Lightning Trainer')
 
     def training_end(self, *args, **kwargs):
         """
@@ -1088,7 +1088,7 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks):
                   }
 
         """
-        warnings.warn('`configure_optimizers` must be implemented to be used with the Lightning Trainer')
+        rank_zero_warn('`configure_optimizers` must be implemented to be used with the Lightning Trainer')
 
     def optimizer_step(
             self,
@@ -1291,7 +1291,7 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks):
                     return loader
 
         """
-        warnings.warn('`train_dataloader` must be implemented to be used with the Lightning Trainer')
+        rank_zero_warn('`train_dataloader` must be implemented to be used with the Lightning Trainer')
 
     def tng_dataloader(self):  # todo: remove in v1.0.0
         """
@@ -1299,8 +1299,8 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks):
             Deprecated in v0.5.0. Use :meth:`train_dataloader` instead. Will be removed in 1.0.0.
         """
         output = self.train_dataloader()
-        warnings.warn("`tng_dataloader` has been renamed to `train_dataloader` since v0.5.0."
-                      " and this method will be removed in v1.0.0", DeprecationWarning)
+        rank_zero_warn("`tng_dataloader` has been renamed to `train_dataloader` since v0.5.0."
+                       " and this method will be removed in v1.0.0", DeprecationWarning)
         return output
 
     def test_dataloader(self) -> Union[DataLoader, List[DataLoader]]:
@@ -1407,7 +1407,7 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks):
             Deprecated in version 0.7.0. You should use :meth:`load_from_checkpoint` instead.
             Will be removed in v0.9.0.
         """
-        warnings.warn(
+        rank_zero_warn(
             "`load_from_metrics` method has been unified with `load_from_checkpoint` in v0.7.0."
             " The deprecated method will be removed in v0.9.0.", DeprecationWarning
         )
@@ -1519,7 +1519,7 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks):
                 is_namespace = checkpoint.get('hparams_type', 'namespace') == 'namespace'
                 hparams = Namespace(**ckpt_hparams) if is_namespace else ckpt_hparams
             else:
-                warnings.warn(
+                rank_zero_warn(
                     f"Checkpoint does not contain hyperparameters but {cls.__name__}'s __init__ "
                     f"contains argument 'hparams'. Will pass in an empty Namespace instead."
                     " Did you forget to store your model hyperparameters in self.hparams?"
diff --git a/pytorch_lightning/core/model_saving.py b/pytorch_lightning/core/model_saving.py
index 54f8fbc4..8c363023 100644
--- a/pytorch_lightning/core/model_saving.py
+++ b/pytorch_lightning/core/model_saving.py
@@ -3,9 +3,9 @@
   The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`model_saving` module has been renamed to `saving` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`model_saving` module has been renamed to `saving` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.core.saving import *  # noqa: F403
diff --git a/pytorch_lightning/core/root_module.py b/pytorch_lightning/core/root_module.py
index af9e89d4..b8e602da 100644
--- a/pytorch_lightning/core/root_module.py
+++ b/pytorch_lightning/core/root_module.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
+
+rank_zero_warn("`root_module` module has been renamed to `lightning` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.core.lightning import *  # noqa: F403
-
-warnings.warn("`root_module` module has been renamed to `lightning` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
diff --git a/pytorch_lightning/logging/__init__.py b/pytorch_lightning/logging/__init__.py
index 2058fffa..1cbccf37 100644
--- a/pytorch_lightning/logging/__init__.py
+++ b/pytorch_lightning/logging/__init__.py
@@ -3,10 +3,10 @@
  The deprecated package name will be removed in v0.9.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`logging` package has been renamed to `loggers` since v0.7.0"
-              " The deprecated package name will be removed in v0.9.0.", DeprecationWarning)
+rank_zero_warn("`logging` package has been renamed to `loggers` since v0.7.0"
+               " The deprecated package name will be removed in v0.9.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers import *  # noqa: F403
 from pytorch_lightning.loggers import base, tensorboard  # noqa: F403
diff --git a/pytorch_lightning/logging/comet.py b/pytorch_lightning/logging/comet.py
index 3e09a1cf..ce854292 100644
--- a/pytorch_lightning/logging/comet.py
+++ b/pytorch_lightning/logging/comet.py
@@ -2,9 +2,9 @@
 .. warning:: `logging` package has been renamed to `loggers` since v0.7.0 and will be removed in v0.9.0
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`logging.comet` module has been renamed to `loggers.comet` since v0.7.0."
-              " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
+rank_zero_warn("`logging.comet` module has been renamed to `loggers.comet` since v0.7.0."
+               " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers.comet import CometLogger  # noqa: F403
diff --git a/pytorch_lightning/logging/comet_logger.py b/pytorch_lightning/logging/comet_logger.py
index 47a524da..83360b36 100644
--- a/pytorch_lightning/logging/comet_logger.py
+++ b/pytorch_lightning/logging/comet_logger.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`comet_logger` module has been renamed to `comet` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`comet_logger` module has been renamed to `comet` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers.comet import CometLogger  # noqa: E402
diff --git a/pytorch_lightning/logging/mlflow.py b/pytorch_lightning/logging/mlflow.py
index c91faec4..15b7fd81 100644
--- a/pytorch_lightning/logging/mlflow.py
+++ b/pytorch_lightning/logging/mlflow.py
@@ -2,9 +2,9 @@
 .. warning:: `logging` package has been renamed to `loggers` since v0.7.0 and will be removed in v0.9.0
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`logging.mlflow` module has been renamed to `loggers.mlflow` since v0.7.0."
-              " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
+rank_zero_warn("`logging.mlflow` module has been renamed to `loggers.mlflow` since v0.7.0."
+               " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers.mlflow import MLFlowLogger  # noqa: F403
diff --git a/pytorch_lightning/logging/mlflow_logger.py b/pytorch_lightning/logging/mlflow_logger.py
index d8fc6359..2e1b5212 100644
--- a/pytorch_lightning/logging/mlflow_logger.py
+++ b/pytorch_lightning/logging/mlflow_logger.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`mlflow_logger` module has been renamed to `mlflow` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`mlflow_logger` module has been renamed to `mlflow` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers.mlflow import MLFlowLogger  # noqa: E402
diff --git a/pytorch_lightning/logging/neptune.py b/pytorch_lightning/logging/neptune.py
index f1e8a81b..af6e18c1 100644
--- a/pytorch_lightning/logging/neptune.py
+++ b/pytorch_lightning/logging/neptune.py
@@ -2,9 +2,9 @@
 .. warning:: `logging` package has been renamed to `loggers` since v0.7.0 and will be removed in v0.9.0
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`logging.neptune` module has been renamed to `loggers.neptune` since v0.7.0."
-              " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
+rank_zero_warn("`logging.neptune` module has been renamed to `loggers.neptune` since v0.7.0."
+               " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers.neptune import NeptuneLogger  # noqa: F403
diff --git a/pytorch_lightning/logging/test_tube.py b/pytorch_lightning/logging/test_tube.py
index c40b7d18..3648db61 100644
--- a/pytorch_lightning/logging/test_tube.py
+++ b/pytorch_lightning/logging/test_tube.py
@@ -2,9 +2,9 @@
 .. warning:: `logging` package has been renamed to `loggers` since v0.7.0 and will be removed in v0.9.0
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`logging.test_tube` module has been renamed to `loggers.test_tube` since v0.7.0."
-              " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
+rank_zero_warn("`logging.test_tube` module has been renamed to `loggers.test_tube` since v0.7.0."
+               " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers.test_tube import TestTubeLogger  # noqa: F403
diff --git a/pytorch_lightning/logging/test_tube_logger.py b/pytorch_lightning/logging/test_tube_logger.py
index cdd06823..3280ac8d 100644
--- a/pytorch_lightning/logging/test_tube_logger.py
+++ b/pytorch_lightning/logging/test_tube_logger.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`test_tube_logger` module has been renamed to `test_tube` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`test_tube_logger` module has been renamed to `test_tube` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers.test_tube import TestTubeLogger  # noqa: E402
diff --git a/pytorch_lightning/logging/wandb.py b/pytorch_lightning/logging/wandb.py
index 0ce86792..98a753c0 100644
--- a/pytorch_lightning/logging/wandb.py
+++ b/pytorch_lightning/logging/wandb.py
@@ -2,9 +2,9 @@
 .. warning:: `logging` package has been renamed to `loggers` since v0.7.0 and will be removed in v0.9.0
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`logging.wandb` module has been renamed to `loggers.wandb` since v0.7.0."
-              " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
+rank_zero_warn("`logging.wandb` module has been renamed to `loggers.wandb` since v0.7.0."
+               " The deprecated module name will be removed in v0.9.0.", DeprecationWarning)
 
 from pytorch_lightning.loggers.wandb import WandbLogger  # noqa: F403
diff --git a/pytorch_lightning/overrides/override_data_parallel.py b/pytorch_lightning/overrides/override_data_parallel.py
index 7685c4de..bf08b1a5 100644
--- a/pytorch_lightning/overrides/override_data_parallel.py
+++ b/pytorch_lightning/overrides/override_data_parallel.py
@@ -3,10 +3,10 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`override_data_parallel` module has been renamed to `data_parallel` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`override_data_parallel` module has been renamed to `data_parallel` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.overrides.data_parallel import (  # noqa: E402
     get_a_var, parallel_apply, LightningDataParallel, LightningDistributedDataParallel)
diff --git a/pytorch_lightning/pt_overrides/__init__.py b/pytorch_lightning/pt_overrides/__init__.py
index b68986d5..5e2b3ddf 100644
--- a/pytorch_lightning/pt_overrides/__init__.py
+++ b/pytorch_lightning/pt_overrides/__init__.py
@@ -3,7 +3,7 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`pt_overrides` package has been renamed to `overrides` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`pt_overrides` package has been renamed to `overrides` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
diff --git a/pytorch_lightning/pt_overrides/override_data_parallel.py b/pytorch_lightning/pt_overrides/override_data_parallel.py
index bc435b7d..34a65e3c 100644
--- a/pytorch_lightning/pt_overrides/override_data_parallel.py
+++ b/pytorch_lightning/pt_overrides/override_data_parallel.py
@@ -3,10 +3,10 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`override_data_parallel` module has been renamed to `data_parallel` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`override_data_parallel` module has been renamed to `data_parallel` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.overrides.data_parallel import (  # noqa: F402
     get_a_var, parallel_apply, LightningDataParallel, LightningDistributedDataParallel)
diff --git a/pytorch_lightning/root_module/__init__.py b/pytorch_lightning/root_module/__init__.py
index f5f66d95..41f741de 100644
--- a/pytorch_lightning/root_module/__init__.py
+++ b/pytorch_lightning/root_module/__init__.py
@@ -3,7 +3,7 @@
  The deprecated package name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`root_module` package has been renamed to `core` since v0.6.0."
-              " The deprecated package name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`root_module` package has been renamed to `core` since v0.6.0."
+               " The deprecated package name will be removed in v0.8.0.", DeprecationWarning)
diff --git a/pytorch_lightning/root_module/decorators.py b/pytorch_lightning/root_module/decorators.py
index 88afe093..7031273b 100644
--- a/pytorch_lightning/root_module/decorators.py
+++ b/pytorch_lightning/root_module/decorators.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`root_module.decorators` module has been renamed to `core.decorators` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`root_module.decorators` module has been renamed to `core.decorators` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.core.decorators import *  # noqa: F403
diff --git a/pytorch_lightning/root_module/grads.py b/pytorch_lightning/root_module/grads.py
index 1f961738..81811411 100644
--- a/pytorch_lightning/root_module/grads.py
+++ b/pytorch_lightning/root_module/grads.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`root_module.grads` module has been renamed to `core.grads` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`root_module.grads` module has been renamed to `core.grads` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.core.grads import *  # noqa: F403
diff --git a/pytorch_lightning/root_module/hooks.py b/pytorch_lightning/root_module/hooks.py
index e4beaee9..0214a391 100644
--- a/pytorch_lightning/root_module/hooks.py
+++ b/pytorch_lightning/root_module/hooks.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`root_module.hooks` module has been renamed to `core.hooks` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`root_module.hooks` module has been renamed to `core.hooks` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.core.hooks import *  # noqa: F403
diff --git a/pytorch_lightning/root_module/memory.py b/pytorch_lightning/root_module/memory.py
index ef739ac2..89d3d281 100644
--- a/pytorch_lightning/root_module/memory.py
+++ b/pytorch_lightning/root_module/memory.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`root_module.memory` module has been renamed to `core.memory` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`root_module.memory` module has been renamed to `core.memory` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.core.memory import *  # noqa: F403
diff --git a/pytorch_lightning/root_module/model_saving.py b/pytorch_lightning/root_module/model_saving.py
index 5af97abf..67cf6a6a 100644
--- a/pytorch_lightning/root_module/model_saving.py
+++ b/pytorch_lightning/root_module/model_saving.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`root_module.model_saving` module has been renamed to `core.saving` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`root_module.model_saving` module has been renamed to `core.saving` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.core.saving import *  # noqa: F403
diff --git a/pytorch_lightning/root_module/root_module.py b/pytorch_lightning/root_module/root_module.py
index 4dd47247..3f3e9fad 100644
--- a/pytorch_lightning/root_module/root_module.py
+++ b/pytorch_lightning/root_module/root_module.py
@@ -3,9 +3,9 @@
  The deprecated module name will be removed in v0.8.0.
 """
 
-import warnings
+from pytorch_lightning.utilities import rank_zero_warn
 
-warnings.warn("`root_module.root_module` module has been renamed to `core.lightning` since v0.6.0."
-              " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
+rank_zero_warn("`root_module.root_module` module has been renamed to `core.lightning` since v0.6.0."
+               " The deprecated module name will be removed in v0.8.0.", DeprecationWarning)
 
 from pytorch_lightning.core.lightning import *  # noqa: F403
diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py
index cddd5f90..ca9e0993 100644
--- a/pytorch_lightning/trainer/data_loading.py
+++ b/pytorch_lightning/trainer/data_loading.py
@@ -1,4 +1,3 @@
-import warnings
 import platform
 from abc import ABC, abstractmethod
 from typing import Union, List, Tuple, Callable
@@ -8,6 +7,7 @@ from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
 
 from pytorch_lightning.core import LightningModule
+from pytorch_lightning.utilities import rank_zero_warn
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 
 try:
@@ -79,9 +79,9 @@ class TrainerDataLoadingMixin(ABC):
         on_windows = platform.system() == 'Windows'
 
         if isinstance(dataloader, DataLoader) and dataloader.num_workers <= 2 and not on_windows:
-            warnings.warn(f'The dataloader, {name}, does not have many workers which may be a bottleneck.'
-                          ' Consider increasing the value of the `num_workers` argument`'
-                          ' in the `DataLoader` init to improve performance.')
+            rank_zero_warn(f'The dataloader, {name}, does not have many workers which may be a bottleneck.'
+                           ' Consider increasing the value of the `num_workers` argument`'
+                           ' in the `DataLoader` init to improve performance.')
 
     def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:
 
diff --git a/pytorch_lightning/trainer/deprecated_api.py b/pytorch_lightning/trainer/deprecated_api.py
index 4cf556ac..e67bc7be 100644
--- a/pytorch_lightning/trainer/deprecated_api.py
+++ b/pytorch_lightning/trainer/deprecated_api.py
@@ -1,8 +1,9 @@
 """Mirroring deprecated API"""
 
-import warnings
 from abc import ABC
 
+from pytorch_lightning.utilities import rank_zero_warn
+
 
 class TrainerDeprecatedAPITillVer0_8(ABC):
 
@@ -12,80 +13,80 @@ class TrainerDeprecatedAPITillVer0_8(ABC):
     @property
     def nb_gpu_nodes(self):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `nb_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `nb_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         return self.num_nodes
 
     @property
     def num_gpu_nodes(self):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `num_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `num_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         return self.num_nodes
 
     @num_gpu_nodes.setter
     def num_gpu_nodes(self, num_nodes):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `num_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `num_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         self.num_nodes = num_nodes
 
     @property
     def gradient_clip(self):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `gradient_clip` has renamed to `gradient_clip_val` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `gradient_clip` has renamed to `gradient_clip_val` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         return self.gradient_clip_val
 
     @gradient_clip.setter
     def gradient_clip(self, gradient_clip):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `gradient_clip` has renamed to `gradient_clip_val` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `gradient_clip` has renamed to `gradient_clip_val` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         self.gradient_clip_val = gradient_clip
 
     @property
     def max_nb_epochs(self):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `max_nb_epochs` has renamed to `max_epochs` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `max_nb_epochs` has renamed to `max_epochs` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         return self.max_epochs
 
     @max_nb_epochs.setter
     def max_nb_epochs(self, max_epochs):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `max_nb_epochs` has renamed to `max_epochs` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `max_nb_epochs` has renamed to `max_epochs` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         self.max_epochs = max_epochs
 
     @property
     def min_nb_epochs(self):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `min_nb_epochs` has renamed to `min_epochs` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `min_nb_epochs` has renamed to `min_epochs` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         return self.min_epochs
 
     @min_nb_epochs.setter
     def min_nb_epochs(self, min_epochs):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `min_nb_epochs` has renamed to `min_epochs` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `min_nb_epochs` has renamed to `min_epochs` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         self.min_epochs = min_epochs
 
     @property
     def nb_sanity_val_steps(self):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `nb_sanity_val_steps` has renamed to "
-                      "`num_sanity_val_steps` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `nb_sanity_val_steps` has renamed to "
+                       "`num_sanity_val_steps` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         return self.num_sanity_val_steps
 
     @nb_sanity_val_steps.setter
     def nb_sanity_val_steps(self, nb):
         """Back compatibility, will be removed in v0.8.0"""
-        warnings.warn("Attribute `nb_sanity_val_steps` has renamed to "
-                      "`num_sanity_val_steps` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("Attribute `nb_sanity_val_steps` has renamed to "
+                       "`num_sanity_val_steps` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         self.num_sanity_val_steps = nb
 
 
@@ -97,12 +98,12 @@ class TrainerDeprecatedAPITillVer0_9(ABC):
     @property
     def show_progress_bar(self):
         """Back compatibility, will be removed in v0.9.0"""
-        warnings.warn("Argument `show_progress_bar` is now set by `progress_bar_refresh_rate` since v0.7.2"
-                      " and this method will be removed in v0.9.0", DeprecationWarning)
+        rank_zero_warn("Argument `show_progress_bar` is now set by `progress_bar_refresh_rate` since v0.7.2"
+                       " and this method will be removed in v0.9.0", DeprecationWarning)
         return self.progress_bar_refresh_rate >= 1
 
     @show_progress_bar.setter
     def show_progress_bar(self, tf):
         """Back compatibility, will be removed in v0.9.0"""
-        warnings.warn("Argument `show_progress_bar` is now set by `progress_bar_refresh_rate` since v0.7.2"
-                      " and this method will be removed in v0.9.0", DeprecationWarning)
+        rank_zero_warn("Argument `show_progress_bar` is now set by `progress_bar_refresh_rate` since v0.7.2"
+                       " and this method will be removed in v0.9.0", DeprecationWarning)
diff --git a/pytorch_lightning/trainer/distrib_data_parallel.py b/pytorch_lightning/trainer/distrib_data_parallel.py
index 98e5f1e7..bfba6aa2 100644
--- a/pytorch_lightning/trainer/distrib_data_parallel.py
+++ b/pytorch_lightning/trainer/distrib_data_parallel.py
@@ -115,7 +115,6 @@ When the script starts again, Lightning will:
 
 import os
 import re
-import warnings
 from abc import ABC, abstractmethod
 from typing import Union
 
@@ -123,6 +122,7 @@ import torch
 from pytorch_lightning import _logger as log
 from pytorch_lightning.loggers import LightningLoggerBase
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
+from pytorch_lightning.utilities.warnings import set_proc_rank, rank_zero_warn
 
 try:
     from apex import amp
@@ -203,20 +203,19 @@ class TrainerDDPMixin(ABC):
                 self.use_ddp2 = distributed_backend == 'ddp2'
 
             elif distributed_backend is None:
-                warnings.warn('You requested multiple GPUs but did not specify a backend, e.g.'
-                              ' Trainer(distributed_backend=dp) (or ddp, ddp2).'
-                              ' Setting distributed_backend=dp for you.')
+                rank_zero_warn('You requested multiple GPUs but did not specify a backend, e.g.'
+                               ' Trainer(distributed_backend=dp) (or ddp, ddp2).'
+                               ' Setting distributed_backend=dp for you.')
                 self.use_dp = True
                 self.use_ddp = False
                 self.use_ddp2 = False
 
         # throw error to force user ddp or ddp2 choice
         if num_gpu_nodes > 1 and not (self.use_ddp2 or self.use_ddp):
-            w = 'DataParallel does not support num_nodes > 1. ' \
-                'Switching to DistributedDataParallel for you. ' \
-                'To silence this warning set distributed_backend=ddp' \
-                'or distributed_backend=ddp2'
-            raise MisconfigurationException(w)
+            raise MisconfigurationException(
+                'DataParallel does not support num_nodes > 1. Switching to DistributedDataParallel for you. '
+                'To silence this warning set distributed_backend=ddp or distributed_backend=ddp2'
+            )
 
         log.info(f'GPU available: {torch.cuda.is_available()}, used: {self.on_gpu}')
 
@@ -295,6 +294,8 @@ class TrainerDDPMixin(ABC):
         elif self.use_ddp2:
             self.proc_rank = self.node_rank
             self.world_size = self.num_gpu_nodes
+        # set warning rank
+        set_proc_rank(self.proc_rank)
 
         # let the exp know the rank to avoid overwriting logs
         if self.logger is not None:
diff --git a/pytorch_lightning/trainer/distrib_parts.py b/pytorch_lightning/trainer/distrib_parts.py
index 084b4a67..95588a2b 100644
--- a/pytorch_lightning/trainer/distrib_parts.py
+++ b/pytorch_lightning/trainer/distrib_parts.py
@@ -348,6 +348,7 @@ from pytorch_lightning.overrides.data_parallel import (
     LightningDataParallel,
 )
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
+from pytorch_lightning.utilities.warnings import set_proc_rank
 
 try:
     from apex import amp
@@ -489,6 +490,7 @@ class TrainerDPMixin(ABC):
         # track current tpu
         self.current_tpu_idx = tpu_core_idx
         self.proc_rank = self.tpu_local_core_rank
+        set_proc_rank(self.proc_rank)
 
         # CHOOSE OPTIMIZER
         # allow for lr schedulers as well
diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py
index b10173ee..ee1ac4ad 100644
--- a/pytorch_lightning/trainer/evaluation_loop.py
+++ b/pytorch_lightning/trainer/evaluation_loop.py
@@ -124,7 +124,6 @@ In this second case, the options you pass to trainer will be used when running
 """
 
 import sys
-import warnings
 from abc import ABC, abstractmethod
 from pprint import pprint
 from typing import Callable
@@ -136,6 +135,7 @@ from tqdm.auto import tqdm
 from pytorch_lightning.core.lightning import LightningModule
 from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel, LightningDataParallel
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
+from pytorch_lightning.utilities import rank_zero_warn
 
 try:
     import torch_xla.distributed.parallel_loader as xla_pl
@@ -299,8 +299,8 @@ class TrainerEvaluationLoopMixin(ABC):
             if self.is_overriden('test_end', model=model):
                 # TODO: remove in v1.0.0
                 eval_results = model.test_end(outputs)
-                warnings.warn('Method `test_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
-                              ' Use `test_epoch_end` instead.', DeprecationWarning)
+                rank_zero_warn('Method `test_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
+                               ' Use `test_epoch_end` instead.', DeprecationWarning)
 
             elif self.is_overriden('test_epoch_end', model=model):
                 eval_results = model.test_epoch_end(outputs)
@@ -309,8 +309,8 @@ class TrainerEvaluationLoopMixin(ABC):
             if self.is_overriden('validation_end', model=model):
                 # TODO: remove in v1.0.0
                 eval_results = model.validation_end(outputs)
-                warnings.warn('Method `validation_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
-                              ' Use `validation_epoch_end` instead.', DeprecationWarning)
+                rank_zero_warn('Method `validation_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
+                               ' Use `validation_epoch_end` instead.', DeprecationWarning)
 
             elif self.is_overriden('validation_epoch_end', model=model):
                 eval_results = model.validation_epoch_end(outputs)
diff --git a/pytorch_lightning/trainer/optimizers.py b/pytorch_lightning/trainer/optimizers.py
index f0905931..2c4f0ed5 100644
--- a/pytorch_lightning/trainer/optimizers.py
+++ b/pytorch_lightning/trainer/optimizers.py
@@ -1,4 +1,3 @@
-import warnings
 from abc import ABC
 from typing import List, Tuple
 
@@ -7,6 +6,7 @@ from torch import optim
 from torch.optim.optimizer import Optimizer
 
 from pytorch_lightning.core.lightning import LightningModule
+from pytorch_lightning.utilities import rank_zero_warn
 
 
 class TrainerOptimizersMixin(ABC):
@@ -18,8 +18,8 @@ class TrainerOptimizersMixin(ABC):
         optim_conf = model.configure_optimizers()
 
         if optim_conf is None:
-            warnings.warn('`LightningModule.configure_optimizers` returned `None`, '
-                          'this fit will run with no optimizer', UserWarning)
+            rank_zero_warn('`LightningModule.configure_optimizers` returned `None`, '
+                           'this fit will run with no optimizer', UserWarning)
             optim_conf = _MockOptimizer()
 
         # single output, single optimizer
diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py
index 5b140064..9cf6678c 100644
--- a/pytorch_lightning/trainer/trainer.py
+++ b/pytorch_lightning/trainer/trainer.py
@@ -2,7 +2,6 @@ import distutils
 import inspect
 import os
 import sys
-import warnings
 from argparse import ArgumentParser
 from typing import Union, Optional, List, Dict, Tuple, Iterable, Any
 
@@ -33,6 +32,7 @@ from pytorch_lightning.trainer.training_io import TrainerIOMixin
 from pytorch_lightning.trainer.training_loop import TrainerTrainLoopMixin
 from pytorch_lightning.trainer.training_tricks import TrainerTrainingTricksMixin
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
+from pytorch_lightning.utilities import rank_zero_warn
 
 try:
     from apex import amp
@@ -266,16 +266,16 @@ class Trainer(
         self.num_nodes = num_nodes
         # Backward compatibility, TODO: remove in v0.8.0
         if nb_gpu_nodes is not None:
-            warnings.warn("Argument `nb_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
-                          " and this method will be removed in v0.8.0", DeprecationWarning)
+            rank_zero_warn("Argument `nb_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
+                           " and this method will be removed in v0.8.0", DeprecationWarning)
             self.num_gpu_nodes = nb_gpu_nodes
         self.log_gpu_memory = log_gpu_memory
 
         self.gradient_clip_val = gradient_clip_val
         # Backward compatibility, TODO: remove in v0.8.0
         if gradient_clip is not None:
-            warnings.warn("Argument `gradient_clip` has renamed to `gradient_clip_val` since v0.5.0"
-                          " and this method will be removed in v0.8.0", DeprecationWarning)
+            rank_zero_warn("Argument `gradient_clip` has renamed to `gradient_clip_val` since v0.5.0"
+                           " and this method will be removed in v0.8.0", DeprecationWarning)
             self.gradient_clip = gradient_clip
 
         self.progress_bar_refresh_rate = progress_bar_refresh_rate
@@ -294,15 +294,15 @@ class Trainer(
         self.max_epochs = max_epochs
         # Backward compatibility, TODO: remove in v0.8.0
         if max_nb_epochs is not None:
-            warnings.warn("Argument `max_nb_epochs` has renamed to `max_epochs` since v0.5.0"
-                          " and this method will be removed in v0.8.0", DeprecationWarning)
+            rank_zero_warn("Argument `max_nb_epochs` has renamed to `max_epochs` since v0.5.0"
+                           " and this method will be removed in v0.8.0", DeprecationWarning)
             self.max_nb_epochs = max_nb_epochs
 
         self.min_epochs = min_epochs
         # Backward compatibility, TODO: remove in v0.8.0
         if min_nb_epochs is not None:
-            warnings.warn("Argument `min_nb_epochs` has renamed to `min_epochs` since v0.5.0"
-                          " and this method will be removed in v0.8.0", DeprecationWarning)
+            rank_zero_warn("Argument `min_nb_epochs` has renamed to `min_epochs` since v0.5.0"
+                           " and this method will be removed in v0.8.0", DeprecationWarning)
             self.min_nb_epochs = min_nb_epochs
 
         self.max_steps = max_steps
@@ -311,16 +311,15 @@ class Trainer(
         self.num_sanity_val_steps = num_sanity_val_steps
         # Backward compatibility, TODO: remove in v0.8.0
         if nb_sanity_val_steps is not None:
-            warnings.warn("Argument `nb_sanity_val_steps` has renamed to "
-                          "`num_sanity_val_steps` since v0.5.0"
-                          " and this method will be removed in v0.8.0", DeprecationWarning)
+            rank_zero_warn("Argument `nb_sanity_val_steps` has renamed to "
+                           "`num_sanity_val_steps` since v0.5.0"
+                           " and this method will be removed in v0.8.0", DeprecationWarning)
             self.nb_sanity_val_steps = nb_sanity_val_steps
 
         # Backward compatibility, TODO: remove in v0.9.0
         if print_nan_grads:
-            warnings.warn("Argument `print_nan_grads` has no effect and will be removed in v0.9.0."
-                          " NaN grads will be printed automatically when detected.",
-                          DeprecationWarning)
+            rank_zero_warn("Argument `print_nan_grads` has no effect and will be removed in v0.9.0."
+                           " NaN grads will be printed automatically when detected.", DeprecationWarning)
 
         self.reload_dataloaders_every_epoch = reload_dataloaders_every_epoch
 
@@ -430,8 +429,8 @@ class Trainer(
 
         # backward compatibility
         if add_row_log_interval is not None:
-            warnings.warn("`add_row_log_interval` has renamed to `row_log_interval` since v0.5.0"
-                          " and this method will be removed in v0.8.0", DeprecationWarning)
+            rank_zero_warn("`add_row_log_interval` has renamed to `row_log_interval` since v0.5.0"
+                           " and this method will be removed in v0.8.0", DeprecationWarning)
             if not row_log_interval:  # in case you did not set the proper value
                 row_log_interval = add_row_log_interval
         self.row_log_interval = row_log_interval
@@ -447,8 +446,8 @@ class Trainer(
 
         # Backward compatibility, TODO: remove in v0.9.0
         if use_amp is not None:
-            warnings.warn("`use_amp` has been replaced by `precision` since v0.7.0"
-                          " and this argument will be removed in v0.9.0", DeprecationWarning)
+            rank_zero_warn("`use_amp` has been replaced by `precision` since v0.7.0"
+                           " and this argument will be removed in v0.9.0", DeprecationWarning)
             self.precision = 16 if use_amp else 32
 
         assert self.precision in (16, 32), 'only 32 or 16 bit precision supported'
@@ -602,8 +601,8 @@ class Trainer(
             Use `training_tqdm_dict` instead. Will remove 0.8.0.
 
         """
-        warnings.warn("`tng_tqdm_dic` has renamed to `training_tqdm_dict` since v0.5.0"
-                      " and this method will be removed in v0.8.0", DeprecationWarning)
+        rank_zero_warn("`tng_tqdm_dic` has renamed to `training_tqdm_dict` since v0.5.0"
+                       " and this method will be removed in v0.8.0", DeprecationWarning)
         return self.training_tqdm_dict
 
     # -----------------------------
@@ -937,10 +936,11 @@ class Trainer(
                                                 ' but have not defined `validation_step()`.')
             else:
                 if not self.is_overriden('validation_epoch_end', model):
-                    warnings.warn('You have defined a `val_dataloader()` and have'
-                                  ' defined a `validation_step()`, you may also want to'
-                                  ' define `validation_epoch_end()` for accumulating stats.',
-                                  RuntimeWarning)
+                    rank_zero_warn(
+                        'You have defined a `val_dataloader()` and have defined a `validation_step()`,'
+                        ' you may also want to define `validation_epoch_end()` for accumulating stats.',
+                        RuntimeWarning
+                    )
         else:
             if self.is_overriden('validation_step', model):
                 raise MisconfigurationException('You have defined `validation_step()`,'
@@ -953,10 +953,10 @@ class Trainer(
                                                 ' but have not defined `test_step()`.')
             else:
                 if not self.is_overriden('test_epoch_end', model):
-                    warnings.warn('You have defined a `test_dataloader()` and'
-                                  ' have defined a `test_step()`, you may also want to'
-                                  ' define `test_epoch_end()` for accumulating stats.',
-                                  RuntimeWarning)
+                    rank_zero_warn(
+                        'You have defined a `test_dataloader()` and have defined a `test_step()`, you may also want to'
+                        ' define `test_epoch_end()` for accumulating stats.', RuntimeWarning
+                    )
         else:
             if self.is_overriden('test_step', model):
                 raise MisconfigurationException('You have defined `test_step()`,'
diff --git a/pytorch_lightning/trainer/training_io.py b/pytorch_lightning/trainer/training_io.py
index 636491f6..fd2d0600 100644
--- a/pytorch_lightning/trainer/training_io.py
+++ b/pytorch_lightning/trainer/training_io.py
@@ -86,7 +86,6 @@ At a rough level, here's what happens inside Trainer :py:mod:`pytorch_lightning.
 import os
 import re
 import signal
-import warnings
 from abc import ABC
 from argparse import Namespace
 from subprocess import call
@@ -102,6 +101,7 @@ from pytorch_lightning.overrides.data_parallel import (
     LightningDistributedDataParallel,
     LightningDataParallel,
 )
+from pytorch_lightning.utilities import rank_zero_warn
 
 try:
     import torch_xla
@@ -321,9 +321,8 @@ class TrainerIOMixin(ABC):
             checkpoint['hparams'] = vars(model.hparams) if is_namespace else model.hparams
             checkpoint['hparams_type'] = 'namespace' if is_namespace else 'dict'
         else:
-            warnings.warn(
-                "Did not find hyperparameters at model.hparams. Saving checkpoint without"
-                " hyperparameters"
+            rank_zero_warn(
+                "Did not find hyperparameters at model.hparams. Saving checkpoint without hyperparameters."
             )
 
         # give the model a chance to add a few things
@@ -372,7 +371,7 @@ class TrainerIOMixin(ABC):
         n_accum = 1 if self.accumulate_grad_batches is None else self.accumulate_grad_batches
         expected_steps = self.num_training_batches / n_accum
         if self.num_training_batches != 0 and self.global_step % expected_steps > 1:
-            warnings.warn(
+            rank_zero_warn(
                 "You're resuming from a checkpoint that ended mid-epoch. "
                 "This can cause unreliable results if further training is done, "
                 "consider using an end of epoch checkpoint. "
diff --git a/pytorch_lightning/trainer/training_loop.py b/pytorch_lightning/trainer/training_loop.py
index b6ec1a27..050c1d28 100644
--- a/pytorch_lightning/trainer/training_loop.py
+++ b/pytorch_lightning/trainer/training_loop.py
@@ -133,7 +133,6 @@ in your model.
 """
 
 import copy
-import warnings
 from abc import ABC, abstractmethod
 from typing import Callable
 from typing import Union, List
@@ -148,6 +147,7 @@ from pytorch_lightning.loggers import LightningLoggerBase
 from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel, LightningDataParallel
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 from pytorch_lightning.trainer.supporters import TensorRunningAccum
+from pytorch_lightning.utilities import rank_zero_warn
 
 try:
     from apex import amp
@@ -284,8 +284,8 @@ class TrainerTrainLoopMixin(ABC):
         """Warning: this is just empty shell for code implemented in other class."""
 
     def train(self):
-        warnings.warn('Displayed epoch numbers in the progress bar start from "1" until v0.6.x,'
-                      ' but will start from "0" in v0.8.0.', RuntimeWarning)
+        rank_zero_warn('Displayed epoch numbers in the progress bar start from "1" until v0.6.x,'
+                       ' but will start from "0" in v0.8.0.', RuntimeWarning)
 
         # get model
         model = self.get_model()
@@ -750,8 +750,8 @@ class TrainerTrainLoopMixin(ABC):
             with self.profiler.profile('training_end'):
                 output = model_ref.training_end(output)
 
-            warnings.warn('`training_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
-                          ' Use training_epoch_end instead', DeprecationWarning)
+            rank_zero_warn('`training_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
+                           ' Use training_epoch_end instead', DeprecationWarning)
 
         return output
 
diff --git a/pytorch_lightning/utilities/__init__.py b/pytorch_lightning/utilities/__init__.py
index e69de29b..469ae3ca 100644
--- a/pytorch_lightning/utilities/__init__.py
+++ b/pytorch_lightning/utilities/__init__.py
@@ -0,0 +1,3 @@
+"""General utilities"""
+
+from pytorch_lightning.utilities.warnings import rank_zero_warn
diff --git a/pytorch_lightning/utilities/warnings.py b/pytorch_lightning/utilities/warnings.py
new file mode 100644
index 00000000..c1fa6fce
--- /dev/null
+++ b/pytorch_lightning/utilities/warnings.py
@@ -0,0 +1,18 @@
+"""Custom Lightning warnings"""
+
+import warnings
+
+_proc_rank = 0
+
+
+def set_proc_rank(value: int) -> None:
+    """Set the (sub)process rank."""
+    global _proc_rank
+    _proc_rank = value
+
+
+def rank_zero_warn(*args, **kwargs) -> None:
+    """Warning only if (sub)process has rank 0."""
+    global _proc_rank
+    if _proc_rank == 0:
+        warnings.warn(*args, **kwargs)
diff --git a/tests/base/utils.py b/tests/base/utils.py
index e5639362..d55e08a5 100644
--- a/tests/base/utils.py
+++ b/tests/base/utils.py
@@ -1,5 +1,4 @@
 import os
-import warnings
 from argparse import Namespace
 
 import numpy as np
@@ -92,7 +91,7 @@ def run_model_test(trainer_options, model, on_gpu=True):
 
 
 def get_default_hparams(continue_training=False, hpc_exp_number=0):
-    tests_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    _ = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
     args = {
         'drop_prob': 0.2,
diff --git a/tests/models/test_cpu.py b/tests/models/test_cpu.py
index 0d8dcba2..a7d112b0 100644
--- a/tests/models/test_cpu.py
+++ b/tests/models/test_cpu.py
@@ -331,14 +331,11 @@ def test_tbptt_cpu_model(tmpdir):
     assert result == 1, 'training failed to complete'
 
 
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine")
 def test_single_gpu_model(tmpdir):
     """Make sure single GPU works (DP mode)."""
     tutils.reset_seed()
 
-    if not torch.cuda.is_available():
-        warnings.warn('test_single_gpu_model cannot run.'
-                      ' Rerun on a GPU node to run this test')
-        return
     model, hparams = tutils.get_default_model()
 
     trainer_options = dict(