CI: split tests-examples (#990)

* CI: split tests-examples

* tests without template

* comment depends

* CircleCI typo

* add doctest

* update test req.

* CI tests

* setup macOS

* longer train

* lover pred acc

* fix model

* rename default model

* lower tests acc

* typo

* imports

* fix test optimizer

* update calls

* fix Win

* lower Drone image

* fix call

* pytorch image

* fix test

* add dev image

* add dev image

* update image

* drone volume

* lint

* update test notes

* rename tests/models >> tests/base

* group models

* conftest

* optim imports

* typos

* fix import

* fix tests

* install AMP

* tests

* fix import

.circleci/config.yml

@@ -11,7 +11,6 @@ references:
       name: Install Dependences
       command: |
         pip install "$TORCH_VERSION" --user
-        # this is temporal fix til test-tube is not merged and released
         pip install -r requirements.txt --user
         sudo pip install pytest pytest-cov pytest-flake8
         pip install -r ./tests/requirements.txt --user
@@ -21,7 +20,16 @@ references:
      name: Testing
      command: |
        python --version ; pip --version ; pip list
-       py.test pytorch_lightning tests pl_examples -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
+       py.test pytorch_lightning tests -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
+     no_output_timeout: 15m
+
+  examples: &examples
+   run:
+     name: PL Examples
+     command: |
+       pip install -r ./pl_examples/requirements.txt --user
+       python --version ; pip --version ; pip list
+       py.test pl_examples -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
      no_output_timeout: 15m
 
   install_pkg: &install_pkg
@@ -84,10 +92,8 @@ jobs:
       - TORCH_VERSION: "torch"
     steps: &steps
       - checkout
-
       - *install_deps
       - *tests
-
       - store_test_results:
           path: test-reports
       - store_artifacts:
@@ -121,6 +127,16 @@ jobs:
       - TORCH_VERSION: "torch>=1.4, <1.5"
     steps: *steps
 
+  Examples:
+    docker:
+      - image: circleci/python:3.7
+    environment:
+        - TORCH_VERSION: "torch"
+    steps:
+      - checkout
+      - *install_deps
+      - *examples
+
   Install-pkg:
     docker:
       - image: circleci/python:3.7
@@ -141,3 +157,4 @@ workflows:
       - PyTorch-v1.3
       - PyTorch-v1.4
       - Install-pkg
+      - Examples

.drone.yml

@@ -6,7 +6,7 @@ name: torch-GPU
 
 steps:
 - name: testing
-  image: nvcr.io/nvidia/pytorch:20.02-py3
+  image: pytorch/pytorch:1.4-cuda10.1-cudnn7-runtime
   environment:
     SLURM_LOCALID: 0
     CODECOV_TOKEN:
@@ -16,12 +16,12 @@ steps:
     - pip install pip -U
     - pip --version
     - nvidia-smi
-    #- pip install torch==1.3
+    - bash ./tests/install_AMP.sh
     - pip install -r requirements.txt --user
     - pip install coverage pytest pytest-cov pytest-flake8 codecov
     - pip install -r ./tests/requirements.txt --user
     - pip list
     - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')"
-    - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules #  --flake8
+    - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --doctest-modules #  --flake8
     - coverage report
     - codecov --token $CODECOV_TOKEN  # --pr $DRONE_PULL_REQUEST --build $DRONE_BUILD_NUMBER --branch $DRONE_BRANCH --commit $DRONE_COMMIT --tag $DRONE_TAG

.github/workflows/ci-testing.yml

@@ -23,7 +23,7 @@ jobs:
         python-version: [3.6, 3.7]
         requires: ['minimal', 'latest']
 
-    # https://stackoverflow.com/a/59076067/4521646
+    # Timeout: https://stackoverflow.com/a/59076067/4521646
     timeout-minutes: 20
     steps:
     - uses: actions/checkout@v2
@@ -32,6 +32,12 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
+    - name: Setup macOS
+      if: runner.os == 'macOS'
+      run: |
+        brew install libomp  # https://github.com/pytorch/pytorch/issues/20030
+
     - name: Set min. dependencies
       if: matrix.requires == 'minimal'
       run: |
@@ -71,7 +77,7 @@ jobs:
       run: |
         # tox --sitepackages
         # flake8 .
-        coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml
+        coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --doctest-modules --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml
         coverage report
 
     - name: Upload pytest test results

.markdownlint.yml

@@ -1,2 +0,0 @@
-MD013: false  # headers with the same names
-MD024: false  # line length

.run_local_tests.sh

@@ -12,5 +12,5 @@ rm -rf ./tests/cometruns*
 rm -rf ./tests/wandb*
 rm -rf ./tests/tests/*
 rm -rf ./lightning_logs
-coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules --flake8
-coverage report -m
+python -m coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules --flake8
+python -m coverage report -m

environment.yml

@@ -1,4 +1,6 @@
+# This is Conda environment file
 # Usage: `conda env update -f environment.yml`
+
 channels:
     - conda-forge
     - pytorch

pl_examples/basic_examples/lightning_module_template.py

@@ -19,7 +19,24 @@ from pytorch_lightning.core import LightningModule
 
 class LightningTemplateModel(LightningModule):
     """
-    Sample model to show how to define a template
+    Sample model to show how to define a template.
+
+    Example:
+
+        >>> # define simple Net for MNIST dataset
+        >>> params = dict(
+        ...     drop_prob=0.2,
+        ...     batch_size=2,
+        ...     in_features=28 * 28,
+        ...     learning_rate=0.001 * 8,
+        ...     optimizer_name='adam',
+        ...     data_root='./datasets',
+        ...     out_features=10,
+        ...     hidden_dim=1000,
+        ... )
+        >>> from argparse import Namespace
+        >>> hparams = Namespace(**params)
+        >>> model = LightningTemplateModel(hparams)
     """
 
     def __init__(self, hparams):

pl_examples/full_examples/semantic_segmentation/models/unet/model.py

@@ -9,9 +9,9 @@ class UNet(nn.Module):
     Link - https://arxiv.org/abs/1505.04597
 
     Parameters:
-    num_classes (int) - Number of output classes required (default 19 for KITTI dataset)
-    bilinear (bool) - Whether to use bilinear interpolation or transposed
-    convolutions for upsampling.
+        num_classes (int) - Number of output classes required (default 19 for KITTI dataset)
+        bilinear (bool) - Whether to use bilinear interpolation or transposed
+        convolutions for upsampling.
     '''
 
     def __init__(self, num_classes=19, bilinear=False):

pytorch_lightning/core/lightning.py

@@ -7,8 +7,8 @@ from argparse import Namespace
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import torch
-from torch import Tensor
 import torch.distributed as torch_distrib
+from torch import Tensor
 from torch.nn.parallel import DistributedDataParallel
 from torch.optim import Adam
 from torch.optim.optimizer import Optimizer

requirements-extra.txt

@@ -1,3 +1,5 @@
+# extended list of package dependencies to reach full functionality
+
 neptune-client>=0.4.4
 comet-ml>=1.0.56
 mlflow>=1.0.0

requirements.txt

@@ -1,3 +1,5 @@
+# the default package dependencies
+
 tqdm>=4.41.0
 numpy>=1.16.4
 torch>=1.1

setup.cfg

@@ -5,7 +5,7 @@ norecursedirs =
     build
 python_files =
     test_*.py
-doctest_plus = disabled
+# doctest_plus = disabled
 addopts = --strict
 markers =
     slow
@@ -41,7 +41,7 @@ ignore =
 # setup.cfg or tox.ini
 [check-manifest]
 ignore =
-    .travis.yml
+    *.yml
     tox.ini
     .github
     .github/*
@@ -51,3 +51,9 @@ ignore =
 license_file = LICENSE
 # long_description = file:README.md
 # long_description_content_type = text/markdown
+
+[pydocstyle]
+convention = pep257
+# D104, D107: Ignore missing docstrings in __init__ files and methods.
+# D202: Ignore a blank line after docstring (collision with Python Black in decorators)
+add-ignore = D104, D107, D202

tests/Dockerfile

@@ -0,0 +1,7 @@
+ARG TORCH_VERSION=1.4
+ARG CUDA_VERSION=10.1
+
+FROM pytorch/pytorch:${TORCH_VERSION}-cuda${CUDA_VERSION}-cudnn7-runtime
+
+# Install AMP
+RUN bash ./tests/install_AMP.sh

tests/README.md

@@ -13,8 +13,8 @@ To run all tests do the following:
 git clone https://github.com/PyTorchLightning/pytorch-lightning
 cd pytorch-lightning
 
-# install module locally
-pip install -e .
+# install AMP support
+bash tests/install_AMP.sh
 
 # install dev deps
 pip install -r tests/requirements.txt
@@ -36,15 +36,13 @@ Make sure to run coverage on a GPU machine with at least 2 GPUs and NVIDIA apex
 cd pytorch-lightning
 
 # generate coverage (coverage is also installed as part of dev dependencies under tests/requirements.txt)
-pip install coverage
 coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v --doctest-modules
 
 # print coverage stats
 coverage report -m
 
-# exporting resulys
+# exporting results
 coverage xml
-codecov -t 17327163-8cca-4a5d-86c8-ca5f2ef700bc  -v
 ```
 
 

tests/base/__init__.py

@@ -0,0 +1,57 @@
+"""Models for testing."""
+
+import torch
+
+from tests.base.models import TestModelBase, DictHparamsModel
+from tests.base.mixins import (
+    LightEmptyTestStep,
+    LightValidationStepMixin,
+    LightValidationMixin,
+    LightValidationStepMultipleDataloadersMixin,
+    LightValidationMultipleDataloadersMixin,
+    LightTestStepMixin,
+    LightTestMixin,
+    LightTestStepMultipleDataloadersMixin,
+    LightTestMultipleDataloadersMixin,
+    LightTestFitSingleTestDataloadersMixin,
+    LightTestFitMultipleTestDataloadersMixin,
+    LightValStepFitSingleDataloaderMixin,
+    LightValStepFitMultipleDataloadersMixin,
+    LightTrainDataloader,
+    LightTestDataloader,
+    LightInfTrainDataloader,
+    LightInfValDataloader,
+    LightInfTestDataloader,
+    LightTestOptimizerWithSchedulingMixin,
+    LightTestMultipleOptimizersWithSchedulingMixin,
+    LightTestOptimizersWithMixedSchedulingMixin,
+    LightTestReduceLROnPlateauMixin
+)
+
+
+class LightningTestModel(LightTrainDataloader,
+                         LightValidationMixin,
+                         LightTestMixin,
+                         TestModelBase):
+    """Most common test case. Validation and test dataloaders."""
+
+    def on_training_metrics(self, logs):
+        logs['some_tensor_to_test'] = torch.rand(1)
+
+
+class LightningTestModelWithoutHyperparametersArg(LightningTestModel):
+    """ without hparams argument in constructor """
+
+    def __init__(self):
+        import tests.base.utils as tutils
+
+        # the user loads the hparams in some other way
+        hparams = tutils.get_default_hparams()
+        super().__init__(hparams)
+
+
+class LightningTestModelWithUnusedHyperparametersArg(LightningTestModelWithoutHyperparametersArg):
+    """ has hparams argument in constructor but is not used """
+
+    def __init__(self, hparams):
+        super().__init__()

tests/base/debug.py

@@ -7,7 +7,7 @@ import pytorch_lightning as pl
 
 
 # from test_models import assert_ok_test_acc, load_model, \
-#     clear_save_dir, get_test_tube_logger, get_hparams, init_save_dir, \
+#     clear_save_dir, get_default_testtube_logger, get_default_hparams, init_save_dir, \
 #     init_checkpoint_callback, reset_seed, set_random_master_port
 
 

tests/base/models.py

@@ -1,5 +1,6 @@
 import os
 from collections import OrderedDict
+from typing import Dict
 
 import torch
 import torch.nn as nn
@@ -8,7 +9,6 @@ from torch import optim
 from torch.utils.data import DataLoader
 from torchvision import transforms
 from torchvision.datasets import MNIST
-from typing import Dict
 
 try:
     from test_tube import HyperOptArgumentParser
@@ -174,9 +174,8 @@ class TestModelBase(LightningModule):
             optimizer = optim.LBFGS(self.parameters(), lr=self.hparams.learning_rate)
         else:
             optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
-
-        # test returning only 1 list instead of 2
-        return optimizer
+        scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
+        return [optimizer], [scheduler]
 
     def prepare_data(self):
         transform = transforms.Compose([transforms.ToTensor(),
@@ -201,36 +200,3 @@ class TestModelBase(LightningModule):
         )
 
         return loader
-
-    @staticmethod
-    def add_model_specific_args(parent_parser, root_dir):  # pragma: no-cover
-        """
-        Parameters you define here will be available to your model through self.hparams
-        :param parent_parser:
-        :param root_dir:
-        :return:
-        """
-        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
-
-        # param overwrites
-        # parser.set_defaults(gradient_clip_val=5.0)
-
-        # network params
-        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
-        parser.add_argument('--in_features', default=28 * 28, type=int)
-        parser.add_argument('--out_features', default=10, type=int)
-        # use 500 for CPU, 50000 for GPU to see speed difference
-        parser.add_argument('--hidden_dim', default=50000, type=int)
-        # data
-        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
-        # training params (opt)
-        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
-                        options=[0.0001, 0.0005, 0.001, 0.005], tunable=False)
-        parser.opt_list('--optimizer_name', default='adam', type=str,
-                        options=['adam'], tunable=False)
-        # if using 2 nodes with 4 gpus each the batch size here
-        #  (256) will be 256 / (2*8) = 16 per gpu
-        parser.opt_list('--batch_size', default=256 * 8, type=int,
-                        options=[32, 64, 128, 256], tunable=False,
-                        help='batch size will be divided over all GPUs being used across all nodes')
-        return parser

tests/base/utils.py

@@ -5,11 +5,11 @@ from argparse import Namespace
 import numpy as np
 import torch
 
-from pl_examples import LightningTemplateModel
+# from pl_examples import LightningTemplateModel
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks import ModelCheckpoint
 from pytorch_lightning.loggers import TestTubeLogger, TensorBoardLogger
-from tests.models import LightningTestModel
+from tests.base import LightningTestModel
 
 # generate a list of random seeds for each test
 RANDOM_PORTS = list(np.random.randint(12000, 19000, 1000))
@@ -21,7 +21,7 @@ ROOT_PATH = os.path.abspath(os.path.dirname(__file__))
 
 
 def run_model_test_no_loggers(trainer_options, model, min_acc=0.50):
-    save_dir = trainer_options['default_save_path']
+    # save_dir = trainer_options['default_save_path']
 
     # fit model
     trainer = Trainer(**trainer_options)
@@ -53,7 +53,7 @@ def run_model_test(trainer_options, model, on_gpu=True):
     save_dir = trainer_options['default_save_path']
 
     # logger file to get meta
-    logger = get_test_tube_logger(save_dir, False)
+    logger = get_default_testtube_logger(save_dir, False)
 
     # logger file to get weights
     checkpoint = init_checkpoint_callback(logger)
@@ -89,7 +89,7 @@ def run_model_test(trainer_options, model, on_gpu=True):
     trainer.hpc_load(save_dir, on_gpu=on_gpu)
 
 
-def get_hparams(continue_training=False, hpc_exp_number=0):
+def get_default_hparams(continue_training=False, hpc_exp_number=0):
     tests_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
     args = {
@@ -111,22 +111,19 @@ def get_hparams(continue_training=False, hpc_exp_number=0):
     return hparams
 
 
-def get_model(use_test_model=False, lbfgs=False):
+def get_default_model(lbfgs=False):
     # set up model with these hyperparams
-    hparams = get_hparams()
+    hparams = get_default_hparams()
     if lbfgs:
         setattr(hparams, 'optimizer_name', 'lbfgs')
         setattr(hparams, 'learning_rate', 0.002)
 
-    if use_test_model:
-        model = LightningTestModel(hparams)
-    else:
-        model = LightningTemplateModel(hparams)
+    model = LightningTestModel(hparams)
 
     return model, hparams
 
 
-def get_test_tube_logger(save_dir, debug=True, version=None):
+def get_default_testtube_logger(save_dir, debug=True, version=None):
     # set up logger object without actually saving logs
     logger = TestTubeLogger(save_dir, name='lightning_logs', debug=debug, version=version)
     return logger
@@ -150,7 +147,7 @@ def get_data_path(expt_logger, path_dir=None):
     return path_expt
 
 
-def load_model(exp, root_weights_dir, module_class=LightningTemplateModel, path_expt=None):
+def load_model(exp, root_weights_dir, module_class=LightningTestModel, path_expt=None):
     # load trained model
     path_expt_dir = get_data_path(exp, path_dir=path_expt)
     tags_path = os.path.join(path_expt_dir, TensorBoardLogger.NAME_CSV_TAGS)
@@ -168,7 +165,7 @@ def load_model(exp, root_weights_dir, module_class=LightningTemplateModel, path_
     return trained_model
 
 
-def load_model_from_checkpoint(root_weights_dir, module_class=LightningTemplateModel):
+def load_model_from_checkpoint(root_weights_dir, module_class=LightningTestModel):
     # load trained model
     checkpoints = [x for x in os.listdir(root_weights_dir) if '.ckpt' in x]
     weights_dir = os.path.join(root_weights_dir, checkpoints[0])
@@ -182,7 +179,7 @@ def load_model_from_checkpoint(root_weights_dir, module_class=LightningTemplateM
     return trained_model
 
 
-def run_prediction(dataloader, trained_model, dp=False, min_acc=0.45):
+def run_prediction(dataloader, trained_model, dp=False, min_acc=0.35):
     # run prediction on 1 batch
     for batch in dataloader:
         break

tests/install_AMP.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+ROOT=$PWD
+git clone https://github.com/NVIDIA/apex
+cd apex
+pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./
+pip install -v --no-cache-dir ./
+cd $ROOT
+rm -rf apex

tests/loggers/test_base.py

@@ -1,10 +1,10 @@
 import pickle
 from unittest.mock import MagicMock
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import LightningLoggerBase, rank_zero_only, LoggerCollection
-from tests.models import LightningTestModel
+from tests.base import LightningTestModel
 
 
 def test_logger_collection():
@@ -57,7 +57,7 @@ class CustomLogger(LightningLoggerBase):
 
 
 def test_custom_logger(tmpdir):
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     logger = CustomLogger()
@@ -78,7 +78,7 @@ def test_custom_logger(tmpdir):
 
 
 def test_multiple_loggers(tmpdir):
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     logger1 = CustomLogger()
@@ -137,7 +137,7 @@ def test_adding_step_key(tmpdir):
 
         return decorated
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     model.validation_epoch_end = _validation_end
     trainer_options = dict(
         max_epochs=4,

tests/loggers/test_comet.py

@@ -5,11 +5,11 @@ from unittest.mock import patch
 import pytest
 import torch
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import CometLogger
 from pytorch_lightning.utilities.debugging import MisconfigurationException
-from tests.models import LightningTestModel
+from tests.base import LightningTestModel
 
 
 def test_comet_logger(tmpdir, monkeypatch):
@@ -22,7 +22,7 @@ def test_comet_logger(tmpdir, monkeypatch):
 
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     comet_dir = os.path.join(tmpdir, 'cometruns')
@@ -132,7 +132,7 @@ def test_comet_pickle(tmpdir, monkeypatch):
 
     tutils.reset_seed()
 
-    # hparams = tutils.get_hparams()
+    # hparams = tutils.get_default_hparams()
     # model = LightningTestModel(hparams)
 
     comet_dir = os.path.join(tmpdir, 'cometruns')

tests/loggers/test_mlflow.py

@@ -1,17 +1,17 @@
 import os
 import pickle
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import MLFlowLogger
-from tests.models import LightningTestModel
+from tests.base import LightningTestModel
 
 
 def test_mlflow_logger(tmpdir):
     """Verify that basic functionality of mlflow logger works."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     mlflow_dir = os.path.join(tmpdir, 'mlruns')

tests/loggers/test_neptune.py

@@ -1,20 +1,19 @@
 import pickle
-
 from unittest.mock import patch, MagicMock
 
 import torch
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import NeptuneLogger
-from tests.models import LightningTestModel
+from tests.base import LightningTestModel
 
 
 def test_neptune_logger(tmpdir):
     """Verify that basic functionality of neptune logger works."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
     logger = NeptuneLogger(offline_mode=True)
 
@@ -103,7 +102,7 @@ def test_neptune_leave_open_experiment_after_fit(tmpdir):
     """Verify that neptune experiment was closed after training"""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     def _run_training(logger):

tests/loggers/test_tensorboard.py

@@ -4,16 +4,16 @@ from argparse import Namespace
 import pytest
 import torch
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import TensorBoardLogger
-from tests.models import LightningTestModel
+from tests.base import LightningTestModel
 
 
 def test_tensorboard_logger(tmpdir):
     """Verify that basic functionality of Tensorboard logger works."""
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     logger = TensorBoardLogger(save_dir=tmpdir, name="tensorboard_logger_test")

tests/loggers/test_test_tube.py

@@ -1,17 +1,17 @@
 import pickle
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
-from tests.models import LightningTestModel
+from tests.base import LightningTestModel
 
 
 def test_testtube_logger(tmpdir):
     """Verify that basic functionality of test tube logger works."""
     tutils.reset_seed()
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     assert logger.name == 'lightning_logs'
 
@@ -32,9 +32,9 @@ def test_testtube_pickle(tmpdir):
     """Verify that pickling a trainer containing a test tube logger works."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
     logger.log_hyperparams(hparams)
     logger.save()
 

tests/loggers/test_trains.py

@@ -1,16 +1,16 @@
 import pickle
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import TrainsLogger
-from tests.models import LightningTestModel
+from tests.base import LightningTestModel
 
 
 def test_trains_logger(tmpdir):
     """Verify that basic functionality of TRAINS logger works."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
     TrainsLogger.set_bypass_mode(True)
     TrainsLogger.set_credentials(api_host='http://integration.trains.allegro.ai:8008',
@@ -36,7 +36,7 @@ def test_trains_pickle(tmpdir):
     """Verify that pickling trainer with TRAINS logger works."""
     tutils.reset_seed()
 
-    # hparams = tutils.get_hparams()
+    # hparams = tutils.get_default_hparams()
     # model = LightningTestModel(hparams)
     TrainsLogger.set_bypass_mode(True)
     TrainsLogger.set_credentials(api_host='http://integration.trains.allegro.ai:8008',

tests/loggers/test_wandb.py

@@ -4,7 +4,7 @@ from unittest.mock import patch
 
 import pytest
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import WandbLogger
 

tests/models/__init__.py

@@ -1,57 +0,0 @@
-"""Models for testing."""
-
-import torch
-
-from .base import TestModelBase, DictHparamsModel
-from .mixins import (
-    LightEmptyTestStep,
-    LightValidationStepMixin,
-    LightValidationMixin,
-    LightValidationStepMultipleDataloadersMixin,
-    LightValidationMultipleDataloadersMixin,
-    LightTestStepMixin,
-    LightTestMixin,
-    LightTestStepMultipleDataloadersMixin,
-    LightTestMultipleDataloadersMixin,
-    LightTestFitSingleTestDataloadersMixin,
-    LightTestFitMultipleTestDataloadersMixin,
-    LightValStepFitSingleDataloaderMixin,
-    LightValStepFitMultipleDataloadersMixin,
-    LightTrainDataloader,
-    LightTestDataloader,
-    LightInfTrainDataloader,
-    LightInfValDataloader,
-    LightInfTestDataloader,
-    LightTestOptimizerWithSchedulingMixin,
-    LightTestMultipleOptimizersWithSchedulingMixin,
-    LightTestOptimizersWithMixedSchedulingMixin,
-    LightTestReduceLROnPlateauMixin
-)
-
-
-class LightningTestModel(LightTrainDataloader,
-                         LightValidationMixin,
-                         LightTestMixin,
-                         TestModelBase):
-    """Most common test case. Validation and test dataloaders."""
-
-    def on_training_metrics(self, logs):
-        logs['some_tensor_to_test'] = torch.rand(1)
-
-
-class LightningTestModelWithoutHyperparametersArg(LightningTestModel):
-    """ without hparams argument in constructor """
-
-    def __init__(self):
-        import tests.models.utils as tutils
-
-        # the user loads the hparams in some other way
-        hparams = tutils.get_hparams()
-        super().__init__(hparams)
-
-
-class LightningTestModelWithUnusedHyperparametersArg(LightningTestModelWithoutHyperparametersArg):
-    """ has hparams argument in constructor but is not used """
-
-    def __init__(self, hparams):
-        super().__init__()

tests/models/test_amp.py

@@ -2,10 +2,10 @@ import os
 
 import pytest
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.utilities.debugging import MisconfigurationException
-from tests.models import (
+from tests.base import (
     LightningTestModel,
 )
 
@@ -17,7 +17,7 @@ def test_amp_single_gpu(tmpdir):
     if not tutils.can_run_gpu_test():
         return
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     trainer_options = dict(
@@ -40,7 +40,7 @@ def test_no_amp_single_gpu(tmpdir):
     if not tutils.can_run_gpu_test():
         return
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     trainer_options = dict(
@@ -66,7 +66,7 @@ def test_amp_gpu_ddp(tmpdir):
     tutils.reset_seed()
     tutils.set_random_master_port()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     trainer_options = dict(
@@ -93,7 +93,7 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir):
     tutils.set_random_master_port()
     os.environ['SLURM_LOCALID'] = str(0)
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     trainer_options = dict(
@@ -105,7 +105,7 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir):
     )
 
     # exp file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     # exp file to get weights
     checkpoint = tutils.init_checkpoint_callback(logger)
@@ -136,14 +136,14 @@ def test_cpu_model_with_amp(tmpdir):
     trainer_options = dict(
         default_save_path=tmpdir,
         show_progress_bar=False,
-        logger=tutils.get_test_tube_logger(tmpdir),
+        logger=tutils.get_default_testtube_logger(tmpdir),
         max_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.4,
         precision=16
     )
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
 
     with pytest.raises((MisconfigurationException, ModuleNotFoundError)):
         tutils.run_model_test(trainer_options, model, on_gpu=False)
@@ -157,7 +157,7 @@ def test_amp_gpu_dp(tmpdir):
     if not tutils.can_run_gpu_test():
         return
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     trainer_options = dict(
         default_save_path=tmpdir,
         max_epochs=1,

tests/models/test_cpu.py

@@ -4,12 +4,12 @@ import warnings
 import pytest
 import torch
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks import (
     EarlyStopping,
 )
-from tests.models import (
+from tests.base import (
     TestModelBase,
     LightTrainDataloader,
     LightningTestModel,
@@ -29,12 +29,12 @@ def test_early_stopping_cpu_model(tmpdir):
         overfit_pct=0.20,
         track_grad_norm=2,
         show_progress_bar=True,
-        logger=tutils.get_test_tube_logger(tmpdir),
+        logger=tutils.get_default_testtube_logger(tmpdir),
         train_percent_check=0.1,
         val_percent_check=0.1,
     )
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
     # test freeze on cpu
@@ -55,7 +55,7 @@ def test_lbfgs_cpu_model(tmpdir):
         val_percent_check=0.2,
     )
 
-    model, hparams = tutils.get_model(use_test_model=True, lbfgs=True)
+    model, hparams = tutils.get_default_model(lbfgs=True)
     tutils.run_model_test_no_loggers(trainer_options, model, min_acc=0.30)
 
 
@@ -73,7 +73,7 @@ def test_default_logger_callbacks_cpu_model(tmpdir):
         val_percent_check=0.01,
     )
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     tutils.run_model_test_no_loggers(trainer_options, model)
 
     # test freeze on cpu
@@ -85,11 +85,11 @@ def test_running_test_after_fitting(tmpdir):
     """Verify test() on fitted model."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     # logger file to get weights
     checkpoint = tutils.init_checkpoint_callback(logger)
@@ -97,7 +97,7 @@ def test_running_test_after_fitting(tmpdir):
     trainer_options = dict(
         default_save_path=tmpdir,
         show_progress_bar=False,
-        max_epochs=4,
+        max_epochs=8,
         train_percent_check=0.4,
         val_percent_check=0.2,
         test_percent_check=0.2,
@@ -114,7 +114,7 @@ def test_running_test_after_fitting(tmpdir):
     trainer.test()
 
     # test we have good test accuracy
-    tutils.assert_ok_model_acc(trainer)
+    tutils.assert_ok_model_acc(trainer, thr=0.35)
 
 
 def test_running_test_without_val(tmpdir):
@@ -124,11 +124,11 @@ def test_running_test_without_val(tmpdir):
     class CurrentTestModel(LightTrainDataloader, LightTestMixin, TestModelBase):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     # logger file to get weights
     checkpoint = tutils.init_checkpoint_callback(logger)
@@ -204,7 +204,7 @@ def test_simple_cpu(tmpdir):
     """Verify continue training session on CPU."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     # logger file to get meta
@@ -230,13 +230,13 @@ def test_cpu_model(tmpdir):
     trainer_options = dict(
         default_save_path=tmpdir,
         show_progress_bar=False,
-        logger=tutils.get_test_tube_logger(tmpdir),
+        logger=tutils.get_default_testtube_logger(tmpdir),
         max_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.4
     )
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
 
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
@@ -251,14 +251,14 @@ def test_all_features_cpu_model(tmpdir):
         overfit_pct=0.20,
         track_grad_norm=2,
         show_progress_bar=False,
-        logger=tutils.get_test_tube_logger(tmpdir),
+        logger=tutils.get_default_testtube_logger(tmpdir),
         accumulate_grad_batches=2,
         max_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.4
     )
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
 
@@ -320,7 +320,7 @@ def test_tbptt_cpu_model(tmpdir):
         early_stop_callback=False
     )
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     hparams.batch_size = batch_size
     hparams.in_features = truncated_bptt_steps
     hparams.hidden_dim = truncated_bptt_steps
@@ -343,7 +343,7 @@ def test_single_gpu_model(tmpdir):
         warnings.warn('test_single_gpu_model cannot run.'
                       ' Rerun on a GPU node to run this test')
         return
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
 
     trainer_options = dict(
         default_save_path=tmpdir,
@@ -371,7 +371,7 @@ def test_nan_loss_detection(tmpdir):
                     output /= 0
             return output
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = InfLossModel(hparams)
 
     # fit model
@@ -398,7 +398,7 @@ def test_nan_params_detection(tmpdir):
                 # simulate parameter that became nan
                 torch.nn.init.constant_(self.c_d1.bias, math.nan)
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     model = NanParamModel(hparams)
     trainer = Trainer(

tests/models/test_gpu.py

@@ -3,20 +3,16 @@ import os
 import pytest
 import torch
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import (
-    ModelCheckpoint,
-)
+from pytorch_lightning.callbacks import ModelCheckpoint
 from pytorch_lightning.core import memory
 from pytorch_lightning.trainer.distrib_parts import (
     parse_gpu_ids,
     determine_root_gpu_device,
 )
 from pytorch_lightning.utilities.debugging import MisconfigurationException
-from tests.models import (
-    LightningTestModel,
-)
+from tests.base import LightningTestModel
 
 PRETEND_N_OF_GPUS = 16
 
@@ -29,7 +25,7 @@ def test_multi_gpu_model_ddp2(tmpdir):
     tutils.reset_seed()
     tutils.set_random_master_port()
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     trainer_options = dict(
         default_save_path=tmpdir,
         show_progress_bar=True,
@@ -52,7 +48,7 @@ def test_multi_gpu_model_ddp(tmpdir):
     tutils.reset_seed()
     tutils.set_random_master_port()
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     trainer_options = dict(
         default_save_path=tmpdir,
         show_progress_bar=False,
@@ -74,7 +70,7 @@ def test_ddp_all_dataloaders_passed_to_fit(tmpdir):
     tutils.reset_seed()
     tutils.set_random_master_port()
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     trainer_options = dict(default_save_path=tmpdir,
                            show_progress_bar=False,
                            max_epochs=1,
@@ -95,7 +91,7 @@ def test_optimizer_return_options():
     tutils.reset_seed()
 
     trainer = Trainer()
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
 
     # single optimizer
     opt_a = torch.optim.Adam(model.parameters(), lr=0.002)
@@ -130,11 +126,11 @@ def test_cpu_slurm_save_load(tmpdir):
     """Verify model save/load/checkpoint on CPU."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
     version = logger.version
 
     trainer_options = dict(
@@ -173,7 +169,7 @@ def test_cpu_slurm_save_load(tmpdir):
     assert os.path.exists(saved_filepath)
 
     # new logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False, version=version)
+    logger = tutils.get_default_testtube_logger(tmpdir, False, version=version)
 
     trainer_options = dict(
         max_epochs=1,
@@ -206,7 +202,7 @@ def test_multi_gpu_none_backend(tmpdir):
     if not tutils.can_run_gpu_test():
         return
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     trainer_options = dict(
         default_save_path=tmpdir,
         show_progress_bar=False,
@@ -227,7 +223,7 @@ def test_multi_gpu_model_dp(tmpdir):
     if not tutils.can_run_gpu_test():
         return
 
-    model, hparams = tutils.get_model()
+    model, hparams = tutils.get_default_model()
     trainer_options = dict(
         default_save_path=tmpdir,
         show_progress_bar=False,

tests/models/test_restore.py

@@ -5,11 +5,11 @@ import os
 import pytest
 import torch
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks import ModelCheckpoint
 from pytorch_lightning.utilities.debugging import MisconfigurationException
-from tests.models import (
+from tests.base import (
     LightningTestModel,
     LightningTestModelWithoutHyperparametersArg,
     LightningTestModelWithUnusedHyperparametersArg
@@ -24,11 +24,11 @@ def test_running_test_pretrained_model_ddp(tmpdir):
     tutils.reset_seed()
     tutils.set_random_master_port()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     # exp file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     # exp file to get weights
     checkpoint = tutils.init_checkpoint_callback(logger)
@@ -72,11 +72,11 @@ def test_running_test_pretrained_model(tmpdir):
     """Verify test() on pretrained model."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     # logger file to get weights
     checkpoint = tutils.init_checkpoint_callback(logger)
@@ -111,7 +111,7 @@ def test_load_model_from_checkpoint(tmpdir):
     """Verify test() on pretrained model."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     trainer_options = dict(
@@ -158,11 +158,11 @@ def test_running_test_pretrained_model_dp(tmpdir):
     if not tutils.can_run_gpu_test():
         return
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     # logger file to get weights
     checkpoint = tutils.init_checkpoint_callback(logger)
@@ -202,7 +202,7 @@ def test_dp_resume(tmpdir):
 
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     trainer_options = dict(
@@ -213,7 +213,7 @@ def test_dp_resume(tmpdir):
     )
 
     # get logger
-    logger = tutils.get_test_tube_logger(tmpdir, debug=False)
+    logger = tutils.get_default_testtube_logger(tmpdir, debug=False)
 
     # exp file to get weights
     # logger file to get weights
@@ -241,7 +241,7 @@ def test_dp_resume(tmpdir):
     trainer.hpc_save(tmpdir, logger)
 
     # init new trainer
-    new_logger = tutils.get_test_tube_logger(tmpdir, version=logger.version)
+    new_logger = tutils.get_default_testtube_logger(tmpdir, version=logger.version)
     trainer_options['logger'] = new_logger
     trainer_options['checkpoint_callback'] = ModelCheckpoint(tmpdir)
     trainer_options['train_percent_check'] = 0.5
@@ -277,11 +277,11 @@ def test_model_saving_loading(tmpdir):
     """Tests use case where trainer saves the model, and user loads it from tags independently."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     trainer_options = dict(
         max_epochs=1,

tests/requirements.txt

@@ -1,3 +1,7 @@
+# install all extra dependencies for full package testing
+-r ../requirements-extra.txt
+
+# extended list of dependencies dor development and run lint and tests
 torchvision>=0.4.0, < 0.5  # the 0.5. has some issues with torch JIT
 tox
 coverage
@@ -8,5 +12,4 @@ pytest-flake8
 flake8
 check-manifest
 twine==1.13.0
-pillow<7.0.0
--r ../requirements-extra.txt
+pillow<7.0.0

tests/test_deprecated.py

@@ -2,8 +2,8 @@
 
 from pytorch_lightning import Trainer
 
-import tests.models.utils as tutils
-from tests.models import TestModelBase, LightTrainDataloader, LightEmptyTestStep
+import tests.base.utils as tutils
+from tests.base import TestModelBase, LightTrainDataloader, LightEmptyTestStep
 
 
 def test_tbd_remove_in_v0_8_0_module_imports():
@@ -85,7 +85,7 @@ class ModelVer0_7(LightTrainDataloader, LightEmptyTestStep, TestModelBase):
 
 
 def test_tbd_remove_in_v1_0_0_model_hooks():
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     model = ModelVer0_6(hparams)
 

tests/trainer/test_callbacks.py

@@ -1,10 +1,7 @@
-import os
-
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Callback
 from pytorch_lightning import Trainer, LightningModule
-from pytorch_lightning.callbacks import ModelCheckpoint
-from tests.models import (
+from tests.base import (
     TestModelBase,
     LightTrainDataloader,
     LightValidationMixin,
@@ -23,7 +20,7 @@ def test_trainer_callback_system(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     def _check_args(trainer, pl_module):

tests/trainer/test_dataloaders.py

@@ -1,9 +1,9 @@
 import pytest
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.utilities.debugging import MisconfigurationException
-from tests.models import (
+from tests.base import (
     TestModelBase,
     LightningTestModel,
     LightEmptyTestStep,
@@ -29,7 +29,7 @@ def test_dataloader_config_errors(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # percent check < 0
@@ -104,7 +104,7 @@ def test_multiple_val_dataloader(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
@@ -143,7 +143,7 @@ def test_multiple_test_dataloader(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
@@ -178,7 +178,7 @@ def test_train_dataloaders_passed_to_fit(tmpdir):
     class CurrentTestModel(LightTrainDataloader, TestModelBase):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     # logger file to get meta
     trainer_options = dict(
@@ -208,7 +208,7 @@ def test_train_val_dataloaders_passed_to_fit(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     # logger file to get meta
     trainer_options = dict(
@@ -243,7 +243,7 @@ def test_all_dataloaders_passed_to_fit(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     # logger file to get meta
     trainer_options = dict(
@@ -282,7 +282,7 @@ def test_multiple_dataloaders_passed_to_fit(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     # logger file to get meta
     trainer_options = dict(
@@ -321,7 +321,7 @@ def test_mixing_of_dataloader_options(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
@@ -360,7 +360,7 @@ def test_inf_train_dataloader(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # fit model
@@ -394,7 +394,7 @@ def test_inf_val_dataloader(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # fit model
@@ -428,7 +428,7 @@ def test_inf_test_dataloader(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # fit model

tests/trainer/test_optimizers.py

@@ -1,13 +1,7 @@
-import math
-import os
-
-import pytest
-import torch
-
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 
-from tests.models import (
+from tests.base import (
     TestModelBase,
     LightTrainDataloader,
     LightValidationStepMixin,
@@ -29,7 +23,7 @@ def test_optimizer_with_scheduling(tmpdir):
             TestModelBase):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
@@ -68,7 +62,7 @@ def test_multi_optimizer_with_scheduling(tmpdir):
             TestModelBase):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
@@ -111,7 +105,7 @@ def test_multi_optimizer_with_scheduling_stepping(tmpdir):
             TestModelBase):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
@@ -160,7 +154,7 @@ def test_reduce_lr_on_plateau_scheduling(tmpdir):
             TestModelBase):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta

tests/trainer/test_trainer.py

@@ -6,7 +6,7 @@ from argparse import Namespace
 import pytest
 import torch
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks import (
     EarlyStopping,
@@ -15,7 +15,7 @@ from pytorch_lightning.callbacks import (
 from pytorch_lightning.core.lightning import load_hparams_from_tags_csv
 from pytorch_lightning.trainer.logging import TrainerLoggingMixin
 from pytorch_lightning.utilities.debugging import MisconfigurationException
-from tests.models import (
+from tests.base import (
     TestModelBase,
     DictHparamsModel,
     LightningTestModel,
@@ -53,7 +53,7 @@ def test_no_val_module(tmpdir):
     """Tests use case where trainer saves the model, and user loads it from tags independently."""
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     class CurrentTestModel(LightTrainDataloader, TestModelBase):
         pass
@@ -61,7 +61,7 @@ def test_no_val_module(tmpdir):
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     trainer_options = dict(
         max_epochs=1,
@@ -97,11 +97,11 @@ def test_no_val_end_module(tmpdir):
     class CurrentTestModel(LightTrainDataloader, LightValidationStepMixin, TestModelBase):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     trainer_options = dict(
         max_epochs=1,
@@ -189,7 +189,7 @@ def test_gradient_accumulation_scheduling(tmpdir):
         # clear gradients
         optimizer.zero_grad()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
     schedule = {1: 2, 3: 4}
 
@@ -209,10 +209,10 @@ def test_gradient_accumulation_scheduling(tmpdir):
 def test_loading_meta_tags(tmpdir):
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     # save tags
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
     logger.log_hyperparams(Namespace(some_str='a_str', an_int=1, a_float=2.0))
     logger.log_hyperparams(hparams)
     logger.save()
@@ -254,7 +254,7 @@ def test_model_checkpoint_options(tmpdir):
     def mock_save_function(filepath):
         open(filepath, 'a').close()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     _ = LightningTestModel(hparams)
 
     # simulated losses
@@ -355,7 +355,7 @@ def test_model_checkpoint_options(tmpdir):
     os.mkdir(save_dir)
 
     # -----------------
-    # CASE K=4 (save all 4 models)
+    # CASE K=4 (save all 4 base)
     # multiple checkpoints within same epoch
 
     checkpoint_callback = ModelCheckpoint(save_dir, save_top_k=4, verbose=1)
@@ -401,7 +401,7 @@ def test_model_checkpoint_options(tmpdir):
 def test_model_freeze_unfreeze():
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
     model.freeze()
@@ -414,7 +414,7 @@ def test_resume_from_checkpoint_epoch_restored(tmpdir):
 
     tutils.reset_seed()
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     def _new_model():
         # Create a model that tracks epochs and batches seen
@@ -474,7 +474,7 @@ def test_resume_from_checkpoint_epoch_restored(tmpdir):
 def _init_steps_model():
     """private method for initializing a model with 5% train epochs"""
     tutils.reset_seed()
-    model, _ = tutils.get_model()
+    model, _ = tutils.get_default_model()
 
     # define train epoch to 5% of data
     train_percent = 0.05
@@ -530,7 +530,7 @@ def test_trainer_min_steps_and_epochs(tmpdir):
     trainer_options.update(dict(
         default_save_path=tmpdir,
         early_stop_callback=EarlyStopping(monitor='val_loss', min_delta=1.0),
-        val_check_interval=20,
+        val_check_interval=2,
         min_epochs=1,
         max_epochs=10
     ))
@@ -571,7 +571,7 @@ def test_benchmark_option(tmpdir):
     ):
         pass
 
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
     model = CurrentTestModel(hparams)
 
     # verify torch.backends.cudnn.benchmark is not turned on
@@ -596,7 +596,7 @@ def test_benchmark_option(tmpdir):
 
 
 def test_testpass_overrides(tmpdir):
-    hparams = tutils.get_hparams()
+    hparams = tutils.get_default_hparams()
 
     class LocalModel(LightTrainDataloader, TestModelBase):
         pass

tests/trainer/test_trainer_cli.py

@@ -4,7 +4,7 @@ from unittest import mock
 
 import pytest
 
-import tests.models.utils as tutils
+import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 
 
@@ -15,7 +15,7 @@ def test_default_args(tmpdir):
     tutils.reset_seed()
 
     # logger file to get meta
-    logger = tutils.get_test_tube_logger(tmpdir, False)
+    logger = tutils.get_default_testtube_logger(tmpdir, False)
 
     parser = ArgumentParser(add_help=False)
     args = parser.parse_args()