diff --git a/.circleci/config.yml b/.circleci/config.yml index d13c73e0..22d6981b 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,6 @@ references: name: Install Dependences command: | pip install "$TORCH_VERSION" --user - # this is temporal fix til test-tube is not merged and released pip install -r requirements.txt --user sudo pip install pytest pytest-cov pytest-flake8 pip install -r ./tests/requirements.txt --user @@ -21,7 +20,16 @@ references: name: Testing command: | python --version ; pip --version ; pip list - py.test pytorch_lightning tests pl_examples -v --doctest-modules --junitxml=test-reports/pytest_junit.xml + py.test pytorch_lightning tests -v --doctest-modules --junitxml=test-reports/pytest_junit.xml + no_output_timeout: 15m + + examples: &examples + run: + name: PL Examples + command: | + pip install -r ./pl_examples/requirements.txt --user + python --version ; pip --version ; pip list + py.test pl_examples -v --doctest-modules --junitxml=test-reports/pytest_junit.xml no_output_timeout: 15m install_pkg: &install_pkg @@ -84,10 +92,8 @@ jobs: - TORCH_VERSION: "torch" steps: &steps - checkout - - *install_deps - *tests - - store_test_results: path: test-reports - store_artifacts: @@ -121,6 +127,16 @@ jobs: - TORCH_VERSION: "torch>=1.4, <1.5" steps: *steps + Examples: + docker: + - image: circleci/python:3.7 + environment: + - TORCH_VERSION: "torch" + steps: + - checkout + - *install_deps + - *examples + Install-pkg: docker: - image: circleci/python:3.7 @@ -141,3 +157,4 @@ workflows: - PyTorch-v1.3 - PyTorch-v1.4 - Install-pkg + - Examples diff --git a/.drone.yml b/.drone.yml index 43cba1e7..53138df6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6,7 +6,7 @@ name: torch-GPU steps: - name: testing - image: nvcr.io/nvidia/pytorch:20.02-py3 + image: pytorch/pytorch:1.4-cuda10.1-cudnn7-runtime environment: SLURM_LOCALID: 0 CODECOV_TOKEN: @@ -16,12 +16,12 @@ steps: - pip install pip -U - pip --version - nvidia-smi - #- pip install torch==1.3 + - bash ./tests/install_AMP.sh - pip install -r requirements.txt --user - pip install coverage pytest pytest-cov pytest-flake8 codecov - pip install -r ./tests/requirements.txt --user - pip list - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')" - - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules # --flake8 + - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --doctest-modules # --flake8 - coverage report - codecov --token $CODECOV_TOKEN # --pr $DRONE_PULL_REQUEST --build $DRONE_BUILD_NUMBER --branch $DRONE_BRANCH --commit $DRONE_COMMIT --tag $DRONE_TAG diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 6371e9e4..2e4c00cb 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -23,7 +23,7 @@ jobs: python-version: [3.6, 3.7] requires: ['minimal', 'latest'] - # https://stackoverflow.com/a/59076067/4521646 + # Timeout: https://stackoverflow.com/a/59076067/4521646 timeout-minutes: 20 steps: - uses: actions/checkout@v2 @@ -32,6 +32,12 @@ jobs: with: python-version: ${{ matrix.python-version }} + # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646 + - name: Setup macOS + if: runner.os == 'macOS' + run: | + brew install libomp # https://github.com/pytorch/pytorch/issues/20030 + - name: Set min. dependencies if: matrix.requires == 'minimal' run: | @@ -71,7 +77,7 @@ jobs: run: | # tox --sitepackages # flake8 . - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml + coverage run --source pytorch_lightning -m py.test pytorch_lightning tests -v --doctest-modules --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}.xml coverage report - name: Upload pytest test results diff --git a/.markdownlint.yml b/.markdownlint.yml deleted file mode 100644 index bc310daa..00000000 --- a/.markdownlint.yml +++ /dev/null @@ -1,2 +0,0 @@ -MD013: false # headers with the same names -MD024: false # line length diff --git a/.run_local_tests.sh b/.run_local_tests.sh index ce2a9208..20fe84ff 100644 --- a/.run_local_tests.sh +++ b/.run_local_tests.sh @@ -12,5 +12,5 @@ rm -rf ./tests/cometruns* rm -rf ./tests/wandb* rm -rf ./tests/tests/* rm -rf ./lightning_logs -coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules --flake8 -coverage report -m +python -m coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules --flake8 +python -m coverage report -m diff --git a/environment.yml b/environment.yml index 1aa7c8f1..80b3fca2 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,6 @@ +# This is Conda environment file # Usage: `conda env update -f environment.yml` + channels: - conda-forge - pytorch diff --git a/pl_examples/basic_examples/lightning_module_template.py b/pl_examples/basic_examples/lightning_module_template.py index 1880bffa..effd750d 100644 --- a/pl_examples/basic_examples/lightning_module_template.py +++ b/pl_examples/basic_examples/lightning_module_template.py @@ -19,7 +19,24 @@ from pytorch_lightning.core import LightningModule class LightningTemplateModel(LightningModule): """ - Sample model to show how to define a template + Sample model to show how to define a template. + + Example: + + >>> # define simple Net for MNIST dataset + >>> params = dict( + ... drop_prob=0.2, + ... batch_size=2, + ... in_features=28 * 28, + ... learning_rate=0.001 * 8, + ... optimizer_name='adam', + ... data_root='./datasets', + ... out_features=10, + ... hidden_dim=1000, + ... ) + >>> from argparse import Namespace + >>> hparams = Namespace(**params) + >>> model = LightningTemplateModel(hparams) """ def __init__(self, hparams): diff --git a/pl_examples/full_examples/semantic_segmentation/models/unet/model.py b/pl_examples/full_examples/semantic_segmentation/models/unet/model.py index 36890aa9..484c6982 100644 --- a/pl_examples/full_examples/semantic_segmentation/models/unet/model.py +++ b/pl_examples/full_examples/semantic_segmentation/models/unet/model.py @@ -9,9 +9,9 @@ class UNet(nn.Module): Link - https://arxiv.org/abs/1505.04597 Parameters: - num_classes (int) - Number of output classes required (default 19 for KITTI dataset) - bilinear (bool) - Whether to use bilinear interpolation or transposed - convolutions for upsampling. + num_classes (int) - Number of output classes required (default 19 for KITTI dataset) + bilinear (bool) - Whether to use bilinear interpolation or transposed + convolutions for upsampling. ''' def __init__(self, num_classes=19, bilinear=False): diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 38a4953c..2a67d327 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -7,8 +7,8 @@ from argparse import Namespace from typing import Any, Callable, Dict, List, Optional, Tuple, Union import torch -from torch import Tensor import torch.distributed as torch_distrib +from torch import Tensor from torch.nn.parallel import DistributedDataParallel from torch.optim import Adam from torch.optim.optimizer import Optimizer diff --git a/requirements-extra.txt b/requirements-extra.txt index c3d1d6b2..a1c3c9b3 100644 --- a/requirements-extra.txt +++ b/requirements-extra.txt @@ -1,3 +1,5 @@ +# extended list of package dependencies to reach full functionality + neptune-client>=0.4.4 comet-ml>=1.0.56 mlflow>=1.0.0 diff --git a/requirements.txt b/requirements.txt index 6d99913a..81441b36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +# the default package dependencies + tqdm>=4.41.0 numpy>=1.16.4 torch>=1.1 diff --git a/setup.cfg b/setup.cfg index 26470605..1db2a972 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ norecursedirs = build python_files = test_*.py -doctest_plus = disabled +# doctest_plus = disabled addopts = --strict markers = slow @@ -41,7 +41,7 @@ ignore = # setup.cfg or tox.ini [check-manifest] ignore = - .travis.yml + *.yml tox.ini .github .github/* @@ -51,3 +51,9 @@ ignore = license_file = LICENSE # long_description = file:README.md # long_description_content_type = text/markdown + +[pydocstyle] +convention = pep257 +# D104, D107: Ignore missing docstrings in __init__ files and methods. +# D202: Ignore a blank line after docstring (collision with Python Black in decorators) +add-ignore = D104, D107, D202 diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 00000000..d0d2f4ca --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,7 @@ +ARG TORCH_VERSION=1.4 +ARG CUDA_VERSION=10.1 + +FROM pytorch/pytorch:${TORCH_VERSION}-cuda${CUDA_VERSION}-cudnn7-runtime + +# Install AMP +RUN bash ./tests/install_AMP.sh diff --git a/tests/README.md b/tests/README.md index 8835ab93..0773c717 100644 --- a/tests/README.md +++ b/tests/README.md @@ -13,8 +13,8 @@ To run all tests do the following: git clone https://github.com/PyTorchLightning/pytorch-lightning cd pytorch-lightning -# install module locally -pip install -e . +# install AMP support +bash tests/install_AMP.sh # install dev deps pip install -r tests/requirements.txt @@ -36,15 +36,13 @@ Make sure to run coverage on a GPU machine with at least 2 GPUs and NVIDIA apex cd pytorch-lightning # generate coverage (coverage is also installed as part of dev dependencies under tests/requirements.txt) -pip install coverage coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v --doctest-modules # print coverage stats coverage report -m -# exporting resulys +# exporting results coverage xml -codecov -t 17327163-8cca-4a5d-86c8-ca5f2ef700bc -v ``` diff --git a/tests/base/__init__.py b/tests/base/__init__.py new file mode 100644 index 00000000..1e684698 --- /dev/null +++ b/tests/base/__init__.py @@ -0,0 +1,57 @@ +"""Models for testing.""" + +import torch + +from tests.base.models import TestModelBase, DictHparamsModel +from tests.base.mixins import ( + LightEmptyTestStep, + LightValidationStepMixin, + LightValidationMixin, + LightValidationStepMultipleDataloadersMixin, + LightValidationMultipleDataloadersMixin, + LightTestStepMixin, + LightTestMixin, + LightTestStepMultipleDataloadersMixin, + LightTestMultipleDataloadersMixin, + LightTestFitSingleTestDataloadersMixin, + LightTestFitMultipleTestDataloadersMixin, + LightValStepFitSingleDataloaderMixin, + LightValStepFitMultipleDataloadersMixin, + LightTrainDataloader, + LightTestDataloader, + LightInfTrainDataloader, + LightInfValDataloader, + LightInfTestDataloader, + LightTestOptimizerWithSchedulingMixin, + LightTestMultipleOptimizersWithSchedulingMixin, + LightTestOptimizersWithMixedSchedulingMixin, + LightTestReduceLROnPlateauMixin +) + + +class LightningTestModel(LightTrainDataloader, + LightValidationMixin, + LightTestMixin, + TestModelBase): + """Most common test case. Validation and test dataloaders.""" + + def on_training_metrics(self, logs): + logs['some_tensor_to_test'] = torch.rand(1) + + +class LightningTestModelWithoutHyperparametersArg(LightningTestModel): + """ without hparams argument in constructor """ + + def __init__(self): + import tests.base.utils as tutils + + # the user loads the hparams in some other way + hparams = tutils.get_default_hparams() + super().__init__(hparams) + + +class LightningTestModelWithUnusedHyperparametersArg(LightningTestModelWithoutHyperparametersArg): + """ has hparams argument in constructor but is not used """ + + def __init__(self, hparams): + super().__init__() diff --git a/tests/models/debug.py b/tests/base/debug.py similarity index 94% rename from tests/models/debug.py rename to tests/base/debug.py index 3c200a52..64f8067e 100644 --- a/tests/models/debug.py +++ b/tests/base/debug.py @@ -7,7 +7,7 @@ import pytorch_lightning as pl # from test_models import assert_ok_test_acc, load_model, \ -# clear_save_dir, get_test_tube_logger, get_hparams, init_save_dir, \ +# clear_save_dir, get_default_testtube_logger, get_default_hparams, init_save_dir, \ # init_checkpoint_callback, reset_seed, set_random_master_port diff --git a/tests/models/mixins.py b/tests/base/mixins.py similarity index 100% rename from tests/models/mixins.py rename to tests/base/mixins.py diff --git a/tests/models/base.py b/tests/base/models.py similarity index 78% rename from tests/models/base.py rename to tests/base/models.py index 0e8e6039..2b9fc27f 100644 --- a/tests/models/base.py +++ b/tests/base/models.py @@ -1,5 +1,6 @@ import os from collections import OrderedDict +from typing import Dict import torch import torch.nn as nn @@ -8,7 +9,6 @@ from torch import optim from torch.utils.data import DataLoader from torchvision import transforms from torchvision.datasets import MNIST -from typing import Dict try: from test_tube import HyperOptArgumentParser @@ -174,9 +174,8 @@ class TestModelBase(LightningModule): optimizer = optim.LBFGS(self.parameters(), lr=self.hparams.learning_rate) else: optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) - - # test returning only 1 list instead of 2 - return optimizer + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10) + return [optimizer], [scheduler] def prepare_data(self): transform = transforms.Compose([transforms.ToTensor(), @@ -201,36 +200,3 @@ class TestModelBase(LightningModule): ) return loader - - @staticmethod - def add_model_specific_args(parent_parser, root_dir): # pragma: no-cover - """ - Parameters you define here will be available to your model through self.hparams - :param parent_parser: - :param root_dir: - :return: - """ - parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) - - # param overwrites - # parser.set_defaults(gradient_clip_val=5.0) - - # network params - parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) - parser.add_argument('--in_features', default=28 * 28, type=int) - parser.add_argument('--out_features', default=10, type=int) - # use 500 for CPU, 50000 for GPU to see speed difference - parser.add_argument('--hidden_dim', default=50000, type=int) - # data - parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) - # training params (opt) - parser.opt_list('--learning_rate', default=0.001 * 8, type=float, - options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) - parser.opt_list('--optimizer_name', default='adam', type=str, - options=['adam'], tunable=False) - # if using 2 nodes with 4 gpus each the batch size here - # (256) will be 256 / (2*8) = 16 per gpu - parser.opt_list('--batch_size', default=256 * 8, type=int, - options=[32, 64, 128, 256], tunable=False, - help='batch size will be divided over all GPUs being used across all nodes') - return parser diff --git a/tests/models/utils.py b/tests/base/utils.py similarity index 91% rename from tests/models/utils.py rename to tests/base/utils.py index 2f971162..c6b8e3ce 100644 --- a/tests/models/utils.py +++ b/tests/base/utils.py @@ -5,11 +5,11 @@ from argparse import Namespace import numpy as np import torch -from pl_examples import LightningTemplateModel +# from pl_examples import LightningTemplateModel from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.loggers import TestTubeLogger, TensorBoardLogger -from tests.models import LightningTestModel +from tests.base import LightningTestModel # generate a list of random seeds for each test RANDOM_PORTS = list(np.random.randint(12000, 19000, 1000)) @@ -21,7 +21,7 @@ ROOT_PATH = os.path.abspath(os.path.dirname(__file__)) def run_model_test_no_loggers(trainer_options, model, min_acc=0.50): - save_dir = trainer_options['default_save_path'] + # save_dir = trainer_options['default_save_path'] # fit model trainer = Trainer(**trainer_options) @@ -53,7 +53,7 @@ def run_model_test(trainer_options, model, on_gpu=True): save_dir = trainer_options['default_save_path'] # logger file to get meta - logger = get_test_tube_logger(save_dir, False) + logger = get_default_testtube_logger(save_dir, False) # logger file to get weights checkpoint = init_checkpoint_callback(logger) @@ -89,7 +89,7 @@ def run_model_test(trainer_options, model, on_gpu=True): trainer.hpc_load(save_dir, on_gpu=on_gpu) -def get_hparams(continue_training=False, hpc_exp_number=0): +def get_default_hparams(continue_training=False, hpc_exp_number=0): tests_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) args = { @@ -111,22 +111,19 @@ def get_hparams(continue_training=False, hpc_exp_number=0): return hparams -def get_model(use_test_model=False, lbfgs=False): +def get_default_model(lbfgs=False): # set up model with these hyperparams - hparams = get_hparams() + hparams = get_default_hparams() if lbfgs: setattr(hparams, 'optimizer_name', 'lbfgs') setattr(hparams, 'learning_rate', 0.002) - if use_test_model: - model = LightningTestModel(hparams) - else: - model = LightningTemplateModel(hparams) + model = LightningTestModel(hparams) return model, hparams -def get_test_tube_logger(save_dir, debug=True, version=None): +def get_default_testtube_logger(save_dir, debug=True, version=None): # set up logger object without actually saving logs logger = TestTubeLogger(save_dir, name='lightning_logs', debug=debug, version=version) return logger @@ -150,7 +147,7 @@ def get_data_path(expt_logger, path_dir=None): return path_expt -def load_model(exp, root_weights_dir, module_class=LightningTemplateModel, path_expt=None): +def load_model(exp, root_weights_dir, module_class=LightningTestModel, path_expt=None): # load trained model path_expt_dir = get_data_path(exp, path_dir=path_expt) tags_path = os.path.join(path_expt_dir, TensorBoardLogger.NAME_CSV_TAGS) @@ -168,7 +165,7 @@ def load_model(exp, root_weights_dir, module_class=LightningTemplateModel, path_ return trained_model -def load_model_from_checkpoint(root_weights_dir, module_class=LightningTemplateModel): +def load_model_from_checkpoint(root_weights_dir, module_class=LightningTestModel): # load trained model checkpoints = [x for x in os.listdir(root_weights_dir) if '.ckpt' in x] weights_dir = os.path.join(root_weights_dir, checkpoints[0]) @@ -182,7 +179,7 @@ def load_model_from_checkpoint(root_weights_dir, module_class=LightningTemplateM return trained_model -def run_prediction(dataloader, trained_model, dp=False, min_acc=0.45): +def run_prediction(dataloader, trained_model, dp=False, min_acc=0.35): # run prediction on 1 batch for batch in dataloader: break diff --git a/tests/install_AMP.sh b/tests/install_AMP.sh new file mode 100644 index 00000000..2c56bb25 --- /dev/null +++ b/tests/install_AMP.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +ROOT=$PWD +git clone https://github.com/NVIDIA/apex +cd apex +pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./ +pip install -v --no-cache-dir ./ +cd $ROOT +rm -rf apex diff --git a/tests/loggers/test_base.py b/tests/loggers/test_base.py index 6f386ed3..9217e1c2 100644 --- a/tests/loggers/test_base.py +++ b/tests/loggers/test_base.py @@ -1,10 +1,10 @@ import pickle from unittest.mock import MagicMock -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import LightningLoggerBase, rank_zero_only, LoggerCollection -from tests.models import LightningTestModel +from tests.base import LightningTestModel def test_logger_collection(): @@ -57,7 +57,7 @@ class CustomLogger(LightningLoggerBase): def test_custom_logger(tmpdir): - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) logger = CustomLogger() @@ -78,7 +78,7 @@ def test_custom_logger(tmpdir): def test_multiple_loggers(tmpdir): - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) logger1 = CustomLogger() @@ -137,7 +137,7 @@ def test_adding_step_key(tmpdir): return decorated - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() model.validation_epoch_end = _validation_end trainer_options = dict( max_epochs=4, diff --git a/tests/loggers/test_comet.py b/tests/loggers/test_comet.py index 69f434c0..1aaf4cb7 100644 --- a/tests/loggers/test_comet.py +++ b/tests/loggers/test_comet.py @@ -5,11 +5,11 @@ from unittest.mock import patch import pytest import torch -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import CometLogger from pytorch_lightning.utilities.debugging import MisconfigurationException -from tests.models import LightningTestModel +from tests.base import LightningTestModel def test_comet_logger(tmpdir, monkeypatch): @@ -22,7 +22,7 @@ def test_comet_logger(tmpdir, monkeypatch): tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) comet_dir = os.path.join(tmpdir, 'cometruns') @@ -132,7 +132,7 @@ def test_comet_pickle(tmpdir, monkeypatch): tutils.reset_seed() - # hparams = tutils.get_hparams() + # hparams = tutils.get_default_hparams() # model = LightningTestModel(hparams) comet_dir = os.path.join(tmpdir, 'cometruns') diff --git a/tests/loggers/test_mlflow.py b/tests/loggers/test_mlflow.py index 6e49a9fe..54e57c7d 100644 --- a/tests/loggers/test_mlflow.py +++ b/tests/loggers/test_mlflow.py @@ -1,17 +1,17 @@ import os import pickle -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import MLFlowLogger -from tests.models import LightningTestModel +from tests.base import LightningTestModel def test_mlflow_logger(tmpdir): """Verify that basic functionality of mlflow logger works.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) mlflow_dir = os.path.join(tmpdir, 'mlruns') diff --git a/tests/loggers/test_neptune.py b/tests/loggers/test_neptune.py index 5c2ab5b5..0e586c33 100644 --- a/tests/loggers/test_neptune.py +++ b/tests/loggers/test_neptune.py @@ -1,20 +1,19 @@ import pickle - from unittest.mock import patch, MagicMock import torch -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import NeptuneLogger -from tests.models import LightningTestModel +from tests.base import LightningTestModel def test_neptune_logger(tmpdir): """Verify that basic functionality of neptune logger works.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) logger = NeptuneLogger(offline_mode=True) @@ -103,7 +102,7 @@ def test_neptune_leave_open_experiment_after_fit(tmpdir): """Verify that neptune experiment was closed after training""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) def _run_training(logger): diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index 220cdeb5..b938be4d 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -4,16 +4,16 @@ from argparse import Namespace import pytest import torch -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import TensorBoardLogger -from tests.models import LightningTestModel +from tests.base import LightningTestModel def test_tensorboard_logger(tmpdir): """Verify that basic functionality of Tensorboard logger works.""" - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) logger = TensorBoardLogger(save_dir=tmpdir, name="tensorboard_logger_test") diff --git a/tests/loggers/test_test_tube.py b/tests/loggers/test_test_tube.py index 0788e0cd..68ac8d93 100644 --- a/tests/loggers/test_test_tube.py +++ b/tests/loggers/test_test_tube.py @@ -1,17 +1,17 @@ import pickle -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer -from tests.models import LightningTestModel +from tests.base import LightningTestModel def test_testtube_logger(tmpdir): """Verify that basic functionality of test tube logger works.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) assert logger.name == 'lightning_logs' @@ -32,9 +32,9 @@ def test_testtube_pickle(tmpdir): """Verify that pickling a trainer containing a test tube logger works.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) logger.log_hyperparams(hparams) logger.save() diff --git a/tests/loggers/test_trains.py b/tests/loggers/test_trains.py index 384d6be8..858ac64a 100644 --- a/tests/loggers/test_trains.py +++ b/tests/loggers/test_trains.py @@ -1,16 +1,16 @@ import pickle -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import TrainsLogger -from tests.models import LightningTestModel +from tests.base import LightningTestModel def test_trains_logger(tmpdir): """Verify that basic functionality of TRAINS logger works.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) TrainsLogger.set_bypass_mode(True) TrainsLogger.set_credentials(api_host='http://integration.trains.allegro.ai:8008', @@ -36,7 +36,7 @@ def test_trains_pickle(tmpdir): """Verify that pickling trainer with TRAINS logger works.""" tutils.reset_seed() - # hparams = tutils.get_hparams() + # hparams = tutils.get_default_hparams() # model = LightningTestModel(hparams) TrainsLogger.set_bypass_mode(True) TrainsLogger.set_credentials(api_host='http://integration.trains.allegro.ai:8008', diff --git a/tests/loggers/test_wandb.py b/tests/loggers/test_wandb.py index abb49544..8e9d6c49 100644 --- a/tests/loggers/test_wandb.py +++ b/tests/loggers/test_wandb.py @@ -4,7 +4,7 @@ from unittest.mock import patch import pytest -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import WandbLogger diff --git a/tests/models/__init__.py b/tests/models/__init__.py index 67206a63..e69de29b 100644 --- a/tests/models/__init__.py +++ b/tests/models/__init__.py @@ -1,57 +0,0 @@ -"""Models for testing.""" - -import torch - -from .base import TestModelBase, DictHparamsModel -from .mixins import ( - LightEmptyTestStep, - LightValidationStepMixin, - LightValidationMixin, - LightValidationStepMultipleDataloadersMixin, - LightValidationMultipleDataloadersMixin, - LightTestStepMixin, - LightTestMixin, - LightTestStepMultipleDataloadersMixin, - LightTestMultipleDataloadersMixin, - LightTestFitSingleTestDataloadersMixin, - LightTestFitMultipleTestDataloadersMixin, - LightValStepFitSingleDataloaderMixin, - LightValStepFitMultipleDataloadersMixin, - LightTrainDataloader, - LightTestDataloader, - LightInfTrainDataloader, - LightInfValDataloader, - LightInfTestDataloader, - LightTestOptimizerWithSchedulingMixin, - LightTestMultipleOptimizersWithSchedulingMixin, - LightTestOptimizersWithMixedSchedulingMixin, - LightTestReduceLROnPlateauMixin -) - - -class LightningTestModel(LightTrainDataloader, - LightValidationMixin, - LightTestMixin, - TestModelBase): - """Most common test case. Validation and test dataloaders.""" - - def on_training_metrics(self, logs): - logs['some_tensor_to_test'] = torch.rand(1) - - -class LightningTestModelWithoutHyperparametersArg(LightningTestModel): - """ without hparams argument in constructor """ - - def __init__(self): - import tests.models.utils as tutils - - # the user loads the hparams in some other way - hparams = tutils.get_hparams() - super().__init__(hparams) - - -class LightningTestModelWithUnusedHyperparametersArg(LightningTestModelWithoutHyperparametersArg): - """ has hparams argument in constructor but is not used """ - - def __init__(self, hparams): - super().__init__() diff --git a/tests/test_amp.py b/tests/models/test_amp.py similarity index 89% rename from tests/test_amp.py rename to tests/models/test_amp.py index 832c7ba7..13d6ff0b 100644 --- a/tests/test_amp.py +++ b/tests/models/test_amp.py @@ -2,10 +2,10 @@ import os import pytest -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.utilities.debugging import MisconfigurationException -from tests.models import ( +from tests.base import ( LightningTestModel, ) @@ -17,7 +17,7 @@ def test_amp_single_gpu(tmpdir): if not tutils.can_run_gpu_test(): return - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) trainer_options = dict( @@ -40,7 +40,7 @@ def test_no_amp_single_gpu(tmpdir): if not tutils.can_run_gpu_test(): return - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) trainer_options = dict( @@ -66,7 +66,7 @@ def test_amp_gpu_ddp(tmpdir): tutils.reset_seed() tutils.set_random_master_port() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) trainer_options = dict( @@ -93,7 +93,7 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir): tutils.set_random_master_port() os.environ['SLURM_LOCALID'] = str(0) - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) trainer_options = dict( @@ -105,7 +105,7 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir): ) # exp file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) # exp file to get weights checkpoint = tutils.init_checkpoint_callback(logger) @@ -136,14 +136,14 @@ def test_cpu_model_with_amp(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, - logger=tutils.get_test_tube_logger(tmpdir), + logger=tutils.get_default_testtube_logger(tmpdir), max_epochs=1, train_percent_check=0.4, val_percent_check=0.4, precision=16 ) - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() with pytest.raises((MisconfigurationException, ModuleNotFoundError)): tutils.run_model_test(trainer_options, model, on_gpu=False) @@ -157,7 +157,7 @@ def test_amp_gpu_dp(tmpdir): if not tutils.can_run_gpu_test(): return - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() trainer_options = dict( default_save_path=tmpdir, max_epochs=1, diff --git a/tests/test_cpu_models.py b/tests/models/test_cpu.py similarity index 91% rename from tests/test_cpu_models.py rename to tests/models/test_cpu.py index 38fc7904..f5f5095d 100644 --- a/tests/test_cpu_models.py +++ b/tests/models/test_cpu.py @@ -4,12 +4,12 @@ import warnings import pytest import torch -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ( EarlyStopping, ) -from tests.models import ( +from tests.base import ( TestModelBase, LightTrainDataloader, LightningTestModel, @@ -29,12 +29,12 @@ def test_early_stopping_cpu_model(tmpdir): overfit_pct=0.20, track_grad_norm=2, show_progress_bar=True, - logger=tutils.get_test_tube_logger(tmpdir), + logger=tutils.get_default_testtube_logger(tmpdir), train_percent_check=0.1, val_percent_check=0.1, ) - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() tutils.run_model_test(trainer_options, model, on_gpu=False) # test freeze on cpu @@ -55,7 +55,7 @@ def test_lbfgs_cpu_model(tmpdir): val_percent_check=0.2, ) - model, hparams = tutils.get_model(use_test_model=True, lbfgs=True) + model, hparams = tutils.get_default_model(lbfgs=True) tutils.run_model_test_no_loggers(trainer_options, model, min_acc=0.30) @@ -73,7 +73,7 @@ def test_default_logger_callbacks_cpu_model(tmpdir): val_percent_check=0.01, ) - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() tutils.run_model_test_no_loggers(trainer_options, model) # test freeze on cpu @@ -85,11 +85,11 @@ def test_running_test_after_fitting(tmpdir): """Verify test() on fitted model.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) # logger file to get weights checkpoint = tutils.init_checkpoint_callback(logger) @@ -97,7 +97,7 @@ def test_running_test_after_fitting(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, - max_epochs=4, + max_epochs=8, train_percent_check=0.4, val_percent_check=0.2, test_percent_check=0.2, @@ -114,7 +114,7 @@ def test_running_test_after_fitting(tmpdir): trainer.test() # test we have good test accuracy - tutils.assert_ok_model_acc(trainer) + tutils.assert_ok_model_acc(trainer, thr=0.35) def test_running_test_without_val(tmpdir): @@ -124,11 +124,11 @@ def test_running_test_without_val(tmpdir): class CurrentTestModel(LightTrainDataloader, LightTestMixin, TestModelBase): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) # logger file to get weights checkpoint = tutils.init_checkpoint_callback(logger) @@ -204,7 +204,7 @@ def test_simple_cpu(tmpdir): """Verify continue training session on CPU.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) # logger file to get meta @@ -230,13 +230,13 @@ def test_cpu_model(tmpdir): trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, - logger=tutils.get_test_tube_logger(tmpdir), + logger=tutils.get_default_testtube_logger(tmpdir), max_epochs=1, train_percent_check=0.4, val_percent_check=0.4 ) - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() tutils.run_model_test(trainer_options, model, on_gpu=False) @@ -251,14 +251,14 @@ def test_all_features_cpu_model(tmpdir): overfit_pct=0.20, track_grad_norm=2, show_progress_bar=False, - logger=tutils.get_test_tube_logger(tmpdir), + logger=tutils.get_default_testtube_logger(tmpdir), accumulate_grad_batches=2, max_epochs=1, train_percent_check=0.4, val_percent_check=0.4 ) - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() tutils.run_model_test(trainer_options, model, on_gpu=False) @@ -320,7 +320,7 @@ def test_tbptt_cpu_model(tmpdir): early_stop_callback=False ) - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() hparams.batch_size = batch_size hparams.in_features = truncated_bptt_steps hparams.hidden_dim = truncated_bptt_steps @@ -343,7 +343,7 @@ def test_single_gpu_model(tmpdir): warnings.warn('test_single_gpu_model cannot run.' ' Rerun on a GPU node to run this test') return - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() trainer_options = dict( default_save_path=tmpdir, @@ -371,7 +371,7 @@ def test_nan_loss_detection(tmpdir): output /= 0 return output - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = InfLossModel(hparams) # fit model @@ -398,7 +398,7 @@ def test_nan_params_detection(tmpdir): # simulate parameter that became nan torch.nn.init.constant_(self.c_d1.bias, math.nan) - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = NanParamModel(hparams) trainer = Trainer( diff --git a/tests/test_gpu_models.py b/tests/models/test_gpu.py similarity index 95% rename from tests/test_gpu_models.py rename to tests/models/test_gpu.py index a95e4d42..9c684ca6 100644 --- a/tests/test_gpu_models.py +++ b/tests/models/test_gpu.py @@ -3,20 +3,16 @@ import os import pytest import torch -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer -from pytorch_lightning.callbacks import ( - ModelCheckpoint, -) +from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.core import memory from pytorch_lightning.trainer.distrib_parts import ( parse_gpu_ids, determine_root_gpu_device, ) from pytorch_lightning.utilities.debugging import MisconfigurationException -from tests.models import ( - LightningTestModel, -) +from tests.base import LightningTestModel PRETEND_N_OF_GPUS = 16 @@ -29,7 +25,7 @@ def test_multi_gpu_model_ddp2(tmpdir): tutils.reset_seed() tutils.set_random_master_port() - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() trainer_options = dict( default_save_path=tmpdir, show_progress_bar=True, @@ -52,7 +48,7 @@ def test_multi_gpu_model_ddp(tmpdir): tutils.reset_seed() tutils.set_random_master_port() - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, @@ -74,7 +70,7 @@ def test_ddp_all_dataloaders_passed_to_fit(tmpdir): tutils.reset_seed() tutils.set_random_master_port() - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() trainer_options = dict(default_save_path=tmpdir, show_progress_bar=False, max_epochs=1, @@ -95,7 +91,7 @@ def test_optimizer_return_options(): tutils.reset_seed() trainer = Trainer() - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() # single optimizer opt_a = torch.optim.Adam(model.parameters(), lr=0.002) @@ -130,11 +126,11 @@ def test_cpu_slurm_save_load(tmpdir): """Verify model save/load/checkpoint on CPU.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) version = logger.version trainer_options = dict( @@ -173,7 +169,7 @@ def test_cpu_slurm_save_load(tmpdir): assert os.path.exists(saved_filepath) # new logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False, version=version) + logger = tutils.get_default_testtube_logger(tmpdir, False, version=version) trainer_options = dict( max_epochs=1, @@ -206,7 +202,7 @@ def test_multi_gpu_none_backend(tmpdir): if not tutils.can_run_gpu_test(): return - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, @@ -227,7 +223,7 @@ def test_multi_gpu_model_dp(tmpdir): if not tutils.can_run_gpu_test(): return - model, hparams = tutils.get_model() + model, hparams = tutils.get_default_model() trainer_options = dict( default_save_path=tmpdir, show_progress_bar=False, diff --git a/tests/test_restore_models.py b/tests/models/test_restore.py similarity index 93% rename from tests/test_restore_models.py rename to tests/models/test_restore.py index cf3a6773..d0088c26 100644 --- a/tests/test_restore_models.py +++ b/tests/models/test_restore.py @@ -5,11 +5,11 @@ import os import pytest import torch -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.utilities.debugging import MisconfigurationException -from tests.models import ( +from tests.base import ( LightningTestModel, LightningTestModelWithoutHyperparametersArg, LightningTestModelWithUnusedHyperparametersArg @@ -24,11 +24,11 @@ def test_running_test_pretrained_model_ddp(tmpdir): tutils.reset_seed() tutils.set_random_master_port() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) # exp file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) # exp file to get weights checkpoint = tutils.init_checkpoint_callback(logger) @@ -72,11 +72,11 @@ def test_running_test_pretrained_model(tmpdir): """Verify test() on pretrained model.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) # logger file to get weights checkpoint = tutils.init_checkpoint_callback(logger) @@ -111,7 +111,7 @@ def test_load_model_from_checkpoint(tmpdir): """Verify test() on pretrained model.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) trainer_options = dict( @@ -158,11 +158,11 @@ def test_running_test_pretrained_model_dp(tmpdir): if not tutils.can_run_gpu_test(): return - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) # logger file to get weights checkpoint = tutils.init_checkpoint_callback(logger) @@ -202,7 +202,7 @@ def test_dp_resume(tmpdir): tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) trainer_options = dict( @@ -213,7 +213,7 @@ def test_dp_resume(tmpdir): ) # get logger - logger = tutils.get_test_tube_logger(tmpdir, debug=False) + logger = tutils.get_default_testtube_logger(tmpdir, debug=False) # exp file to get weights # logger file to get weights @@ -241,7 +241,7 @@ def test_dp_resume(tmpdir): trainer.hpc_save(tmpdir, logger) # init new trainer - new_logger = tutils.get_test_tube_logger(tmpdir, version=logger.version) + new_logger = tutils.get_default_testtube_logger(tmpdir, version=logger.version) trainer_options['logger'] = new_logger trainer_options['checkpoint_callback'] = ModelCheckpoint(tmpdir) trainer_options['train_percent_check'] = 0.5 @@ -277,11 +277,11 @@ def test_model_saving_loading(tmpdir): """Tests use case where trainer saves the model, and user loads it from tags independently.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) trainer_options = dict( max_epochs=1, diff --git a/tests/requirements.txt b/tests/requirements.txt index 82676510..b82220b7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,7 @@ +# install all extra dependencies for full package testing +-r ../requirements-extra.txt + +# extended list of dependencies dor development and run lint and tests torchvision>=0.4.0, < 0.5 # the 0.5. has some issues with torch JIT tox coverage @@ -8,5 +12,4 @@ pytest-flake8 flake8 check-manifest twine==1.13.0 -pillow<7.0.0 --r ../requirements-extra.txt \ No newline at end of file +pillow<7.0.0 \ No newline at end of file diff --git a/tests/test_deprecated.py b/tests/test_deprecated.py index a79eb745..ddaae354 100644 --- a/tests/test_deprecated.py +++ b/tests/test_deprecated.py @@ -2,8 +2,8 @@ from pytorch_lightning import Trainer -import tests.models.utils as tutils -from tests.models import TestModelBase, LightTrainDataloader, LightEmptyTestStep +import tests.base.utils as tutils +from tests.base import TestModelBase, LightTrainDataloader, LightEmptyTestStep def test_tbd_remove_in_v0_8_0_module_imports(): @@ -85,7 +85,7 @@ class ModelVer0_7(LightTrainDataloader, LightEmptyTestStep, TestModelBase): def test_tbd_remove_in_v1_0_0_model_hooks(): - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = ModelVer0_6(hparams) diff --git a/tests/trainer/test_callbacks.py b/tests/trainer/test_callbacks.py index 55a84633..377dce76 100644 --- a/tests/trainer/test_callbacks.py +++ b/tests/trainer/test_callbacks.py @@ -1,10 +1,7 @@ -import os - -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Callback from pytorch_lightning import Trainer, LightningModule -from pytorch_lightning.callbacks import ModelCheckpoint -from tests.models import ( +from tests.base import ( TestModelBase, LightTrainDataloader, LightValidationMixin, @@ -23,7 +20,7 @@ def test_trainer_callback_system(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) def _check_args(trainer, pl_module): diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index 40670daf..6f0ee15a 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -1,9 +1,9 @@ import pytest -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.utilities.debugging import MisconfigurationException -from tests.models import ( +from tests.base import ( TestModelBase, LightningTestModel, LightEmptyTestStep, @@ -29,7 +29,7 @@ def test_dataloader_config_errors(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # percent check < 0 @@ -104,7 +104,7 @@ def test_multiple_val_dataloader(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta @@ -143,7 +143,7 @@ def test_multiple_test_dataloader(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta @@ -178,7 +178,7 @@ def test_train_dataloaders_passed_to_fit(tmpdir): class CurrentTestModel(LightTrainDataloader, TestModelBase): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() # logger file to get meta trainer_options = dict( @@ -208,7 +208,7 @@ def test_train_val_dataloaders_passed_to_fit(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() # logger file to get meta trainer_options = dict( @@ -243,7 +243,7 @@ def test_all_dataloaders_passed_to_fit(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() # logger file to get meta trainer_options = dict( @@ -282,7 +282,7 @@ def test_multiple_dataloaders_passed_to_fit(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() # logger file to get meta trainer_options = dict( @@ -321,7 +321,7 @@ def test_mixing_of_dataloader_options(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta @@ -360,7 +360,7 @@ def test_inf_train_dataloader(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # fit model @@ -394,7 +394,7 @@ def test_inf_val_dataloader(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # fit model @@ -428,7 +428,7 @@ def test_inf_test_dataloader(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # fit model diff --git a/tests/trainer/test_optimizers.py b/tests/trainer/test_optimizers.py index 3ea0e3ff..4de3580e 100644 --- a/tests/trainer/test_optimizers.py +++ b/tests/trainer/test_optimizers.py @@ -1,13 +1,7 @@ -import math -import os - -import pytest -import torch - -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer -from tests.models import ( +from tests.base import ( TestModelBase, LightTrainDataloader, LightValidationStepMixin, @@ -29,7 +23,7 @@ def test_optimizer_with_scheduling(tmpdir): TestModelBase): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta @@ -68,7 +62,7 @@ def test_multi_optimizer_with_scheduling(tmpdir): TestModelBase): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta @@ -111,7 +105,7 @@ def test_multi_optimizer_with_scheduling_stepping(tmpdir): TestModelBase): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta @@ -160,7 +154,7 @@ def test_reduce_lr_on_plateau_scheduling(tmpdir): TestModelBase): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 47a1eb3d..89d849d7 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -6,7 +6,7 @@ from argparse import Namespace import pytest import torch -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ( EarlyStopping, @@ -15,7 +15,7 @@ from pytorch_lightning.callbacks import ( from pytorch_lightning.core.lightning import load_hparams_from_tags_csv from pytorch_lightning.trainer.logging import TrainerLoggingMixin from pytorch_lightning.utilities.debugging import MisconfigurationException -from tests.models import ( +from tests.base import ( TestModelBase, DictHparamsModel, LightningTestModel, @@ -53,7 +53,7 @@ def test_no_val_module(tmpdir): """Tests use case where trainer saves the model, and user loads it from tags independently.""" tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() class CurrentTestModel(LightTrainDataloader, TestModelBase): pass @@ -61,7 +61,7 @@ def test_no_val_module(tmpdir): model = CurrentTestModel(hparams) # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) trainer_options = dict( max_epochs=1, @@ -97,11 +97,11 @@ def test_no_val_end_module(tmpdir): class CurrentTestModel(LightTrainDataloader, LightValidationStepMixin, TestModelBase): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) trainer_options = dict( max_epochs=1, @@ -189,7 +189,7 @@ def test_gradient_accumulation_scheduling(tmpdir): # clear gradients optimizer.zero_grad() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) schedule = {1: 2, 3: 4} @@ -209,10 +209,10 @@ def test_gradient_accumulation_scheduling(tmpdir): def test_loading_meta_tags(tmpdir): tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() # save tags - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) logger.log_hyperparams(Namespace(some_str='a_str', an_int=1, a_float=2.0)) logger.log_hyperparams(hparams) logger.save() @@ -254,7 +254,7 @@ def test_model_checkpoint_options(tmpdir): def mock_save_function(filepath): open(filepath, 'a').close() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() _ = LightningTestModel(hparams) # simulated losses @@ -355,7 +355,7 @@ def test_model_checkpoint_options(tmpdir): os.mkdir(save_dir) # ----------------- - # CASE K=4 (save all 4 models) + # CASE K=4 (save all 4 base) # multiple checkpoints within same epoch checkpoint_callback = ModelCheckpoint(save_dir, save_top_k=4, verbose=1) @@ -401,7 +401,7 @@ def test_model_checkpoint_options(tmpdir): def test_model_freeze_unfreeze(): tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) model.freeze() @@ -414,7 +414,7 @@ def test_resume_from_checkpoint_epoch_restored(tmpdir): tutils.reset_seed() - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() def _new_model(): # Create a model that tracks epochs and batches seen @@ -474,7 +474,7 @@ def test_resume_from_checkpoint_epoch_restored(tmpdir): def _init_steps_model(): """private method for initializing a model with 5% train epochs""" tutils.reset_seed() - model, _ = tutils.get_model() + model, _ = tutils.get_default_model() # define train epoch to 5% of data train_percent = 0.05 @@ -530,7 +530,7 @@ def test_trainer_min_steps_and_epochs(tmpdir): trainer_options.update(dict( default_save_path=tmpdir, early_stop_callback=EarlyStopping(monitor='val_loss', min_delta=1.0), - val_check_interval=20, + val_check_interval=2, min_epochs=1, max_epochs=10 )) @@ -571,7 +571,7 @@ def test_benchmark_option(tmpdir): ): pass - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() model = CurrentTestModel(hparams) # verify torch.backends.cudnn.benchmark is not turned on @@ -596,7 +596,7 @@ def test_benchmark_option(tmpdir): def test_testpass_overrides(tmpdir): - hparams = tutils.get_hparams() + hparams = tutils.get_default_hparams() class LocalModel(LightTrainDataloader, TestModelBase): pass diff --git a/tests/trainer/test_trainer_cli.py b/tests/trainer/test_trainer_cli.py index 294ac9bc..92bbd647 100644 --- a/tests/trainer/test_trainer_cli.py +++ b/tests/trainer/test_trainer_cli.py @@ -4,7 +4,7 @@ from unittest import mock import pytest -import tests.models.utils as tutils +import tests.base.utils as tutils from pytorch_lightning import Trainer @@ -15,7 +15,7 @@ def test_default_args(tmpdir): tutils.reset_seed() # logger file to get meta - logger = tutils.get_test_tube_logger(tmpdir, False) + logger = tutils.get_default_testtube_logger(tmpdir, False) parser = ArgumentParser(add_help=False) args = parser.parse_args()