From 2096306362bdf9e6dcecb4770f24d9e9192f350b Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sun, 26 Apr 2020 12:49:43 -0400 Subject: [PATCH 01/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 30ecaa48..65035e45 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ [![codecov](https://codecov.io/gh/PyTorchLightning/pytorch-lightning/branch/master/graph/badge.svg)](https://codecov.io/gh/PyTorchLightning/pytorch-lightning) [![CodeFactor](https://www.codefactor.io/repository/github/pytorchlightning/pytorch-lightning/badge)](https://www.codefactor.io/repository/github/pytorchlightning/pytorch-lightning) -[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=0.7.4)](https://pytorch-lightning.readthedocs.io/en/0.7.4/) +[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=0.7.4)](https://pytorch-lightning.readthedocs.io/en/latest/) [![Slack](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://join.slack.com/t/pytorch-lightning/shared_invite/enQtODU5ODIyNTUzODQwLTFkMDg5Mzc1MDBmNjEzMDgxOTVmYTdhYjA1MDdmODUyOTg2OGQ1ZWZkYTQzODhhNzdhZDA3YmNhMDhlMDY4YzQ) [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/PytorchLightning/pytorch-lightning/blob/master/LICENSE) [![Next Release](https://img.shields.io/badge/Next%20Release-May%2006-.svg)](https://shields.io/) From 0f019d7a944980447943f735ceb22c04e981d6ee Mon Sep 17 00:00:00 2001 From: Peter Yu <2057325+yukw777@users.noreply.github.com> Date: Sun, 26 Apr 2020 16:10:58 -0400 Subject: [PATCH 02/11] pin actions/checkout version to v2 (#1617) --- .github/workflows/docker_builds.yml | 12 +++++------- .github/workflows/pypi-release.yml | 4 ++-- .github/workflows/rebase.yml | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker_builds.yml b/.github/workflows/docker_builds.yml index 9ba099e5..1ac289ba 100644 --- a/.github/workflows/docker_builds.yml +++ b/.github/workflows/docker_builds.yml @@ -1,10 +1,10 @@ name: Publish Docker on: - push: + push: branches: - - master - release: - types: + - master + release: + types: - created jobs: @@ -19,7 +19,7 @@ jobs: if: contains(github.ref, 'refs/tags/') id: get_version run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - uses: actions/checkout@2.14 + - uses: actions/checkout@v2 - name: Publish Releases to Docker # only on releases uses: elgohr/Publish-Docker-Github-Action@master @@ -42,5 +42,3 @@ jobs: dockerfile: docker/Dockerfile buildargs: PYTHON_VERSION=${{ matrix.python_version }},PYTORCH_VERSION=${{ matrix.pytorch_version }},LIGHTNING_VERSION=${{ steps.get_version.outputs.VERSION }} tags: "latest_py${{ matrix.python_version }}_torch${{ matrix.pytorch_version }}" - - diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 96535064..041f67bc 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 - name: Set up Python 3.7 uses: actions/setup-python@v1 with: @@ -30,7 +30,7 @@ jobs: - name: Build run: >- python setup.py sdist bdist_wheel - + # We do this, since failures on test.pypi aren't that bad - name: Publish to Test PyPI if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml index 3db59b60..2aa94bea 100644 --- a/.github/workflows/rebase.yml +++ b/.github/workflows/rebase.yml @@ -10,7 +10,7 @@ jobs: if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v2 with: fetch-depth: 0 - name: Automatic Rebase @@ -23,4 +23,4 @@ jobs: runs-on: ubuntu-latest steps: - name: Always run - run: echo "This job is used to prevent the workflow to fail when all other jobs are skipped." \ No newline at end of file + run: echo "This job is used to prevent the workflow to fail when all other jobs are skipped." From 68c9a110a200f1344adc8fef4dcf7e1a9f87ad7d Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sun, 26 Apr 2020 22:11:22 +0200 Subject: [PATCH 03/11] changelog (#1616) * changelog * warning * pull * typo * typo --- CHANGELOG.md | 86 +++++++------------ .../gradient_accumulation_scheduler.py | 7 +- 2 files changed, 36 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61eeb435..5b8e2f29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,86 +8,66 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added -- Added flag `replace_sampler_ddp` to manually disaple sampler replacement in ddp ([#1513](https://github.com/PyTorchLightning/pytorch-lightning/pull/1513)) +### Changed +### Deprecated + +### Removed + +### Fixed + + +## [0.7.4] - 2020-04-26 + +### Added + +- Added flag `replace_sampler_ddp` to manually disable sampler replacement in DDP ([#1513](https://github.com/PyTorchLightning/pytorch-lightning/pull/1513)) - Added speed parity tests (max 1 sec difference per epoch)([#1482](https://github.com/PyTorchLightning/pytorch-lightning/pull/1482)) - - Added `auto_select_gpus` flag to trainer that enables automatic selection of available GPUs on exclusive mode systems. - -- Added learining rate finder ([#1347](https://github.com/PyTorchLightning/pytorch-lightning/pull/1347)) - -- Added support for ddp mode in clusters without SLURM ([#1345](https://github.com/PyTorchLightning/pytorch-lightning/issues/1345)) - -- Added `test_dataloaders` parameter to `Trainer.test()` ([#1393](https://github.com/PyTorchLightning/pytorch-lightning/issues/1393)) - +- Added learning rate finder ([#1347](https://github.com/PyTorchLightning/pytorch-lightning/pull/1347)) +- Added support for ddp mode in clusters without SLURM ([#1387](https://github.com/PyTorchLightning/pytorch-lightning/pull/1387)) +- Added `test_dataloaders` parameter to `Trainer.test()` ([#1434](https://github.com/PyTorchLightning/pytorch-lightning/pull/1434)) - Added `terminate_on_nan` flag to trainer that performs a NaN check with each training iteration when set to `True` ([#1475](https://github.com/PyTorchLightning/pytorch-lightning/pull/1475)) - - Added speed parity tests (max 1 sec difference per epoch)([#1482](https://github.com/PyTorchLightning/pytorch-lightning/pull/1482)) - - Added `terminate_on_nan` flag to trainer that performs a NaN check with each training iteration when set to `True`. ([#1475](https://github.com/PyTorchLightning/pytorch-lightning/pull/1475)) - - Added `ddp_cpu` backend for testing ddp without GPUs ([#1158](https://github.com/PyTorchLightning/pytorch-lightning/pull/1158)) - - Added [Horovod](http://horovod.ai) support as a distributed backend `Trainer(distributed_backend='horovod')` ([#1529](https://github.com/PyTorchLightning/pytorch-lightning/pull/1529)) - - Added support for 8 core distributed training on Kaggle TPU's ([#1568](https://github.com/PyTorchLightning/pytorch-lightning/pull/1568)) - - Added support for native AMP ([#1561](https://github.com/PyTorchLightning/pytorch-lightning/pull/1561), [#1580](https://github.com/PyTorchLightning/pytorch-lightning/pull/1580)) ### Changed - Changed the default behaviour to no longer include a NaN check with each training iteration. ([#1475](https://github.com/PyTorchLightning/pytorch-lightning/pull/1475)) - -- Decoupled the progress bar from trainer. It is a callback now and can be customized or even be replaced entirely ([#1450](https://github.com/PyTorchLightning/pytorch-lightning/pull/1450)). - -- Changed lr schedule step interval behavior to update every backwards pass instead of every forwards pass ([#1476](https://github.com/PyTorchLightning/pytorch-lightning/issues/1476)) - +- Decoupled the progress bar from trainer` it is a callback now and can be customized or even be replaced entirely ([#1450](https://github.com/PyTorchLightning/pytorch-lightning/pull/1450)). +- Changed lr schedule step interval behavior to update every backwards pass instead of every forwards pass ([#1477](https://github.com/PyTorchLightning/pytorch-lightning/pull/1477)) - Defines shared proc. rank, remove rank from instances (e.g. loggers) ([#1408](https://github.com/PyTorchLightning/pytorch-lightning/pull/1408)) - -- Updated semantic segmentation example with custom u-net and logging ([#1371](https://github.com/PyTorchLightning/pytorch-lightning/pull/1371)) - -- Diabled val and test shuffling ([#1600](https://github.com/PyTorchLightning/pytorch-lightning/pull/1600)) - +- Updated semantic segmentation example with custom U-Net and logging ([#1371](https://github.com/PyTorchLightning/pytorch-lightning/pull/1371)) +- Disabled val and test shuffling ([#1600](https://github.com/PyTorchLightning/pytorch-lightning/pull/1600)) ### Deprecated -- Deprecatd `training_tqdm_dict` in favor of `progress_bar_dict` ([#1450](https://github.com/PyTorchLightning/pytorch-lightning/pull/1450)). - +- Deprecated `training_tqdm_dict` in favor of `progress_bar_dict` ([#1450](https://github.com/PyTorchLightning/pytorch-lightning/pull/1450)). ### Removed -- Removed `test_dataloaders` parameter from `Trainer.fit()` ([#1393](https://github.com/PyTorchLightning/pytorch-lightning/issues/1393)) - +- Removed `test_dataloaders` parameter from `Trainer.fit()` ([#1434](https://github.com/PyTorchLightning/pytorch-lightning/pull/1434)) ### Fixed - Added the possibility to pass nested metrics dictionaries to loggers ([#1582](https://github.com/PyTorchLightning/pytorch-lightning/pull/1582)) - - Fixed memory leak from opt return ([#1528](https://github.com/PyTorchLightning/pytorch-lightning/pull/1528)) - - Fixed saving checkpoint before deleting old ones ([#1453](https://github.com/PyTorchLightning/pytorch-lightning/pull/1453)) - - Fixed loggers - flushing last logged metrics even before continue, e.g. `trainer.test()` results ([#1459](https://github.com/PyTorchLightning/pytorch-lightning/pull/1459)) - - Fixed optimizer configuration when `configure_optimizers` returns dict without `lr_scheduler` ([#1443](https://github.com/PyTorchLightning/pytorch-lightning/pull/1443)) - -- Fixed LightningModule - Mixing hparams and arguments in `LightningModule.__init__()` crashes load_from_checkpoint() ([#1505](https://github.com/PyTorchLightning/pytorch-lightning/pull/1505)) - +- Fixed `LightningModule` - mixing hparams and arguments in `LightningModule.__init__()` crashes load_from_checkpoint() ([#1505](https://github.com/PyTorchLightning/pytorch-lightning/pull/1505)) - Added a missing call to the `on_before_zero_grad` model hook ([#1493](https://github.com/PyTorchLightning/pytorch-lightning/pull/1493)). - -- Allow use of sweeps with WandbLogger ([#1512](https://github.com/PyTorchLightning/pytorch-lightning/pull/1512)) - +- Allow use of sweeps with `WandbLogger` ([#1512](https://github.com/PyTorchLightning/pytorch-lightning/pull/1512)) - Fixed a bug that caused the `callbacks` Trainer argument to reference a global variable ([#1534](https://github.com/PyTorchLightning/pytorch-lightning/pull/1534)). - -- Fixed a bug that set all boolean CLI arguments from Trainer.add_argparse_args always to True ([#1570](https://github.com/PyTorchLightning/pytorch-lightning/issues/1570)) - -- Fixed do not copy the batch when training on a single GPU ([#1576](https://github.com/PyTorchLightning/pytorch-lightning/issues/1576), [#1579](https://github.com/PyTorchLightning/pytorch-lightning/issues/1579)) - +- Fixed a bug that set all boolean CLI arguments from `Trainer.add_argparse_args` always to True ([#1571](https://github.com/PyTorchLightning/pytorch-lightning/pull/1571)) +- Fixed do not copy the batch when training on a single GPU ([#1576](https://github.com/PyTorchLightning/pytorch-lightning/pull/1576), [#1579](https://github.com/PyTorchLightning/pytorch-lightning/pull/1579)) - Fixed soft checkpoint removing on DDP ([#1408](https://github.com/PyTorchLightning/pytorch-lightning/pull/1408)) - -- Fixes automatic parser bug ([#1585](https://github.com/PyTorchLightning/pytorch-lightning/issues/1585)) - -- Fixed bool conversion from string ([#1606](https://github.com/PyTorchLightning/pytorch-lightning/issues/1606)) +- Fixed automatic parser bug ([#1585](https://github.com/PyTorchLightning/pytorch-lightning/pull/1585)) +- Fixed bool conversion from string ([#1606](https://github.com/PyTorchLightning/pytorch-lightning/pull/1606)) ## [0.7.3] - 2020-04-09 @@ -156,17 +136,17 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Removed - Removed test for no test dataloader in .fit ([#1495](https://github.com/PyTorchLightning/pytorch-lightning/pull/1495)) -- Removed duplicated module `pytorch_lightning.utilities.arg_parse` for loading CLI arguments ([#1167](https://github.com/PyTorchLightning/pytorch-lightning/issues/1167)) +- Removed duplicated module `pytorch_lightning.utilities.arg_parse` for loading CLI arguments ([#1167](https://github.com/PyTorchLightning/pytorch-lightning/pull/1167)) - Removed wandb logger's `finalize` method ([#1193](https://github.com/PyTorchLightning/pytorch-lightning/pull/1193)) -- Dropped `torchvision` dependency in tests and added own MNIST dataset class instead ([#986](https://github.com/PyTorchLightning/pytorch-lightning/issues/986)) +- Dropped `torchvision` dependency in tests and added own MNIST dataset class instead ([#986](https://github.com/PyTorchLightning/pytorch-lightning/pull/986)) ### Fixed - Fixed `model_checkpoint` when saving all models ([#1359](https://github.com/PyTorchLightning/pytorch-lightning/pull/1359)) - `Trainer.add_argparse_args` classmethod fixed. Now it adds a type for the arguments ([#1147](https://github.com/PyTorchLightning/pytorch-lightning/pull/1147)) -- Fixed bug related to type checking of `ReduceLROnPlateau` lr schedulers([#1114](https://github.com/PyTorchLightning/pytorch-lightning/issues/1114)) +- Fixed bug related to type checking of `ReduceLROnPlateau` lr schedulers([#1126](https://github.com/PyTorchLightning/pytorch-lightning/pull/1126)) - Fixed a bug to ensure lightning checkpoints to be backward compatible ([#1132](https://github.com/PyTorchLightning/pytorch-lightning/pull/1132)) -- Fixed a bug that created an extra dataloader with active `reload_dataloaders_every_epoch` ([#1181](https://github.com/PyTorchLightning/pytorch-lightning/issues/1181)) +- Fixed a bug that created an extra dataloader with active `reload_dataloaders_every_epoch` ([#1196](https://github.com/PyTorchLightning/pytorch-lightning/pull/1196)) - Fixed all warnings and errors in the docs build process ([#1191](https://github.com/PyTorchLightning/pytorch-lightning/pull/1191)) - Fixed an issue where `val_percent_check=0` would not disable validation ([#1251](https://github.com/PyTorchLightning/pytorch-lightning/pull/1251)) - Fixed average of incomplete `TensorRunningMean` ([#1309](https://github.com/PyTorchLightning/pytorch-lightning/pull/1309)) @@ -185,7 +165,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Fixed `load_spawn_weights` only in proc rank 0 ([#1385](https://github.com/PyTorchLightning/pytorch-lightning/pull/1385)) - Fixes `use_amp` issue ([#1145](https://github.com/PyTorchLightning/pytorch-lightning/pull/1145)) - Fixes using deprecated `use_amp` attribute ([#1145](https://github.com/PyTorchLightning/pytorch-lightning/pull/1145)) -- Fixed Tensorboard logger error: lightning_logs directory not exists in multi-node DDP on nodes with rank != 0 ([#1375](https://github.com/PyTorchLightning/pytorch-lightning/issues/1375)) +- Fixed Tensorboard logger error: lightning_logs directory not exists in multi-node DDP on nodes with rank != 0 ([#1377](https://github.com/PyTorchLightning/pytorch-lightning/pull/1377)) - Fixed `Unimplemented backend XLA` error on TPU ([#1387](https://github.com/PyTorchLightning/pytorch-lightning/pull/1387)) ## [0.7.1] - 2020-03-07 diff --git a/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py b/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py index 0d2fa63b..bc1cd79e 100644 --- a/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py +++ b/pytorch_lightning/callbacks/gradient_accumulation_scheduler.py @@ -45,11 +45,10 @@ class GradientAccumulationScheduler(Callback): raise TypeError("All epoches and accumulation factor must be integers") minimal_epoch = min(scheduling.keys()) - rank_zero_warn('Epochs indexing of `scheduling` starts from "1" until v0.6.x,' - ' but will start from "0" in v0.8.0.', DeprecationWarning) + # rank_zero_warn('Epochs indexing of `scheduling` starts from "1" until v0.6.x,' + # ' but will start from "0" in v0.8.0.', DeprecationWarning) if minimal_epoch < 1: - msg = f"Epochs indexing from 1, epoch {minimal_epoch} cannot be interpreted correct" - raise IndexError(msg) + raise IndexError(f"Epochs indexing from 1, epoch {minimal_epoch} cannot be interpreted correct") if minimal_epoch != 1: # if user didnt define first epoch accumulation factor scheduling.update({1: 1}) From 813e37916d9b17224be3d6c4d1672876bfb88a54 Mon Sep 17 00:00:00 2001 From: "J. Borovec" Date: Wed, 22 Apr 2020 16:13:55 +0200 Subject: [PATCH 04/11] try GH actions cache (#1558, #1624) spec cache spec cache trigger extras revert refactor cache cache cache cache --- .github/workflows/ci-testing.yml | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 4f94ba04..36e49e0e 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -60,22 +60,20 @@ jobs: python -c "req = open('requirements.txt').read().replace('>', '=') ; open('requirements.txt', 'w').write(req)" python -c "req = open('requirements-extra.txt').read().replace('>', '=') ; open('requirements-extra.txt', 'w').write(req)" + # Note: This uses an internal pip API and may not always work + # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow + - name: Get pip cache + id: pip-cache + run: | + python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" + - name: Cache pip uses: actions/cache@v1 with: - path: ~/.cache/pip # This path is specific to Ubuntu - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements-extra.txt') }} restore-keys: | - ${{ runner.os }}-pip-${{ matrix.python-version }}- - ${{ runner.os }}-${{ matrix.python-version }}- - - - name: Cache datasets - uses: actions/cache@v1 - with: - path: tests/Datasets # This path is specific to Ubuntu - # Look to see if there is a cache hit for the corresponding requirements file - key: mnist-dataset + ${{ runner.os }}-${{ matrix.python-version }}-pip- - name: Install dependencies run: | @@ -88,6 +86,13 @@ jobs: pip list shell: bash + - name: Cache datasets + uses: actions/cache@v1 + with: + path: tests/Datasets # This path is specific to Ubuntu + # Look to see if there is a cache hit for the corresponding requirements file + key: mnist-dataset + - name: Tests # env: # TOXENV: py${{ matrix.python-version }} @@ -111,7 +116,6 @@ jobs: python setup.py check --metadata --strict python setup.py sdist twine check dist/* - #- name: Try install package # if: ! startsWith(matrix.os, 'windows') # run: | @@ -122,4 +126,4 @@ jobs: - name: Statistics if: success() run: | - coverage report + coverage report \ No newline at end of file From afd0e5d02f334c262ed144a879b295b9c6abbee7 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sun, 26 Apr 2020 17:27:45 -0400 Subject: [PATCH 05/11] fix hparams issue (#1623) fix hparams issue fix hparams issue --- pytorch_lightning/trainer/training_io.py | 26 ++++++++++++++++++++++++ tests/trainer/test_trainer.py | 11 +++++----- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/trainer/training_io.py b/pytorch_lightning/trainer/training_io.py index 0e9d00c6..4bb3c406 100644 --- a/pytorch_lightning/trainer/training_io.py +++ b/pytorch_lightning/trainer/training_io.py @@ -325,6 +325,7 @@ class TrainerIOMixin(ABC): checkpoint['native_amp_scaling_state'] = self.scaler.state_dict() if hasattr(model, "hparams"): + self.__clean_namespace(model.hparams) is_namespace = isinstance(model.hparams, Namespace) checkpoint['hparams'] = vars(model.hparams) if is_namespace else model.hparams checkpoint['hparams_type'] = 'namespace' if is_namespace else 'dict' @@ -338,6 +339,31 @@ class TrainerIOMixin(ABC): return checkpoint + def __clean_namespace(self, hparams): + """ + Removes all functions from hparams so we can pickle + :param hparams: + :return: + """ + + if isinstance(hparams, Namespace): + del_attrs = [] + for k in hparams.__dict__: + if callable(getattr(hparams, k)): + del_attrs.append(k) + + for k in del_attrs: + delattr(hparams, k) + + elif isinstance(hparams, dict): + del_attrs = [] + for k, v in hparams.items(): + if callable(v): + del_attrs.append(k) + + for k in del_attrs: + del hparams[k] + # -------------------- # HPC IO # -------------------- diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 2d34be24..6876a693 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -27,7 +27,7 @@ from tests.base import ( def test_hparams_save_load(tmpdir): - model = DictHparamsModel({'in_features': 28 * 28, 'out_features': 10}) + model = DictHparamsModel({'in_features': 28 * 28, 'out_features': 10, 'failed_key': lambda x: x}) # logger file to get meta trainer_options = dict( @@ -79,12 +79,13 @@ def test_no_val_module(tmpdir): new_weights_path = os.path.join(tmpdir, 'save_test.ckpt') trainer.save_checkpoint(new_weights_path) - # load new model - tags_path = tutils.get_data_path(logger, path_dir=tmpdir) - tags_path = os.path.join(tags_path, 'meta_tags.csv') + # assert ckpt has hparams + ckpt = torch.load(new_weights_path) + assert 'hparams' in ckpt.keys(), 'hparams missing from checkpoints' + + # won't load without hparams in the ckpt model_2 = LightningTestModel.load_from_checkpoint( checkpoint_path=new_weights_path, - tags_csv=tags_path ) model_2.eval() From 2180aa19ad2a1fc70e217a19964592d1bc7db62e Mon Sep 17 00:00:00 2001 From: "J. Borovec" Date: Wed, 22 Apr 2020 16:13:55 +0200 Subject: [PATCH 06/11] Remove warning (#1634) Remove warning Update CHANGELOG.md --- .github/workflows/ci-testing.yml | 3 ++- CHANGELOG.md | 2 ++ pytorch_lightning/__init__.py | 2 +- pytorch_lightning/trainer/training_loop.py | 3 --- tests/trainer/test_trainer.py | 9 +++++---- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 36e49e0e..7426a229 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -1,6 +1,6 @@ name: CI testing -# https://help.github.com/en/actions/reference/events-that-trigger-workflows +# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows on: # Trigger the workflow on push or pull request, # but only for the master branch @@ -116,6 +116,7 @@ jobs: python setup.py check --metadata --strict python setup.py sdist twine check dist/* + #- name: Try install package # if: ! startsWith(matrix.os, 'windows') # run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b8e2f29..9aff6450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Removed +- Removed Warning from trainer loop ([#1634](https://github.com/PyTorchLightning/pytorch-lightning/pull/1634)) + ### Fixed diff --git a/pytorch_lightning/__init__.py b/pytorch_lightning/__init__.py index 1aa8eb23..2a656ce2 100644 --- a/pytorch_lightning/__init__.py +++ b/pytorch_lightning/__init__.py @@ -1,6 +1,6 @@ """Root package info.""" -__version__ = '0.7.4' +__version__ = '0.7.5rc1' __author__ = 'William Falcon et al.' __author_email__ = 'waf2107@columbia.edu' __license__ = 'Apache-2.0' diff --git a/pytorch_lightning/trainer/training_loop.py b/pytorch_lightning/trainer/training_loop.py index 8697270c..37bac3d9 100644 --- a/pytorch_lightning/trainer/training_loop.py +++ b/pytorch_lightning/trainer/training_loop.py @@ -299,9 +299,6 @@ class TrainerTrainLoopMixin(ABC): """Warning: this is just empty shell for code implemented in other class.""" def train(self): - rank_zero_warn('Displayed epoch numbers in the progress bar start from "1" until v0.6.x,' - ' but will start from "0" in v0.8.0.', RuntimeWarning) - # get model model = self.get_model() diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 6876a693..5edf017c 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -120,12 +120,13 @@ def test_no_val_end_module(tmpdir): new_weights_path = os.path.join(tmpdir, 'save_test.ckpt') trainer.save_checkpoint(new_weights_path) - # load new model - tags_path = tutils.get_data_path(logger, path_dir=tmpdir) - tags_path = os.path.join(tags_path, 'meta_tags.csv') + # assert ckpt has hparams + ckpt = torch.load(new_weights_path) + assert 'hparams' in ckpt.keys(), 'hparams missing from checkpoints' + + # won't load without hparams in the ckpt model_2 = LightningTestModel.load_from_checkpoint( checkpoint_path=new_weights_path, - tags_csv=tags_path ) model_2.eval() From 9604d7bf8994615431af9d86c8de154677237b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Jab=C5=82o=C5=84ski?= Date: Mon, 27 Apr 2020 11:02:33 +0200 Subject: [PATCH 07/11] Fix ModelCheckpoint not being fixable (#1632) --- pytorch_lightning/callbacks/model_checkpoint.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py index 1d5d6d14..44d240b5 100644 --- a/pytorch_lightning/callbacks/model_checkpoint.py +++ b/pytorch_lightning/callbacks/model_checkpoint.py @@ -116,10 +116,10 @@ class ModelCheckpoint(Callback): torch_inf = torch.tensor(np.Inf) mode_dict = { - 'min': (torch.lt, torch_inf, 'min'), - 'max': (torch.gt, -torch_inf, 'max'), - 'auto': (torch.gt, -torch_inf, 'max') if 'acc' in self.monitor or self.monitor.startswith('fmeasure') - else (torch.lt, torch_inf, 'min'), + 'min': (torch_inf, 'min'), + 'max': (-torch_inf, 'max'), + 'auto': (-torch_inf, 'max') if 'acc' in self.monitor or self.monitor.startswith('fmeasure') + else (torch_inf, 'min'), } if mode not in mode_dict: @@ -127,7 +127,7 @@ class ModelCheckpoint(Callback): f'fallback to auto mode.', RuntimeWarning) mode = 'auto' - self.monitor_op, self.kth_value, self.mode = mode_dict[mode] + self.kth_value, self.mode = mode_dict[mode] def _del_model(self, filepath): if os.path.isfile(filepath): @@ -151,7 +151,12 @@ class ModelCheckpoint(Callback): if not isinstance(current, torch.Tensor): current = torch.tensor(current) - return self.monitor_op(current, self.best_k_models[self.kth_best_model]) + monitor_op = { + "min": torch.lt, + "max": torch.gt, + }[self.mode] + + return monitor_op(current, self.best_k_models[self.kth_best_model]) def format_checkpoint_name(self, epoch, metrics, ver=None): """Generate a filename according to the defined template. From b83b8005f9eaf0ff19edc6cbef1cc95aea280be1 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Mon, 27 Apr 2020 09:44:26 +0200 Subject: [PATCH 08/11] Allow metrics logged together with hparams (#1630) Update tensorboard.py Update CHANGELOG.md Update tensorboard.py Update test_tensorboard.py Update test_tensorboard.py tests pep8 --- CHANGELOG.md | 2 ++ pytorch_lightning/loggers/tensorboard.py | 7 +++++-- tests/loggers/test_tensorboard.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aff6450..0978f344 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added ### Changed + +- Allow logging of metrics togther with hparams ([#1630](https://github.com/PyTorchLightning/pytorch-lightning/pull/1630)) ### Deprecated diff --git a/pytorch_lightning/loggers/tensorboard.py b/pytorch_lightning/loggers/tensorboard.py index 613262dd..fc33c9e9 100644 --- a/pytorch_lightning/loggers/tensorboard.py +++ b/pytorch_lightning/loggers/tensorboard.py @@ -101,7 +101,8 @@ class TensorBoardLogger(LightningLoggerBase): return self._experiment @rank_zero_only - def log_hyperparams(self, params: Union[Dict[str, Any], Namespace]) -> None: + def log_hyperparams(self, params: Union[Dict[str, Any], Namespace], + metrics: Optional[Dict[str, Any]] = None) -> None: params = self._convert_params(params) params = self._flatten_dict(params) sanitized_params = self._sanitize_params(params) @@ -114,7 +115,9 @@ class TensorBoardLogger(LightningLoggerBase): ) else: from torch.utils.tensorboard.summary import hparams - exp, ssi, sei = hparams(sanitized_params, {}) + if metrics is None: + metrics = {} + exp, ssi, sei = hparams(sanitized_params, metrics) writer = self.experiment._get_file_writer() writer.add_summary(exp) writer.add_summary(ssi) diff --git a/tests/loggers/test_tensorboard.py b/tests/loggers/test_tensorboard.py index 937a233c..a17cedc4 100644 --- a/tests/loggers/test_tensorboard.py +++ b/tests/loggers/test_tensorboard.py @@ -77,3 +77,19 @@ def test_tensorboard_log_hyperparams(tmpdir): "layer": torch.nn.BatchNorm1d } logger.log_hyperparams(hparams) + + +def test_tensorboard_log_hparams_and_metrics(tmpdir): + logger = TensorBoardLogger(tmpdir) + hparams = { + "float": 0.3, + "int": 1, + "string": "abc", + "bool": True, + "dict": {'a': {'b': 'c'}}, + "list": [1, 2, 3], + "namespace": Namespace(foo=Namespace(bar='buzz')), + "layer": torch.nn.BatchNorm1d + } + metrics = {'abc': torch.tensor([0.54])} + logger.log_hyperparams(hparams, metrics) From 9557b73cadea19015640910e54ce6ac7f5d404cb Mon Sep 17 00:00:00 2001 From: William Falcon Date: Mon, 27 Apr 2020 06:51:42 -0400 Subject: [PATCH 09/11] Fixes CPU DDP breaking change and DDP change (#1635) ddp pickle ddp pickle ddp pickle ddp pickle ddp pickle ddp pickle ddp pickle --- pytorch_lightning/trainer/data_loading.py | 1 + pytorch_lightning/trainer/trainer.py | 29 ++++++++++++++--------- pytorch_lightning/trainer/training_io.py | 29 ++--------------------- pytorch_lightning/utilities/parsing.py | 29 +++++++++++++++++++++++ tests/trainer/test_trainer_cli.py | 5 ++++ 5 files changed, 55 insertions(+), 38 deletions(-) diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index 7d0b8180..b3e15024 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -133,6 +133,7 @@ class TrainerDataLoadingMixin(ABC): world_size = { 'ddp': self.num_nodes * self.num_processes, 'ddp2': self.num_nodes, + 'ddp_cpu': self.num_processes * self.num_nodes } sampler = DistributedSampler( dataloader.dataset, diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index eeafb73f..deed0098 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -614,17 +614,8 @@ class Trainer( return bool(parsing.strtobool(x)) if arg == 'gpus': - def allowed_type(x): - if ',' in x: - return str(x) - else: - return int(x) - - def arg_default(x): - if ',' in x: - return str(x) - else: - return int(x) + allowed_type = Trainer.allowed_type + arg_default = Trainer.arg_default parser.add_argument( f'--{arg}', @@ -637,6 +628,18 @@ class Trainer( return parser + def allowed_type(x): + if ',' in x: + return str(x) + else: + return int(x) + + def arg_default(x): + if ',' in x: + return str(x) + else: + return int(x) + @classmethod def from_argparse_args(cls, args, **kwargs): @@ -711,6 +714,10 @@ class Trainer( model.logger = self.logger self.copy_trainer_model_properties(model) + # clean hparams + if hasattr(model, 'hparams'): + parsing.clean_namespace(model.hparams) + # set up the passed in dataloaders (if needed) self.__attach_dataloaders(model, train_dataloader, val_dataloaders) diff --git a/pytorch_lightning/trainer/training_io.py b/pytorch_lightning/trainer/training_io.py index 4bb3c406..82bc0829 100644 --- a/pytorch_lightning/trainer/training_io.py +++ b/pytorch_lightning/trainer/training_io.py @@ -101,7 +101,7 @@ from pytorch_lightning.overrides.data_parallel import ( LightningDistributedDataParallel, LightningDataParallel, ) -from pytorch_lightning.utilities import rank_zero_warn +from pytorch_lightning.utilities import rank_zero_warn, parsing try: import torch_xla @@ -325,7 +325,7 @@ class TrainerIOMixin(ABC): checkpoint['native_amp_scaling_state'] = self.scaler.state_dict() if hasattr(model, "hparams"): - self.__clean_namespace(model.hparams) + parsing.clean_namespace(model.hparams) is_namespace = isinstance(model.hparams, Namespace) checkpoint['hparams'] = vars(model.hparams) if is_namespace else model.hparams checkpoint['hparams_type'] = 'namespace' if is_namespace else 'dict' @@ -339,31 +339,6 @@ class TrainerIOMixin(ABC): return checkpoint - def __clean_namespace(self, hparams): - """ - Removes all functions from hparams so we can pickle - :param hparams: - :return: - """ - - if isinstance(hparams, Namespace): - del_attrs = [] - for k in hparams.__dict__: - if callable(getattr(hparams, k)): - del_attrs.append(k) - - for k in del_attrs: - delattr(hparams, k) - - elif isinstance(hparams, dict): - del_attrs = [] - for k, v in hparams.items(): - if callable(v): - del_attrs.append(k) - - for k in del_attrs: - del hparams[k] - # -------------------- # HPC IO # -------------------- diff --git a/pytorch_lightning/utilities/parsing.py b/pytorch_lightning/utilities/parsing.py index 26fc410d..2549e485 100644 --- a/pytorch_lightning/utilities/parsing.py +++ b/pytorch_lightning/utilities/parsing.py @@ -1,3 +1,6 @@ +from argparse import Namespace + + def strtobool(val): """Convert a string representation of truth to true (1) or false (0). Copied from the python implementation distutils.utils.strtobool @@ -18,3 +21,29 @@ def strtobool(val): return 0 else: raise ValueError(f'invalid truth value {val}') + + +def clean_namespace(hparams): + """ + Removes all functions from hparams so we can pickle + :param hparams: + :return: + """ + + if isinstance(hparams, Namespace): + del_attrs = [] + for k in hparams.__dict__: + if callable(getattr(hparams, k)): + del_attrs.append(k) + + for k in del_attrs: + delattr(hparams, k) + + elif isinstance(hparams, dict): + del_attrs = [] + for k, v in hparams.items(): + if callable(v): + del_attrs.append(k) + + for k in del_attrs: + del hparams[k] diff --git a/tests/trainer/test_trainer_cli.py b/tests/trainer/test_trainer_cli.py index bfc67111..93cbb8e2 100644 --- a/tests/trainer/test_trainer_cli.py +++ b/tests/trainer/test_trainer_cli.py @@ -47,6 +47,11 @@ def test_add_argparse_args_redefined(cli_args): assert depr_name not in args trainer = Trainer.from_argparse_args(args=args) + + # make sure trainer can be pickled + import pickle + pickle.dumps(trainer) + assert isinstance(trainer, Trainer) From 0782c6e8c4c278ca382e66e29d4f2f288324a0fd Mon Sep 17 00:00:00 2001 From: William Falcon Date: Sun, 26 Apr 2020 17:27:45 -0400 Subject: [PATCH 10/11] test pickling (#1636) fix hparams issue cache cache cache cache ddp pickle ddp pickle ddp pickle ddp pickle ddp pickle ddp pickle ddp pickle ddp fix pep8 --- pytorch_lightning/callbacks/early_stopping.py | 13 +++++++++++-- tests/callbacks/test_callbacks.py | 9 +++++++++ tests/loggers/test_all.py | 3 +++ tests/trainer/test_trainer.py | 7 +++++++ tests/trainer/test_trainer_cli.py | 7 ++++--- 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/callbacks/early_stopping.py b/pytorch_lightning/callbacks/early_stopping.py index 4c30b677..d383a2fb 100644 --- a/pytorch_lightning/callbacks/early_stopping.py +++ b/pytorch_lightning/callbacks/early_stopping.py @@ -57,6 +57,7 @@ class EarlyStopping(Callback): self.min_delta = min_delta self.wait = 0 self.stopped_epoch = 0 + self.mode = mode mode_dict = { 'min': torch.lt, @@ -67,9 +68,8 @@ class EarlyStopping(Callback): if mode not in mode_dict: if self.verbose > 0: log.info(f'EarlyStopping mode {mode} is unknown, fallback to auto mode.') - mode = 'auto' + self.mode = 'auto' - self.monitor_op = mode_dict[mode] self.min_delta *= 1 if self.monitor_op == torch.gt else -1 def _validate_condition_metric(self, logs): @@ -94,6 +94,15 @@ class EarlyStopping(Callback): return True + @property + def monitor_op(self): + mode_dict = { + 'min': torch.lt, + 'max': torch.gt, + 'auto': torch.gt if 'acc' in self.monitor else torch.lt + } + return mode_dict[self.mode] + def on_train_start(self, trainer, pl_module): # Allow instances to be re-used self.wait = 0 diff --git a/tests/callbacks/test_callbacks.py b/tests/callbacks/test_callbacks.py index 4731d435..c6c36ca5 100644 --- a/tests/callbacks/test_callbacks.py +++ b/tests/callbacks/test_callbacks.py @@ -240,6 +240,15 @@ def test_early_stopping_no_val_step(tmpdir): assert trainer.current_epoch < trainer.max_epochs +def test_pickling(tmpdir): + import pickle + early_stopping = EarlyStopping() + ckpt = ModelCheckpoint(tmpdir) + + pickle.dumps(ckpt) + pickle.dumps(early_stopping) + + def test_model_checkpoint_with_non_string_input(tmpdir): """ Test that None in checkpoint callback is valid and that chkp_path is set correctly """ diff --git a/tests/loggers/test_all.py b/tests/loggers/test_all.py index d9bb804b..383ca263 100644 --- a/tests/loggers/test_all.py +++ b/tests/loggers/test_all.py @@ -88,6 +88,9 @@ def test_loggers_pickle(tmpdir, monkeypatch, logger_class): logger_args = _get_logger_args(logger_class, tmpdir) logger = logger_class(**logger_args) + # test pickling loggers + pickle.dumps(logger) + trainer = Trainer( max_epochs=1, logger=logger diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index 5edf017c..f89f6b3f 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -26,6 +26,13 @@ from tests.base import ( ) +def test_model_pickle(tmpdir): + import pickle + + model = TestModelBase(tutils.get_default_hparams()) + pickle.dumps(model) + + def test_hparams_save_load(tmpdir): model = DictHparamsModel({'in_features': 28 * 28, 'out_features': 10, 'failed_key': lambda x: x}) diff --git a/tests/trainer/test_trainer_cli.py b/tests/trainer/test_trainer_cli.py index 93cbb8e2..b2d1da95 100644 --- a/tests/trainer/test_trainer_cli.py +++ b/tests/trainer/test_trainer_cli.py @@ -1,6 +1,7 @@ import inspect from argparse import ArgumentParser, Namespace from unittest import mock +import pickle import pytest @@ -42,14 +43,14 @@ def test_add_argparse_args_redefined(cli_args): args = parser.parse_args(cli_args) + # make sure we can pickle args + pickle.dumps(args) + # Check few deprecated args are not in namespace: for depr_name in ('gradient_clip', 'nb_gpu_nodes', 'max_nb_epochs'): assert depr_name not in args trainer = Trainer.from_argparse_args(args=args) - - # make sure trainer can be pickled - import pickle pickle.dumps(trainer) assert isinstance(trainer, Trainer) From cac8f0250cc92b8d1c7f30a6abf0ce4f874c7678 Mon Sep 17 00:00:00 2001 From: "J. Borovec" Date: Mon, 27 Apr 2020 16:20:32 +0200 Subject: [PATCH 11/11] releasing --- CHANGELOG.md | 13 +++++++------ pytorch_lightning/__init__.py | 2 +- tests/trainer/test_trainer.py | 9 ++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0978f344..1d990dce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). -## [unreleased] - YYYY-MM-DD - -### Added +## [0.7.5] - 2020-04-27 ### Changed -- Allow logging of metrics togther with hparams ([#1630](https://github.com/PyTorchLightning/pytorch-lightning/pull/1630)) - -### Deprecated +- Allow logging of metrics together with `hparams` ([#1630](https://github.com/PyTorchLightning/pytorch-lightning/pull/1630)) +- Allow metrics logged together with hparams ([#1630](https://github.com/PyTorchLightning/pytorch-lightning/pull/1630)) ### Removed @@ -20,6 +17,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Fixed +- Fixed ModelCheckpoint not being fixable ([#1632](https://github.com/PyTorchLightning/pytorch-lightning/pull/1632)) +- Fixed CPU DDP breaking change and DDP change ([#1635](https://github.com/PyTorchLightning/pytorch-lightning/pull/1635)) +- Tested pickling ([#1636](https://github.com/PyTorchLightning/pytorch-lightning/pull/1636)) + ## [0.7.4] - 2020-04-26 diff --git a/pytorch_lightning/__init__.py b/pytorch_lightning/__init__.py index 2a656ce2..705bdfc6 100644 --- a/pytorch_lightning/__init__.py +++ b/pytorch_lightning/__init__.py @@ -1,6 +1,6 @@ """Root package info.""" -__version__ = '0.7.5rc1' +__version__ = '0.7.5' __author__ = 'William Falcon et al.' __author_email__ = 'waf2107@columbia.edu' __license__ = 'Apache-2.0' diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index f89f6b3f..18cc2586 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -127,13 +127,12 @@ def test_no_val_end_module(tmpdir): new_weights_path = os.path.join(tmpdir, 'save_test.ckpt') trainer.save_checkpoint(new_weights_path) - # assert ckpt has hparams - ckpt = torch.load(new_weights_path) - assert 'hparams' in ckpt.keys(), 'hparams missing from checkpoints' - - # won't load without hparams in the ckpt + # load new model + tags_path = tutils.get_data_path(logger, path_dir=tmpdir) + tags_path = os.path.join(tags_path, 'meta_tags.csv') model_2 = LightningTestModel.load_from_checkpoint( checkpoint_path=new_weights_path, + tags_csv=tags_path ) model_2.eval()