Replaces ddp .spawn with subprocess (#2029)

* replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * replace ddp spawn with subprocess * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix * hot fix
2026-09-09 11:32:07 +08:00 · 2020-06-01 11:00:32 -04:00
parent fd38f52e55
commit 82a20296e3
19 changed files with 283 additions and 174 deletions
@@ -1,3 +1,4 @@
+import os
 import platform
 from collections import namedtuple

@@ -9,6 +10,77 @@ import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks import EarlyStopping
 from tests.base import EvalModelTemplate
+from pytorch_lightning.callbacks import ModelCheckpoint
+
+
+def test_cpu_slurm_save_load(tmpdir):
+    """Verify model save/load/checkpoint on CPU."""
+    hparams = EvalModelTemplate.get_default_hparams()
+    model = EvalModelTemplate(**hparams)
+
+    # logger file to get meta
+    logger = tutils.get_default_logger(tmpdir)
+    version = logger.version
+
+    # fit model
+    trainer = Trainer(
+        max_epochs=1,
+        logger=logger,
+        train_percent_check=0.2,
+        val_percent_check=0.2,
+        checkpoint_callback=ModelCheckpoint(tmpdir)
+    )
+    result = trainer.fit(model)
+    real_global_step = trainer.global_step
+
+    # traning complete
+    assert result == 1, 'cpu model failed to complete'
+
+    # predict with trained model before saving
+    # make a prediction
+    dataloaders = model.test_dataloader()
+    if not isinstance(dataloaders, list):
+        dataloaders = [dataloaders]
+
+    for dataloader in dataloaders:
+        for batch in dataloader:
+            break
+
+    x, y = batch
+    x = x.view(x.size(0), -1)
+
+    model.eval()
+    pred_before_saving = model(x)
+
+    # test HPC saving
+    # simulate snapshot on slurm
+    saved_filepath = trainer.hpc_save(tmpdir, logger)
+    assert os.path.exists(saved_filepath)
+
+    # new logger file to get meta
+    logger = tutils.get_default_logger(tmpdir, version=version)
+
+    trainer = Trainer(
+        max_epochs=1,
+        logger=logger,
+        checkpoint_callback=ModelCheckpoint(tmpdir),
+    )
+    model = EvalModelTemplate(**hparams)
+
+    # set the epoch start hook so we can predict before the model does the full training
+    def assert_pred_same():
+        assert trainer.global_step == real_global_step and trainer.global_step > 0
+
+        # predict with loaded model to make sure answers are the same
+        trainer.model.eval()
+        new_pred = trainer.model(x)
+        assert torch.all(torch.eq(pred_before_saving, new_pred)).item() == 1
+
+    model.on_epoch_start = assert_pred_same
+
+    # by calling fit again, we trigger training, loading weights from the cluster
+    # and our hook to predict using current model before any more weight updates
+    trainer.fit(model)


 def test_early_stopping_cpu_model(tmpdir):
@@ -17,6 +89,7 @@ def test_early_stopping_cpu_model(tmpdir):
    trainer_options = dict(
        default_root_dir=tmpdir,
        early_stop_callback=stopping,
+        max_epochs=2,
        gradient_clip_val=1.0,
        overfit_pct=0.20,
        track_grad_norm=2,
@@ -39,6 +112,7 @@ def test_early_stopping_cpu_model(tmpdir):
                     version_parse(torch.__version__) < version_parse("1.3.0")),
                    reason="Distributed training is not supported on MacOS before Torch 1.3.0")
 def test_multi_cpu_model_ddp(tmpdir):
+    print('in ddp test')
    """Make sure DDP works."""
    tutils.set_random_master_port()

@@ -61,19 +135,19 @@ def test_lbfgs_cpu_model(tmpdir):
    """Test each of the trainer options."""
    trainer_options = dict(
        default_root_dir=tmpdir,
-        max_epochs=2,
+        max_epochs=1,
        progress_bar_refresh_rate=0,
        weights_summary='top',
-        train_percent_check=1.0,
+        train_percent_check=0.2,
        val_percent_check=0.2,
    )

    hparams = EvalModelTemplate.get_default_hparams()
    hparams.update(optimizer_name='lbfgs',
-                   learning_rate=0.002)
+                   learning_rate=0.004)
    model = EvalModelTemplate(**hparams)
    model.configure_optimizers = model.configure_optimizers__lbfgs
-    tutils.run_model_test_without_loggers(trainer_options, model, min_acc=0.5)
+    tutils.run_model_test_without_loggers(trainer_options, model, min_acc=0.25)


 def test_default_logger_callbacks_cpu_model(tmpdir):
@@ -110,7 +184,7 @@ def test_running_test_after_fitting(tmpdir):
    trainer = Trainer(
        default_root_dir=tmpdir,
        progress_bar_refresh_rate=0,
-        max_epochs=8,
+        max_epochs=2,
        train_percent_check=0.4,
        val_percent_check=0.2,
        test_percent_check=0.2,
@@ -324,19 +398,3 @@ def test_tbptt_cpu_model(tmpdir):
    result = trainer.fit(model)

    assert result == 1, 'training failed to complete'
-
-
-@pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine")
-def test_single_gpu_model(tmpdir):
-    """Make sure single GPU works (DP mode)."""
-    trainer_options = dict(
-        default_root_dir=tmpdir,
-        progress_bar_refresh_rate=0,
-        max_epochs=1,
-        train_percent_check=0.1,
-        val_percent_check=0.1,
-        gpus=1
-    )
-
-    model = EvalModelTemplate()
-    tutils.run_model_test(trainer_options, model)
@@ -5,7 +5,6 @@ import torch

 import tests.base.utils as tutils
 from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import ModelCheckpoint
 from pytorch_lightning.core import memory
 from pytorch_lightning.trainer.distrib_parts import parse_gpu_ids, determine_root_gpu_device
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
@@ -14,6 +13,23 @@ from tests.base import EvalModelTemplate
 PRETEND_N_OF_GPUS = 16


+@pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine")
+@pytest.mark.parametrize('gpus', [1, [0], [1]])
+def test_single_gpu_model(tmpdir, gpus):
+    """Make sure single GPU works (DP mode)."""
+    trainer_options = dict(
+        default_root_dir=tmpdir,
+        progress_bar_refresh_rate=0,
+        max_epochs=1,
+        train_percent_check=0.1,
+        val_percent_check=0.1,
+        gpus=gpus
+    )
+
+    model = EvalModelTemplate()
+    tutils.run_model_test(trainer_options, model)
+
+
@pytest.mark.spawn
@pytest.mark.parametrize("backend", ['dp', 'ddp', 'ddp2'])
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
@@ -40,6 +56,7 @@ def test_multi_gpu_model(tmpdir, backend):
    memory.get_memory_profile('min_max')


+@pytest.mark.spawn
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_ddp_all_dataloaders_passed_to_fit(tmpdir):
    """Make sure DDP works with dataloaders passed to fit()"""
@@ -48,8 +65,8 @@ def test_ddp_all_dataloaders_passed_to_fit(tmpdir):
    trainer_options = dict(default_root_dir=tmpdir,
                           progress_bar_refresh_rate=0,
                           max_epochs=1,
-                           train_percent_check=0.4,
-                           val_percent_check=0.2,
+                           train_percent_check=0.1,
+                           val_percent_check=0.1,
                           gpus=[0, 1],
                           distributed_backend='ddp')

@@ -62,74 +79,6 @@ def test_ddp_all_dataloaders_passed_to_fit(tmpdir):
    assert result == 1, "DDP doesn't work with dataloaders passed to fit()."


-def test_cpu_slurm_save_load(tmpdir):
-    """Verify model save/load/checkpoint on CPU."""
-    hparams = EvalModelTemplate.get_default_hparams()
-    model = EvalModelTemplate(**hparams)
-
-    # logger file to get meta
-    logger = tutils.get_default_logger(tmpdir)
-    version = logger.version
-
-    # fit model
-    trainer = Trainer(
-        max_epochs=1,
-        logger=logger,
-        checkpoint_callback=ModelCheckpoint(tmpdir)
-    )
-    result = trainer.fit(model)
-    real_global_step = trainer.global_step
-
-    # traning complete
-    assert result == 1, 'cpu model failed to complete'
-
-    # predict with trained model before saving
-    # make a prediction
-    dataloaders = model.test_dataloader()
-    if not isinstance(dataloaders, list):
-        dataloaders = [dataloaders]
-
-    for dataloader in dataloaders:
-        for batch in dataloader:
-            break
-
-    x, y = batch
-    x = x.view(x.size(0), -1)
-
-    model.eval()
-    pred_before_saving = model(x)
-
-    # test HPC saving
-    # simulate snapshot on slurm
-    saved_filepath = trainer.hpc_save(tmpdir, logger)
-    assert os.path.exists(saved_filepath)
-
-    # new logger file to get meta
-    logger = tutils.get_default_logger(tmpdir, version=version)
-
-    trainer = Trainer(
-        max_epochs=1,
-        logger=logger,
-        checkpoint_callback=ModelCheckpoint(tmpdir),
-    )
-    model = EvalModelTemplate(**hparams)
-
-    # set the epoch start hook so we can predict before the model does the full training
-    def assert_pred_same():
-        assert trainer.global_step == real_global_step and trainer.global_step > 0
-
-        # predict with loaded model to make sure answers are the same
-        trainer.model.eval()
-        new_pred = trainer.model(x)
-        assert torch.all(torch.eq(pred_before_saving, new_pred)).item() == 1
-
-    model.on_epoch_start = assert_pred_same
-
-    # by calling fit again, we trigger training, loading weights from the cluster
-    # and our hook to predict using current model before any more weight updates
-    trainer.fit(model)
-
-
@pytest.mark.spawn
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_multi_gpu_none_backend(tmpdir):
@@ -76,7 +76,7 @@ def test_running_test_pretrained_model_cpu(tmpdir):

    trainer_options = dict(
        progress_bar_refresh_rate=0,
-        max_epochs=4,
+        max_epochs=3,
        train_percent_check=0.4,
        val_percent_check=0.2,
        checkpoint_callback=checkpoint,