When running DDP without DistributedSampler, throw warning instead of exception (#91)

This commit is contained in:
Nic Eggert
2019-08-10 15:58:12 -04:00
committed by William Falcon
parent c1434f0a3e
commit 996b1f9a6d
2 changed files with 7 additions and 4 deletions
+6 -3
View File
@@ -444,8 +444,9 @@ class Trainer(TrainerIO):
if self.use_ddp and not isinstance(self.tng_dataloader.sampler, DistributedSampler):
msg = """
when using multiple gpus and multiple nodes you must pass
a DistributedSampler to DataLoader(sampler).
You're using multiple gpus and multiple nodes without using a DistributedSampler
to assign a subset of your data to each process. To silence this warning, pass a
DistributedSampler to your DataLoader.
ie: this:
dataset = myDataset()
@@ -455,8 +456,10 @@ becomes:
dataset = myDataset()
dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
dataloader = Dataloader(dataset, sampler=dist_sampler)
If you want each process to load the full dataset, ignore this warning.
"""
raise MisconfigurationException(msg)
warnings.warn(msg)
# -----------------------------
# MODEL TRAINING
+1 -1
View File
@@ -665,7 +665,7 @@ def test_ddp_sampler_error():
use_amp=True
)
with pytest.raises(MisconfigurationException):
with pytest.warns(UserWarning):
trainer.get_dataloaders(model)
clear_save_dir()