generalize reinstantiation of dataloader (#1346)

* generalize reinstantiation of dataloader

* fix condition

* add test

* update changelog

* fix changelog

Co-authored-by: J. Borovec <jirka.borovec@seznam.cz>
This commit is contained in:
Justus Schock
2020-04-03 17:55:08 -04:00
committed by GitHub
co-authored by J. Borovec
parent e68ba1c836
commit f6a86e8551
3 changed files with 50 additions and 19 deletions
+4 -12
View File
@@ -84,16 +84,10 @@ class TrainerDataLoadingMixin(ABC):
if need_dist_sampler and no_sampler_added:
skip_keys = ['sampler', 'batch_sampler', 'dataset_kind']
dl_args = {
'dataset': dataloader.dataset,
'batch_size': dataloader.batch_size,
'shuffle': False,
'num_workers': dataloader.num_workers,
'collate_fn': dataloader.collate_fn,
'pin_memory': dataloader.pin_memory,
'drop_last': dataloader.drop_last,
'timeout': dataloader.timeout,
'worker_init_fn': dataloader.worker_init_fn
k: v for k, v in dataloader.__dict__.items() if not k.startswith('_') and k not in skip_keys
}
if self.use_tpu:
@@ -102,13 +96,11 @@ class TrainerDataLoadingMixin(ABC):
num_replicas=xm.xrt_world_size(),
rank=xm.get_ordinal()
)
dl_args['shuffle'] = False
else:
sampler = DistributedSampler(dataloader.dataset)
dl_args['shuffle'] = False
dl_args['sampler'] = sampler
dataloader = DataLoader(**dl_args)
dataloader = type(dataloader)(**dl_args)
return dataloader