cleaning up demos (#313)

* cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos * cleaning up demos
2026-09-09 11:32:07 +08:00 · 2019-10-05 16:39:05 -04:00
parent f7d762416c
commit 07c5d22ae3
9 changed files with 194 additions and 99 deletions
@@ -51,14 +51,14 @@ if __name__ == '__main__':
    # gpu args
    parent_parser.add_argument(
        '--gpus',
-        type=str,
-        default='-1',
-        help='any integer (number of GPUs to use) or -1 for all'
+        type=int,
+        default=2,
+        help='how many gpus'
    )
    parent_parser.add_argument(
        '--distributed_backend',
        type=str,
-        default=None,
+        default='dp',
        help='supports three options dp, ddp, ddp2'
    )
    parent_parser.add_argument(
@@ -8,7 +8,7 @@ from torchvision.datasets import MNIST
 import torchvision.transforms as transforms
 import torch
 import torch.nn.functional as F
-from test_tube import HyperOptArgumentParser
+from argparse import ArgumentParser
 from torch import optim
 from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
@@ -95,7 +95,7 @@ class LightningTemplateModel(LightningModule):
        loss_val = self.loss(y, y_hat)

        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
-        if self.trainer.use_dp:
+        if self.trainer.use_dp or self.trainer.use_ddp2:
            loss_val = loss_val.unsqueeze(0)

        output = OrderedDict({
@@ -126,7 +126,7 @@ class LightningTemplateModel(LightningModule):
            val_acc = val_acc.cuda(loss_val.device.index)

        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
-        if self.trainer.use_dp:
+        if self.trainer.use_dp or self.trainer.use_ddp2:
            loss_val = loss_val.unsqueeze(0)
            val_acc = val_acc.unsqueeze(0)

@@ -168,7 +168,7 @@ class LightningTemplateModel(LightningModule):
        val_loss_mean /= len(outputs)
        val_acc_mean /= len(outputs)
        tqdm_dict = {'val_loss': val_loss_mean, 'val_acc': val_acc_mean}
-        result = {'progress_bar': tqdm_dict}
+        result = {'progress_bar': tqdm_dict, 'logs': tqdm_dict}
        return result

    # ---------------------
@@ -190,20 +190,20 @@ class LightningTemplateModel(LightningModule):
        dataset = MNIST(root=self.hparams.data_root, train=train,
                        transform=transform, download=True)

-        # when using multi-node (ddp) we need to add the datasampler
+        # when using multi-node (ddp) we need to add the  datasampler
        train_sampler = None
        batch_size = self.hparams.batch_size

        if self.use_ddp:
-            train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
-            batch_size = batch_size // self.trainer.world_size  # scale batch size
+            train_sampler = DistributedSampler(dataset)

        should_shuffle = train_sampler is None
        loader = DataLoader(
            dataset=dataset,
            batch_size=batch_size,
            shuffle=should_shuffle,
-            sampler=train_sampler
+            sampler=train_sampler,
+            num_workers=0
        )

        return loader
@@ -231,7 +231,7 @@ class LightningTemplateModel(LightningModule):
        :param root_dir:
        :return:
        """
-        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+        parser = ArgumentParser(parents=[parent_parser])

        # param overwrites
        # parser.set_defaults(gradient_clip_val=5.0)
@@ -241,21 +241,13 @@ class LightningTemplateModel(LightningModule):
        parser.add_argument('--out_features', default=10, type=int)
        # use 500 for CPU, 50000 for GPU to see speed difference
        parser.add_argument('--hidden_dim', default=50000, type=int)
-        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=True)
-        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
-                        options=[0.0001, 0.0005, 0.001],
-                        tunable=True)
+        parser.add_argument('--drop_prob', default=0.2, type=float)
+        parser.add_argument('--learning_rate', default=0.001, type=float)

        # data
        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)

        # training params (opt)
-        parser.opt_list('--optimizer_name', default='adam', type=str,
-                        options=['adam'], tunable=False)
-
-        # if using 2 nodes with 4 gpus each the batch size here
-        #  (256) will be 256 / (2*8) = 16 per gpu
-        parser.opt_list('--batch_size', default=256 * 8, type=int,
-                        options=[32, 64, 128, 256], tunable=False,
-                        help='batch size will be divided over all gpus being used across all nodes')
+        parser.add_argument('--optimizer_name', default='adam', type=str)
+        parser.add_argument('--batch_size', default=64, type=int)
        return parser