diff --git a/pl_examples/README.md b/pl_examples/README.md index 443a15f7..5757d64f 100644 --- a/pl_examples/README.md +++ b/pl_examples/README.md @@ -1,11 +1,14 @@ # Examples -This folder has 3 sections: - -### Domain templates -These are templates to show common approaches such as GANs and RL. +This folder has 4 sections: ### Basic examples These show the most common use of Lightning for either CPU or GPU training. +### Domain templates +These are templates to show common approaches such as GANs and RL. + +### Full examples +Contains examples demonstrating ImageNet training, Semantic Segmentation, etc. + ### Multi-node examples These show how to run jobs on a GPU cluster using lightning. \ No newline at end of file diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index 4e8e9b47..9faae5a3 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -31,7 +31,7 @@ python gpu_template.py --gpus 2 --distributed_backend ddp --- #### DistributedDataParallel+DP (ddp2) -Train on multiple GPUs using DistributedDataParallel + dataparallel. +Train on multiple GPUs using DistributedDataParallel + DataParallel. On a single node, uses all GPUs for 1 model. Then shares gradient information across nodes. ```bash diff --git a/pl_examples/basic_examples/cpu_template.py b/pl_examples/basic_examples/cpu_template.py index 7781ba32..537c4cdc 100644 --- a/pl_examples/basic_examples/cpu_template.py +++ b/pl_examples/basic_examples/cpu_template.py @@ -1,5 +1,5 @@ """ -Runs a model on a single node across N-gpus. +Runs a model on the CPU on a single node. """ import os from argparse import ArgumentParser diff --git a/pl_examples/basic_examples/gpu_template.py b/pl_examples/basic_examples/gpu_template.py index 090b2adc..b51c3770 100644 --- a/pl_examples/basic_examples/gpu_template.py +++ b/pl_examples/basic_examples/gpu_template.py @@ -1,5 +1,5 @@ """ -Runs a model on a single node across N-gpus. +Runs a model on a single node across multiple gpus. """ import os from argparse import ArgumentParser diff --git a/pl_examples/basic_examples/lightning_module_template.py b/pl_examples/basic_examples/lightning_module_template.py index b6444cee..c5a7f939 100644 --- a/pl_examples/basic_examples/lightning_module_template.py +++ b/pl_examples/basic_examples/lightning_module_template.py @@ -1,5 +1,5 @@ """ -Example template for defining a system +Example template for defining a system. """ import os from argparse import ArgumentParser @@ -41,8 +41,7 @@ class LightningTemplateModel(LightningModule): def __init__(self, hparams): """ - Pass in parsed HyperOptArgumentParser to the model - :param hparams: + Pass in hyperparameters as a `argparse.Namespace` or a `dict` to the model. """ # init superclass super().__init__() @@ -61,8 +60,7 @@ class LightningTemplateModel(LightningModule): # --------------------- def __build_model(self): """ - Layout model - :return: + Layout the model. """ self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim) @@ -77,11 +75,9 @@ class LightningTemplateModel(LightningModule): # --------------------- def forward(self, x): """ - No special modification required for lightning, define as you normally would - :param x: - :return: + No special modification required for Lightning, define it as you normally would + in the `nn.Module` in vanilla PyTorch. """ - x = self.c_d1(x) x = torch.tanh(x) x = self.c_d1_bn(x) @@ -98,9 +94,8 @@ class LightningTemplateModel(LightningModule): def training_step(self, batch, batch_idx): """ - Lightning calls this inside the training loop - :param batch: - :return: + Lightning calls this inside the training loop with the data from the training dataloader + passed in as `batch`. """ # forward pass x, y = batch @@ -123,9 +118,8 @@ class LightningTemplateModel(LightningModule): def validation_step(self, batch, batch_idx): """ - Lightning calls this inside the validation loop - :param batch: - :return: + Lightning calls this inside the validation loop with the data from the validation dataloader + passed in as `batch`. """ x, y = batch x = x.view(x.size(0), -1) @@ -151,9 +145,8 @@ class LightningTemplateModel(LightningModule): def validation_epoch_end(self, outputs): """ - Called at the end of validation to aggregate outputs - :param outputs: list of individual outputs of each validation step - :return: + Called at the end of validation to aggregate outputs. + :param outputs: list of individual outputs of each validation step. """ # if returned a scalar from validation_step, outputs is a list of tensor scalars # we return just the average in this case (if we want) @@ -187,8 +180,8 @@ class LightningTemplateModel(LightningModule): # --------------------- def configure_optimizers(self): """ - return whatever optimizers we want here - :return: list of optimizers + Return whatever optimizers and learning rate schedulers you want here. + At least one optimizer is required. """ optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10) @@ -234,9 +227,8 @@ class LightningTemplateModel(LightningModule): def test_step(self, batch, batch_idx): """ - Lightning calls this during testing, similar to val_step - :param batch: - :return:val + Lightning calls this during testing, similar to `validation_step`, + with the data from the test dataloader passed in as `batch`. """ output = self.validation_step(batch, batch_idx) # Rename output keys @@ -247,9 +239,8 @@ class LightningTemplateModel(LightningModule): def test_epoch_end(self, outputs): """ - Called at the end of test to aggregate outputs, similar to validation_epoch_end - :param outputs: list of individual outputs of each validation step - :return: + Called at the end of test to aggregate outputs, similar to `validation_epoch_end`. + :param outputs: list of individual outputs of each test step """ results = self.validation_step_end(outputs) @@ -266,10 +257,7 @@ class LightningTemplateModel(LightningModule): @staticmethod def add_model_specific_args(parent_parser, root_dir): # pragma: no-cover """ - Parameters you define here will be available to your model through self.hparams - :param parent_parser: - :param root_dir: - :return: + Parameters you define here will be available to your model through `self.hparams`. """ parser = ArgumentParser(parents=[parent_parser]) diff --git a/pl_examples/domain_templates/gan.py b/pl_examples/domain_templates/gan.py index 3661b363..795876e2 100644 --- a/pl_examples/domain_templates/gan.py +++ b/pl_examples/domain_templates/gan.py @@ -2,7 +2,7 @@ To run this template just do: python gan.py -After a few epochs, launch tensorboard to see the images being generated at every batch. +After a few epochs, launch TensorBoard to see the images being generated at every batch: tensorboard --logdir default """ diff --git a/pl_examples/domain_templates/reinforse_learn_Qnet.py b/pl_examples/domain_templates/reinforce_learn_Qnet.py similarity index 95% rename from pl_examples/domain_templates/reinforse_learn_Qnet.py rename to pl_examples/domain_templates/reinforce_learn_Qnet.py index 5a797d7d..4301957a 100644 --- a/pl_examples/domain_templates/reinforse_learn_Qnet.py +++ b/pl_examples/domain_templates/reinforce_learn_Qnet.py @@ -1,17 +1,17 @@ """ -# Deep Reinforcement Learning: Deep Q-network (DQN) +Deep Reinforcement Learning: Deep Q-network (DQN) -this example is based off https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On- +This example is based on https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On- Second-Edition/blob/master/Chapter06/02_dqn_pong.py The template illustrates using Lightning for Reinforcement Learning. The example builds a basic DQN using the classic CartPole environment. -to run the template just run: -python dqn.py +To run the template just run: +python reinforce_learn_Qnet.py -After ~1500 steps, you will see the total_reward hitting the max score of 200. Open up tensor boards to -see the metrics. +After ~1500 steps, you will see the total_reward hitting the max score of 200. Open up TensorBoard to +see the metrics: tensorboard --logdir default """ @@ -72,7 +72,7 @@ class ReplayBuffer: def __init__(self, capacity: int) -> None: self.buffer = deque(maxlen=capacity) - def __len__(self) -> None: + def __len__(self) -> int: return len(self.buffer) def append(self, experience: Experience) -> None: @@ -128,7 +128,7 @@ class Agent: self.state = self.env.reset() def reset(self) -> None: - """ Resents the environment and updates the state""" + """Resets the environment and updates the state""" self.state = self.env.reset() def get_action(self, net: nn.Module, epsilon: float, device: str) -> int: @@ -220,7 +220,7 @@ class DQNLightning(pl.LightningModule): def forward(self, x: torch.Tensor) -> torch.Tensor: """ - Passes in a state x through the network and gets the q_values of each action as an output + Passes in a state `x` through the network and gets the `q_values` of each action as an output Args: x: environment state @@ -292,7 +292,7 @@ class DQNLightning(pl.LightningModule): return OrderedDict({'loss': loss, 'log': log, 'progress_bar': log}) def configure_optimizers(self) -> List[Optimizer]: - """ Initialize Adam optimizer""" + """Initialize Adam optimizer""" optimizer = optim.Adam(self.net.parameters(), lr=self.hparams.lr) return [optimizer] diff --git a/pl_examples/full_examples/semantic_segmentation/models/unet/model.py b/pl_examples/full_examples/semantic_segmentation/models/unet/model.py index 484c6982..e858ed2f 100644 --- a/pl_examples/full_examples/semantic_segmentation/models/unet/model.py +++ b/pl_examples/full_examples/semantic_segmentation/models/unet/model.py @@ -4,15 +4,15 @@ from models.unet.parts import DoubleConv, Down, Up class UNet(nn.Module): - ''' + """ Architecture based on U-Net: Convolutional Networks for Biomedical Image Segmentation Link - https://arxiv.org/abs/1505.04597 Parameters: - num_classes (int) - Number of output classes required (default 19 for KITTI dataset) - bilinear (bool) - Whether to use bilinear interpolation or transposed - convolutions for upsampling. - ''' + num_classes (int): Number of output classes required (default 19 for KITTI dataset) + bilinear (bool): Whether to use bilinear interpolation or transposed + convolutions for upsampling. + """ def __init__(self, num_classes=19, bilinear=False): super().__init__() diff --git a/pl_examples/full_examples/semantic_segmentation/models/unet/parts.py b/pl_examples/full_examples/semantic_segmentation/models/unet/parts.py index 6828c3f7..8036ea05 100644 --- a/pl_examples/full_examples/semantic_segmentation/models/unet/parts.py +++ b/pl_examples/full_examples/semantic_segmentation/models/unet/parts.py @@ -4,10 +4,10 @@ import torch.nn.functional as F class DoubleConv(nn.Module): - ''' + """ Double Convolution and BN and ReLU (3x3 conv -> BN -> ReLU) ** 2 - ''' + """ def __init__(self, in_ch, out_ch): super().__init__() @@ -25,9 +25,9 @@ class DoubleConv(nn.Module): class Down(nn.Module): - ''' + """ Combination of MaxPool2d and DoubleConv in series - ''' + """ def __init__(self, in_ch, out_ch): super().__init__() @@ -41,11 +41,11 @@ class Down(nn.Module): class Up(nn.Module): - ''' + """ Upsampling (by either bilinear interpolation or transpose convolutions) followed by concatenation of feature map from contracting path, followed by double 3x3 convolution. - ''' + """ def __init__(self, in_ch, out_ch, bilinear=False): super().__init__() diff --git a/pl_examples/full_examples/semantic_segmentation/semseg.py b/pl_examples/full_examples/semantic_segmentation/semseg.py index dfac9e9c..5426dfa2 100644 --- a/pl_examples/full_examples/semantic_segmentation/semseg.py +++ b/pl_examples/full_examples/semantic_segmentation/semseg.py @@ -13,8 +13,8 @@ import pytorch_lightning as pl class KITTI(Dataset): - ''' - Dataset Class for KITTI Semantic Segmentation Benchmark dataset + """ + Class for KITTI Semantic Segmentation Benchmark dataset Dataset link - http://www.cvlibs.net/datasets/kitti/eval_semseg.php?benchmark=semantics2015 There are 34 classes in the given labels. However, not all of them are useful for training @@ -33,7 +33,7 @@ class KITTI(Dataset): In the `get_item` function, images and masks are resized to the given `img_size`, masks are encoded using `encode_segmap`, and given `transform` (if any) are applied to the image only (mask does not usually require transforms, but they can be implemented in a similar way). - ''' + """ def __init__( self, @@ -67,7 +67,7 @@ class KITTI(Dataset): self.mask_list = None def __len__(self): - return(len(self.img_list)) + return len(self.img_list) def __getitem__(self, idx): img = Image.open(self.img_list[idx]) @@ -89,9 +89,9 @@ class KITTI(Dataset): return img def encode_segmap(self, mask): - ''' + """ Sets void classes to zero so they won't be considered for training - ''' + """ for voidc in self.void_labels: mask[mask == voidc] = self.ignore_index for validc in self.valid_labels: @@ -99,9 +99,9 @@ class KITTI(Dataset): return mask def get_filenames(self, path): - ''' + """ Returns a list of absolute paths to images inside given `path` - ''' + """ files_list = list() for filename in os.listdir(path): files_list.append(os.path.join(path, filename)) @@ -109,7 +109,7 @@ class KITTI(Dataset): class SegModel(pl.LightningModule): - ''' + """ Semantic Segmentation Module This is a basic semantic segmentation module implemented with Lightning. @@ -120,7 +120,7 @@ class SegModel(pl.LightningModule): It uses the FCN ResNet50 model as an example. Adam optimizer is used along with Cosine Annealing learning rate scheduler. - ''' + """ def __init__(self, hparams): super().__init__() diff --git a/pl_examples/multi_node_examples/README.md b/pl_examples/multi_node_examples/README.md index 63315ff9..f4bb719d 100644 --- a/pl_examples/multi_node_examples/README.md +++ b/pl_examples/multi_node_examples/README.md @@ -8,13 +8,13 @@ To run this demo do the following: 3. Choose a script to submit #### DDP -Submit this job to run with distributedDataParallel (2 nodes, 2 gpus each) +Submit this job to run with DistributedDataParallel (2 nodes, 2 gpus each) ```bash sbatch ddp_job_submit.sh YourEnv ``` #### DDP2 -Submit this job to run with a different implementation of distributedDataParallel. +Submit this job to run with a different implementation of DistributedDataParallel. In this version, each node acts like DataParallel but syncs across nodes like DDP. ```bash sbatch ddp2_job_submit.sh YourEnv diff --git a/pl_examples/multi_node_examples/multi_node_ddp2_demo.py b/pl_examples/multi_node_examples/multi_node_ddp2_demo.py index d88ee08a..6abf8739 100644 --- a/pl_examples/multi_node_examples/multi_node_ddp2_demo.py +++ b/pl_examples/multi_node_examples/multi_node_ddp2_demo.py @@ -16,11 +16,7 @@ np.random.seed(SEED) def main(hparams): - """ - Main training routine specific for this project - :param hparams: - :return: - """ + """Main training routine specific for this project.""" # ------------------------ # 1 INIT LIGHTNING MODEL # ------------------------