From 6a414195fd1f99f285cd355afee6314a20c3f762 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 16 Jan 2020 13:36:48 -0500 Subject: [PATCH] working on trainer docs --- pytorch_lightning/core/__init__.py | 304 ++++++++++++++-------------- pytorch_lightning/core/lightning.py | 119 +++++------ 2 files changed, 211 insertions(+), 212 deletions(-) diff --git a/pytorch_lightning/core/__init__.py b/pytorch_lightning/core/__init__.py index c2694eab..9b20b562 100644 --- a/pytorch_lightning/core/__init__.py +++ b/pytorch_lightning/core/__init__.py @@ -1,150 +1,158 @@ """ -Lightning Module interface -========================== - -A lightning module is a strict superclass of nn.Module, it provides a standard interface - for the trainer to interact with the model. - -The easiest thing to do is copy the minimal example below and modify accordingly. - -Otherwise, to Define a Lightning Module, implement the following methods: - - -Minimal example ---------------- - -.. code-block:: python - - import os - import torch - from torch.nn import functional as F - from torch.utils.data import DataLoader - from torchvision.datasets import MNIST - import torchvision.transforms as transforms - - import pytorch_lightning as pl - - class CoolModel(pl.LightningModule): - - def __init__(self): - super(CoolModel, self).__init__() - # not the best model... - self.l1 = torch.nn.Linear(28 * 28, 10) - - def forward(self, x): - return torch.relu(self.l1(x.view(x.size(0), -1))) - - def training_step(self, batch, batch_idx): - # REQUIRED - x, y = batch - y_hat = self.forward(x) - return {'loss': F.cross_entropy(y_hat, y)} - - def validation_step(self, batch, batch_idx): - # OPTIONAL - x, y = batch - y_hat = self.forward(x) - return {'val_loss': F.cross_entropy(y_hat, y)} - - def validation_end(self, outputs): - # OPTIONAL - val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() - return {'val_loss': val_loss_mean} - - def test_step(self, batch, batch_idx): - # OPTIONAL - x, y = batch - y_hat = self.forward(x) - return {'test_loss': F.cross_entropy(y_hat, y)} - - def test_end(self, outputs): - # OPTIONAL - test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() - return {'test_loss': test_loss_mean} - - def configure_optimizers(self): - # REQUIRED - return torch.optim.Adam(self.parameters(), lr=0.02) - - @pl.data_loader - def train_dataloader(self): - return DataLoader(MNIST(os.getcwd(), train=True, download=True, - transform=transforms.ToTensor()), batch_size=32) - - @pl.data_loader - def val_dataloader(self): - # OPTIONAL - # can also return a list of val dataloaders - return DataLoader(MNIST(os.getcwd(), train=True, download=True, - transform=transforms.ToTensor()), batch_size=32) - - @pl.data_loader - def test_dataloader(self): - # OPTIONAL - # can also return a list of test dataloaders - return DataLoader(MNIST(os.getcwd(), train=False, download=True, - transform=transforms.ToTensor()), batch_size=32) - - -How do these methods fit into the broader training? ---------------------------------------------------- - -The LightningModule interface is on the right. Each method corresponds - to a part of a research project. Lightning automates everything not in blue. - -.. figure:: docs/source/_static/images/overview_flat.jpg - :align: center - - Overview. - - -Optional Methods ----------------- - -**add_model_specific_args** - -.. code-block:: python - - @staticmethod - def add_model_specific_args(parent_parser, root_dir) - -Lightning has a list of default argparse commands. - This method is your chance to add or modify commands specific to your model. - The `hyperparameter argument parser - `_ - is available anywhere in your model by calling self.hparams. - -**Return** -An argument parser - -**Example** - -.. code-block:: python - - @staticmethod - def add_model_specific_args(parent_parser, root_dir): - parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) - - # param overwrites - # parser.set_defaults(gradient_clip_val=5.0) - - # network params - parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) - parser.add_argument('--in_features', default=28*28) - parser.add_argument('--out_features', default=10) - # use 500 for CPU, 50000 for GPU to see speed difference - parser.add_argument('--hidden_dim', default=50000) - - # data - parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) - - # training params (opt) - parser.opt_list('--learning_rate', default=0.001, type=float, - options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) - parser.opt_list('--batch_size', default=256, type=int, - options=[32, 64, 128, 256], tunable=False) - parser.opt_list('--optimizer_name', default='adam', type=str, - options=['adam'], tunable=False) - return parser - +Test """ +# """ +# Lightning Module interface +# ========================== +# +# A lightning module is a strict superclass of nn.Module, it provides a standard interface +# for the trainer to interact with the model. +# +# The easiest thing to do is copy the minimal example below and modify accordingly. +# +# Otherwise, to Define a Lightning Module, implement the following methods: +# +# +# Minimal example +# --------------- +# +# .. code-block:: python +# +# import os +# import torch +# from torch.nn import functional as F +# from torch.utils.data import DataLoader +# from torchvision.datasets import MNIST +# import torchvision.transforms as transforms +# +# import pytorch_lightning as pl +# +# class CoolModel(pl.LightningModule): +# +# def __init__(self): +# super(CoolModel, self).__init__() +# # not the best model... +# self.l1 = torch.nn.Linear(28 * 28, 10) +# +# def forward(self, x): +# return torch.relu(self.l1(x.view(x.size(0), -1))) +# +# def training_step(self, batch, batch_idx): +# # REQUIRED +# x, y = batch +# y_hat = self.forward(x) +# return {'loss': F.cross_entropy(y_hat, y)} +# +# def validation_step(self, batch, batch_idx): +# # OPTIONAL +# x, y = batch +# y_hat = self.forward(x) +# return {'val_loss': F.cross_entropy(y_hat, y)} +# +# def validation_end(self, outputs): +# # OPTIONAL +# val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() +# return {'val_loss': val_loss_mean} +# +# def test_step(self, batch, batch_idx): +# # OPTIONAL +# x, y = batch +# y_hat = self.forward(x) +# return {'test_loss': F.cross_entropy(y_hat, y)} +# +# def test_end(self, outputs): +# # OPTIONAL +# test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() +# return {'test_loss': test_loss_mean} +# +# def configure_optimizers(self): +# # REQUIRED +# return torch.optim.Adam(self.parameters(), lr=0.02) +# +# @pl.data_loader +# def train_dataloader(self): +# return DataLoader(MNIST(os.getcwd(), train=True, download=True, +# transform=transforms.ToTensor()), batch_size=32) +# +# @pl.data_loader +# def val_dataloader(self): +# # OPTIONAL +# # can also return a list of val dataloaders +# return DataLoader(MNIST(os.getcwd(), train=True, download=True, +# transform=transforms.ToTensor()), batch_size=32) +# +# @pl.data_loader +# def test_dataloader(self): +# # OPTIONAL +# # can also return a list of test dataloaders +# return DataLoader(MNIST(os.getcwd(), train=False, download=True, +# transform=transforms.ToTensor()), batch_size=32) +# +# +# How do these methods fit into the broader training? +# --------------------------------------------------- +# +# The LightningModule interface is on the right. Each method corresponds +# to a part of a research project. Lightning automates everything not in blue. +# +# .. figure:: docs/source/_static/images/overview_flat.jpg +# :align: center +# +# Overview. +# +# +# Optional Methods +# ---------------- +# +# **add_model_specific_args** +# +# .. code-block:: python +# +# @staticmethod +# def add_model_specific_args(parent_parser, root_dir) +# +# Lightning has a list of default argparse commands. +# This method is your chance to add or modify commands specific to your model. +# The `hyperparameter argument parser +# `_ +# is available anywhere in your model by calling self.hparams. +# +# **Return** +# An argument parser +# +# **Example** +# +# .. code-block:: python +# +# @staticmethod +# def add_model_specific_args(parent_parser, root_dir): +# parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) +# +# # param overwrites +# # parser.set_defaults(gradient_clip_val=5.0) +# +# # network params +# parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) +# parser.add_argument('--in_features', default=28*28) +# parser.add_argument('--out_features', default=10) +# # use 500 for CPU, 50000 for GPU to see speed difference +# parser.add_argument('--hidden_dim', default=50000) +# +# # data +# parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) +# +# # training params (opt) +# parser.opt_list('--learning_rate', default=0.001, type=float, +# options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) +# parser.opt_list('--batch_size', default=256, type=int, +# options=[32, 64, 128, 256], tunable=False) +# parser.opt_list('--optimizer_name', default='adam', type=str, +# options=['adam'], tunable=False) +# return parser +# +# """ + +from .test_b import TestB +from .lightning import LightningModule + +__all__ = ['TestB'] diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 95345aad..fdaba821 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1,13 +1,4 @@ -""" -The LightningModule is the "system recipe." It groups the following in one file: - - computational system definition - - computations done on forward - - training loop - - validation loop - - testing loop - - train, val, test dataloaders - - optimizers -""" + import os import warnings @@ -18,69 +9,69 @@ from argparse import Namespace import torch import torch.distributed as dist - +# from pytorch_lightning.core.decorators import data_loader from pytorch_lightning.core.grads import GradInformation from pytorch_lightning.core.hooks import ModelHooks -from pytorch_lightning.core.memory import ModelSummary from pytorch_lightning.core.saving import ModelIO +from pytorch_lightning.core.memory import ModelSummary from pytorch_lightning.trainer.training_io import load_hparams_from_tags_csv from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel class LightningModule(ABC, GradInformation, ModelIO, ModelHooks): - """ - A LightningModule has the following properties which you can access at any time - - **logger** - A reference to the logger you passed into trainer. - Passing a logger is optional. If you don't pass one in, Lightning will create one - for you automatically. This logger saves logs to `/os.getcwd()/lightning_logs`:: - - Trainer(logger=your_logger) - - - Call it from anywhere in your LightningModule to add metrics, images, etc... - whatever your logger supports. - - Here is an example using the TestTubeLogger (which is a wrapper - on 'PyTorch SummaryWriter `_ - with versioned folder structure). - - .. code-block:: python - - # if logger is a tensorboard logger or TestTubeLogger - self.logger.experiment.add_embedding(...) - self.logger.experiment.log({'val_loss': 0.9}) - self.logger.experiment.add_scalars(...) - - - **trainer** - Last resort access to any state the trainer has. - Changing certain properties here could affect your training run. - - .. code-block:: python - - self.trainer.optimizers - self.trainer.current_epoch - ... - - Debugging - --------- - - The LightningModule also offers these tricks to help debug. - - **example_input_array** - - In the LightningModule init, you can set a dummy tensor for this property - to get a print out of sizes coming into and out of every layer. - - .. code-block:: python - - def __init__(self): - # put the dimensions of the first input to your system - self.example_input_array = torch.rand(5, 28 * 28) - """ + # """ + # A LightningModule has the following properties which you can access at any time + # + # **logger** + # A reference to the logger you passed into trainer. + # Passing a logger is optional. If you don't pass one in, Lightning will create one + # for you automatically. This logger saves logs to `/os.getcwd()/lightning_logs`:: + # + # Trainer(logger=your_logger) + # + # + # Call it from anywhere in your LightningModule to add metrics, images, etc... + # whatever your logger supports. + # + # Here is an example using the TestTubeLogger (which is a wrapper + # on 'PyTorch SummaryWriter `_ + # with versioned folder structure). + # + # .. code-block:: python + # + # # if logger is a tensorboard logger or TestTubeLogger + # self.logger.experiment.add_embedding(...) + # self.logger.experiment.log({'val_loss': 0.9}) + # self.logger.experiment.add_scalars(...) + # + # + # **trainer** + # Last resort access to any state the trainer has. + # Changing certain properties here could affect your training run. + # + # .. code-block:: python + # + # self.trainer.optimizers + # self.trainer.current_epoch + # ... + # + # Debugging + # --------- + # + # The LightningModule also offers these tricks to help debug. + # + # **example_input_array** + # + # In the LightningModule init, you can set a dummy tensor for this property + # to get a print out of sizes coming into and out of every layer. + # + # .. code-block:: python + # + # def __init__(self): + # # put the dimensions of the first input to your system + # self.example_input_array = torch.rand(5, 28 * 28) + # """ def __init__(self, *args, **kwargs): super(LightningModule, self).__init__(*args, **kwargs)