diff --git a/examples/__init__.py b/examples/__init__.py index 6743d7f9..0d456dac 100644 --- a/examples/__init__.py +++ b/examples/__init__.py @@ -1 +1,5 @@ -from .new_project_templates.lightning_module_template import LightningTemplateModel \ No newline at end of file +from .new_project_templates.lightning_module_template import LightningTemplateModel + +__all__ = [ + 'LightningTemplateModel' +] diff --git a/examples/new_project_templates/lightning_module_template.py b/examples/new_project_templates/lightning_module_template.py index 483a4e3a..a6550035 100644 --- a/examples/new_project_templates/lightning_module_template.py +++ b/examples/new_project_templates/lightning_module_template.py @@ -182,7 +182,7 @@ class LightningTemplateModel(LightningModule): if self.on_gpu: train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank) batch_size = batch_size // self.trainer.world_size # scale batch size - except Exception as e: + except Exception: pass should_shuffle = train_sampler is None @@ -211,7 +211,7 @@ class LightningTemplateModel(LightningModule): return self.__dataloader(train=False) @staticmethod - def add_model_specific_args(parent_parser, root_dir): # pragma: no cover + def add_model_specific_args(parent_parser, root_dir): # pragma: no cover """ Parameters you define here will be available to your model through self.hparams :param parent_parser: @@ -224,20 +224,21 @@ class LightningTemplateModel(LightningModule): # parser.set_defaults(gradient_clip=5.0) # network params - parser.add_argument('--in_features', default=28*28, type=int) + parser.add_argument('--in_features', default=28 * 28, type=int) parser.add_argument('--out_features', default=10, type=int) - parser.add_argument('--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference + # use 500 for CPU, 50000 for GPU to see speed difference + parser.add_argument('--hidden_dim', default=50000, type=int) parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) - parser.opt_list('--learning_rate', default=0.001*8, type=float, options=[0.0001, 0.0005, 0.001, 0.005], + parser.opt_list('--learning_rate', default=0.001 * 8, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu - parser.opt_list('--batch_size', default=256*8, type=int, options=[32, 64, 128, 256], tunable=False, + parser.opt_list('--batch_size', default=256 * 8, type=int, options=[32, 64, 128, 256], tunable=False, help='batch size will be divided over all the gpus being used across all nodes') return parser diff --git a/examples/new_project_templates/trainer_cpu_template.py b/examples/new_project_templates/trainer_cpu_template.py index de6ba7c4..21d705c6 100644 --- a/examples/new_project_templates/trainer_cpu_template.py +++ b/examples/new_project_templates/trainer_cpu_template.py @@ -67,7 +67,7 @@ if __name__ == '__main__': add_default_args(parent_parser, root_dir) # allow model to overwrite or extend args - parser = ExampleModel.add_model_specific_args(parent_parser) + parser = LightningTemplateModel.add_model_specific_args(parent_parser) hyperparams = parser.parse_args() # train model diff --git a/pytorch_lightning/__init__.py b/pytorch_lightning/__init__.py index 235590f2..145d3bd0 100644 --- a/pytorch_lightning/__init__.py +++ b/pytorch_lightning/__init__.py @@ -11,3 +11,9 @@ __copyright__ = 'Copyright (c) 2018-2019, %s.' % __author__ __doc__ = """ The Keras for ML researchers using PyTorch """ + +__all__ = [ + 'Trainer', + 'LightningModule', + 'data_loader', +] diff --git a/pytorch_lightning/callbacks/__init__.py b/pytorch_lightning/callbacks/__init__.py index f180c254..035deb06 100644 --- a/pytorch_lightning/callbacks/__init__.py +++ b/pytorch_lightning/callbacks/__init__.py @@ -1 +1,6 @@ -from .pt_callbacks import EarlyStopping, ModelCheckpoint \ No newline at end of file +from .pt_callbacks import EarlyStopping, ModelCheckpoint + +__all__ = [ + 'EarlyStopping', + 'ModelCheckpoint', +] diff --git a/pytorch_lightning/callbacks/pt_callbacks.py b/pytorch_lightning/callbacks/pt_callbacks.py index 89c8f6b2..f07c6a28 100644 --- a/pytorch_lightning/callbacks/pt_callbacks.py +++ b/pytorch_lightning/callbacks/pt_callbacks.py @@ -122,9 +122,9 @@ class EarlyStopping(Callback): current = logs.get(self.monitor) stop_training = False if current is None: - print('Early stopping conditioned on metric `%s` ''which is not available. Available metrics are: %s' % - (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning - ) + print('Early stopping conditioned on metric `%s` ' + 'which is not available. Available metrics are: %s' % + (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning) exit(-1) if self.monitor_op(current - self.min_delta, self.best): @@ -188,8 +188,7 @@ class ModelCheckpoint(Callback): if mode not in ['auto', 'min', 'max']: print('ModelCheckpoint mode %s is unknown, ' - 'fallback to auto mode.' % (mode), - RuntimeWarning) + 'fallback to auto mode.' % (mode), RuntimeWarning) mode = 'auto' if mode == 'min': @@ -233,8 +232,8 @@ class ModelCheckpoint(Callback): if self.save_best_only: current = logs.get(self.monitor) if current is None: - print('Can save best model only with %s available, ' - 'skipping.' % (self.monitor), RuntimeWarning) + print('Can save best model only with %s available,' + ' skipping.' % (self.monitor), RuntimeWarning) else: if self.monitor_op(current, self.best): if self.verbose > 0: diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index 2a73d867..c0f67d7a 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -1,12 +1,10 @@ """ The trainer handles all the logic for running a val loop, training loop, distributing, etc... """ -import subprocess -import traceback -import warnings + import os -import pdb import re +import warnings import numpy as np import tqdm @@ -201,7 +199,7 @@ class Trainer(TrainerIO): try: self.nb_slurm_tasks = int(os.environ['SLURM_NTASKS']) self.is_slurm_managing_tasks = self.nb_slurm_tasks == self.nb_requested_gpus - except Exception as e: + except Exception: # likely not on slurm, so set the slurm managed flag to false self.is_slurm_managing_tasks = False @@ -235,13 +233,13 @@ class Trainer(TrainerIO): print('using 16bit precision') if use_amp and not APEX_AVAILABLE: # pragma: no cover - msg = ''' + msg = """ You set use_amp=True but do not have apex installed. - Install apex first using this guide and rerun with use_amp=True: + Install apex first using this guide and rerun with use_amp=True: https://github.com/NVIDIA/apex#linux - + this run will NOT use 16 bit precision - ''' + """ raise ModuleNotFoundError(msg) @property @@ -275,7 +273,7 @@ class Trainer(TrainerIO): 'tng_loss': '{0:.3f}'.format(self.avg_loss), 'v_nb': '{}'.format(self.experiment.version), 'epoch': '{}'.format(self.current_epoch), - 'batch_nb':'{}'.format(self.batch_nb), + 'batch_nb': '{}'.format(self.batch_nb), } tqdm_dic.update(self.tqdm_metrics) @@ -389,18 +387,18 @@ class Trainer(TrainerIO): self.val_dataloader = model.val_dataloader if self.use_ddp and not isinstance(self.tng_dataloader.sampler, DistributedSampler): - msg = ''' + msg = """ when using multiple gpus and multiple nodes you must pass a DistributedSampler to DataLoader(sampler). - + ie: this: dataset = myDataset() dataloader = Dataloader(dataset) - + becomes: dataset = myDataset() dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) dataloader = Dataloader(dataset, sampler=dist_sampler) - ''' + """ raise MisconfigurationException(msg) # ----------------------------- @@ -418,8 +416,8 @@ class Trainer(TrainerIO): self.ddp_train(task, model) else: msg = f""" - You requested {self.nb_requested_gpus} GPUs but launched {self.nb_slurm_tasks} slurm tasks. - We will launch {self.nb_requested_gpus} processes for you. + You requested {self.nb_requested_gpus} GPUs but launched {self.nb_slurm_tasks} slurm tasks. + We will launch {self.nb_requested_gpus} processes for you. We recommend you let slurm manage the processes by setting: --ntasks-per-node={self.nb_requested_gpus} If you're not using SLURM, ignore this message! """ @@ -484,7 +482,7 @@ class Trainer(TrainerIO): try: node_id = os.environ['SLURM_NODEID'] self.node_rank = int(node_id) - except Exception as e: + except Exception: self.node_rank = 0 # recover original exp before went into process @@ -543,14 +541,14 @@ class Trainer(TrainerIO): # sets the appropriate port try: port = os.environ['MASTER_PORT'] - except Exception as e: + except Exception: port = 12910 os.environ['MASTER_PORT'] = f'{port}' # figure out the root node addr try: root_node = os.environ['SLURM_NODELIST'].split(' ')[0] - except Exception as e: + except Exception: root_node = '127.0.0.2' root_node = self.resolve_root_node_address(root_node) @@ -773,14 +771,14 @@ class Trainer(TrainerIO): try: model_specific_tqdm_metrics_dic = output['prog'] - except Exception as e: + except Exception: model_specific_tqdm_metrics_dic = {} # if output dict doesn't have the keyword loss # then assume the output=loss if scalar try: loss = output['loss'] - except Exception as e: + except Exception: if type(output) is torch.Tensor: loss = output diff --git a/pytorch_lightning/pt_overrides/override_data_parallel.py b/pytorch_lightning/pt_overrides/override_data_parallel.py index f06f8030..ab88d286 100644 --- a/pytorch_lightning/pt_overrides/override_data_parallel.py +++ b/pytorch_lightning/pt_overrides/override_data_parallel.py @@ -63,7 +63,6 @@ class LightningDataParallel(DataParallel): outputs = self.parallel_apply(replicas, inputs, kwargs) return self.gather(outputs, self.output_device) - def parallel_apply(self, replicas, inputs, kwargs): return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)]) diff --git a/pytorch_lightning/root_module/grads.py b/pytorch_lightning/root_module/grads.py index d50fa450..b28cf21c 100644 --- a/pytorch_lightning/root_module/grads.py +++ b/pytorch_lightning/root_module/grads.py @@ -4,6 +4,7 @@ Module to describe gradients from torch import nn + class GradInformation(nn.Module): def grad_norm(self, norm_type): @@ -17,11 +18,10 @@ class GradInformation(nn.Module): norm = param_norm ** (1 / norm_type) results['grad_{}_norm_{}'.format(norm_type, i)] = round(norm.data.cpu().numpy().flatten()[0], 3) - except Exception as e: + except Exception: # this param had no grad pass total_norm = total_norm ** (1. / norm_type) results['grad_{}_norm_total'.format(norm_type)] = round(total_norm.data.cpu().numpy().flatten()[0], 3) return results - diff --git a/pytorch_lightning/root_module/hooks.py b/pytorch_lightning/root_module/hooks.py index 849826a8..00ece234 100644 --- a/pytorch_lightning/root_module/hooks.py +++ b/pytorch_lightning/root_module/hooks.py @@ -43,4 +43,3 @@ class ModelHooks(torch.nn.Module): :return: """ pass - diff --git a/pytorch_lightning/root_module/memory.py b/pytorch_lightning/root_module/memory.py index 128fb16f..a0ec9aa0 100644 --- a/pytorch_lightning/root_module/memory.py +++ b/pytorch_lightning/root_module/memory.py @@ -94,7 +94,7 @@ class ModelSummary(object): mods = list(self.model.modules()) sizes = [] - for i in range(1,len(mods)): + for i in range(1, len(mods)): m = mods[i] p = list(m.parameters()) modsz = [] @@ -127,7 +127,7 @@ class ModelSummary(object): if self.model.example_input_array is not None: cols.extend(['In_sizes', 'Out_sizes']) - df = pd.DataFrame(np.zeros( (len(self.layer_names), len(cols)))) + df = pd.DataFrame(np.zeros((len(self.layer_names), len(cols)))) df.columns = cols df['Name'] = self.layer_names @@ -152,16 +152,16 @@ class ModelSummary(object): self.make_summary() -def print_mem_stack(): # pragma: no cover +def print_mem_stack(): # pragma: no cover for obj in gc.get_objects(): try: if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)): print(type(obj), obj.size()) - except Exception as e: + except Exception: pass -def count_mem_items(): # pragma: no cover +def count_mem_items(): # pragma: no cover nb_params = 0 nb_tensors = 0 for obj in gc.get_objects(): @@ -172,7 +172,7 @@ def count_mem_items(): # pragma: no cover nb_params += 1 else: nb_tensors += 1 - except Exception as e: + except Exception: pass return nb_params, nb_tensors diff --git a/pytorch_lightning/root_module/root_module.py b/pytorch_lightning/root_module/root_module.py index 1421a72d..96dbfdcd 100644 --- a/pytorch_lightning/root_module/root_module.py +++ b/pytorch_lightning/root_module/root_module.py @@ -129,6 +129,3 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): def unfreeze(self): for param in self.parameters(): param.requires_grad = True - - - diff --git a/pytorch_lightning/testing/lm_test_module.py b/pytorch_lightning/testing/lm_test_module.py index 9861810e..24995c7f 100644 --- a/pytorch_lightning/testing/lm_test_module.py +++ b/pytorch_lightning/testing/lm_test_module.py @@ -202,7 +202,7 @@ class LightningTestModel(LightningModule): if self.on_gpu and not self.force_remove_distributed_sampler: train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank) batch_size = batch_size // self.trainer.world_size # scale batch size - except Exception as e: + except Exception: pass should_shuffle = train_sampler is None @@ -242,19 +242,20 @@ class LightningTestModel(LightningModule): # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) - parser.add_argument('--in_features', default=28*28, type=int) + parser.add_argument('--in_features', default=28 * 28, type=int) parser.add_argument('--out_features', default=10, type=int) - parser.add_argument('--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference + # use 500 for CPU, 50000 for GPU to see speed difference + parser.add_argument('--hidden_dim', default=50000, type=int) # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) - parser.opt_list('--learning_rate', default=0.001*8, type=float, options=[0.0001, 0.0005, 0.001, 0.005], + parser.opt_list('--learning_rate', default=0.001 * 8, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu - parser.opt_list('--batch_size', default=256*8, type=int, options=[32, 64, 128, 256], tunable=False, + parser.opt_list('--batch_size', default=256 * 8, type=int, options=[32, 64, 128, 256], tunable=False, help='batch size will be divided over all the gpus being used across all nodes') return parser diff --git a/pytorch_lightning/utilities/arg_parse.py b/pytorch_lightning/utilities/arg_parse.py index f274e751..44399c39 100644 --- a/pytorch_lightning/utilities/arg_parse.py +++ b/pytorch_lightning/utilities/arg_parse.py @@ -3,6 +3,9 @@ List of default args which mught be useful for all the available flags Might need to update with the new flags """ +import os + + def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None): # tng, test, val check intervals @@ -44,7 +47,7 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None # test_tube settings parser.add_argument('-en', '--tt_name', default='pt_test') parser.add_argument('-td', '--tt_description', default='pytorch lightning test') - parser.add_argument('--tt_save_path', default=root_dir + '/test_tube_logs', help='logging dir') + parser.add_argument('--tt_save_path', default=os.path.join(root_dir, 'test_tube_logs'), help='logging dir') parser.add_argument('--enable_single_run', dest='single_run', action='store_true') parser.add_argument('--nb_hopt_trials', default=1, type=int) parser.add_argument('--log_stdout', dest='log_stdout', action='store_true') @@ -55,8 +58,7 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None parser.add_argument('--default_tensor_type', default='torch.cuda.FloatTensor', type=str) parser.add_argument('--use_amp', dest='use_amp', action='store_true') parser.add_argument('--check_grad_nans', dest='check_grad_nans', action='store_true') - parser.add_argument('--amp_level', default='O2',type=str) - + parser.add_argument('--amp_level', default='O2', type=str) # run on hpc parser.add_argument('--on_cluster', dest='on_cluster', action='store_true') diff --git a/tests/debug.py b/tests/debug.py index 09e9186b..fb0109a0 100644 --- a/tests/debug.py +++ b/tests/debug.py @@ -61,8 +61,8 @@ def get_model(): root_dir = os.path.dirname(os.path.realpath(__file__)) hparams = Namespace(**{'drop_prob': 0.2, 'batch_size': 32, - 'in_features': 28*28, - 'learning_rate': 0.001*8, + 'in_features': 28 * 28, + 'learning_rate': 0.001 * 8, 'optimizer_name': 'adam', 'data_root': os.path.join(root_dir, 'mnist'), 'out_features': 10, diff --git a/tests/test_models.py b/tests/test_models.py index 73ba2e43..fbf36442 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -136,11 +136,9 @@ def test_cpu_slurm_save_load(): def test_loading_meta_tags(): hparams = get_hparams() - save_dir = init_save_dir() - # save tags exp = get_exp(False) - exp.tag({'some_str':'a_str', 'an_int': 1, 'a_float': 2.0}) + exp.tag({'some_str': 'a_str', 'an_int': 1, 'a_float': 2.0}) exp.argparse(hparams) exp.save() @@ -502,7 +500,6 @@ def test_multi_gpu_model_ddp(): run_gpu_model_test(trainer_options, model, hparams) - def test_ddp_sampler_error(): """ Make sure DDP + AMP work @@ -587,8 +584,8 @@ def get_hparams(continue_training=False, hpc_exp_number=0): args = { 'drop_prob': 0.2, 'batch_size': 32, - 'in_features': 28*28, - 'learning_rate': 0.001*8, + 'in_features': 28 * 28, + 'learning_rate': 0.001 * 8, 'optimizer_name': 'adam', 'data_root': os.path.join(root_dir, 'mnist'), 'out_features': 10,