From 480eed5cb61cb67037cac8b8eb6df9e0fe9286d6 Mon Sep 17 00:00:00 2001 From: Nic Eggert Date: Fri, 27 Sep 2019 11:05:29 -0500 Subject: [PATCH] Enable any ML experiment tracking framework (#223) * Implement generic loggers for experiment tracking * Add tests for loggers * Get model tests passing * Test and fix logger pickling * Expand pickle test and fix bug * Missed exp -> logger conversion * Remove commented code * Add docstrings * Update logging docs * Add mlflow to test requirements * Make linter happy * Fix mlflow timestamp * Update Logging.md * Update test_models.py * Update test_models.py * Update test_models.py * Update properties.md * Fix tests * Line length --- docs/LightningModule/properties.md | 18 +- docs/Trainer/Logging.md | 142 ++++++----- pytorch_lightning/logging/__init__.py | 7 + pytorch_lightning/logging/base.py | 71 ++++++ pytorch_lightning/logging/mlflow_logger.py | 44 ++++ pytorch_lightning/logging/test_tube_logger.py | 66 +++++ pytorch_lightning/trainer/trainer.py | 43 ++-- pytorch_lightning/trainer/trainer_io.py | 6 +- tests/requirements.txt | 3 +- tests/test_logging.py | 116 +++++++++ tests/test_models.py | 226 +++++++++--------- 11 files changed, 540 insertions(+), 202 deletions(-) create mode 100644 pytorch_lightning/logging/__init__.py create mode 100644 pytorch_lightning/logging/base.py create mode 100644 pytorch_lightning/logging/mlflow_logger.py create mode 100644 pytorch_lightning/logging/test_tube_logger.py create mode 100644 tests/test_logging.py diff --git a/docs/LightningModule/properties.md b/docs/LightningModule/properties.md index f8730ac9..1d513a37 100644 --- a/docs/LightningModule/properties.md +++ b/docs/LightningModule/properties.md @@ -9,12 +9,20 @@ The current epoch Current dtype --- -#### experiment -An instance of test-tube Experiment which you can use to log anything for tensorboard (subclass of [PyTorch SummaryWriter](https://pytorch.org/docs/stable/tensorboard.html)). +#### logger +A reference to the logger you passed into trainer. +```python +Trainer(logger=your_logger) +``` + +Call it from anywhere in your LightningModule to add metrics, images, etc... whatever your logger supports. + +Here is an example using the Test-tube logger (which is a wrapper on [PyTorch SummaryWriter](https://pytorch.org/docs/stable/tensorboard.html) with versioned folder structure). ```{.python} -self.experiment.add_embedding(...) -self.experiment.log({'val_loss': 0.9}) -self.experiment.add_scalars(...) +# if logger is a tensorboard logger or test-tube experiment +self.logger.add_embedding(...) +self.logger.log({'val_loss': 0.9}) +self.logger.add_scalars(...) ``` --- diff --git a/docs/Trainer/Logging.md b/docs/Trainer/Logging.md index 85cdfce1..765e1afd 100644 --- a/docs/Trainer/Logging.md +++ b/docs/Trainer/Logging.md @@ -1,7 +1,88 @@ -Lighting offers a few options for logging information about model, gpu usage, etc (via test-tube). It also offers printing options for training monitoring. +Lighting offers options for logging information about model, gpu usage, etc, via several different logging frameworks. It also offers printing options for training monitoring. --- +### Setting up logging + +Initialize your logger, which should inherit from `LightningBaseLogger`, and pass +it to `Trainer`. +```{.python} +my_logger = MyLightningLogger(...) +trainer = Trainer(logger=my_logger) +``` + +Lightning supports several common experiment tracking frameworks out of the box + +--- +#### Test tube + +Log using [test tube](https://williamfalcon.github.io/test-tube/). + +```{.python} +from pytorch_lightning.logging import TestTubeLogger +tt_logger = TestTubeLogger( + save_dir=".", + name="default", + debug=False, + create_git_tag=False +) +trainer = Trainer(logger=tt_logger) +``` + +--- +#### MLFlow + +Log using [mlflow](https://mlflow.org) + +```{.python} +from pytorch_lightning.logging import MLFlowLogger +mlf_logger = MLFlowLogger( + experiment_name="default", + tracking_uri="file:/." +) +trainer = Trainer(logger=mlf_logger) +``` + +--- +#### Custom logger + +You can implement your own logger by writing a class that inherits from +`LightningLoggerBase`. Use the `rank_zero_only` decorator to make sure that +only the first process in DDP training logs data. + +```{.python} +from pytorch_lightning.logging import LightningLoggerBase, rank_zero_only + +class MyLogger(LightningLoggerBase): + + @rank_zero_only + def log_hyperparams(self, params): + # params is an argparse.Namespace + # your code to record hyperparameters goes here + pass + + @rank_zero_only + def log_metrics(self, metrics, step_num): + # metrics is a dictionary of metric names and values + # your code to record metrics goes here + pass + + def save(self): + # Optional. Any code necessary to save logger data goes here + pass + + @rank_zero_only + def finalize(self, status): + # Optional. Any code that needs to be run after training + # finishes goes here +``` + +If you write a logger than may be useful to others, please send +a pull request to add it to Lighting! + +--- +### Using loggers + #### Display metrics in progress bar ``` {.python} # DEFAULT @@ -17,7 +98,7 @@ trainer = Trainer(row_log_interval=10) ``` --- -#### Log metric row every k batches +#### Log GPU memory Logs GPU memory when metrics are logged. ``` {.python} # DEFAULT @@ -38,61 +119,12 @@ trainer = Trainer(process_position=1) --- #### Save a snapshot of all hyperparameters -Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use. -Give lightning a test-tube Experiment object to automate this for you. +Log hyperparameters using the logger ``` {.python} -from test_tube import Experiment +logger = TestTubeLogger(...) +logger.log_hyperparams(args) -exp = Experiment(...) -Trainer(experiment=exp) -``` - ---- -#### Snapshot code for a training run -Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag. -Give lightning a test-tube Experiment object to automate this for you. -``` {.python} -from test_tube import Experiment - -exp = Experiment(create_git_tag=True) -Trainer(experiment=exp) -``` - ---- -### Tensorboard support -In the LightningModule you can access the experiment logger by doing: -```python -self.experiment - -# add image -# Look at PyTorch SummaryWriter docs for what you can do. -self.experiment.add_image(...) -``` - -The experiment object is a strict subclass of PyTorch SummaryWriter. However, this class -also snapshots every detail about the experiment (data folder paths, code, hyperparams), -and allows you to visualize it using tensorboard. -``` {.python} -from test_tube import Experiment, HyperOptArgumentParser - -# exp hyperparams -args = HyperOptArgumentParser() -hparams = args.parse_args() - -# this is a summaryWriter with nicer logging structure -exp = Experiment(save_dir='/some/path', create_git_tag=True) - -# track experiment details (must be ArgumentParser or HyperOptArgumentParser). -# each option in the parser is tracked -exp.argparse(hparams) -exp.tag({'description': 'running demo'}) - -# trainer uses the exp object to log exp data -trainer = Trainer(experiment=exp) -trainer.fit(model) - -# view logs at: -# tensorboard --logdir /some/path +Trainer(logger=logger) ``` --- diff --git a/pytorch_lightning/logging/__init__.py b/pytorch_lightning/logging/__init__.py new file mode 100644 index 00000000..89e3d435 --- /dev/null +++ b/pytorch_lightning/logging/__init__.py @@ -0,0 +1,7 @@ +from .base import LightningLoggerBase, rank_zero_only +from .test_tube_logger import TestTubeLogger + +try: + from .mlflow_logger import MLFlowLogger +except ModuleNotFoundError: + pass diff --git a/pytorch_lightning/logging/base.py b/pytorch_lightning/logging/base.py new file mode 100644 index 00000000..0545eafd --- /dev/null +++ b/pytorch_lightning/logging/base.py @@ -0,0 +1,71 @@ +from functools import wraps + + +def rank_zero_only(fn): + """Decorate a logger method to run it only on the process with rank 0 + + :param fn: Function to decorate + """ + + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + if self.rank == 0: + fn(self, *args, **kwargs) + + return wrapped_fn + + +class LightningLoggerBase: + """Base class for experiment loggers""" + + def __init__(self): + self._rank = 0 + + def log_metrics(self, metrics, step_num): + """Record metrics + + :param metric: Dictionary with metric names as keys and measured + quanties as values + :param step_num: Step number at which the metrics should be recorded + """ + raise NotImplementedError() + + def log_hyperparams(self, params): + """Record hyperparameters + + :param params: argparse.Namespace containing the hyperparameters + """ + raise NotImplementedError() + + def save(self): + """Save log data""" + pass + + def finalize(self, status): + """Do any processing that is necessary to finalize an experiment + + :param status: Status that the experiment finished with (e.g. success, failed, aborted) + """ + pass + + def close(self): + """Do any cleanup that is necessary to close an experiment""" + pass + + @property + def rank(self): + """ + Process rank. In general, metrics should only be logged by the process + with rank 0 + """ + return self._rank + + @rank.setter + def rank(self, value): + """Set the process rank""" + self._rank = value + + @property + def version(self): + """Return the experiment version""" + return None diff --git a/pytorch_lightning/logging/mlflow_logger.py b/pytorch_lightning/logging/mlflow_logger.py new file mode 100644 index 00000000..c9cf3940 --- /dev/null +++ b/pytorch_lightning/logging/mlflow_logger.py @@ -0,0 +1,44 @@ +from time import time +from logging import getLogger + +import mlflow + +from .base import LightningLoggerBase, rank_zero_only + +logger = getLogger(__name__) + + +class MLFlowLogger(LightningLoggerBase): + + def __init__(self, experiment_name, tracking_uri=None): + super().__init__() + self.client = mlflow.tracking.MlflowClient(tracking_uri) + + experiment = self.client.get_experiment_by_name(experiment_name) + if experiment is None: + logger.warning( + f"Experiment with name f{experiment_name} not found. Creating it." + ) + self.client.create_experiment(experiment_name) + experiment = self.client.get_experiment_by_name(experiment_name) + + run = self.client.create_run(experiment.experiment_id) + self.run_id = run.info.run_id + + @rank_zero_only + def log_hyperparams(self, params): + for k, v in vars(params).items(): + self.client.log_param(self.run_id, k, v) + + @rank_zero_only + def log_metrics(self, metrics, step_num=None): + timestamp_ms = int(time() * 1000) + for k, v in metrics.items(): + self.client.log_metric(self.run_id, k, v, timestamp_ms, step_num) + + def save(self): + pass + + @rank_zero_only + def finalize(self, status="FINISHED"): + self.client.set_terminated(self.run_id, status) diff --git a/pytorch_lightning/logging/test_tube_logger.py b/pytorch_lightning/logging/test_tube_logger.py new file mode 100644 index 00000000..51222efe --- /dev/null +++ b/pytorch_lightning/logging/test_tube_logger.py @@ -0,0 +1,66 @@ +import os.path +from copy import copy + +from .base import LightningLoggerBase, rank_zero_only + +from test_tube import Experiment + + +class TestTubeLogger(LightningLoggerBase): + def __init__( + self, save_dir, name="default", debug=False, version=None, create_git_tag=False + ): + super().__init__() + self.experiment = Experiment( + save_dir=save_dir, + name=name, + debug=debug, + version=version, + create_git_tag=create_git_tag, + ) + + @rank_zero_only + def log_hyperparams(self, params): + self.experiment.argparse(params) + + @rank_zero_only + def log_metrics(self, metrics, step_num=None): + self.experiment.log(metrics, global_step=step_num) + + @rank_zero_only + def save(self): + self.experiment.save() + + @rank_zero_only + def finalize(self, status): + self.save() + self.close() + + def close(self): + self.experiment.close() + + @property + def rank(self): + return self.experiment.rank + + @rank.setter + def rank(self, value): + self.experiment.rank = value + + @property + def version(self): + return self.experiment.version + + # Test tube experiments are not pickleable, so we need to override a few + # methods to get DDP working. See + # https://docs.python.org/3/library/pickle.html#handling-stateful-objects + # for more info. + def __getstate__(self): + state = self.__dict__.copy() + state["experiment"] = self.experiment.get_meta_copy() + return state + + def __setstate__(self, state): + self.experiment = state["experiment"].get_non_ddp_exp() + del state['experiment'] + self.__dict__.update(state) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index d806bc3b..6c02d9b3 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -55,7 +55,7 @@ def reduce_distributed_output(output, nb_gpus): class Trainer(TrainerIO): def __init__(self, - experiment=None, + logger=None, early_stop_callback=None, checkpoint_callback=None, gradient_clip_val=0, @@ -86,7 +86,7 @@ class Trainer(TrainerIO): nb_sanity_val_steps=5): """ - :param experiment: Test-tube experiment + :param logger: Logger for experiment tracking :param early_stop_callback: Callback for early stopping :param checkpoint_callback: Callback for checkpointing :param gradient_clip_val: int. 0 means don't clip. @@ -160,11 +160,8 @@ class Trainer(TrainerIO): # configure weights save path self.__configure_weights_path(checkpoint_callback, weights_save_path) - # configure experiment - self.experiment = experiment - self.exp_save_path = None - if self.experiment is not None: - self.exp_save_path = experiment.get_data_path(experiment.name, experiment.version) + # configure logger + self.logger = logger # accumulated grads self.__configure_accumulated_gradients(accumulate_grad_batches) @@ -423,8 +420,8 @@ class Trainer(TrainerIO): 'batch_nb': '{}'.format(self.batch_nb), } - if self.experiment is not None: - tqdm_dict['v_nb'] = self.experiment.version + if self.logger is not None and self.logger.version is not None: + tqdm_dict['v_nb'] = self.logger.version tqdm_dict.update(self.tqdm_metrics) @@ -658,10 +655,6 @@ class Trainer(TrainerIO): def fit(self, model): # when using multi-node or DDP within a node start each module in a separate process if self.use_ddp: - # must copy only the meta of the exp so it survives pickle/unpickle - # when going to new process - if self.experiment is not None: - self.experiment = self.experiment.get_meta_copy() if self.is_slurm_managing_tasks: task = int(os.environ['SLURM_LOCALID']) @@ -786,12 +779,8 @@ class Trainer(TrainerIO): self.world_size = self.nb_gpu_nodes * self.num_gpus # let the exp know the rank to avoid overwriting logs - # recover original exp before went into process - # init in write mode only on proc 0 - if self.experiment is not None: - self.experiment = self.experiment.get_non_ddp_exp() - self.experiment.debug = self.proc_rank > 0 - self.experiment.rank = self.proc_rank + if self.logger is not None: + self.logger.rank = self.proc_rank # set up server using proc 0's ip address # try to init for 20 times at max in case ports are taken @@ -897,12 +886,12 @@ class Trainer(TrainerIO): ref_model.summarize() # link up experiment object - if self.experiment is not None: - ref_model.experiment = self.experiment + if self.logger is not None: + ref_model.logger = self.logger # save exp to get started if self.proc_rank == 0: - self.experiment.save() + self.logger.save() # track model now. # if cluster resets state, the model will update with the saved weights @@ -1016,8 +1005,8 @@ class Trainer(TrainerIO): # when batch should be saved if (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch: - if self.proc_rank == 0 and self.experiment is not None: - self.experiment.save() + if self.proc_rank == 0 and self.logger is not None: + self.logger.save() # when metrics should be logged if batch_nb % self.row_log_interval == 0 or early_stop_epoch: @@ -1044,9 +1033,9 @@ class Trainer(TrainerIO): # log metrics scalar_metrics = self.__metrics_to_scalars( metrics, blacklist=self.__log_vals_blacklist()) - if self.proc_rank == 0 and self.experiment is not None: - self.experiment.log(scalar_metrics, global_step=self.global_step) - self.experiment.save() + if self.proc_rank == 0 and self.logger is not None: + self.logger.log_metrics(scalar_metrics, step_num=self.global_step) + self.logger.save() # end epoch early if early_stop_epoch: diff --git a/pytorch_lightning/trainer/trainer_io.py b/pytorch_lightning/trainer/trainer_io.py index 71b677cc..d64b08f9 100644 --- a/pytorch_lightning/trainer/trainer_io.py +++ b/pytorch_lightning/trainer/trainer_io.py @@ -233,12 +233,12 @@ class TrainerIO(object): # ---------------------------------- # PRIVATE OPS # ---------------------------------- - def hpc_save(self, folderpath, experiment): + def hpc_save(self, folderpath, logger): # make sure the checkpoint folder exists os.makedirs(folderpath, exist_ok=True) - # save exp to make sure we get all the metrics - experiment.save() + # save logger to make sure we get all the metrics + logger.save() ckpt_number = self.max_ckpt_in_folder(folderpath) + 1 diff --git a/tests/requirements.txt b/tests/requirements.txt index 076bfd65..ead06828 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,4 +5,5 @@ pytest>=3.0.5 pytest-cov flake8 check-manifest -test_tube \ No newline at end of file +test_tube +mlflow diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 00000000..1a787207 --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,116 @@ +import os.path +import pickle +import shutil + +import numpy as np +from pytorch_lightning import Trainer + +from pytorch_lightning.testing import LightningTestModel + +from .test_models import get_hparams, get_test_tube_logger, init_save_dir, clear_save_dir + + +def test_testtube_logger(): + """verify that basic functionality of test tube logger works""" + + hparams = get_hparams() + model = LightningTestModel(hparams) + + save_dir = init_save_dir() + + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() + + trainer_options = dict( + max_nb_epochs=1, + logger=logger + ) + + trainer = Trainer(**trainer_options) + result = trainer.fit(model) + + assert result == 1, "Training failed" + + clear_save_dir() + + +def test_testtube_pickle(): + """Verify that pickling a trainer containing a test tube logger works""" + hparams = get_hparams() + model = LightningTestModel(hparams) + + save_dir = init_save_dir() + + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() + + trainer_options = dict( + max_nb_epochs=1, + logger=logger + ) + + trainer = Trainer(**trainer_options) + pkl_bytes = pickle.dumps(trainer) + trainer2 = pickle.loads(pkl_bytes) + trainer2.logger.log_metrics({"acc": 1.0}) + + +def test_mlflow_logger(): + """verify that basic functionality of mlflow logger works""" + try: + from pytorch_lightning.logging import MLFlowLogger + except ModuleNotFoundError: + return + + hparams = get_hparams() + model = LightningTestModel(hparams) + + root_dir = os.path.dirname(os.path.realpath(__file__)) + mlflow_dir = os.path.join(root_dir, "mlruns") + + logger = MLFlowLogger("test", f"file://{mlflow_dir}") + logger.log_hyperparams(hparams) + logger.save() + + trainer_options = dict( + max_nb_epochs=1, + logger=logger + ) + + trainer = Trainer(**trainer_options) + result = trainer.fit(model) + + assert result == 1, "Training failed" + + n = np.random.randint(0, 10000000, 1)[0] + shutil.move(mlflow_dir, mlflow_dir + f'_{n}') + + +def test_mlflow_pickle(): + """verify that pickling trainer with mlflow logger works""" + try: + from pytorch_lightning.logging import MLFlowLogger + except ModuleNotFoundError: + return + + hparams = get_hparams() + model = LightningTestModel(hparams) + + root_dir = os.path.dirname(os.path.realpath(__file__)) + mlflow_dir = os.path.join(root_dir, "mlruns") + + logger = MLFlowLogger("test", f"file://{mlflow_dir}") + logger.log_hyperparams(hparams) + logger.save() + + trainer_options = dict( + max_nb_epochs=1, + logger=logger + ) + + trainer = Trainer(**trainer_options) + pkl_bytes = pickle.dumps(trainer) + trainer2 = pickle.loads(pkl_bytes) + trainer2.logger.log_metrics({"acc": 1.0}) diff --git a/tests/test_models.py b/tests/test_models.py index 3c82492a..33abe410 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -6,7 +6,6 @@ from argparse import Namespace import pytest import numpy as np import torch -from test_tube import Experiment # sys.path += [os.path.abspath('..'), os.path.abspath('../..')] from pytorch_lightning import Trainer @@ -29,6 +28,7 @@ from pytorch_lightning.root_module import memory from pytorch_lightning.trainer.trainer import reduce_distributed_output from pytorch_lightning.root_module import model_saving from pytorch_lightning.trainer import trainer_io +from pytorch_lightning.logging import TestTubeLogger from examples import LightningTemplateModel SEED = 2334 @@ -59,16 +59,15 @@ def test_dp_resume(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # get logger + logger = get_test_tube_logger(debug=False) + logger.log_hyperparams(hparams) # exp file to get weights checkpoint = ModelCheckpoint(save_dir) # add these to the trainer options - trainer_options['experiment'] = exp + trainer_options['logger'] = logger trainer_options['checkpoint_callback'] = checkpoint # fit model @@ -86,11 +85,11 @@ def test_dp_resume(): # HPC LOAD/SAVE # --------------------------- # save - trainer.hpc_save(save_dir, exp) + trainer.hpc_save(save_dir, logger) # init new trainer - new_exp = get_exp(False, version=exp.version) - trainer_options['experiment'] = new_exp + new_logger = get_test_tube_logger(version=logger.version) + trainer_options['logger'] = new_logger trainer_options['checkpoint_callback'] = ModelCheckpoint(save_dir) trainer_options['train_percent_check'] = 0.2 trainer_options['val_percent_check'] = 0.2 @@ -133,9 +132,9 @@ def test_running_test_pretrained_model_ddp(): save_dir = init_save_dir() # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() # exp file to get weights checkpoint = ModelCheckpoint(save_dir) @@ -146,7 +145,7 @@ def test_running_test_pretrained_model_ddp(): train_percent_check=0.4, val_percent_check=0.2, checkpoint_callback=checkpoint, - experiment=exp, + logger=logger, gpus=[0, 1], distributed_backend='ddp' ) @@ -157,7 +156,8 @@ def test_running_test_pretrained_model_ddp(): # correct result and ok accuracy assert result == 1, 'training failed to complete' - pretrained_model = load_model(exp, save_dir, on_gpu=True, module_class=LightningTestModel) + pretrained_model = load_model(logger.experiment, save_dir, on_gpu=True, + module_class=LightningTestModel) # run test set new_trainer = Trainer(**trainer_options) @@ -176,12 +176,12 @@ def test_running_test_after_fitting(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() - # exp file to get weights + # logger file to get weights checkpoint = ModelCheckpoint(save_dir) trainer_options = dict( @@ -191,7 +191,7 @@ def test_running_test_after_fitting(): val_percent_check=0.2, test_percent_check=0.2, checkpoint_callback=checkpoint, - experiment=exp + logger=logger ) # fit model @@ -217,12 +217,12 @@ def test_running_test_without_val(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() - # exp file to get weights + # logger file to get weights checkpoint = ModelCheckpoint(save_dir) trainer_options = dict( @@ -232,7 +232,7 @@ def test_running_test_without_val(): val_percent_check=0.2, test_percent_check=0.2, checkpoint_callback=checkpoint, - experiment=exp + logger=logger ) # fit model @@ -256,12 +256,12 @@ def test_running_test_pretrained_model(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() - # exp file to get weights + # logger file to get weights checkpoint = ModelCheckpoint(save_dir) trainer_options = dict( @@ -270,7 +270,7 @@ def test_running_test_pretrained_model(): train_percent_check=0.4, val_percent_check=0.2, checkpoint_callback=checkpoint, - experiment=exp + logger=logger ) # fit model @@ -280,7 +280,7 @@ def test_running_test_pretrained_model(): # correct result and ok accuracy assert result == 1, 'training failed to complete' pretrained_model = load_model( - exp, save_dir, on_gpu=False, module_class=LightningTestModel + logger.experiment, save_dir, on_gpu=False, module_class=LightningTestModel ) new_trainer = Trainer(**trainer_options) @@ -301,12 +301,12 @@ def test_running_test_pretrained_model_dp(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() - # exp file to get weights + # logger file to get weights checkpoint = ModelCheckpoint(save_dir) trainer_options = dict( @@ -315,7 +315,7 @@ def test_running_test_pretrained_model_dp(): train_percent_check=0.4, val_percent_check=0.2, checkpoint_callback=checkpoint, - experiment=exp, + logger=logger, gpus=[0, 1], distributed_backend='dp' ) @@ -326,7 +326,8 @@ def test_running_test_pretrained_model_dp(): # correct result and ok accuracy assert result == 1, 'training failed to complete' - pretrained_model = load_model(exp, save_dir, on_gpu=True, module_class=LightningTestModel) + pretrained_model = load_model(logger.experiment, save_dir, on_gpu=True, + module_class=LightningTestModel) new_trainer = Trainer(**trainer_options) new_trainer.test(pretrained_model) @@ -516,7 +517,7 @@ def test_early_stopping_cpu_model(): track_grad_norm=2, print_nan_grads=True, show_progress_bar=False, - experiment=get_exp(), + logger=get_test_tube_logger(), train_percent_check=0.1, val_percent_check=0.1 ) @@ -542,14 +543,14 @@ def test_no_val_module(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() trainer_options = dict( max_nb_epochs=1, - experiment=exp, + logger=logger, checkpoint_callback=ModelCheckpoint(save_dir) ) @@ -565,7 +566,7 @@ def test_no_val_module(): trainer.save_checkpoint(new_weights_path) # load new model - tags_path = exp.get_data_path(exp.name, exp.version) + tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version) tags_path = os.path.join(tags_path, 'meta_tags.csv') model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path, tags_csv=tags_path, on_gpu=False) @@ -588,14 +589,14 @@ def test_no_val_end_module(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() trainer_options = dict( max_nb_epochs=1, - experiment=exp, + logger=logger, checkpoint_callback=ModelCheckpoint(save_dir) ) @@ -611,7 +612,7 @@ def test_no_val_end_module(): trainer.save_checkpoint(new_weights_path) # load new model - tags_path = exp.get_data_path(exp.name, exp.version) + tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version) tags_path = os.path.join(tags_path, 'meta_tags.csv') model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path, tags_csv=tags_path, on_gpu=False) @@ -631,7 +632,7 @@ def test_simple_cpu(): save_dir = init_save_dir() - # exp file to get meta + # logger file to get meta trainer_options = dict( max_nb_epochs=1, val_percent_check=0.1, @@ -715,18 +716,18 @@ def test_cpu_restore_training(): save_dir = init_save_dir() - # exp file to get meta - test_exp_version = 10 - exp = get_exp(False, version=test_exp_version) - exp.argparse(hparams) - exp.save() + # logger file to get meta + test_logger_version = 10 + logger = get_test_tube_logger(False, version=test_logger_version) + logger.log_hyperparams(hparams) + logger.save() trainer_options = dict( max_nb_epochs=2, val_check_interval=0.50, val_percent_check=0.2, train_percent_check=0.2, - experiment=exp, + logger=logger, checkpoint_callback=ModelCheckpoint(save_dir) ) @@ -741,13 +742,13 @@ def test_cpu_restore_training(): # wipe-out trainer and model # retrain with not much data... this simulates picking training back up after slurm # we want to see if the weights come back correctly - new_exp = get_exp(False, version=test_exp_version) + new_logger = get_test_tube_logger(False, version=test_logger_version) trainer_options = dict( max_nb_epochs=2, val_check_interval=0.50, val_percent_check=0.2, train_percent_check=0.2, - experiment=new_exp, + logger=new_logger, checkpoint_callback=ModelCheckpoint(save_dir), ) trainer = Trainer(**trainer_options) @@ -805,16 +806,16 @@ def test_cpu_slurm_save_load(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() - version = exp.version + version = logger.version trainer_options = dict( max_nb_epochs=1, - experiment=exp, + logger=logger, checkpoint_callback=ModelCheckpoint(save_dir) ) @@ -839,17 +840,17 @@ def test_cpu_slurm_save_load(): # test HPC saving # simulate snapshot on slurm - saved_filepath = trainer.hpc_save(save_dir, exp) + saved_filepath = trainer.hpc_save(save_dir, logger) assert os.path.exists(saved_filepath) - # new exp file to get meta - exp = get_exp(False, version=version) - exp.argparse(hparams) - exp.save() + # new logger file to get meta + logger = get_test_tube_logger(False, version=version) + logger.log_hyperparams(hparams) + logger.save() trainer_options = dict( max_nb_epochs=1, - experiment=exp, + logger=logger, checkpoint_callback=ModelCheckpoint(save_dir), ) trainer = Trainer(**trainer_options) @@ -874,16 +875,19 @@ def test_cpu_slurm_save_load(): def test_loading_meta_tags(): + from argparse import Namespace hparams = get_hparams() # save tags - exp = get_exp(False) - exp.tag({'some_str': 'a_str', 'an_int': 1, 'a_float': 2.0}) - exp.argparse(hparams) - exp.save() + logger = get_test_tube_logger(False) + logger.log_hyperparams(Namespace(some_str='a_str', an_int=1, a_float=2.0)) + logger.log_hyperparams(hparams) + logger.save() # load tags - tags_path = exp.get_data_path(exp.name, exp.version) + '/meta_tags.csv' + tags_path = logger.experiment.get_data_path( + logger.experiment.name, logger.experiment.version + ) + '/meta_tags.csv' tags = trainer_io.load_hparams_from_tags_csv(tags_path) assert tags.batch_size == 32 and tags.hidden_dim == 1000 @@ -922,14 +926,14 @@ def test_model_saving_loading(): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() trainer_options = dict( max_nb_epochs=1, - experiment=exp, + logger=logger, checkpoint_callback=ModelCheckpoint(save_dir) ) @@ -956,7 +960,7 @@ def test_model_saving_loading(): trainer.save_checkpoint(new_weights_path) # load new model - tags_path = exp.get_data_path(exp.name, exp.version) + tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version) tags_path = os.path.join(tags_path, 'meta_tags.csv') model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path, tags_csv=tags_path, on_gpu=False) @@ -1004,16 +1008,16 @@ def test_amp_gpu_ddp_slurm_managed(): save_dir = init_save_dir() # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() # exp file to get weights checkpoint = ModelCheckpoint(save_dir) # add these to the trainer options trainer_options['checkpoint_callback'] = checkpoint - trainer_options['experiment'] = exp + trainer_options['logger'] = logger # fit model trainer = Trainer(**trainer_options) @@ -1030,7 +1034,7 @@ def test_amp_gpu_ddp_slurm_managed(): assert trainer.resolve_root_node_address('abc[23-24, 45-40, 40]') == 'abc23' # test model loading with a map_location - pretrained_model = load_model(exp, save_dir, True) + pretrained_model = load_model(logger.experiment, save_dir, True) # test model preds run_prediction(model.test_dataloader, pretrained_model) @@ -1041,7 +1045,7 @@ def test_amp_gpu_ddp_slurm_managed(): trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers() # test HPC loading / saving - trainer.hpc_save(save_dir, exp) + trainer.hpc_save(save_dir, logger) trainer.hpc_load(save_dir, on_gpu=True) # test freeze on gpu @@ -1059,7 +1063,7 @@ def test_cpu_model_with_amp(): trainer_options = dict( show_progress_bar=False, - experiment=get_exp(), + logger=get_test_tube_logger(), max_nb_epochs=1, train_percent_check=0.4, val_percent_check=0.4, @@ -1080,7 +1084,7 @@ def test_cpu_model(): trainer_options = dict( show_progress_bar=False, - experiment=get_exp(), + logger=get_test_tube_logger(), max_nb_epochs=1, train_percent_check=0.4, val_percent_check=0.4 @@ -1103,7 +1107,7 @@ def test_all_features_cpu_model(): track_grad_norm=2, print_nan_grads=True, show_progress_bar=False, - experiment=get_exp(), + logger=get_test_tube_logger(), accumulate_grad_batches=2, max_nb_epochs=1, train_percent_check=0.4, @@ -1214,11 +1218,11 @@ def test_ddp_sampler_error(): hparams = get_hparams() model = LightningTestModel(hparams, force_remove_distributed_sampler=True) - exp = get_exp(True) - exp.save() + logger = get_test_tube_logger(True) + logger.save() trainer = Trainer( - experiment=exp, + logger=logger, show_progress_bar=False, max_nb_epochs=1, gpus=[0, 1], @@ -1245,7 +1249,7 @@ def test_multiple_val_dataloader(): hparams = get_hparams() model = CurrentTestModel(hparams) - # exp file to get meta + # logger file to get meta trainer_options = dict( max_nb_epochs=1, val_percent_check=0.1, @@ -1279,7 +1283,7 @@ def test_multiple_test_dataloader(): hparams = get_hparams() model = CurrentTestModel(hparams) - # exp file to get meta + # logger file to get meta trainer_options = dict( max_nb_epochs=1, val_percent_check=0.1, @@ -1306,17 +1310,17 @@ def test_multiple_test_dataloader(): def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True): save_dir = init_save_dir() - # exp file to get meta - exp = get_exp(False) - exp.argparse(hparams) - exp.save() + # logger file to get meta + logger = get_test_tube_logger(False) + logger.log_hyperparams(hparams) + logger.save() - # exp file to get weights + # logger file to get weights checkpoint = ModelCheckpoint(save_dir) # add these to the trainer options trainer_options['checkpoint_callback'] = checkpoint - trainer_options['experiment'] = exp + trainer_options['logger'] = logger # fit model trainer = Trainer(**trainer_options) @@ -1326,7 +1330,7 @@ def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True): assert result == 1, 'amp + ddp model failed to complete' # test model loading - pretrained_model = load_model(exp, save_dir, on_gpu) + pretrained_model = load_model(logger.experiment, save_dir, on_gpu) # test new model accuracy run_prediction(model.test_dataloader, pretrained_model) @@ -1337,7 +1341,7 @@ def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True): trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers() # test HPC loading / saving - trainer.hpc_save(save_dir, exp) + trainer.hpc_save(save_dir, logger) trainer.hpc_load(save_dir, on_gpu=on_gpu) clear_save_dir() @@ -1376,12 +1380,12 @@ def get_model(use_test_model=False): return model, hparams -def get_exp(debug=True, version=None): - # set up exp object without actually saving logs +def get_test_tube_logger(debug=True, version=None): + # set up logger object without actually saving logs root_dir = os.path.dirname(os.path.realpath(__file__)) save_dir = os.path.join(root_dir, 'save_dir') - exp = Experiment(debug=debug, save_dir=save_dir, name='tests_tt_dir', version=version) - return exp + logger = TestTubeLogger(save_dir, name='test_tt_dir', debug=debug, version=version) + return logger def init_save_dir():