mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-09 11:32:07 +08:00
Enable any ML experiment tracking framework (#223)
* Implement generic loggers for experiment tracking * Add tests for loggers * Get model tests passing * Test and fix logger pickling * Expand pickle test and fix bug * Missed exp -> logger conversion * Remove commented code * Add docstrings * Update logging docs * Add mlflow to test requirements * Make linter happy * Fix mlflow timestamp * Update Logging.md * Update test_models.py * Update test_models.py * Update test_models.py * Update properties.md * Fix tests * Line length
This commit is contained in:
Nic Eggert
committed by
William Falcon
William Falcon
parent
e9c5aff7ba
commit
480eed5cb6
@@ -9,12 +9,20 @@ The current epoch
|
||||
Current dtype
|
||||
|
||||
---
|
||||
#### experiment
|
||||
An instance of test-tube Experiment which you can use to log anything for tensorboard (subclass of [PyTorch SummaryWriter](https://pytorch.org/docs/stable/tensorboard.html)).
|
||||
#### logger
|
||||
A reference to the logger you passed into trainer.
|
||||
```python
|
||||
Trainer(logger=your_logger)
|
||||
```
|
||||
|
||||
Call it from anywhere in your LightningModule to add metrics, images, etc... whatever your logger supports.
|
||||
|
||||
Here is an example using the Test-tube logger (which is a wrapper on [PyTorch SummaryWriter](https://pytorch.org/docs/stable/tensorboard.html) with versioned folder structure).
|
||||
```{.python}
|
||||
self.experiment.add_embedding(...)
|
||||
self.experiment.log({'val_loss': 0.9})
|
||||
self.experiment.add_scalars(...)
|
||||
# if logger is a tensorboard logger or test-tube experiment
|
||||
self.logger.add_embedding(...)
|
||||
self.logger.log({'val_loss': 0.9})
|
||||
self.logger.add_scalars(...)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
+87
-55
@@ -1,7 +1,88 @@
|
||||
Lighting offers a few options for logging information about model, gpu usage, etc (via test-tube). It also offers printing options for training monitoring.
|
||||
Lighting offers options for logging information about model, gpu usage, etc, via several different logging frameworks. It also offers printing options for training monitoring.
|
||||
|
||||
|
||||
---
|
||||
### Setting up logging
|
||||
|
||||
Initialize your logger, which should inherit from `LightningBaseLogger`, and pass
|
||||
it to `Trainer`.
|
||||
```{.python}
|
||||
my_logger = MyLightningLogger(...)
|
||||
trainer = Trainer(logger=my_logger)
|
||||
```
|
||||
|
||||
Lightning supports several common experiment tracking frameworks out of the box
|
||||
|
||||
---
|
||||
#### Test tube
|
||||
|
||||
Log using [test tube](https://williamfalcon.github.io/test-tube/).
|
||||
|
||||
```{.python}
|
||||
from pytorch_lightning.logging import TestTubeLogger
|
||||
tt_logger = TestTubeLogger(
|
||||
save_dir=".",
|
||||
name="default",
|
||||
debug=False,
|
||||
create_git_tag=False
|
||||
)
|
||||
trainer = Trainer(logger=tt_logger)
|
||||
```
|
||||
|
||||
---
|
||||
#### MLFlow
|
||||
|
||||
Log using [mlflow](https://mlflow.org)
|
||||
|
||||
```{.python}
|
||||
from pytorch_lightning.logging import MLFlowLogger
|
||||
mlf_logger = MLFlowLogger(
|
||||
experiment_name="default",
|
||||
tracking_uri="file:/."
|
||||
)
|
||||
trainer = Trainer(logger=mlf_logger)
|
||||
```
|
||||
|
||||
---
|
||||
#### Custom logger
|
||||
|
||||
You can implement your own logger by writing a class that inherits from
|
||||
`LightningLoggerBase`. Use the `rank_zero_only` decorator to make sure that
|
||||
only the first process in DDP training logs data.
|
||||
|
||||
```{.python}
|
||||
from pytorch_lightning.logging import LightningLoggerBase, rank_zero_only
|
||||
|
||||
class MyLogger(LightningLoggerBase):
|
||||
|
||||
@rank_zero_only
|
||||
def log_hyperparams(self, params):
|
||||
# params is an argparse.Namespace
|
||||
# your code to record hyperparameters goes here
|
||||
pass
|
||||
|
||||
@rank_zero_only
|
||||
def log_metrics(self, metrics, step_num):
|
||||
# metrics is a dictionary of metric names and values
|
||||
# your code to record metrics goes here
|
||||
pass
|
||||
|
||||
def save(self):
|
||||
# Optional. Any code necessary to save logger data goes here
|
||||
pass
|
||||
|
||||
@rank_zero_only
|
||||
def finalize(self, status):
|
||||
# Optional. Any code that needs to be run after training
|
||||
# finishes goes here
|
||||
```
|
||||
|
||||
If you write a logger than may be useful to others, please send
|
||||
a pull request to add it to Lighting!
|
||||
|
||||
---
|
||||
### Using loggers
|
||||
|
||||
#### Display metrics in progress bar
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
@@ -17,7 +98,7 @@ trainer = Trainer(row_log_interval=10)
|
||||
```
|
||||
|
||||
---
|
||||
#### Log metric row every k batches
|
||||
#### Log GPU memory
|
||||
Logs GPU memory when metrics are logged.
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
@@ -38,61 +119,12 @@ trainer = Trainer(process_position=1)
|
||||
|
||||
---
|
||||
#### Save a snapshot of all hyperparameters
|
||||
Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use.
|
||||
Give lightning a test-tube Experiment object to automate this for you.
|
||||
Log hyperparameters using the logger
|
||||
``` {.python}
|
||||
from test_tube import Experiment
|
||||
logger = TestTubeLogger(...)
|
||||
logger.log_hyperparams(args)
|
||||
|
||||
exp = Experiment(...)
|
||||
Trainer(experiment=exp)
|
||||
```
|
||||
|
||||
---
|
||||
#### Snapshot code for a training run
|
||||
Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag.
|
||||
Give lightning a test-tube Experiment object to automate this for you.
|
||||
``` {.python}
|
||||
from test_tube import Experiment
|
||||
|
||||
exp = Experiment(create_git_tag=True)
|
||||
Trainer(experiment=exp)
|
||||
```
|
||||
|
||||
---
|
||||
### Tensorboard support
|
||||
In the LightningModule you can access the experiment logger by doing:
|
||||
```python
|
||||
self.experiment
|
||||
|
||||
# add image
|
||||
# Look at PyTorch SummaryWriter docs for what you can do.
|
||||
self.experiment.add_image(...)
|
||||
```
|
||||
|
||||
The experiment object is a strict subclass of PyTorch SummaryWriter. However, this class
|
||||
also snapshots every detail about the experiment (data folder paths, code, hyperparams),
|
||||
and allows you to visualize it using tensorboard.
|
||||
``` {.python}
|
||||
from test_tube import Experiment, HyperOptArgumentParser
|
||||
|
||||
# exp hyperparams
|
||||
args = HyperOptArgumentParser()
|
||||
hparams = args.parse_args()
|
||||
|
||||
# this is a summaryWriter with nicer logging structure
|
||||
exp = Experiment(save_dir='/some/path', create_git_tag=True)
|
||||
|
||||
# track experiment details (must be ArgumentParser or HyperOptArgumentParser).
|
||||
# each option in the parser is tracked
|
||||
exp.argparse(hparams)
|
||||
exp.tag({'description': 'running demo'})
|
||||
|
||||
# trainer uses the exp object to log exp data
|
||||
trainer = Trainer(experiment=exp)
|
||||
trainer.fit(model)
|
||||
|
||||
# view logs at:
|
||||
# tensorboard --logdir /some/path
|
||||
Trainer(logger=logger)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
from .base import LightningLoggerBase, rank_zero_only
|
||||
from .test_tube_logger import TestTubeLogger
|
||||
|
||||
try:
|
||||
from .mlflow_logger import MLFlowLogger
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
@@ -0,0 +1,71 @@
|
||||
from functools import wraps
|
||||
|
||||
|
||||
def rank_zero_only(fn):
|
||||
"""Decorate a logger method to run it only on the process with rank 0
|
||||
|
||||
:param fn: Function to decorate
|
||||
"""
|
||||
|
||||
@wraps(fn)
|
||||
def wrapped_fn(self, *args, **kwargs):
|
||||
if self.rank == 0:
|
||||
fn(self, *args, **kwargs)
|
||||
|
||||
return wrapped_fn
|
||||
|
||||
|
||||
class LightningLoggerBase:
|
||||
"""Base class for experiment loggers"""
|
||||
|
||||
def __init__(self):
|
||||
self._rank = 0
|
||||
|
||||
def log_metrics(self, metrics, step_num):
|
||||
"""Record metrics
|
||||
|
||||
:param metric: Dictionary with metric names as keys and measured
|
||||
quanties as values
|
||||
:param step_num: Step number at which the metrics should be recorded
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def log_hyperparams(self, params):
|
||||
"""Record hyperparameters
|
||||
|
||||
:param params: argparse.Namespace containing the hyperparameters
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def save(self):
|
||||
"""Save log data"""
|
||||
pass
|
||||
|
||||
def finalize(self, status):
|
||||
"""Do any processing that is necessary to finalize an experiment
|
||||
|
||||
:param status: Status that the experiment finished with (e.g. success, failed, aborted)
|
||||
"""
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
"""Do any cleanup that is necessary to close an experiment"""
|
||||
pass
|
||||
|
||||
@property
|
||||
def rank(self):
|
||||
"""
|
||||
Process rank. In general, metrics should only be logged by the process
|
||||
with rank 0
|
||||
"""
|
||||
return self._rank
|
||||
|
||||
@rank.setter
|
||||
def rank(self, value):
|
||||
"""Set the process rank"""
|
||||
self._rank = value
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
"""Return the experiment version"""
|
||||
return None
|
||||
@@ -0,0 +1,44 @@
|
||||
from time import time
|
||||
from logging import getLogger
|
||||
|
||||
import mlflow
|
||||
|
||||
from .base import LightningLoggerBase, rank_zero_only
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class MLFlowLogger(LightningLoggerBase):
|
||||
|
||||
def __init__(self, experiment_name, tracking_uri=None):
|
||||
super().__init__()
|
||||
self.client = mlflow.tracking.MlflowClient(tracking_uri)
|
||||
|
||||
experiment = self.client.get_experiment_by_name(experiment_name)
|
||||
if experiment is None:
|
||||
logger.warning(
|
||||
f"Experiment with name f{experiment_name} not found. Creating it."
|
||||
)
|
||||
self.client.create_experiment(experiment_name)
|
||||
experiment = self.client.get_experiment_by_name(experiment_name)
|
||||
|
||||
run = self.client.create_run(experiment.experiment_id)
|
||||
self.run_id = run.info.run_id
|
||||
|
||||
@rank_zero_only
|
||||
def log_hyperparams(self, params):
|
||||
for k, v in vars(params).items():
|
||||
self.client.log_param(self.run_id, k, v)
|
||||
|
||||
@rank_zero_only
|
||||
def log_metrics(self, metrics, step_num=None):
|
||||
timestamp_ms = int(time() * 1000)
|
||||
for k, v in metrics.items():
|
||||
self.client.log_metric(self.run_id, k, v, timestamp_ms, step_num)
|
||||
|
||||
def save(self):
|
||||
pass
|
||||
|
||||
@rank_zero_only
|
||||
def finalize(self, status="FINISHED"):
|
||||
self.client.set_terminated(self.run_id, status)
|
||||
@@ -0,0 +1,66 @@
|
||||
import os.path
|
||||
from copy import copy
|
||||
|
||||
from .base import LightningLoggerBase, rank_zero_only
|
||||
|
||||
from test_tube import Experiment
|
||||
|
||||
|
||||
class TestTubeLogger(LightningLoggerBase):
|
||||
def __init__(
|
||||
self, save_dir, name="default", debug=False, version=None, create_git_tag=False
|
||||
):
|
||||
super().__init__()
|
||||
self.experiment = Experiment(
|
||||
save_dir=save_dir,
|
||||
name=name,
|
||||
debug=debug,
|
||||
version=version,
|
||||
create_git_tag=create_git_tag,
|
||||
)
|
||||
|
||||
@rank_zero_only
|
||||
def log_hyperparams(self, params):
|
||||
self.experiment.argparse(params)
|
||||
|
||||
@rank_zero_only
|
||||
def log_metrics(self, metrics, step_num=None):
|
||||
self.experiment.log(metrics, global_step=step_num)
|
||||
|
||||
@rank_zero_only
|
||||
def save(self):
|
||||
self.experiment.save()
|
||||
|
||||
@rank_zero_only
|
||||
def finalize(self, status):
|
||||
self.save()
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
self.experiment.close()
|
||||
|
||||
@property
|
||||
def rank(self):
|
||||
return self.experiment.rank
|
||||
|
||||
@rank.setter
|
||||
def rank(self, value):
|
||||
self.experiment.rank = value
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
return self.experiment.version
|
||||
|
||||
# Test tube experiments are not pickleable, so we need to override a few
|
||||
# methods to get DDP working. See
|
||||
# https://docs.python.org/3/library/pickle.html#handling-stateful-objects
|
||||
# for more info.
|
||||
def __getstate__(self):
|
||||
state = self.__dict__.copy()
|
||||
state["experiment"] = self.experiment.get_meta_copy()
|
||||
return state
|
||||
|
||||
def __setstate__(self, state):
|
||||
self.experiment = state["experiment"].get_non_ddp_exp()
|
||||
del state['experiment']
|
||||
self.__dict__.update(state)
|
||||
@@ -55,7 +55,7 @@ def reduce_distributed_output(output, nb_gpus):
|
||||
class Trainer(TrainerIO):
|
||||
|
||||
def __init__(self,
|
||||
experiment=None,
|
||||
logger=None,
|
||||
early_stop_callback=None,
|
||||
checkpoint_callback=None,
|
||||
gradient_clip_val=0,
|
||||
@@ -86,7 +86,7 @@ class Trainer(TrainerIO):
|
||||
nb_sanity_val_steps=5):
|
||||
"""
|
||||
|
||||
:param experiment: Test-tube experiment
|
||||
:param logger: Logger for experiment tracking
|
||||
:param early_stop_callback: Callback for early stopping
|
||||
:param checkpoint_callback: Callback for checkpointing
|
||||
:param gradient_clip_val: int. 0 means don't clip.
|
||||
@@ -160,11 +160,8 @@ class Trainer(TrainerIO):
|
||||
# configure weights save path
|
||||
self.__configure_weights_path(checkpoint_callback, weights_save_path)
|
||||
|
||||
# configure experiment
|
||||
self.experiment = experiment
|
||||
self.exp_save_path = None
|
||||
if self.experiment is not None:
|
||||
self.exp_save_path = experiment.get_data_path(experiment.name, experiment.version)
|
||||
# configure logger
|
||||
self.logger = logger
|
||||
|
||||
# accumulated grads
|
||||
self.__configure_accumulated_gradients(accumulate_grad_batches)
|
||||
@@ -423,8 +420,8 @@ class Trainer(TrainerIO):
|
||||
'batch_nb': '{}'.format(self.batch_nb),
|
||||
}
|
||||
|
||||
if self.experiment is not None:
|
||||
tqdm_dict['v_nb'] = self.experiment.version
|
||||
if self.logger is not None and self.logger.version is not None:
|
||||
tqdm_dict['v_nb'] = self.logger.version
|
||||
|
||||
tqdm_dict.update(self.tqdm_metrics)
|
||||
|
||||
@@ -658,10 +655,6 @@ class Trainer(TrainerIO):
|
||||
def fit(self, model):
|
||||
# when using multi-node or DDP within a node start each module in a separate process
|
||||
if self.use_ddp:
|
||||
# must copy only the meta of the exp so it survives pickle/unpickle
|
||||
# when going to new process
|
||||
if self.experiment is not None:
|
||||
self.experiment = self.experiment.get_meta_copy()
|
||||
|
||||
if self.is_slurm_managing_tasks:
|
||||
task = int(os.environ['SLURM_LOCALID'])
|
||||
@@ -786,12 +779,8 @@ class Trainer(TrainerIO):
|
||||
self.world_size = self.nb_gpu_nodes * self.num_gpus
|
||||
|
||||
# let the exp know the rank to avoid overwriting logs
|
||||
# recover original exp before went into process
|
||||
# init in write mode only on proc 0
|
||||
if self.experiment is not None:
|
||||
self.experiment = self.experiment.get_non_ddp_exp()
|
||||
self.experiment.debug = self.proc_rank > 0
|
||||
self.experiment.rank = self.proc_rank
|
||||
if self.logger is not None:
|
||||
self.logger.rank = self.proc_rank
|
||||
|
||||
# set up server using proc 0's ip address
|
||||
# try to init for 20 times at max in case ports are taken
|
||||
@@ -897,12 +886,12 @@ class Trainer(TrainerIO):
|
||||
ref_model.summarize()
|
||||
|
||||
# link up experiment object
|
||||
if self.experiment is not None:
|
||||
ref_model.experiment = self.experiment
|
||||
if self.logger is not None:
|
||||
ref_model.logger = self.logger
|
||||
|
||||
# save exp to get started
|
||||
if self.proc_rank == 0:
|
||||
self.experiment.save()
|
||||
self.logger.save()
|
||||
|
||||
# track model now.
|
||||
# if cluster resets state, the model will update with the saved weights
|
||||
@@ -1016,8 +1005,8 @@ class Trainer(TrainerIO):
|
||||
|
||||
# when batch should be saved
|
||||
if (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch:
|
||||
if self.proc_rank == 0 and self.experiment is not None:
|
||||
self.experiment.save()
|
||||
if self.proc_rank == 0 and self.logger is not None:
|
||||
self.logger.save()
|
||||
|
||||
# when metrics should be logged
|
||||
if batch_nb % self.row_log_interval == 0 or early_stop_epoch:
|
||||
@@ -1044,9 +1033,9 @@ class Trainer(TrainerIO):
|
||||
# log metrics
|
||||
scalar_metrics = self.__metrics_to_scalars(
|
||||
metrics, blacklist=self.__log_vals_blacklist())
|
||||
if self.proc_rank == 0 and self.experiment is not None:
|
||||
self.experiment.log(scalar_metrics, global_step=self.global_step)
|
||||
self.experiment.save()
|
||||
if self.proc_rank == 0 and self.logger is not None:
|
||||
self.logger.log_metrics(scalar_metrics, step_num=self.global_step)
|
||||
self.logger.save()
|
||||
|
||||
# end epoch early
|
||||
if early_stop_epoch:
|
||||
|
||||
@@ -233,12 +233,12 @@ class TrainerIO(object):
|
||||
# ----------------------------------
|
||||
# PRIVATE OPS
|
||||
# ----------------------------------
|
||||
def hpc_save(self, folderpath, experiment):
|
||||
def hpc_save(self, folderpath, logger):
|
||||
# make sure the checkpoint folder exists
|
||||
os.makedirs(folderpath, exist_ok=True)
|
||||
|
||||
# save exp to make sure we get all the metrics
|
||||
experiment.save()
|
||||
# save logger to make sure we get all the metrics
|
||||
logger.save()
|
||||
|
||||
ckpt_number = self.max_ckpt_in_folder(folderpath) + 1
|
||||
|
||||
|
||||
@@ -5,4 +5,5 @@ pytest>=3.0.5
|
||||
pytest-cov
|
||||
flake8
|
||||
check-manifest
|
||||
test_tube
|
||||
test_tube
|
||||
mlflow
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import os.path
|
||||
import pickle
|
||||
import shutil
|
||||
|
||||
import numpy as np
|
||||
from pytorch_lightning import Trainer
|
||||
|
||||
from pytorch_lightning.testing import LightningTestModel
|
||||
|
||||
from .test_models import get_hparams, get_test_tube_logger, init_save_dir, clear_save_dir
|
||||
|
||||
|
||||
def test_testtube_logger():
|
||||
"""verify that basic functionality of test tube logger works"""
|
||||
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
logger=logger
|
||||
)
|
||||
|
||||
trainer = Trainer(**trainer_options)
|
||||
result = trainer.fit(model)
|
||||
|
||||
assert result == 1, "Training failed"
|
||||
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_testtube_pickle():
|
||||
"""Verify that pickling a trainer containing a test tube logger works"""
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
logger=logger
|
||||
)
|
||||
|
||||
trainer = Trainer(**trainer_options)
|
||||
pkl_bytes = pickle.dumps(trainer)
|
||||
trainer2 = pickle.loads(pkl_bytes)
|
||||
trainer2.logger.log_metrics({"acc": 1.0})
|
||||
|
||||
|
||||
def test_mlflow_logger():
|
||||
"""verify that basic functionality of mlflow logger works"""
|
||||
try:
|
||||
from pytorch_lightning.logging import MLFlowLogger
|
||||
except ModuleNotFoundError:
|
||||
return
|
||||
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
mlflow_dir = os.path.join(root_dir, "mlruns")
|
||||
|
||||
logger = MLFlowLogger("test", f"file://{mlflow_dir}")
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
logger=logger
|
||||
)
|
||||
|
||||
trainer = Trainer(**trainer_options)
|
||||
result = trainer.fit(model)
|
||||
|
||||
assert result == 1, "Training failed"
|
||||
|
||||
n = np.random.randint(0, 10000000, 1)[0]
|
||||
shutil.move(mlflow_dir, mlflow_dir + f'_{n}')
|
||||
|
||||
|
||||
def test_mlflow_pickle():
|
||||
"""verify that pickling trainer with mlflow logger works"""
|
||||
try:
|
||||
from pytorch_lightning.logging import MLFlowLogger
|
||||
except ModuleNotFoundError:
|
||||
return
|
||||
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
mlflow_dir = os.path.join(root_dir, "mlruns")
|
||||
|
||||
logger = MLFlowLogger("test", f"file://{mlflow_dir}")
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
logger=logger
|
||||
)
|
||||
|
||||
trainer = Trainer(**trainer_options)
|
||||
pkl_bytes = pickle.dumps(trainer)
|
||||
trainer2 = pickle.loads(pkl_bytes)
|
||||
trainer2.logger.log_metrics({"acc": 1.0})
|
||||
+115
-111
@@ -6,7 +6,6 @@ from argparse import Namespace
|
||||
import pytest
|
||||
import numpy as np
|
||||
import torch
|
||||
from test_tube import Experiment
|
||||
|
||||
# sys.path += [os.path.abspath('..'), os.path.abspath('../..')]
|
||||
from pytorch_lightning import Trainer
|
||||
@@ -29,6 +28,7 @@ from pytorch_lightning.root_module import memory
|
||||
from pytorch_lightning.trainer.trainer import reduce_distributed_output
|
||||
from pytorch_lightning.root_module import model_saving
|
||||
from pytorch_lightning.trainer import trainer_io
|
||||
from pytorch_lightning.logging import TestTubeLogger
|
||||
from examples import LightningTemplateModel
|
||||
|
||||
SEED = 2334
|
||||
@@ -59,16 +59,15 @@ def test_dp_resume():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# get logger
|
||||
logger = get_test_tube_logger(debug=False)
|
||||
logger.log_hyperparams(hparams)
|
||||
|
||||
# exp file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
|
||||
# add these to the trainer options
|
||||
trainer_options['experiment'] = exp
|
||||
trainer_options['logger'] = logger
|
||||
trainer_options['checkpoint_callback'] = checkpoint
|
||||
|
||||
# fit model
|
||||
@@ -86,11 +85,11 @@ def test_dp_resume():
|
||||
# HPC LOAD/SAVE
|
||||
# ---------------------------
|
||||
# save
|
||||
trainer.hpc_save(save_dir, exp)
|
||||
trainer.hpc_save(save_dir, logger)
|
||||
|
||||
# init new trainer
|
||||
new_exp = get_exp(False, version=exp.version)
|
||||
trainer_options['experiment'] = new_exp
|
||||
new_logger = get_test_tube_logger(version=logger.version)
|
||||
trainer_options['logger'] = new_logger
|
||||
trainer_options['checkpoint_callback'] = ModelCheckpoint(save_dir)
|
||||
trainer_options['train_percent_check'] = 0.2
|
||||
trainer_options['val_percent_check'] = 0.2
|
||||
@@ -133,9 +132,9 @@ def test_running_test_pretrained_model_ddp():
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
# exp file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
@@ -146,7 +145,7 @@ def test_running_test_pretrained_model_ddp():
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.2,
|
||||
checkpoint_callback=checkpoint,
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
gpus=[0, 1],
|
||||
distributed_backend='ddp'
|
||||
)
|
||||
@@ -157,7 +156,8 @@ def test_running_test_pretrained_model_ddp():
|
||||
|
||||
# correct result and ok accuracy
|
||||
assert result == 1, 'training failed to complete'
|
||||
pretrained_model = load_model(exp, save_dir, on_gpu=True, module_class=LightningTestModel)
|
||||
pretrained_model = load_model(logger.experiment, save_dir, on_gpu=True,
|
||||
module_class=LightningTestModel)
|
||||
|
||||
# run test set
|
||||
new_trainer = Trainer(**trainer_options)
|
||||
@@ -176,12 +176,12 @@ def test_running_test_after_fitting():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
# exp file to get weights
|
||||
# logger file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
|
||||
trainer_options = dict(
|
||||
@@ -191,7 +191,7 @@ def test_running_test_after_fitting():
|
||||
val_percent_check=0.2,
|
||||
test_percent_check=0.2,
|
||||
checkpoint_callback=checkpoint,
|
||||
experiment=exp
|
||||
logger=logger
|
||||
)
|
||||
|
||||
# fit model
|
||||
@@ -217,12 +217,12 @@ def test_running_test_without_val():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
# exp file to get weights
|
||||
# logger file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
|
||||
trainer_options = dict(
|
||||
@@ -232,7 +232,7 @@ def test_running_test_without_val():
|
||||
val_percent_check=0.2,
|
||||
test_percent_check=0.2,
|
||||
checkpoint_callback=checkpoint,
|
||||
experiment=exp
|
||||
logger=logger
|
||||
)
|
||||
|
||||
# fit model
|
||||
@@ -256,12 +256,12 @@ def test_running_test_pretrained_model():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
# exp file to get weights
|
||||
# logger file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
|
||||
trainer_options = dict(
|
||||
@@ -270,7 +270,7 @@ def test_running_test_pretrained_model():
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.2,
|
||||
checkpoint_callback=checkpoint,
|
||||
experiment=exp
|
||||
logger=logger
|
||||
)
|
||||
|
||||
# fit model
|
||||
@@ -280,7 +280,7 @@ def test_running_test_pretrained_model():
|
||||
# correct result and ok accuracy
|
||||
assert result == 1, 'training failed to complete'
|
||||
pretrained_model = load_model(
|
||||
exp, save_dir, on_gpu=False, module_class=LightningTestModel
|
||||
logger.experiment, save_dir, on_gpu=False, module_class=LightningTestModel
|
||||
)
|
||||
|
||||
new_trainer = Trainer(**trainer_options)
|
||||
@@ -301,12 +301,12 @@ def test_running_test_pretrained_model_dp():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
# exp file to get weights
|
||||
# logger file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
|
||||
trainer_options = dict(
|
||||
@@ -315,7 +315,7 @@ def test_running_test_pretrained_model_dp():
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.2,
|
||||
checkpoint_callback=checkpoint,
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
gpus=[0, 1],
|
||||
distributed_backend='dp'
|
||||
)
|
||||
@@ -326,7 +326,8 @@ def test_running_test_pretrained_model_dp():
|
||||
|
||||
# correct result and ok accuracy
|
||||
assert result == 1, 'training failed to complete'
|
||||
pretrained_model = load_model(exp, save_dir, on_gpu=True, module_class=LightningTestModel)
|
||||
pretrained_model = load_model(logger.experiment, save_dir, on_gpu=True,
|
||||
module_class=LightningTestModel)
|
||||
|
||||
new_trainer = Trainer(**trainer_options)
|
||||
new_trainer.test(pretrained_model)
|
||||
@@ -516,7 +517,7 @@ def test_early_stopping_cpu_model():
|
||||
track_grad_norm=2,
|
||||
print_nan_grads=True,
|
||||
show_progress_bar=False,
|
||||
experiment=get_exp(),
|
||||
logger=get_test_tube_logger(),
|
||||
train_percent_check=0.1,
|
||||
val_percent_check=0.1
|
||||
)
|
||||
@@ -542,14 +543,14 @@ def test_no_val_module():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
checkpoint_callback=ModelCheckpoint(save_dir)
|
||||
)
|
||||
|
||||
@@ -565,7 +566,7 @@ def test_no_val_module():
|
||||
trainer.save_checkpoint(new_weights_path)
|
||||
|
||||
# load new model
|
||||
tags_path = exp.get_data_path(exp.name, exp.version)
|
||||
tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version)
|
||||
tags_path = os.path.join(tags_path, 'meta_tags.csv')
|
||||
model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
|
||||
tags_csv=tags_path, on_gpu=False)
|
||||
@@ -588,14 +589,14 @@ def test_no_val_end_module():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
checkpoint_callback=ModelCheckpoint(save_dir)
|
||||
)
|
||||
|
||||
@@ -611,7 +612,7 @@ def test_no_val_end_module():
|
||||
trainer.save_checkpoint(new_weights_path)
|
||||
|
||||
# load new model
|
||||
tags_path = exp.get_data_path(exp.name, exp.version)
|
||||
tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version)
|
||||
tags_path = os.path.join(tags_path, 'meta_tags.csv')
|
||||
model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
|
||||
tags_csv=tags_path, on_gpu=False)
|
||||
@@ -631,7 +632,7 @@ def test_simple_cpu():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
# logger file to get meta
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
val_percent_check=0.1,
|
||||
@@ -715,18 +716,18 @@ def test_cpu_restore_training():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
test_exp_version = 10
|
||||
exp = get_exp(False, version=test_exp_version)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
test_logger_version = 10
|
||||
logger = get_test_tube_logger(False, version=test_logger_version)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=2,
|
||||
val_check_interval=0.50,
|
||||
val_percent_check=0.2,
|
||||
train_percent_check=0.2,
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
checkpoint_callback=ModelCheckpoint(save_dir)
|
||||
)
|
||||
|
||||
@@ -741,13 +742,13 @@ def test_cpu_restore_training():
|
||||
# wipe-out trainer and model
|
||||
# retrain with not much data... this simulates picking training back up after slurm
|
||||
# we want to see if the weights come back correctly
|
||||
new_exp = get_exp(False, version=test_exp_version)
|
||||
new_logger = get_test_tube_logger(False, version=test_logger_version)
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=2,
|
||||
val_check_interval=0.50,
|
||||
val_percent_check=0.2,
|
||||
train_percent_check=0.2,
|
||||
experiment=new_exp,
|
||||
logger=new_logger,
|
||||
checkpoint_callback=ModelCheckpoint(save_dir),
|
||||
)
|
||||
trainer = Trainer(**trainer_options)
|
||||
@@ -805,16 +806,16 @@ def test_cpu_slurm_save_load():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
version = exp.version
|
||||
version = logger.version
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
checkpoint_callback=ModelCheckpoint(save_dir)
|
||||
)
|
||||
|
||||
@@ -839,17 +840,17 @@ def test_cpu_slurm_save_load():
|
||||
|
||||
# test HPC saving
|
||||
# simulate snapshot on slurm
|
||||
saved_filepath = trainer.hpc_save(save_dir, exp)
|
||||
saved_filepath = trainer.hpc_save(save_dir, logger)
|
||||
assert os.path.exists(saved_filepath)
|
||||
|
||||
# new exp file to get meta
|
||||
exp = get_exp(False, version=version)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# new logger file to get meta
|
||||
logger = get_test_tube_logger(False, version=version)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
checkpoint_callback=ModelCheckpoint(save_dir),
|
||||
)
|
||||
trainer = Trainer(**trainer_options)
|
||||
@@ -874,16 +875,19 @@ def test_cpu_slurm_save_load():
|
||||
|
||||
|
||||
def test_loading_meta_tags():
|
||||
from argparse import Namespace
|
||||
hparams = get_hparams()
|
||||
|
||||
# save tags
|
||||
exp = get_exp(False)
|
||||
exp.tag({'some_str': 'a_str', 'an_int': 1, 'a_float': 2.0})
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(Namespace(some_str='a_str', an_int=1, a_float=2.0))
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
# load tags
|
||||
tags_path = exp.get_data_path(exp.name, exp.version) + '/meta_tags.csv'
|
||||
tags_path = logger.experiment.get_data_path(
|
||||
logger.experiment.name, logger.experiment.version
|
||||
) + '/meta_tags.csv'
|
||||
tags = trainer_io.load_hparams_from_tags_csv(tags_path)
|
||||
|
||||
assert tags.batch_size == 32 and tags.hidden_dim == 1000
|
||||
@@ -922,14 +926,14 @@ def test_model_saving_loading():
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
checkpoint_callback=ModelCheckpoint(save_dir)
|
||||
)
|
||||
|
||||
@@ -956,7 +960,7 @@ def test_model_saving_loading():
|
||||
trainer.save_checkpoint(new_weights_path)
|
||||
|
||||
# load new model
|
||||
tags_path = exp.get_data_path(exp.name, exp.version)
|
||||
tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version)
|
||||
tags_path = os.path.join(tags_path, 'meta_tags.csv')
|
||||
model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
|
||||
tags_csv=tags_path, on_gpu=False)
|
||||
@@ -1004,16 +1008,16 @@ def test_amp_gpu_ddp_slurm_managed():
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
# exp file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
|
||||
# add these to the trainer options
|
||||
trainer_options['checkpoint_callback'] = checkpoint
|
||||
trainer_options['experiment'] = exp
|
||||
trainer_options['logger'] = logger
|
||||
|
||||
# fit model
|
||||
trainer = Trainer(**trainer_options)
|
||||
@@ -1030,7 +1034,7 @@ def test_amp_gpu_ddp_slurm_managed():
|
||||
assert trainer.resolve_root_node_address('abc[23-24, 45-40, 40]') == 'abc23'
|
||||
|
||||
# test model loading with a map_location
|
||||
pretrained_model = load_model(exp, save_dir, True)
|
||||
pretrained_model = load_model(logger.experiment, save_dir, True)
|
||||
|
||||
# test model preds
|
||||
run_prediction(model.test_dataloader, pretrained_model)
|
||||
@@ -1041,7 +1045,7 @@ def test_amp_gpu_ddp_slurm_managed():
|
||||
trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers()
|
||||
|
||||
# test HPC loading / saving
|
||||
trainer.hpc_save(save_dir, exp)
|
||||
trainer.hpc_save(save_dir, logger)
|
||||
trainer.hpc_load(save_dir, on_gpu=True)
|
||||
|
||||
# test freeze on gpu
|
||||
@@ -1059,7 +1063,7 @@ def test_cpu_model_with_amp():
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
experiment=get_exp(),
|
||||
logger=get_test_tube_logger(),
|
||||
max_nb_epochs=1,
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.4,
|
||||
@@ -1080,7 +1084,7 @@ def test_cpu_model():
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
experiment=get_exp(),
|
||||
logger=get_test_tube_logger(),
|
||||
max_nb_epochs=1,
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.4
|
||||
@@ -1103,7 +1107,7 @@ def test_all_features_cpu_model():
|
||||
track_grad_norm=2,
|
||||
print_nan_grads=True,
|
||||
show_progress_bar=False,
|
||||
experiment=get_exp(),
|
||||
logger=get_test_tube_logger(),
|
||||
accumulate_grad_batches=2,
|
||||
max_nb_epochs=1,
|
||||
train_percent_check=0.4,
|
||||
@@ -1214,11 +1218,11 @@ def test_ddp_sampler_error():
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams, force_remove_distributed_sampler=True)
|
||||
|
||||
exp = get_exp(True)
|
||||
exp.save()
|
||||
logger = get_test_tube_logger(True)
|
||||
logger.save()
|
||||
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
logger=logger,
|
||||
show_progress_bar=False,
|
||||
max_nb_epochs=1,
|
||||
gpus=[0, 1],
|
||||
@@ -1245,7 +1249,7 @@ def test_multiple_val_dataloader():
|
||||
hparams = get_hparams()
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
# exp file to get meta
|
||||
# logger file to get meta
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
val_percent_check=0.1,
|
||||
@@ -1279,7 +1283,7 @@ def test_multiple_test_dataloader():
|
||||
hparams = get_hparams()
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
# exp file to get meta
|
||||
# logger file to get meta
|
||||
trainer_options = dict(
|
||||
max_nb_epochs=1,
|
||||
val_percent_check=0.1,
|
||||
@@ -1306,17 +1310,17 @@ def test_multiple_test_dataloader():
|
||||
def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True):
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
# logger file to get meta
|
||||
logger = get_test_tube_logger(False)
|
||||
logger.log_hyperparams(hparams)
|
||||
logger.save()
|
||||
|
||||
# exp file to get weights
|
||||
# logger file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
|
||||
# add these to the trainer options
|
||||
trainer_options['checkpoint_callback'] = checkpoint
|
||||
trainer_options['experiment'] = exp
|
||||
trainer_options['logger'] = logger
|
||||
|
||||
# fit model
|
||||
trainer = Trainer(**trainer_options)
|
||||
@@ -1326,7 +1330,7 @@ def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True):
|
||||
assert result == 1, 'amp + ddp model failed to complete'
|
||||
|
||||
# test model loading
|
||||
pretrained_model = load_model(exp, save_dir, on_gpu)
|
||||
pretrained_model = load_model(logger.experiment, save_dir, on_gpu)
|
||||
|
||||
# test new model accuracy
|
||||
run_prediction(model.test_dataloader, pretrained_model)
|
||||
@@ -1337,7 +1341,7 @@ def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True):
|
||||
trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers()
|
||||
|
||||
# test HPC loading / saving
|
||||
trainer.hpc_save(save_dir, exp)
|
||||
trainer.hpc_save(save_dir, logger)
|
||||
trainer.hpc_load(save_dir, on_gpu=on_gpu)
|
||||
|
||||
clear_save_dir()
|
||||
@@ -1376,12 +1380,12 @@ def get_model(use_test_model=False):
|
||||
return model, hparams
|
||||
|
||||
|
||||
def get_exp(debug=True, version=None):
|
||||
# set up exp object without actually saving logs
|
||||
def get_test_tube_logger(debug=True, version=None):
|
||||
# set up logger object without actually saving logs
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
save_dir = os.path.join(root_dir, 'save_dir')
|
||||
exp = Experiment(debug=debug, save_dir=save_dir, name='tests_tt_dir', version=version)
|
||||
return exp
|
||||
logger = TestTubeLogger(save_dir, name='test_tt_dir', debug=debug, version=version)
|
||||
return logger
|
||||
|
||||
|
||||
def init_save_dir():
|
||||
|
||||
Reference in New Issue
Block a user