diff --git a/docs/source/conf.py b/docs/source/conf.py index 4d181c67..9a994868 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -385,7 +385,7 @@ autodoc_default_options = { 'methods': None, # 'attributes': None, 'special-members': '__call__', - # 'exclude-members': '__weakref__', + 'exclude-members': '_abc_impl', 'show-inheritance': True, 'private-members': True, 'noindex': True, diff --git a/docs/source/experiment_logging.rst b/docs/source/experiment_logging.rst index 5aeb15d5..e9ddb472 100644 --- a/docs/source/experiment_logging.rst +++ b/docs/source/experiment_logging.rst @@ -1,201 +1,271 @@ Experiment Logging -=================== +================== Comet.ml ^^^^^^^^ `Comet.ml `_ is a third-party logger. -To use CometLogger as your logger do the following. +To use :class:`~pytorch_lightning.loggers.CometLogger` as your logger do the following. +First, install the package: + +.. code-block:: bash + + pip install comet-ml + +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: + +.. doctest:: + + >>> import os + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import CometLogger + >>> comet_logger = CometLogger( + ... api_key=os.environ.get('COMET_API_KEY'), + ... workspace=os.environ.get('COMET_WORKSPACE'), # Optional + ... save_dir='.', # Optional + ... project_name='default_project', # Optional + ... rest_api_key=os.environ.get('COMET_REST_API_KEY'), # Optional + ... experiment_name='default' # Optional + ... ) + >>> trainer = Trainer(logger=comet_logger) + +The :class:`~pytorch_lightning.loggers.CometLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. doctest:: + + >>> from pytorch_lightning import LightningModule + >>> class MyModule(LightningModule): + ... def any_lightning_module_function_or_hook(self): + ... some_img = fake_image() + ... self.logger.experiment.add_image('generated_images', some_img, 0) .. seealso:: :class:`~pytorch_lightning.loggers.CometLogger` docs. -.. code-block:: python +MLflow +^^^^^^ - from pytorch_lightning.loggers import CometLogger +`MLflow `_ is a third-party logger. +To use :class:`~pytorch_lightning.loggers.MLFlowLogger` as your logger do the following. +First, install the package: - comet_logger = CometLogger( - api_key=os.environ["COMET_KEY"], - workspace=os.environ["COMET_WORKSPACE"], # Optional - project_name="default_project", # Optional - rest_api_key=os.environ["COMET_REST_KEY"], # Optional - experiment_name="default" # Optional - ) - trainer = Trainer(logger=comet_logger) +.. code-block:: bash -The CometLogger is available anywhere except ``__init__`` in your LightningModule + pip install mlflow -.. code-block:: python +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: - class MyModule(pl.LightningModule): +.. doctest:: - def any_lightning_module_function_or_hook(self, ...): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import MLFlowLogger + >>> mlf_logger = MLFlowLogger( + ... experiment_name="default", + ... tracking_uri="file:/." + ... ) + >>> trainer = Trainer(logger=mlf_logger) + +.. seealso:: + :class:`~pytorch_lightning.loggers.MLFlowLogger` docs. Neptune.ai ^^^^^^^^^^ `Neptune.ai `_ is a third-party logger. -To use Neptune.ai as your logger do the following. +To use :class:`~pytorch_lightning.loggers.NeptuneLogger` as your logger do the following. +First, install the package: + +.. code-block:: bash + + pip install neptune-client + +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: + +.. doctest:: + + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import NeptuneLogger + >>> neptune_logger = NeptuneLogger( + ... api_key='ANONYMOUS', # replace with your own + ... project_name='shared/pytorch-lightning-integration', + ... experiment_name='default', # Optional, + ... params={'max_epochs': 10}, # Optional, + ... tags=['pytorch-lightning', 'mlp'], # Optional, + ... ) + >>> trainer = Trainer(logger=neptune_logger) + +The :class:`~pytorch_lightning.loggers.NeptuneLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. doctest:: + + >>> from pytorch_lightning import LightningModule + >>> class MyModule(LightningModule): + ... def any_lightning_module_function_or_hook(self): + ... some_img = fake_image() + ... self.logger.experiment.add_image('generated_images', some_img, 0) .. seealso:: :class:`~pytorch_lightning.loggers.NeptuneLogger` docs. -.. code-block:: python - - from pytorch_lightning.loggers import NeptuneLogger - - neptune_logger = NeptuneLogger( - project_name="USER_NAME/PROJECT_NAME", - experiment_name="default", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-lightning","mlp"] # Optional, - ) - trainer = Trainer(logger=neptune_logger) - -The Neptune.ai is available anywhere except ``__init__`` in your LightningModule - -.. code-block:: python - - class MyModule(pl.LightningModule): - - def any_lightning_module_function_or_hook(self, ...): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) - allegro.ai TRAINS ^^^^^^^^^^^^^^^^^ `allegro.ai `_ is a third-party logger. -To use TRAINS as your logger do the following. +To use :class:`~pytorch_lightning.loggers.TrainsLogger` as your logger do the following. +First, install the package: + +.. code-block:: bash + + pip install trains + +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: + +.. doctest:: + + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import TrainsLogger + >>> trains_logger = TrainsLogger( + ... project_name='examples', + ... task_name='pytorch lightning test', + ... ) # doctest: +ELLIPSIS + TRAINS Task: ... + TRAINS results page: ... + >>> trainer = Trainer(logger=trains_logger) + +The :class:`~pytorch_lightning.loggers.TrainsLogger` is available anywhere in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. doctest:: + + >>> from pytorch_lightning import LightningModule + >>> class MyModule(LightningModule): + ... def __init__(self): + ... some_img = fake_image() + ... self.logger.experiment.log_image('debug', 'generated_image_0', some_img, 0) .. seealso:: :class:`~pytorch_lightning.loggers.TrainsLogger` docs. -.. code-block:: python - - from pytorch_lightning.loggers import TrainsLogger - - trains_logger = TrainsLogger( - project_name="examples", - task_name="pytorch lightning test" - ) - trainer = Trainer(logger=trains_logger) - -The TrainsLogger is available anywhere in your LightningModule - -.. code-block:: python - - class MyModule(pl.LightningModule): - - def __init__(self, ...): - some_img = fake_image() - self.logger.log_image('debug', 'generated_image_0', some_img, 0) - Tensorboard ^^^^^^^^^^^ -To use `Tensorboard `_ as your logger do the following. +To use `TensorBoard `_ as your logger do the following. + +.. doctest:: + + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import TensorBoardLogger + >>> logger = TensorBoardLogger('tb_logs', name='my_model') + >>> trainer = Trainer(logger=logger) + +The :class:`~pytorch_lightning.loggers.TensorBoardLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. doctest:: + + >>> from pytorch_lightning import LightningModule + >>> class MyModule(LightningModule): + ... def any_lightning_module_function_or_hook(self): + ... some_img = fake_image() + ... self.logger.experiment.add_image('generated_images', some_img, 0) .. seealso:: :class:`~pytorch_lightning.loggers.TensorBoardLogger` docs. -.. code-block:: python - - from pytorch_lightning.loggers import TensorBoardLogger - - logger = TensorBoardLogger("tb_logs", name="my_model") - trainer = Trainer(logger=logger) - -The TensorBoardLogger is available anywhere except ``__init__`` in your LightningModule - -.. code-block:: python - - class MyModule(pl.LightningModule): - - def any_lightning_module_function_or_hook(self, ...): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) - - Test Tube ^^^^^^^^^ -`Test Tube `_ is a tensorboard logger but with nicer file structure. -To use TestTube as your logger do the following. +`Test Tube `_ is a +`TensorBoard `_ logger but with nicer file structure. +To use :class:`~pytorch_lightning.loggers.TestTubeLogger` as your logger do the following. +First, install the package: + +.. code-block:: bash + + pip install test_tube + +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: + +.. doctest:: + + >>> from pytorch_lightning.loggers import TestTubeLogger + >>> logger = TestTubeLogger('tb_logs', name='my_model') + >>> trainer = Trainer(logger=logger) + +The :class:`~pytorch_lightning.loggers.TestTubeLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. + +.. doctest:: + + >>> from pytorch_lightning import LightningModule + >>> class MyModule(LightningModule): + ... def any_lightning_module_function_or_hook(self): + ... some_img = fake_image() + ... self.logger.experiment.add_image('generated_images', some_img, 0) .. seealso:: :class:`~pytorch_lightning.loggers.TestTubeLogger` docs. -.. code-block:: python +Weights and Biases +^^^^^^^^^^^^^^^^^^ - from pytorch_lightning.loggers import TestTubeLogger +`Weights and Biases `_ is a third-party logger. +To use :class:`~pytorch_lightning.loggers.WandbLogger` as your logger do the following. +First, install the package: - logger = TestTubeLogger("tb_logs", name="my_model") - trainer = Trainer(logger=logger) +.. code-block:: bash -The TestTubeLogger is available anywhere except ``__init__`` in your LightningModule + pip install wandb -.. code-block:: python +Then configure the logger and pass it to the :class:`~pytorch_lightning.trainer.trainer.Trainer`: - class MyModule(pl.LightningModule): +.. doctest:: - def any_lightning_module_function_or_hook(self, ...): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) + >>> from pytorch_lightning.loggers import WandbLogger + >>> wandb_logger = WandbLogger() + >>> trainer = Trainer(logger=wandb_logger) -Wandb -^^^^^ +The :class:`~pytorch_lightning.loggers.WandbLogger` is available anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. -`Wandb `_ is a third-party logger. -To use Wandb as your logger do the following. +.. doctest:: + + >>> from pytorch_lightning import LightningModule + >>> class MyModule(LightningModule): + ... def any_lightning_module_function_or_hook(self): + ... some_img = fake_image() + ... self.logger.experiment.log({ + ... "generated_images": [wandb.Image(some_img, caption="...")] + ... }) .. seealso:: :class:`~pytorch_lightning.loggers.WandbLogger` docs. -.. code-block:: python - - from pytorch_lightning.loggers import WandbLogger - - wandb_logger = WandbLogger() - trainer = Trainer(logger=wandb_logger) - -The Wandb logger is available anywhere except ``__init__`` in your LightningModule - -.. code-block:: python - - class MyModule(pl.LightningModule): - - def any_lightning_module_function_or_hook(self, ...): - some_img = fake_image() - self.logger.experiment.add_image('generated_images', some_img, 0) - - Multiple Loggers ^^^^^^^^^^^^^^^^ -PyTorch-Lightning supports use of multiple loggers, just pass a list to the `Trainer`. +Lightning supports the use of multiple loggers, just pass a list to the +:class:`~pytorch_lightning.trainer.trainer.Trainer`. -.. code-block:: python +.. doctest:: - from pytorch_lightning.loggers import TensorBoardLogger, TestTubeLogger + >>> from pytorch_lightning.loggers import TensorBoardLogger, TestTubeLogger + >>> logger1 = TensorBoardLogger('tb_logs', name='my_model') + >>> logger2 = TestTubeLogger('tb_logs', name='my_model') + >>> trainer = Trainer(logger=[logger1, logger2]) - logger1 = TensorBoardLogger("tb_logs", name="my_model") - logger2 = TestTubeLogger("tt_logs", name="my_model") - trainer = Trainer(logger=[logger1, logger2]) - -The loggers are available as a list anywhere except ``__init__`` in your LightningModule +The loggers are available as a list anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule`. -.. code-block:: python +.. doctest:: - class MyModule(pl.LightningModule): - - def any_lightning_module_function_or_hook(self, ...): - some_img = fake_image() - - # Option 1 - self.logger.experiment[0].add_image('generated_images', some_img, 0) - - # Option 2 - self.logger[0].experiment.add_image('generated_images', some_img, 0) + >>> from pytorch_lightning import LightningModule + >>> class MyModule(LightningModule): + ... def any_lightning_module_function_or_hook(self): + ... some_img = fake_image() + ... # Option 1 + ... self.logger.experiment[0].add_image('generated_images', some_img, 0) + ... # Option 2 + ... self.logger[0].experiment.add_image('generated_images', some_img, 0) diff --git a/pytorch_lightning/loggers/__init__.py b/pytorch_lightning/loggers/__init__.py index 0c1c62dc..edb3ffec 100644 --- a/pytorch_lightning/loggers/__init__.py +++ b/pytorch_lightning/loggers/__init__.py @@ -1,62 +1,59 @@ """ -Lightning supports most popular logging frameworks (Tensorboard, comet, weights and biases, etc...). -To use a logger, simply pass it into the trainer. To use multiple loggers, simply pass in a ``list`` -or ``tuple`` of loggers. +Lightning supports the most popular logging frameworks (TensorBoard, Comet, Weights and Biases, etc...). +To use a logger, simply pass it into the :class:`~pytorch_lightning.trainer.trainer.Trainer`. +Lightning uses TensorBoard by default. -.. code-block:: python +>>> from pytorch_lightning import Trainer +>>> from pytorch_lightning import loggers +>>> tb_logger = loggers.TensorBoardLogger('logs/') +>>> trainer = Trainer(logger=tb_logger) - from pytorch_lightning import loggers +Choose from any of the others such as MLflow, Comet, Neptune, WandB, ... - # lightning uses tensorboard by default - tb_logger = loggers.TensorBoardLogger() - trainer = Trainer(logger=tb_logger) +>>> comet_logger = loggers.CometLogger(save_dir='logs/') +>>> trainer = Trainer(logger=comet_logger) - # or choose from any of the others such as MLFlow, Comet, Neptune, Wandb - comet_logger = loggers.CometLogger() - trainer = Trainer(logger=comet_logger) +To use multiple loggers, simply pass in a ``list`` or ``tuple`` of loggers ... - # or pass a list - tb_logger = loggers.TensorBoardLogger() - comet_logger = loggers.CometLogger() - trainer = Trainer(logger=[tb_logger, comet_logger]) +>>> tb_logger = loggers.TensorBoardLogger('logs/') +>>> comet_logger = loggers.CometLogger(save_dir='logs/') +>>> trainer = Trainer(logger=[tb_logger, comet_logger]) -.. note:: All loggers log by default to ``os.getcwd()``. To change the path without creating a logger set +Note: + All loggers log by default to ``os.getcwd()``. To change the path without creating a logger set ``Trainer(default_root_dir='/your/path/to/save/checkpoints')`` -Custom logger +Custom Logger ------------- You can implement your own logger by writing a class that inherits from -``LightningLoggerBase``. Use the ``rank_zero_only`` decorator to make sure that -only the first process in DDP training logs data. - -.. code-block:: python - - from pytorch_lightning.loggers import LightningLoggerBase, rank_zero_only - - class MyLogger(LightningLoggerBase): - - @rank_zero_only - def log_hyperparams(self, params): - # params is an argparse.Namespace - # your code to record hyperparameters goes here - pass - - @rank_zero_only - def log_metrics(self, metrics, step): - # metrics is a dictionary of metric names and values - # your code to record metrics goes here - pass - - def save(self): - # Optional. Any code necessary to save logger data goes here - pass - - @rank_zero_only - def finalize(self, status): - # Optional. Any code that needs to be run after training - # finishes goes here +:class:`LightningLoggerBase`. Use the :func:`~pytorch_lightning.loggers.base.rank_zero_only` +decorator to make sure that only the first process in DDP training logs data. +>>> from pytorch_lightning.loggers import LightningLoggerBase, rank_zero_only +>>> class MyLogger(LightningLoggerBase): +... +... @rank_zero_only +... def log_hyperparams(self, params): +... # params is an argparse.Namespace +... # your code to record hyperparameters goes here +... pass +... +... @rank_zero_only +... def log_metrics(self, metrics, step): +... # metrics is a dictionary of metric names and values +... # your code to record metrics goes here +... pass +... +... def save(self): +... # Optional. Any code necessary to save logger data goes here +... pass +... +... @rank_zero_only +... def finalize(self, status): +... # Optional. Any code that needs to be run after training +... # finishes goes here +... pass If you write a logger that may be useful to others, please send a pull request to add it to Lighting! @@ -64,16 +61,17 @@ a pull request to add it to Lighting! Using loggers ------------- -Call the logger anywhere except ``__init__`` in your LightningModule by doing: +Call the logger anywhere except ``__init__`` in your +:class:`~pytorch_lightning.core.lightning.LightningModule` by doing: -.. code-block:: python - - def train_step(...): - # example - self.logger.experiment.whatever_method_summary_writer_supports(...) - - def any_lightning_module_function_or_hook(...): - self.logger.experiment.add_histogram(...) +>>> from pytorch_lightning import LightningModule +>>> class LitModel(LightningModule): +... def training_step(self, batch, batch_idx): +... # example +... self.logger.experiment.whatever_method_summary_writer_supports(...) +... +... def any_lightning_module_function_or_hook(self): +... self.logger.experiment.add_histogram(...) Read more in the `Experiment Logging use case <./experiment_logging.html>`_. @@ -85,7 +83,11 @@ from os import environ from pytorch_lightning.loggers.base import LightningLoggerBase, LoggerCollection, rank_zero_only from pytorch_lightning.loggers.tensorboard import TensorBoardLogger -__all__ = ['TensorBoardLogger'] +__all__ = [ + 'LightningLoggerBase', + 'LoggerCollection', + 'TensorBoardLogger', +] try: # needed to prevent ImportError and duplicated logs. diff --git a/pytorch_lightning/loggers/base.py b/pytorch_lightning/loggers/base.py index be93c2ba..260b5f40 100644 --- a/pytorch_lightning/loggers/base.py +++ b/pytorch_lightning/loggers/base.py @@ -26,27 +26,28 @@ def rank_zero_only(fn: Callable): class LightningLoggerBase(ABC): - """Base class for experiment loggers.""" + """ + Base class for experiment loggers. + + Args: + agg_key_funcs: + Dictionary which maps a metric name to a function, which will + aggregate the metric values for the same steps. + agg_default_func: + Default function to aggregate metric values. If some metric name + is not presented in the `agg_key_funcs` dictionary, then the + `agg_default_func` will be used for aggregation. + + Note: + The `agg_key_funcs` and `agg_default_func` arguments are used only when + one logs metrics with the :meth:`~LightningLoggerBase.agg_and_log_metrics` method. + """ def __init__( self, agg_key_funcs: Optional[Mapping[str, Callable[[Sequence[float]], float]]] = None, agg_default_func: Callable[[Sequence[float]], float] = np.mean ): - """ - Args: - agg_key_funcs: - Dictionary which maps a metric name to a function, which will - aggregate the metric values for the same steps. - agg_default_func: - Default function to aggregate metric values. If some metric name - is not presented in the `agg_key_funcs` dictionary, then the - `agg_default_func` will be used for aggregation. - - Notes: - `agg_key_funcs` and `agg_default_func` are used only when one logs metrics with - `LightningLoggerBase.agg_and_log_metrics` method. - """ self._rank = 0 self._prev_step: int = -1 self._metrics_to_agg: List[Dict[str, float]] = [] @@ -58,7 +59,8 @@ class LightningLoggerBase(ABC): agg_key_funcs: Optional[Mapping[str, Callable[[Sequence[float]], float]]] = None, agg_default_func: Callable[[Sequence[float]], float] = np.mean ): - """Update aggregation methods. + """ + Update aggregation methods. Args: agg_key_funcs: @@ -77,19 +79,20 @@ class LightningLoggerBase(ABC): @property @abstractmethod def experiment(self) -> Any: - """Return the experiment object associated with this logger""" + """Return the experiment object associated with this logger.""" def _aggregate_metrics( self, metrics: Dict[str, float], step: Optional[int] = None ) -> Tuple[int, Optional[Dict[str, float]]]: - """Aggregates metrics. + """ + Aggregates metrics. Args: metrics: Dictionary with metric names as keys and measured quantities as values step: Step number at which the metrics should be recorded Returns: - sStep and aggregated metrics. The return value could be None. In such case, metrics + Step and aggregated metrics. The return value could be ``None``. In such case, metrics are added to the aggregation list, but not aggregated yet. """ # if you still receiving metric from the same step, just accumulate it @@ -125,7 +128,8 @@ class LightningLoggerBase(ABC): self.log_metrics(metrics=metrics_to_log, step=agg_step) def agg_and_log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None): - """Aggregates and records metrics. + """ + Aggregates and records metrics. This method doesn't log the passed metrics instantaneously, but instead it aggregates them and logs only if metrics are ready to be logged. @@ -140,9 +144,11 @@ class LightningLoggerBase(ABC): @abstractmethod def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None): - """Records metrics. + """ + Records metrics. This method logs metrics as as soon as it received them. If you want to aggregate - metrics for one specific `step`, use the `agg_and_log_metrics` method. + metrics for one specific `step`, use the + :meth:`~pytorch_lightning.loggers.base.LightningLoggerBase.agg_and_log_metrics` method. Args: metrics: Dictionary with metric names as keys and measured quantities as values @@ -163,14 +169,15 @@ class LightningLoggerBase(ABC): @staticmethod def _flatten_dict(params: Dict[str, Any], delimiter: str = '/') -> Dict[str, Any]: - """Flatten hierarchical dict e.g. {'a': {'b': 'c'}} -> {'a/b': 'c'}. + """ + Flatten hierarchical dict, e.g. ``{'a': {'b': 'c'}} -> {'a/b': 'c'}``. Args: - params: Dictionary contains hparams - delimiter: Delimiter to express the hierarchy. Defaults to '/'. + params: Dictionary containing the hyperparameters + delimiter: Delimiter to express the hierarchy. Defaults to ``'/'``. Returns: - Flatten dict. + Flattened dict. Examples: >>> LightningLoggerBase._flatten_dict({'a': {'b': 'c'}}) @@ -196,7 +203,8 @@ class LightningLoggerBase(ABC): @staticmethod def _sanitize_params(params: Dict[str, Any]) -> Dict[str, Any]: - """Returns params with non-primitvies converted to strings for logging + """ + Returns params with non-primitvies converted to strings for logging. >>> params = {"float": 0.3, ... "int": 1, @@ -219,10 +227,11 @@ class LightningLoggerBase(ABC): @abstractmethod def log_hyperparams(self, params: argparse.Namespace): - """Record hyperparameters. + """ + Record hyperparameters. Args: - params: argparse.Namespace containing the hyperparameters + params: :class:`~argparse.Namespace` containing the hyperparameters """ def save(self) -> None: @@ -230,7 +239,8 @@ class LightningLoggerBase(ABC): self._finalize_agg_metrics() def finalize(self, status: str) -> None: - """Do any processing that is necessary to finalize an experiment. + """ + Do any processing that is necessary to finalize an experiment. Args: status: Status that the experiment finished with (e.g. success, failed, aborted) @@ -263,12 +273,13 @@ class LightningLoggerBase(ABC): class LoggerCollection(LightningLoggerBase): - """The `LoggerCollection` class is used to iterate all logging actions over the given `logger_iterable`. + """ + The :class:`LoggerCollection` class is used to iterate all logging actions over + the given `logger_iterable`. Args: logger_iterable: An iterable collection of loggers """ - def __init__(self, logger_iterable: Iterable[LightningLoggerBase]): super().__init__() self._logger_iterable = logger_iterable @@ -314,7 +325,8 @@ def merge_dicts( agg_key_funcs: Optional[Mapping[str, Callable[[Sequence[float]], float]]] = None, default_func: Callable[[Sequence[float]], float] = np.mean ) -> Dict: - """Merge a sequence with dictionaries into one dictionary by aggregating the + """ + Merge a sequence with dictionaries into one dictionary by aggregating the same keys with some given function. Args: @@ -324,7 +336,7 @@ def merge_dicts( Mapping from key name to function. This function will aggregate a list of values, obtained from the same key of all dictionaries. If some key has no specified aggregation function, the default one - will be used. Default is: None (all keys will be aggregated by the + will be used. Default is: ``None`` (all keys will be aggregated by the default function). default_func: Default function to aggregate keys, which are not presented in the diff --git a/pytorch_lightning/loggers/comet.py b/pytorch_lightning/loggers/comet.py index 82ef4c92..8cd5dd22 100644 --- a/pytorch_lightning/loggers/comet.py +++ b/pytorch_lightning/loggers/comet.py @@ -1,9 +1,6 @@ -r""" - -.. _comet: - -CometLogger -------------- +""" +Comet +----- """ from argparse import Namespace @@ -33,7 +30,56 @@ from pytorch_lightning.utilities.exceptions import MisconfigurationException class CometLogger(LightningLoggerBase): r""" - Log using `comet.ml `_. + Log using `Comet.ml `_. Install it with pip: + + .. code-block:: bash + + pip install comet-ml + + Comet requires either an API Key (online mode) or a local directory path (offline mode). + + **ONLINE MODE** + + Example: + >>> import os + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import CometLogger + >>> # arguments made to CometLogger are passed on to the comet_ml.Experiment class + >>> comet_logger = CometLogger( + ... api_key=os.environ.get('COMET_API_KEY'), + ... workspace=os.environ.get('COMET_WORKSPACE'), # Optional + ... save_dir='.', # Optional + ... project_name='default_project', # Optional + ... rest_api_key=os.environ.get('COMET_REST_API_KEY'), # Optional + ... experiment_name='default' # Optional + ... ) + >>> trainer = Trainer(logger=comet_logger) + + **OFFLINE MODE** + + Example: + >>> from pytorch_lightning.loggers import CometLogger + >>> # arguments made to CometLogger are passed on to the comet_ml.Experiment class + >>> comet_logger = CometLogger( + ... save_dir='.', + ... workspace=os.environ.get('COMET_WORKSPACE'), # Optional + ... project_name='default_project', # Optional + ... rest_api_key=os.environ.get('COMET_REST_API_KEY'), # Optional + ... experiment_name='default' # Optional + ... ) + >>> trainer = Trainer(logger=comet_logger) + + Args: + api_key: Required in online mode. API key, found on Comet.ml + save_dir: Required in offline mode. The path for the directory to save local comet logs + workspace: Optional. Name of workspace for this user + project_name: Optional. Send your experiment to a specific project. + Otherwise will be sent to Uncategorized Experiments. + If the project name does not already exist, Comet.ml will create a new project. + rest_api_key: Optional. Rest API key found in Comet.ml settings. + This is used to determine version number + experiment_name: Optional. String representing the name for this particular experiment on Comet.ml. + experiment_key: Optional. If set, restores from existing experiment. """ def __init__(self, @@ -45,50 +91,7 @@ class CometLogger(LightningLoggerBase): experiment_name: Optional[str] = None, experiment_key: Optional[str] = None, **kwargs): - r""" - Requires either an API Key (online mode) or a local directory path (offline mode) - - .. code-block:: python - - # ONLINE MODE - from pytorch_lightning.loggers import CometLogger - # arguments made to CometLogger are passed on to the comet_ml.Experiment class - comet_logger = CometLogger( - api_key=os.environ["COMET_API_KEY"], - workspace=os.environ["COMET_WORKSPACE"], # Optional - project_name="default_project", # Optional - rest_api_key=os.environ["COMET_REST_API_KEY"], # Optional - experiment_name="default" # Optional - ) - trainer = Trainer(logger=comet_logger) - - .. code-block:: python - - # OFFLINE MODE - from pytorch_lightning.loggers import CometLogger - # arguments made to CometLogger are passed on to the comet_ml.Experiment class - comet_logger = CometLogger( - save_dir=".", - workspace=os.environ["COMET_WORKSPACE"], # Optional - project_name="default_project", # Optional - rest_api_key=os.environ["COMET_REST_API_KEY"], # Optional - experiment_name="default" # Optional - ) - trainer = Trainer(logger=comet_logger) - - Args: - api_key (str): Required in online mode. API key, found on Comet.ml - save_dir (str): Required in offline mode. The path for the directory to save local comet logs - workspace (str): Optional. Name of workspace for this user - project_name (str): Optional. Send your experiment to a specific project. - Otherwise will be sent to Uncategorized Experiments. - If project name does not already exists Comet.ml will create a new project. - rest_api_key (str): Optional. Rest API key found in Comet.ml settings. - This is used to determine version number - experiment_name (str): Optional. String representing the name for this particular experiment on Comet.ml. - experiment_key (str): Optional. If set, restores from existing experiment. - """ super().__init__() self._experiment = None @@ -128,8 +131,8 @@ class CometLogger(LightningLoggerBase): @property def experiment(self) -> CometBaseExperiment: r""" - - Actual comet object. To use comet features do the following. + Actual Comet object. To use Comet features in your + :class:`~pytorch_lightning.core.lightning.LightningModule` do the following. Example:: @@ -191,12 +194,13 @@ class CometLogger(LightningLoggerBase): @rank_zero_only def finalize(self, status: str) -> None: r""" - When calling self.experiment.end(), that experiment won't log any more data to Comet. That's why, if you need - to log any more data you need to create an ExistingCometExperiment. For example, to log data when testing your - model after training, because when training is finalized CometLogger.finalize is called. + When calling ``self.experiment.end()``, that experiment won't log any more data to Comet. + That's why, if you need to log any more data, you need to create an ExistingCometExperiment. + For example, to log data when testing your model after training, because when training is + finalized :meth:`CometLogger.finalize` is called. - This happens automatically in the CometLogger.experiment property, when self._experiment is set to None - i.e. self.reset_experiment(). + This happens automatically in the :meth:`~CometLogger.experiment` property, when + ``self._experiment`` is set to ``None``, i.e. ``self.reset_experiment()``. """ self.experiment.end() self.reset_experiment() diff --git a/pytorch_lightning/loggers/mlflow.py b/pytorch_lightning/loggers/mlflow.py index 7472e298..6f672c6a 100644 --- a/pytorch_lightning/loggers/mlflow.py +++ b/pytorch_lightning/loggers/mlflow.py @@ -1,27 +1,6 @@ """ -Log using `mlflow `_ - -.. code-block:: python - - from pytorch_lightning.loggers import MLFlowLogger - mlf_logger = MLFlowLogger( - experiment_name="default", - tracking_uri="file:/." - ) - trainer = Trainer(logger=mlf_logger) - - -Use the logger anywhere in you LightningModule as follows: - -.. code-block:: python - - def train_step(...): - # example - self.logger.experiment.whatever_ml_flow_supports(...) - - def any_lightning_module_function_or_hook(...): - self.logger.experiment.whatever_ml_flow_supports(...) - +MLflow +------ """ import os from argparse import Namespace @@ -40,21 +19,45 @@ from pytorch_lightning.loggers.base import LightningLoggerBase, rank_zero_only class MLFlowLogger(LightningLoggerBase): - """MLFLow logger""" + """ + Log using `MLflow `_. Install it with pip: + .. code-block:: bash + + pip install mlflow + + Example: + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import MLFlowLogger + >>> mlf_logger = MLFlowLogger( + ... experiment_name="default", + ... tracking_uri="file:./ml-runs" + ... ) + >>> trainer = Trainer(logger=mlf_logger) + + Use the logger anywhere in you :class:`~pytorch_lightning.core.lightning.LightningModule` as follows: + + >>> from pytorch_lightning import LightningModule + >>> class LitModel(LightningModule): + ... def training_step(self, batch, batch_idx): + ... # example + ... self.logger.experiment.whatever_ml_flow_supports(...) + ... + ... def any_lightning_module_function_or_hook(self): + ... self.logger.experiment.whatever_ml_flow_supports(...) + + Args: + experiment_name: The name of the experiment + tracking_uri: Address of local or remote tracking server. + If not provided, defaults to the service set by ``mlflow.tracking.set_tracking_uri``. + tags: A dictionary tags for the experiment. + + """ def __init__(self, experiment_name: str = 'default', tracking_uri: Optional[str] = None, tags: Optional[Dict[str, Any]] = None, save_dir: Optional[str] = None): - r""" - Logs using MLFlow - - Args: - experiment_name (str): The name of the experiment - tracking_uri (str): where this should track - tags (dict): todo this param - """ super().__init__() if not tracking_uri and save_dir: tracking_uri = f'file:{os.sep * 2}{save_dir}' @@ -66,7 +69,8 @@ class MLFlowLogger(LightningLoggerBase): @property def experiment(self) -> MlflowClient: r""" - Actual mlflow object. To use mlflow features do the following. + Actual MLflow object. To use mlflow features in your + :class:`~pytorch_lightning.core.lightning.LightningModule` do the following. Example:: diff --git a/pytorch_lightning/loggers/neptune.py b/pytorch_lightning/loggers/neptune.py index d71f44ce..e4a41757 100644 --- a/pytorch_lightning/loggers/neptune.py +++ b/pytorch_lightning/loggers/neptune.py @@ -1,14 +1,12 @@ """ -Log using `neptune-logger `_ - -.. _neptune: - -NeptuneLogger --------------- +Neptune +------- """ from argparse import Namespace from typing import Optional, List, Dict, Any, Union, Iterable +from PIL.Image import Image + try: import neptune from neptune.experiments import Experiment @@ -25,10 +23,149 @@ from pytorch_lightning.loggers.base import LightningLoggerBase, rank_zero_only class NeptuneLogger(LightningLoggerBase): r""" - Neptune logger can be used in the online mode or offline (silent) mode. - To log experiment data in online mode, NeptuneLogger requries an API key: - """ + Log using `Neptune `_. Install it with pip: + .. code-block:: bash + + pip install neptune-client + + The Neptune logger can be used in the online mode or offline (silent) mode. + To log experiment data in online mode, :class:`NeptuneLogger` requries an API key. + In offline mode, Neptune will log to a local directory. + + **ONLINE MODE** + + Example: + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import NeptuneLogger + >>> # arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class + >>> # We are using an api_key for the anonymous user "neptuner" but you can use your own. + >>> neptune_logger = NeptuneLogger( + ... api_key='ANONYMOUS', + ... project_name='shared/pytorch-lightning-integration', + ... experiment_name='default', # Optional, + ... params={'max_epochs': 10}, # Optional, + ... tags=['pytorch-lightning', 'mlp'] # Optional, + ... ) + >>> trainer = Trainer(max_epochs=10, logger=neptune_logger) + + **OFFLINE MODE** + + Example: + >>> from pytorch_lightning.loggers import NeptuneLogger + >>> # arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class + >>> neptune_logger = NeptuneLogger( + ... offline_mode=True, + ... project_name='USER_NAME/PROJECT_NAME', + ... experiment_name='default', # Optional, + ... params={'max_epochs': 10}, # Optional, + ... tags=['pytorch-lightning', 'mlp'] # Optional, + ... ) + >>> trainer = Trainer(max_epochs=10, logger=neptune_logger) + + Use the logger anywhere in you :class:`~pytorch_lightning.core.lightning.LightningModule` as follows: + + >>> from pytorch_lightning import LightningModule + >>> class LitModel(LightningModule): + ... def training_step(self, batch, batch_idx): + ... # log metrics + ... self.logger.experiment.log_metric('acc_train', ...) + ... # log images + ... self.logger.experiment.log_image('worse_predictions', ...) + ... # log model checkpoint + ... self.logger.experiment.log_artifact('model_checkpoint.pt', ...) + ... self.logger.experiment.whatever_neptune_supports(...) + ... + ... def any_lightning_module_function_or_hook(self): + ... self.logger.experiment.log_metric('acc_train', ...) + ... self.logger.experiment.log_image('worse_predictions', ...) + ... self.logger.experiment.log_artifact('model_checkpoint.pt', ...) + ... self.logger.experiment.whatever_neptune_supports(...) + + If you want to log objects after the training is finished use ``close_after_train=False``: + + .. code-block:: python + + neptune_logger = NeptuneLogger( + ... + close_after_fit=False, + ... + ) + trainer = Trainer(logger=neptune_logger) + trainer.fit() + + # Log test metrics + trainer.test(model) + + # Log additional metrics + from sklearn.metrics import accuracy_score + + accuracy = accuracy_score(y_true, y_pred) + neptune_logger.experiment.log_metric('test_accuracy', accuracy) + + # Log charts + from scikitplot.metrics import plot_confusion_matrix + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(16, 12)) + plot_confusion_matrix(y_true, y_pred, ax=ax) + neptune_logger.experiment.log_image('confusion_matrix', fig) + + # Save checkpoints folder + neptune_logger.experiment.log_artifact('my/checkpoints') + + # When you are done, stop the experiment + neptune_logger.experiment.stop() + + See Also: + - An `Example experiment `_ showing the UI of Neptune. + - `Tutorial `_ on how to use + Pytorch Lightning with Neptune. + + Args: + api_key: Required in online mode. + Neptune API token, found on https://neptune.ai. + Read how to get your + `API key `_. + It is recommended to keep it in the `NEPTUNE_API_TOKEN` + environment variable and then you can leave ``api_key=None``. + project_name: Required in online mode. Qualified name of a project in a form of + "namespace/project_name" for example "tom/minst-classification". + If ``None``, the value of `NEPTUNE_PROJECT` environment variable will be taken. + You need to create the project in https://neptune.ai first. + offline_mode: Optional default False. If ``True`` no logs will be sent + to Neptune. Usually used for debug purposes. + close_after_fit: Optional default ``True``. If ``False`` the experiment + will not be closed after training and additional metrics, + images or artifacts can be logged. Also, remember to close the experiment explicitly + by running ``neptune_logger.experiment.stop()``. + experiment_name: Optional. Editable name of the experiment. + Name is displayed in the experiment’s Details (Metadata section) and + in experiments view as a column. + upload_source_files: Optional. List of source files to be uploaded. + Must be list of str or single str. Uploaded sources are displayed + in the experiment’s Source code tab. + If ``None`` is passed, the Python file from which the experiment was created will be uploaded. + Pass an empty list (``[]``) to upload no files. + Unix style pathname pattern expansion is supported. + For example, you can pass ``'\*.py'`` + to upload all python source files from the current directory. + For recursion lookup use ``'\**/\*.py'`` (for Python 3.5 and later). + For more information see :mod:`glob` library. + params: Optional. Parameters of the experiment. + After experiment creation params are read-only. + Parameters are displayed in the experiment’s Parameters section and + each key-value pair can be viewed in the experiments view as a column. + properties: Optional. Default is ``{}``. Properties of the experiment. + They are editable after the experiment is created. + Properties are displayed in the experiment’s Details section and + each key-value pair can be viewed in the experiments view as a column. + tags: Optional. Default is ``[]``. Must be list of str. Tags of the experiment. + They are editable after the experiment is created (see: ``append_tag()`` and ``remove_tag()``). + Tags are displayed in the experiment’s Details section and can be viewed + in the experiments view as a column. + """ def __init__(self, api_key: Optional[str] = None, project_name: Optional[str] = None, @@ -40,138 +177,6 @@ class NeptuneLogger(LightningLoggerBase): properties: Optional[Dict[str, Any]] = None, tags: Optional[List[str]] = None, **kwargs): - r""" - Initialize a neptune.ai logger. - - .. note:: Requires either an API Key (online mode) or a local directory path (offline mode) - - .. code-block:: python - - # ONLINE MODE - from pytorch_lightning.loggers import NeptuneLogger - # arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class - # We are using an api_key for the anonymous user "neptuner" but you can use your own. - - neptune_logger = NeptuneLogger( - api_key="ANONYMOUS" - project_name="shared/pytorch-lightning-integration", - experiment_name="default", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-lightning","mlp"] # Optional, - ) - trainer = Trainer(max_epochs=10, logger=neptune_logger) - - .. code-block:: python - - # OFFLINE MODE - from pytorch_lightning.loggers import NeptuneLogger - # arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class - - neptune_logger = NeptuneLogger( - project_name="USER_NAME/PROJECT_NAME", - experiment_name="default", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-lightning","mlp"] # Optional, - ) - trainer = Trainer(max_epochs=10, logger=neptune_logger) - - Use the logger anywhere in you LightningModule as follows: - - .. code-block:: python - - def train_step(...): - # example - self.logger.experiment.log_metric("acc_train", acc_train) # log metrics - self.logger.experiment.log_image("worse_predictions", prediction_image) # log images - self.logger.experiment.log_artifact("model_checkpoint.pt", prediction_image) # log model checkpoint - self.logger.experiment.whatever_neptune_supports(...) - - def any_lightning_module_function_or_hook(...): - self.logger.experiment.log_metric("acc_train", acc_train) # log metrics - self.logger.experiment.log_image("worse_predictions", prediction_image) # log images - self.logger.experiment.log_artifact("model_checkpoint.pt", prediction_image) # log model checkpoint - self.logger.experiment.whatever_neptune_supports(...) - - If you want to log objects after the training is finished use close_after_train=False: - - .. code-block:: python - - neptune_logger = NeptuneLogger( - ... - close_after_fit=False, - ...) - trainer = Trainer(logger=neptune_logger) - trainer.fit() - - # Log test metrics - trainer.test(model) - - # Log additional metrics - from sklearn.metrics import accuracy_score - - accuracy = accuracy_score(y_true, y_pred) - neptune_logger.experiment.log_metric('test_accuracy', accuracy) - - # Log charts - from scikitplot.metrics import plot_confusion_matrix - import matplotlib.pyplot as plt - - fig, ax = plt.subplots(figsize=(16, 12)) - plot_confusion_matrix(y_true, y_pred, ax=ax) - neptune_logger.experiment.log_image('confusion_matrix', fig) - - # Save checkpoints folder - neptune_logger.experiment.log_artifact('my/checkpoints') - - # When you are done, stop the experiment - neptune_logger.experiment.stop() - - You can go and see an example experiment here: - https://ui.neptune.ai/o/shared/org/pytorch-lightning-integration/e/PYTOR-66/charts - - Args: - api_key: Required in online mode. - Neputne API token, found on https://neptune.ai - Read how to get your API key - https://docs.neptune.ai/python-api/tutorials/get-started.html#copy-api-token. - It is recommended to keep it in the `NEPTUNE_API_TOKEN` - environment variable and then you can leave `api_key=None` - project_name: Required in online mode. Qualified name of a project in a form of - "namespace/project_name" for example "tom/minst-classification". - If None, the value of NEPTUNE_PROJECT environment variable will be taken. - You need to create the project in https://neptune.ai first. - offline_mode: Optional default False. If offline_mode=True no logs will be send - to neptune. Usually used for debug and test purposes. - close_after_fit: Optional default True. If close_after_fit=False the experiment - will not be closed after training and additional metrics, - images or artifacts can be logged. Also, remember to close the experiment explicitly - by running neptune_logger.experiment.stop(). - experiment_name: Optional. Editable name of the experiment. - Name is displayed in the experiment’s Details (Metadata section) and - in experiments view as a column. - upload_source_files: Optional. List of source files to be uploaded. - Must be list of str or single str. Uploaded sources are displayed - in the experiment’s Source code tab. - If None is passed, Python file from which experiment was created will be uploaded. - Pass empty list ([]) to upload no files. - Unix style pathname pattern expansion is supported. - For example, you can pass '\*.py' - to upload all python source files from the current directory. - For recursion lookup use '\**/\*.py' (for Python 3.5 and later). - For more information see glob library. - params: Optional. Parameters of the experiment. - After experiment creation params are read-only. - Parameters are displayed in the experiment’s Parameters section and - each key-value pair can be viewed in experiments view as a column. - properties: Optional default is {}. Properties of the experiment. - They are editable after experiment is created. - Properties are displayed in the experiment’s Details and - each key-value pair can be viewed in experiments view as a column. - tags: Optional default []. Must be list of str. Tags of the experiment. - They are editable after experiment is created (see: append_tag() and remove_tag()). - Tags are displayed in the experiment’s Details and can be viewed - in experiments view as a column. - """ super().__init__() self.api_key = api_key self.project_name = project_name @@ -205,8 +210,8 @@ class NeptuneLogger(LightningLoggerBase): @property def experiment(self) -> Experiment: r""" - - Actual neptune object. To use neptune features do the following. + Actual Neptune object. To use neptune features in your + :class:`~pytorch_lightning.core.lightning.LightningModule` do the following. Example:: @@ -237,7 +242,8 @@ class NeptuneLogger(LightningLoggerBase): metrics: Dict[str, Union[torch.Tensor, float]], step: Optional[int] = None ) -> None: - """Log metrics (numeric values) in Neptune experiments + """ + Log metrics (numeric values) in Neptune experiments. Args: metrics: Dictionary with metric names as keys and measured quantities as values @@ -273,10 +279,11 @@ class NeptuneLogger(LightningLoggerBase): metric_value: Union[torch.Tensor, float, str], step: Optional[int] = None ) -> None: - """Log metrics (numeric values) in Neptune experiments + """ + Log metrics (numeric values) in Neptune experiments. Args: - metric_name: The name of log, i.e. mse, loss, accuracy. + metric_name: The name of log, i.e. mse, loss, accuracy. metric_value: The value of the log (data-point). step: Step number at which the metrics should be recorded, must be strictly increasing """ @@ -290,23 +297,29 @@ class NeptuneLogger(LightningLoggerBase): @rank_zero_only def log_text(self, log_name: str, text: str, step: Optional[int] = None) -> None: - """Log text data in Neptune experiment + """ + Log text data in Neptune experiments. Args: - log_name: The name of log, i.e. mse, my_text_data, timing_info. + log_name: The name of log, i.e. mse, my_text_data, timing_info. text: The value of the log (data-point). step: Step number at which the metrics should be recorded, must be strictly increasing """ self.log_metric(log_name, text, step=step) @rank_zero_only - def log_image(self, log_name: str, image: Union[str, Any], step: Optional[int] = None) -> None: - """Log image data in Neptune experiment + def log_image(self, + log_name: str, + image: Union[str, Image, Any], + step: Optional[int] = None) -> None: + """ + Log image data in Neptune experiment Args: log_name: The name of log, i.e. bboxes, visualisations, sample_images. - image (str|PIL.Image|matplotlib.figure.Figure): The value of the log (data-point). - Can be one of the following types: PIL image, matplotlib.figure.Figure, path to image file (str) + image: The value of the log (data-point). + Can be one of the following types: PIL image, `matplotlib.figure.Figure`, + path to image file (str) step: Step number at which the metrics should be recorded, must be strictly increasing """ if step is None: @@ -320,14 +333,15 @@ class NeptuneLogger(LightningLoggerBase): Args: artifact: A path to the file in local filesystem. - destination: Optional default None. A destination path. - If None is passed, an artifact file name will be used. + destination: Optional. Default is ``None``. A destination path. + If ``None`` is passed, an artifact file name will be used. """ self.experiment.log_artifact(artifact, destination) @rank_zero_only def set_property(self, key: str, value: Any) -> None: - """Set key-value pair as Neptune experiment property. + """ + Set key-value pair as Neptune experiment property. Args: key: Property key. @@ -337,10 +351,11 @@ class NeptuneLogger(LightningLoggerBase): @rank_zero_only def append_tags(self, tags: Union[str, Iterable[str]]) -> None: - """appends tags to neptune experiment + """ + Appends tags to the neptune experiment. Args: - tags: Tags to add to the current experiment. If str is passed, singe tag is added. + tags: Tags to add to the current experiment. If str is passed, a single tag is added. If multiple - comma separated - str are passed, all of them are added as tags. If list of str is passed, all elements of the list are added as tags. """ diff --git a/pytorch_lightning/loggers/tensorboard.py b/pytorch_lightning/loggers/tensorboard.py index 187ab882..58e26f09 100644 --- a/pytorch_lightning/loggers/tensorboard.py +++ b/pytorch_lightning/loggers/tensorboard.py @@ -1,3 +1,8 @@ +""" +TensorBoard +----------- +""" + import csv import os from argparse import Namespace @@ -14,26 +19,25 @@ from pytorch_lightning import _logger as log class TensorBoardLogger(LightningLoggerBase): r""" - Log to local file system in TensorBoard format - - Implemented using :class:`torch.utils.tensorboard.SummaryWriter`. Logs are saved to - ``os.path.join(save_dir, name, version)`` + Log to local file system in `TensorBoard `_ format. + Implemented using :class:`~torch.utils.tensorboard.SummaryWriter`. Logs are saved to + ``os.path.join(save_dir, name, version)``. This is the default logger in Lightning, it comes + preinstalled. Example: - .. code-block:: python - - logger = TensorBoardLogger("tb_logs", name="my_model") - trainer = Trainer(logger=logger) - trainer.train(model) + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import TensorBoardLogger + >>> logger = TensorBoardLogger("tb_logs", name="my_model") + >>> trainer = Trainer(logger=logger) Args: save_dir: Save directory - name: Experiment name. Defaults to "default". If it is the empty string then no per-experiment + name: Experiment name. Defaults to ``'default'``. If it is the empty string then no per-experiment subdirectory is used. version: Experiment version. If version is not specified the logger inspects the save directory for existing versions, then automatically assigns the next available version. If it is a string then it is used as the run-specific subdirectory name, - otherwise version_${version} is used. + otherwise ``'version_${version}'`` is used. \**kwargs: Other arguments are passed directly to the :class:`SummaryWriter` constructor. """ @@ -57,8 +61,8 @@ class TensorBoardLogger(LightningLoggerBase): def root_dir(self) -> str: """ Parent directory for all tensorboard checkpoint subdirectories. - If the experiment name parameter is None or the empty string, no experiment subdirectory is used - and checkpoint will be saved in save_dir/version_dir + If the experiment name parameter is ``None`` or the empty string, no experiment subdirectory is used + and the checkpoint will be saved in "save_dir/version_dir" """ if self.name is None or len(self.name) == 0: return self.save_dir @@ -68,9 +72,9 @@ class TensorBoardLogger(LightningLoggerBase): @property def log_dir(self) -> str: """ - The directory for this run's tensorboard checkpoint. By default, it is named 'version_${self.version}' - but it can be overridden by passing a string value for the constructor's version parameter - instead of None or an int + The directory for this run's tensorboard checkpoint. By default, it is named + ``'version_${self.version}'`` but it can be overridden by passing a string value + for the constructor's version parameter instead of ``None`` or an int. """ # create a pseudo standard path ala test-tube version = self.version if isinstance(self.version, str) else f"version_{self.version}" @@ -80,14 +84,14 @@ class TensorBoardLogger(LightningLoggerBase): @property def experiment(self) -> SummaryWriter: r""" + Actual tensorboard object. To use TensorBoard features in your + :class:`~pytorch_lightning.core.lightning.LightningModule` do the following. - Actual tensorboard object. To use tensorboard features do the following. + Example:: - Example:: + self.logger.experiment.some_tensorboard_function() - self.logger.experiment.some_tensorboard_function() - - """ + """ if self._experiment is not None: return self._experiment diff --git a/pytorch_lightning/loggers/test_tube.py b/pytorch_lightning/loggers/test_tube.py index dac8490c..fb81dbd1 100644 --- a/pytorch_lightning/loggers/test_tube.py +++ b/pytorch_lightning/loggers/test_tube.py @@ -1,3 +1,7 @@ +""" +Test Tube +--------- +""" from argparse import Namespace from typing import Optional, Dict, Any, Union @@ -12,8 +16,40 @@ from pytorch_lightning.loggers.base import LightningLoggerBase, rank_zero_only class TestTubeLogger(LightningLoggerBase): r""" - Log to local file system in TensorBoard format but using a nicer folder structure. - (see `full docs `_). + Log to local file system in `TensorBoard `_ format + but using a nicer folder structure (see `full docs `_). + Install it with pip: + + .. code-block:: bash + + pip install test_tube + + Example: + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import TestTubeLogger + >>> logger = TestTubeLogger("tt_logs", name="my_exp_name") + >>> trainer = Trainer(logger=logger) + + Use the logger anywhere in your :class:`~pytorch_lightning.core.lightning.LightningModule` as follows: + + >>> from pytorch_lightning import LightningModule + >>> class LitModel(LightningModule): + ... def training_step(self, batch, batch_idx): + ... # example + ... self.logger.experiment.whatever_method_summary_writer_supports(...) + ... + ... def any_lightning_module_function_or_hook(self): + ... self.logger.experiment.add_histogram(...) + + Args: + save_dir: Save directory + name: Experiment name. Defaults to ``'default'``. + description: A short snippet about this experiment + debug: If ``True``, it doesn't log anything. + version: Experiment version. If version is not specified the logger inspects the save + directory for existing versions, then automatically assigns the next available version. + create_git_tag: If ``True`` creates a git tag to save the code used in this experiment. + """ __test__ = False @@ -25,38 +61,6 @@ class TestTubeLogger(LightningLoggerBase): debug: bool = False, version: Optional[int] = None, create_git_tag: bool = False): - r""" - - Example - ---------- - - .. code-block:: python - - logger = TestTubeLogger("tt_logs", name="my_exp_name") - trainer = Trainer(logger=logger) - trainer.train(model) - - Use the logger anywhere in you LightningModule as follows: - - .. code-block:: python - - def train_step(...): - # example - self.logger.experiment.whatever_method_summary_writer_supports(...) - - def any_lightning_module_function_or_hook(...): - self.logger.experiment.add_histogram(...) - - Args: - save_dir (str): Save directory - name (str): Experiment name. Defaults to "default". - description (str): A short snippet about this experiment - debug (bool): If True, it doesn't log anything - version (int): Experiment version. If version is not specified the logger inspects the save - directory for existing versions, then automatically assigns the next available version. - create_git_tag (bool): If True creates a git tag to save the code used in this experiment - - """ super().__init__() self.save_dir = save_dir self._name = name @@ -70,14 +74,14 @@ class TestTubeLogger(LightningLoggerBase): def experiment(self) -> Experiment: r""" - Actual test-tube object. To use test-tube features do the following. + Actual TestTube object. To use TestTube features in your + :class:`~pytorch_lightning.core.lightning.LightningModule` do the following. - Example:: + Example:: - self.logger.experiment.some_test_tube_function() - - """ + self.logger.experiment.some_test_tube_function() + """ if self._experiment is not None: return self._experiment diff --git a/pytorch_lightning/loggers/trains.py b/pytorch_lightning/loggers/trains.py index 721d8259..ab43bdd3 100644 --- a/pytorch_lightning/loggers/trains.py +++ b/pytorch_lightning/loggers/trains.py @@ -1,27 +1,6 @@ """ -Log using `allegro.ai TRAINS `_ - -.. code-block:: python - - from pytorch_lightning.loggers import TrainsLogger - trains_logger = TrainsLogger( - project_name="pytorch lightning", - task_name="default", - ) - trainer = Trainer(logger=trains_logger) - - -Use the logger anywhere in you LightningModule as follows: - -.. code-block:: python - - def train_step(...): - # example - self.logger.experiment.whatever_trains_supports(...) - - def any_lightning_module_function_or_hook(...): - self.logger.experiment.whatever_trains_supports(...) - +TRAINS +------ """ from argparse import Namespace from os import environ @@ -44,22 +23,50 @@ from pytorch_lightning.loggers.base import LightningLoggerBase, rank_zero_only class TrainsLogger(LightningLoggerBase): - """Logs using TRAINS + """ + Log using `allegro.ai TRAINS `_. Install it with pip: + + .. code-block:: bash + + pip install trains + + Example: + >>> from pytorch_lightning import Trainer + >>> from pytorch_lightning.loggers import TrainsLogger + >>> trains_logger = TrainsLogger( + ... project_name='pytorch lightning', + ... task_name='default', + ... output_uri='.', + ... ) # doctest: +ELLIPSIS + TRAINS Task: ... + TRAINS results page: ... + >>> trainer = Trainer(logger=trains_logger) + + Use the logger anywhere in your :class:`~pytorch_lightning.core.lightning.LightningModule` as follows: + + >>> from pytorch_lightning import LightningModule + >>> class LitModel(LightningModule): + ... def training_step(self, batch, batch_idx): + ... # example + ... self.logger.experiment.whatever_trains_supports(...) + ... + ... def any_lightning_module_function_or_hook(self): + ... self.logger.experiment.whatever_trains_supports(...) Args: - project_name: The name of the experiment's project. Defaults to None. - task_name: The name of the experiment. Defaults to None. - task_type: The name of the experiment. Defaults to 'training'. - reuse_last_task_id: Start with the previously used task id. Defaults to True. - output_uri: Default location for output models. Defaults to None. - auto_connect_arg_parser: Automatically grab the ArgParser - and connect it with the task. Defaults to True. - auto_connect_frameworks: If True, automatically patch to trains backend. Defaults to True. - auto_resource_monitoring: If true, machine vitals will be - sent along side the task scalars. Defaults to True. + project_name: The name of the experiment's project. Defaults to ``None``. + task_name: The name of the experiment. Defaults to ``None``. + task_type: The name of the experiment. Defaults to ``'training'``. + reuse_last_task_id: Start with the previously used task id. Defaults to ``True``. + output_uri: Default location for output models. Defaults to ``None``. + auto_connect_arg_parser: Automatically grab the :class:`~argparse.ArgumentParser` + and connect it with the task. Defaults to ``True``. + auto_connect_frameworks: If ``True``, automatically patch to trains backend. Defaults to ``True``. + auto_resource_monitoring: If ``True``, machine vitals will be + sent along side the task scalars. Defaults to ``True``. Examples: - >>> logger = TrainsLogger("lightning_log", "my-lightning-test", output_uri=".") # doctest: +ELLIPSIS + >>> logger = TrainsLogger("pytorch lightning", "default", output_uri=".") # doctest: +ELLIPSIS TRAINS Task: ... TRAINS results page: ... >>> logger.log_metrics({"val_loss": 1.23}, step=0) @@ -116,12 +123,13 @@ class TrainsLogger(LightningLoggerBase): @property def experiment(self) -> Task: - r"""Actual TRAINS object. To use TRAINS features do the following. + r""" + Actual TRAINS object. To use TRAINS features in your + :class:`~pytorch_lightning.core.lightning.LightningModule` do the following. - Example: - .. code-block:: python + Example:: - self.logger.experiment.some_trains_function() + self.logger.experiment.some_trains_function() """ return self._trains @@ -138,11 +146,11 @@ class TrainsLogger(LightningLoggerBase): @rank_zero_only def log_hyperparams(self, params: Union[Dict[str, Any], Namespace]) -> None: - """Log hyperparameters (numeric values) in TRAINS experiments + """ + Log hyperparameters (numeric values) in TRAINS experiments. Args: - params: - The hyperparameters that passed through the model. + params: The hyperparameters that passed through the model. """ if not self._trains: return @@ -155,15 +163,15 @@ class TrainsLogger(LightningLoggerBase): @rank_zero_only def log_metrics(self, metrics: Dict[str, float], step: Optional[int] = None) -> None: - """Log metrics (numeric values) in TRAINS experiments. - This method will be called by Trainer. + """ + Log metrics (numeric values) in TRAINS experiments. + This method will be called by Trainer. Args: - metrics: - The dictionary of the metrics. + metrics: The dictionary of the metrics. If the key contains "/", it will be split by the delimiter, then the elements will be logged as "title" and "series" respectively. - step: Step number at which the metrics should be recorded. Defaults to None. + step: Step number at which the metrics should be recorded. Defaults to ``None``. """ if not self._trains: return @@ -188,14 +196,15 @@ class TrainsLogger(LightningLoggerBase): @rank_zero_only def log_metric(self, title: str, series: str, value: float, step: Optional[int] = None) -> None: - """Log metrics (numeric values) in TRAINS experiments. - This method will be called by the users. + """ + Log metrics (numeric values) in TRAINS experiments. + This method will be called by the users. Args: title: The title of the graph to log, e.g. loss, accuracy. series: The series name in the graph, e.g. classification, localization. value: The value to log. - step: Step number at which the metrics should be recorded. Defaults to None. + step: Step number at which the metrics should be recorded. Defaults to ``None``. """ if not self._trains: return @@ -210,7 +219,7 @@ class TrainsLogger(LightningLoggerBase): @rank_zero_only def log_text(self, text: str) -> None: - """Log console text data in TRAINS experiment + """Log console text data in TRAINS experiment. Args: text: The value of the log (data-point). @@ -229,20 +238,20 @@ class TrainsLogger(LightningLoggerBase): self, title: str, series: str, image: Union[str, np.ndarray, Image, torch.Tensor], step: Optional[int] = None) -> None: - """Log Debug image in TRAINS experiment + """ + Log Debug image in TRAINS experiment Args: title: The title of the debug image, i.e. "failed", "passed". series: The series name of the debug image, i.e. "Image 0", "Image 1". - image: - Debug image to log. Can be one of the following types: - Torch, Numpy, PIL image, path to image file (str) - If Numpy or Torch, the image is assume to be the following: - shape: CHW - color space: RGB - value range: [0., 1.] (float) or [0, 255] (uint8) - step: - Step number at which the metrics should be recorded. Defaults to None. + image: Debug image to log. If :class:`numpy.ndarray` or :class:`torch.Tensor`, + the image is assumed to be the following: + + - shape: CHW + - color space: RGB + - value range: [0., 1.] (float) or [0, 255] (uint8) + + step: Step number at which the metrics should be recorded. Defaults to None. """ if not self._trains: return @@ -266,26 +275,27 @@ class TrainsLogger(LightningLoggerBase): self, name: str, artifact: Union[str, Path, Dict[str, Any], np.ndarray, Image], metadata: Optional[Dict[str, Any]] = None, delete_after_upload: bool = False) -> None: - """Save an artifact (file/object) in TRAINS experiment storage. + """ + Save an artifact (file/object) in TRAINS experiment storage. - Arguments: - name: Artifact name. Notice! it will override previous artifact - if name already exists + Args: + name: Artifact name. Notice! it will override the previous artifact + if the name already exists. artifact: Artifact object to upload. Currently supports: - - string / pathlib2.Path are treated as path to artifact file to upload - If wildcard or a folder is passed, zip file containing the - local files will be created and uploaded + - string / :class:`pathlib.Path` are treated as path to artifact file to upload + If a wildcard or a folder is passed, a zip file containing the + local files will be created and uploaded. - dict will be stored as .json file and uploaded - - pandas.DataFrame will be stored as .csv.gz (compressed CSV file) and uploaded - - numpy.ndarray will be stored as .npz and uploaded - - PIL.Image will be stored to .png file and uploaded + - :class:`pandas.DataFrame` will be stored as .csv.gz (compressed CSV file) and uploaded + - :class:`numpy.ndarray` will be stored as .npz and uploaded + - :class:`PIL.Image.Image` will be stored to .png file and uploaded metadata: - Simple key/value dictionary to store on the artifact. Defaults to None. + Simple key/value dictionary to store on the artifact. Defaults to ``None``. delete_after_upload: - If True local artifact will be deleted (only applies if artifact_object is a - local file). Defaults to False. + If ``True``, the local artifact will be deleted (only applies if ``artifact`` is a + local file). Defaults to ``False``. """ if not self._trains: return @@ -325,17 +335,19 @@ class TrainsLogger(LightningLoggerBase): def set_credentials(cls, api_host: str = None, web_host: str = None, files_host: str = None, key: str = None, secret: str = None) -> None: """ - Set new default TRAINS-server host and credentials + Set new default TRAINS-server host and credentials. These configurations could be overridden by either OS environment variables - or trains.conf configuration file + or trains.conf configuration file. - Notice! credentials needs to be set *prior* to Logger initialization + Note: + Credentials need to be set *prior* to Logger initialization. - :param api_host: Trains API server url, example: host='http://localhost:8008' - :param web_host: Trains WEB server url, example: host='http://localhost:8080' - :param files_host: Trains Files server url, example: host='http://localhost:8081' - :param key: user key/secret pair, example: key='thisisakey123' - :param secret: user key/secret pair, example: secret='thisisseceret123' + Args: + api_host: Trains API server url, example: ``host='http://localhost:8008'`` + web_host: Trains WEB server url, example: ``host='http://localhost:8080'`` + files_host: Trains Files server url, example: ``host='http://localhost:8081'`` + key: user key/secret pair, example: ``key='thisisakey123'`` + secret: user key/secret pair, example: ``secret='thisisseceret123'`` """ Task.set_credentials(api_host=api_host, web_host=web_host, files_host=files_host, key=key, secret=secret) @@ -343,21 +355,25 @@ class TrainsLogger(LightningLoggerBase): @classmethod def set_bypass_mode(cls, bypass: bool) -> None: """ - set_bypass_mode will bypass all outside communication, and will drop all logs. - Should only be used in "standalone mode", when there is no access to the *trains-server* + Will bypass all outside communication, and will drop all logs. + Should only be used in "standalone mode", when there is no access to the *trains-server*. - :param bypass: If True, all outside communication is skipped + Args: + bypass: If ``True``, all outside communication is skipped. """ cls._bypass = bypass @classmethod def bypass_mode(cls) -> bool: """ - bypass_mode returns the bypass mode state. - Notice GITHUB_ACTIONS env will automatically set bypass_mode to True - unless overridden specifically with set_bypass_mode(False) + Returns the bypass mode state. - :return: If True, all outside communication is skipped + Note: + `GITHUB_ACTIONS` env will automatically set bypass_mode to ``True`` + unless overridden specifically with ``TrainsLogger.set_bypass_mode(False)``. + + Return: + If True, all outside communication is skipped. """ return cls._bypass if cls._bypass is not None else bool(environ.get('CI')) diff --git a/pytorch_lightning/loggers/wandb.py b/pytorch_lightning/loggers/wandb.py index 3b21fc9f..8d4bd0aa 100644 --- a/pytorch_lightning/loggers/wandb.py +++ b/pytorch_lightning/loggers/wandb.py @@ -1,9 +1,6 @@ -r""" - -.. _wandb: - -WandbLogger -------------- +""" +Weights and Biases +------------------ """ import os from argparse import Namespace @@ -23,27 +20,36 @@ from pytorch_lightning.loggers.base import LightningLoggerBase, rank_zero_only class WandbLogger(LightningLoggerBase): """ - Logger for `W&B `_. + Log using `Weights and Biases `_. Install it with pip: + + .. code-block:: bash + + pip install wandb Args: - name (str): display name for the run. - save_dir (str): path where data is saved. - offline (bool): run offline (data can be streamed later to wandb servers). - id or version (str): sets the version, mainly used to resume a previous run. - anonymous (bool): enables or explicitly disables anonymous logging. - project (str): the name of the project to which this run will belong. - tags (list of str): tags associated with this run. - log_model (bool): save checkpoints in wandb dir to upload on W&B servers. + name: Display name for the run. + save_dir: Path where data is saved. + offline: Run offline (data can be streamed later to wandb servers). + id: Sets the version, mainly used to resume a previous run. + anonymous: Enables or explicitly disables anonymous logging. + version: Sets the version, mainly used to resume a previous run. + project: The name of the project to which this run will belong. + tags: Tags associated with this run. + log_model: Save checkpoints in wandb dir to upload on W&B servers. + experiment: WandB experiment object + entity: The team posting this run (default: your username or your default team) - Example - -------- - .. code-block:: python + Example: + >>> from pytorch_lightning.loggers import WandbLogger + >>> from pytorch_lightning import Trainer + >>> wandb_logger = WandbLogger() + >>> trainer = Trainer(logger=wandb_logger) - from pytorch_lightning.loggers import WandbLogger - from pytorch_lightning import Trainer + See Also: + - `Tutorial `__ + on how to use W&B with Pytorch Lightning. - wandb_logger = WandbLogger() - trainer = Trainer(logger=wandb_logger) """ def __init__(self, @@ -83,13 +89,14 @@ class WandbLogger(LightningLoggerBase): def experiment(self) -> Run: r""" - Actual wandb object. To use wandb features do the following. + Actual wandb object. To use wandb features in your + :class:`~pytorch_lightning.core.lightning.LightningModule` do the following. - Example:: + Example:: - self.logger.experiment.some_wandb_function() + self.logger.experiment.some_wandb_function() - """ + """ if self._experiment is None: if self._offline: os.environ['WANDB_MODE'] = 'dryrun' diff --git a/requirements-extra.txt b/requirements-extra.txt index 5ebf92c7..aadce75a 100644 --- a/requirements-extra.txt +++ b/requirements-extra.txt @@ -1,6 +1,6 @@ # extended list of package dependencies to reach full functionality -neptune-client>=0.4.4 +neptune-client>=0.4.109 comet-ml>=1.0.56 mlflow>=1.0.0 test_tube>=0.7.5