[tune] Function API checkpointing (#8471)

Co-authored-by: krfricke <krfricke@users.noreply.github.com>
This commit is contained in:
Richard Liaw
2020-06-15 10:42:54 -07:00
committed by GitHub
co-authored by krfricke
parent 91e57f2e53
commit 6c49c01837
21 changed files with 897 additions and 237 deletions
+13
View File
@@ -3,6 +3,7 @@ import logging
import os
from ray.tune.error import TuneError
from ray.tune.function_runner import detect_checkpoint_function
from ray.tune.registry import register_trainable, get_trainable_cls
from ray.tune.result import DEFAULT_RESULTS_DIR
from ray.tune.sample import sample_from
@@ -92,6 +93,18 @@ class Experiment:
restore=None):
config = config or {}
if callable(run) and detect_checkpoint_function(run):
if checkpoint_at_end:
raise ValueError(
"'checkpoint_at_end' cannot be used with a "
"checkpointable function. You can specify and register "
"checkpoints within your trainable function.")
if checkpoint_freq:
raise ValueError(
"'checkpoint_freq' cannot be used with a "
"checkpointable function. You can specify checkpoints "
"within your trainable function.")
self._run_identifier = Experiment.register_if_needed(run)
self.name = name or self._run_identifier
if upload_dir: