[tune] Test example checkpointing (#4728)

This commit is contained in:
Richard Liaw
2019-07-10 01:58:26 -07:00
committed by GitHub
parent e55c8ca165
commit 0b540ab492
14 changed files with 157 additions and 68 deletions
+3 -7
View File
@@ -216,19 +216,15 @@ For TensorFlow model training, this would look something like this `(full tensor
def _setup(self, config):
self.saver = tf.train.Saver()
self.sess = ...
self.iteration = 0
def _train(self):
self.sess.run(...)
self.iteration += 1
def _save(self, checkpoint_dir):
return self.saver.save(
self.sess, checkpoint_dir + "/save",
global_step=self.iteration)
return self.saver.save(self.sess, os.path.join(checkpoint_dir, save))
def _restore(self, path):
return self.saver.restore(self.sess, path)
def _restore(self, checkpoint_prefix):
self.saver.restore(self.sess, checkpoint_prefix)
Additionally, checkpointing can be used to provide fault-tolerance for experiments. This can be enabled by setting ``checkpoint_freq=N`` and ``max_failures=M`` to checkpoint trials every *N* iterations and recover from up to *M* crashes per trial, e.g.: