wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-12 21:33:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] support rerunning failed trials (#10060)

Browse Source
This commit is contained in:
Richard Liaw
2020-08-22 09:59:05 -07:00
committed by GitHub
parent 32ed1a18b7
commit 24ee496b89
5 changed files with 208 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/tests/test_api.py
+34 -1
View File
@@ -1,5 +1,6 @@
from collections import Counter
import shutil
import tempfile
import copy
import os
import time
@@ -569,6 +570,38 @@ class TrainableFunctionApiTest(unittest.TestCase):
print(trial.last_result)
self.assertEqual(trial.last_result[DONE], True)
def testRerun(self):
tmpdir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(tmpdir))
def test(config):
tid = config["id"]
fail = config["fail"]
marker = os.path.join(tmpdir, f"t{tid}-{fail}.log")
if not os.path.exists(marker) and fail:
open(marker, "w").close()
raise ValueError
for i in range(10):
time.sleep(0.1)
tune.report(hello=123)
config = dict(
name="hi-2",
config={
"fail": tune.grid_search([True, False]),
"id": tune.grid_search(list(range(5)))
},
verbose=1,
local_dir=tmpdir,
loggers=None)
trials = tune.run(test, raise_on_failed_trial=False, **config).trials
self.assertEqual(Counter(t.status for t in trials)["ERROR"], 5)
new_trials = tune.run(
test, resume=True, run_errored_only=True, **config).trials
self.assertEqual(Counter(t.status for t in new_trials)["ERROR"], 0)
self.assertTrue(
all(t.last_result.get("hello") == 123 for t in new_trials))
def testErrorReturn(self):
def train(config, reporter):
raise Exception("uh oh")
python/ray/tune/tests/test_trial_runner_3.py
+95 -18
View File
@@ -22,11 +22,15 @@ from ray.tune.suggest.search_generator import SearchGenerator
class TrialRunnerTest3(unittest.TestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
def tearDown(self):
ray.shutdown()
_register_all() # re-register the evicted objects
if "CUDA_VISIBLE_DEVICES" in os.environ:
del os.environ["CUDA_VISIBLE_DEVICES"]
shutil.rmtree(self.tmpdir)
def testStepHook(self):
ray.init(num_cpus=4, num_gpus=2)
@@ -125,7 +129,7 @@ class TrialRunnerTest3(unittest.TestCase):
def testSearchAlgFinished(self):
"""Checks that SearchAlg is Finished before all trials are done."""
ray.init(num_cpus=4, num_gpus=2)
ray.init(num_cpus=4, local_mode=True, include_dashboard=False)
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 1}}
experiments = [Experiment.from_json("test", experiment_spec)]
searcher = _MockSuggestionAlgorithm()
@@ -150,7 +154,7 @@ class TrialRunnerTest3(unittest.TestCase):
def on_trial_result(self, *args, **kwargs):
return TrialScheduler.STOP
ray.init(num_cpus=4, num_gpus=2)
ray.init(num_cpus=4, local_mode=True, include_dashboard=False)
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 2}}
experiments = [Experiment.from_json("test", experiment_spec)]
searcher = _MockSuggestionAlgorithm()
@@ -241,7 +245,7 @@ class TrialRunnerTest3(unittest.TestCase):
def suggest(self, trial_id):
return {}
ray.init(num_cpus=2)
ray.init(num_cpus=2, local_mode=True, include_dashboard=False)
experiment_spec = {
"run": "__fake",
"num_samples": 2,
@@ -271,7 +275,6 @@ class TrialRunnerTest3(unittest.TestCase):
def testSearcherSaveRestore(self):
ray.init(num_cpus=8, local_mode=True)
tmpdir = tempfile.mkdtemp()
def create_searcher():
class TestSuggestion(Searcher):
@@ -313,7 +316,7 @@ class TrialRunnerTest3(unittest.TestCase):
searcher = create_searcher()
runner = TrialRunner(
search_alg=searcher,
local_checkpoint_dir=tmpdir,
local_checkpoint_dir=self.tmpdir,
checkpoint_period=-1)
for i in range(6):
runner.step()
@@ -331,7 +334,9 @@ class TrialRunnerTest3(unittest.TestCase):
searcher = create_searcher()
runner2 = TrialRunner(
search_alg=searcher, local_checkpoint_dir=tmpdir, resume="LOCAL")
search_alg=searcher,
local_checkpoint_dir=self.tmpdir,
resume="LOCAL")
assert len(runner2.get_trials()) == 6, [
t.config for t in runner2.get_trials()
]
@@ -355,12 +360,87 @@ class TrialRunnerTest3(unittest.TestCase):
count = Counter(evaluated)
assert all(v <= 3 for v in count.values())
def testTrialErrorResumeFalse(self):
ray.init(num_cpus=3, local_mode=True, include_dashboard=False)
runner = TrialRunner(local_checkpoint_dir=self.tmpdir)
kwargs = {
"stopping_criterion": {
"training_iteration": 4
},
"resources": Resources(cpu=1, gpu=0),
}
trials = [
Trial("__fake", config={"mock_error": True}, **kwargs),
Trial("__fake", **kwargs),
Trial("__fake", **kwargs),
]
for t in trials:
runner.add_trial(t)
while not runner.is_finished():
runner.step()
runner.checkpoint(force=True)
assert trials[0].status == Trial.ERROR
del runner
new_runner = TrialRunner(
run_errored_only=False,
resume=True,
local_checkpoint_dir=self.tmpdir)
assert len(new_runner.get_trials()) == 3
assert Trial.ERROR in (t.status for t in new_runner.get_trials())
def testTrialErrorResumeTrue(self):
ray.init(num_cpus=3, local_mode=True, include_dashboard=False)
runner = TrialRunner(local_checkpoint_dir=self.tmpdir)
kwargs = {
"stopping_criterion": {
"training_iteration": 4
},
"resources": Resources(cpu=1, gpu=0),
}
trials = [
Trial("__fake", config={"mock_error": True}, **kwargs),
Trial("__fake", **kwargs),
Trial("__fake", **kwargs),
]
for t in trials:
runner.add_trial(t)
while not runner.is_finished():
runner.step()
runner.checkpoint(force=True)
assert trials[0].status == Trial.ERROR
del runner
new_runner = TrialRunner(
run_errored_only=True,
resume=True,
local_checkpoint_dir=self.tmpdir)
assert len(new_runner.get_trials()) == 3
assert Trial.ERROR not in (t.status for t in new_runner.get_trials())
# The below is just a check for standard behavior.
disable_error = False
for t in new_runner.get_trials():
if t.config.get("mock_error"):
t.config["mock_error"] = False
disable_error = True
assert disable_error
while not new_runner.is_finished():
new_runner.step()
assert Trial.ERROR not in (t.status for t in new_runner.get_trials())
def testTrialSaveRestore(self):
"""Creates different trials to test runner.checkpoint/restore."""
ray.init(num_cpus=3)
tmpdir = tempfile.mkdtemp()
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
runner = TrialRunner(
local_checkpoint_dir=self.tmpdir, checkpoint_period=0)
trials = [
Trial(
"__fake",
@@ -401,7 +481,7 @@ class TrialRunnerTest3(unittest.TestCase):
self.assertEquals(len(runner.trial_executor.get_checkpoints()), 3)
self.assertEquals(trials[2].status, Trial.RUNNING)
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=self.tmpdir)
for tid in ["trial_terminate", "trial_fail"]:
original_trial = runner.get_trial(tid)
restored_trial = runner2.get_trial(tid)
@@ -416,14 +496,13 @@ class TrialRunnerTest3(unittest.TestCase):
runner2.step() # Process result, dispatch save
runner2.step() # Process save
self.assertRaises(TuneError, runner2.step)
shutil.rmtree(tmpdir)
def testTrialNoSave(self):
"""Check that non-checkpointing trials are not saved."""
ray.init(num_cpus=3)
tmpdir = tempfile.mkdtemp()
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
runner = TrialRunner(
local_checkpoint_dir=self.tmpdir, checkpoint_period=0)
runner.add_trial(
Trial(
"__fake",
@@ -454,7 +533,7 @@ class TrialRunnerTest3(unittest.TestCase):
runner.step()
runner.step()
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=self.tmpdir)
new_trials = runner2.get_trials()
self.assertEquals(len(new_trials), 3)
self.assertTrue(
@@ -464,7 +543,6 @@ class TrialRunnerTest3(unittest.TestCase):
self.assertTrue(runner2.get_trial("pending").status == Trial.PENDING)
self.assertTrue(not runner2.get_trial("pending").last_result)
runner2.step()
shutil.rmtree(tmpdir)
def testCheckpointWithFunction(self):
ray.init()
@@ -474,18 +552,17 @@ class TrialRunnerTest3(unittest.TestCase):
"on_episode_start": lambda i: i,
}},
checkpoint_freq=1)
tmpdir = tempfile.mkdtemp()
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
runner = TrialRunner(
local_checkpoint_dir=self.tmpdir, checkpoint_period=0)
runner.add_trial(trial)
for _ in range(5):
runner.step()
# force checkpoint
runner.checkpoint()
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=self.tmpdir)
new_trial = runner2.get_trials()[0]
self.assertTrue("callbacks" in new_trial.config)
self.assertTrue("on_episode_start" in new_trial.config["callbacks"])
shutil.rmtree(tmpdir)
def testCheckpointOverwrite(self):
def count_checkpoints(cdir):