mirror of
https://github.com/wassname/ray.git
synced 2026-07-09 06:06:26 +08:00
Shard unit tests into medium sized files for test stability (#6398)
This commit is contained in:
Eric Liang
+24
-2
@@ -50,12 +50,18 @@ py_test(
|
||||
deps = [":tune_lib"],
|
||||
)
|
||||
|
||||
py_test(
|
||||
name = "test_experiment_analysis_mem",
|
||||
size = "small",
|
||||
srcs = ["tests/test_experiment_analysis_mem.py"],
|
||||
deps = [":tune_lib"],
|
||||
)
|
||||
|
||||
py_test(
|
||||
name = "test_experiment",
|
||||
size = "small",
|
||||
srcs = ["tests/test_experiment.py"],
|
||||
deps = [":tune_lib"],
|
||||
flaky = 1,
|
||||
)
|
||||
|
||||
py_test(
|
||||
@@ -96,6 +102,22 @@ py_test(
|
||||
tags = ["exclusive"],
|
||||
)
|
||||
|
||||
py_test(
|
||||
name = "test_trial_runner_2",
|
||||
size = "medium",
|
||||
srcs = ["tests/test_trial_runner_2.py"],
|
||||
deps = [":tune_lib"],
|
||||
tags = ["exclusive"],
|
||||
)
|
||||
|
||||
py_test(
|
||||
name = "test_trial_runner_3",
|
||||
size = "medium",
|
||||
srcs = ["tests/test_trial_runner_3.py"],
|
||||
deps = [":tune_lib"],
|
||||
tags = ["exclusive"],
|
||||
)
|
||||
|
||||
py_test(
|
||||
name = "test_var",
|
||||
size = "small",
|
||||
@@ -146,7 +168,7 @@ py_test(
|
||||
|
||||
py_test(
|
||||
name = "test_tune_server",
|
||||
size = "medium",
|
||||
size = "small",
|
||||
srcs = ["tests/test_tune_server.py"],
|
||||
deps = [":tune_lib"],
|
||||
tags = ["exclusive"],
|
||||
|
||||
@@ -123,6 +123,8 @@ def test_trial_processed_after_node_failure(start_connected_emptyhead_cluster):
|
||||
|
||||
cluster.remove_node(node)
|
||||
runner.step()
|
||||
if not mock_process_failure.called:
|
||||
runner.step()
|
||||
assert mock_process_failure.called
|
||||
|
||||
|
||||
@@ -259,11 +261,9 @@ def test_trial_migration(start_connected_emptyhead_cluster):
|
||||
cluster.remove_node(node2)
|
||||
cluster.wait_for_nodes()
|
||||
runner.step() # Recovery step
|
||||
assert t2.last_result["training_iteration"] == 2
|
||||
for i in range(1):
|
||||
if t2.status != Trial.TERMINATED:
|
||||
runner.step()
|
||||
|
||||
assert t2.status == Trial.TERMINATED
|
||||
assert t2.status == Trial.TERMINATED, runner.debug_string()
|
||||
|
||||
# Test recovery of trial that won't be checkpointed
|
||||
t3 = Trial("__fake", **{"stopping_criterion": {"training_iteration": 3}})
|
||||
@@ -274,7 +274,9 @@ def test_trial_migration(start_connected_emptyhead_cluster):
|
||||
cluster.remove_node(node3)
|
||||
cluster.wait_for_nodes()
|
||||
runner.step() # Error handling step
|
||||
assert t3.status == Trial.ERROR
|
||||
if t3.status != Trial.ERROR:
|
||||
runner.step()
|
||||
assert t3.status == Trial.ERROR, runner.debug_string()
|
||||
|
||||
with pytest.raises(TuneError):
|
||||
runner.step()
|
||||
@@ -340,9 +342,9 @@ def test_migration_checkpoint_removal(start_connected_emptyhead_cluster):
|
||||
|
||||
runner.step() # Recovery step
|
||||
for i in range(3):
|
||||
runner.step()
|
||||
|
||||
assert t1.status == Trial.TERMINATED
|
||||
if t1.status != Trial.TERMINATED:
|
||||
runner.step()
|
||||
assert t1.status == Trial.TERMINATED, runner.debug_string()
|
||||
|
||||
|
||||
def test_cluster_down_simple(start_connected_cluster, tmpdir):
|
||||
|
||||
@@ -10,67 +10,10 @@ import os
|
||||
import pandas as pd
|
||||
|
||||
import ray
|
||||
from ray.tune import run, Trainable, sample_from, Analysis, grid_search
|
||||
from ray.tune import run, sample_from
|
||||
from ray.tune.examples.async_hyperband_example import MyTrainableClass
|
||||
|
||||
|
||||
class ExperimentAnalysisInMemorySuite(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class MockTrainable(Trainable):
|
||||
def _setup(self, config):
|
||||
self.id = config["id"]
|
||||
self.idx = 0
|
||||
self.scores_dict = {
|
||||
0: [5, 0],
|
||||
1: [4, 1],
|
||||
2: [2, 8],
|
||||
3: [9, 6],
|
||||
4: [7, 3]
|
||||
}
|
||||
|
||||
def _train(self):
|
||||
val = self.scores_dict[self.id][self.idx]
|
||||
self.idx += 1
|
||||
return {"score": val}
|
||||
|
||||
def _save(self, checkpoint_dir):
|
||||
pass
|
||||
|
||||
def _restore(self, checkpoint_path):
|
||||
pass
|
||||
|
||||
self.MockTrainable = MockTrainable
|
||||
ray.init(local_mode=False, num_cpus=1)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
ray.shutdown()
|
||||
|
||||
def testCompareTrials(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
scores_all = [5, 4, 2, 9, 7, 0, 1, 8, 6, 3]
|
||||
scores_last = scores_all[5:]
|
||||
|
||||
ea = run(
|
||||
self.MockTrainable,
|
||||
name="analysis_exp",
|
||||
local_dir=self.test_dir,
|
||||
stop={"training_iteration": 2},
|
||||
num_samples=1,
|
||||
config={"id": grid_search(list(range(5)))})
|
||||
|
||||
max_all = ea.get_best_trial("score",
|
||||
"max").metric_analysis["score"]["max"]
|
||||
min_all = ea.get_best_trial("score",
|
||||
"min").metric_analysis["score"]["min"]
|
||||
max_last = ea.get_best_trial("score", "max",
|
||||
"last").metric_analysis["score"]["last"]
|
||||
self.assertEqual(max_all, max(scores_all))
|
||||
self.assertEqual(min_all, min(scores_all))
|
||||
self.assertEqual(max_last, max(scores_last))
|
||||
self.assertNotEqual(max_last, max(scores_all))
|
||||
|
||||
|
||||
class ExperimentAnalysisSuite(unittest.TestCase):
|
||||
def setUp(self):
|
||||
ray.init(local_mode=False)
|
||||
@@ -155,54 +98,6 @@ class ExperimentAnalysisSuite(unittest.TestCase):
|
||||
self.assertEquals(df.shape[0], 1)
|
||||
|
||||
|
||||
class AnalysisSuite(unittest.TestCase):
|
||||
def setUp(self):
|
||||
ray.init(local_mode=True)
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.num_samples = 10
|
||||
self.metric = "episode_reward_mean"
|
||||
self.run_test_exp(test_name="analysis_exp1")
|
||||
self.run_test_exp(test_name="analysis_exp2")
|
||||
|
||||
def run_test_exp(self, test_name=None):
|
||||
run(MyTrainableClass,
|
||||
name=test_name,
|
||||
local_dir=self.test_dir,
|
||||
return_trials=False,
|
||||
stop={"training_iteration": 1},
|
||||
num_samples=self.num_samples,
|
||||
config={
|
||||
"width": sample_from(
|
||||
lambda spec: 10 + int(90 * random.random())),
|
||||
"height": sample_from(lambda spec: int(100 * random.random())),
|
||||
})
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
ray.shutdown()
|
||||
|
||||
def testDataframe(self):
|
||||
analysis = Analysis(self.test_dir)
|
||||
df = analysis.dataframe()
|
||||
self.assertTrue(isinstance(df, pd.DataFrame))
|
||||
self.assertEquals(df.shape[0], self.num_samples * 2)
|
||||
|
||||
def testBestLogdir(self):
|
||||
analysis = Analysis(self.test_dir)
|
||||
logdir = analysis.get_best_logdir(self.metric)
|
||||
self.assertTrue(logdir.startswith(self.test_dir))
|
||||
logdir2 = analysis.get_best_logdir(self.metric, mode="min")
|
||||
self.assertTrue(logdir2.startswith(self.test_dir))
|
||||
self.assertNotEquals(logdir, logdir2)
|
||||
|
||||
def testBestConfigIsLogdir(self):
|
||||
analysis = Analysis(self.test_dir)
|
||||
for metric, mode in [(self.metric, "min"), (self.metric, "max")]:
|
||||
logdir = analysis.get_best_logdir(metric, mode=mode)
|
||||
best_config = analysis.get_best_config(metric, mode=mode)
|
||||
self.assertEquals(analysis.get_all_configs()[logdir], best_config)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
import sys
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import unittest
|
||||
import shutil
|
||||
import tempfile
|
||||
import random
|
||||
import pandas as pd
|
||||
|
||||
import ray
|
||||
from ray.tune import run, Trainable, sample_from, Analysis, grid_search
|
||||
from ray.tune.examples.async_hyperband_example import MyTrainableClass
|
||||
|
||||
|
||||
class ExperimentAnalysisInMemorySuite(unittest.TestCase):
|
||||
def setUp(self):
|
||||
class MockTrainable(Trainable):
|
||||
def _setup(self, config):
|
||||
self.id = config["id"]
|
||||
self.idx = 0
|
||||
self.scores_dict = {
|
||||
0: [5, 0],
|
||||
1: [4, 1],
|
||||
2: [2, 8],
|
||||
3: [9, 6],
|
||||
4: [7, 3]
|
||||
}
|
||||
|
||||
def _train(self):
|
||||
val = self.scores_dict[self.id][self.idx]
|
||||
self.idx += 1
|
||||
return {"score": val}
|
||||
|
||||
def _save(self, checkpoint_dir):
|
||||
pass
|
||||
|
||||
def _restore(self, checkpoint_path):
|
||||
pass
|
||||
|
||||
self.MockTrainable = MockTrainable
|
||||
ray.init(local_mode=False, num_cpus=1)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
ray.shutdown()
|
||||
|
||||
def testCompareTrials(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
scores_all = [5, 4, 2, 9, 7, 0, 1, 8, 6, 3]
|
||||
scores_last = scores_all[5:]
|
||||
|
||||
ea = run(
|
||||
self.MockTrainable,
|
||||
name="analysis_exp",
|
||||
local_dir=self.test_dir,
|
||||
stop={"training_iteration": 2},
|
||||
num_samples=1,
|
||||
config={"id": grid_search(list(range(5)))})
|
||||
|
||||
max_all = ea.get_best_trial("score",
|
||||
"max").metric_analysis["score"]["max"]
|
||||
min_all = ea.get_best_trial("score",
|
||||
"min").metric_analysis["score"]["min"]
|
||||
max_last = ea.get_best_trial("score", "max",
|
||||
"last").metric_analysis["score"]["last"]
|
||||
self.assertEqual(max_all, max(scores_all))
|
||||
self.assertEqual(min_all, min(scores_all))
|
||||
self.assertEqual(max_last, max(scores_last))
|
||||
self.assertNotEqual(max_last, max(scores_all))
|
||||
|
||||
|
||||
class AnalysisSuite(unittest.TestCase):
|
||||
def setUp(self):
|
||||
ray.init(local_mode=True)
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.num_samples = 10
|
||||
self.metric = "episode_reward_mean"
|
||||
self.run_test_exp(test_name="analysis_exp1")
|
||||
self.run_test_exp(test_name="analysis_exp2")
|
||||
|
||||
def run_test_exp(self, test_name=None):
|
||||
run(MyTrainableClass,
|
||||
name=test_name,
|
||||
local_dir=self.test_dir,
|
||||
return_trials=False,
|
||||
stop={"training_iteration": 1},
|
||||
num_samples=self.num_samples,
|
||||
config={
|
||||
"width": sample_from(
|
||||
lambda spec: 10 + int(90 * random.random())),
|
||||
"height": sample_from(lambda spec: int(100 * random.random())),
|
||||
})
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir, ignore_errors=True)
|
||||
ray.shutdown()
|
||||
|
||||
def testDataframe(self):
|
||||
analysis = Analysis(self.test_dir)
|
||||
df = analysis.dataframe()
|
||||
self.assertTrue(isinstance(df, pd.DataFrame))
|
||||
self.assertEquals(df.shape[0], self.num_samples * 2)
|
||||
|
||||
def testBestLogdir(self):
|
||||
analysis = Analysis(self.test_dir)
|
||||
logdir = analysis.get_best_logdir(self.metric)
|
||||
self.assertTrue(logdir.startswith(self.test_dir))
|
||||
logdir2 = analysis.get_best_logdir(self.metric, mode="min")
|
||||
self.assertTrue(logdir2.startswith(self.test_dir))
|
||||
self.assertNotEquals(logdir, logdir2)
|
||||
|
||||
def testBestConfigIsLogdir(self):
|
||||
analysis = Analysis(self.test_dir)
|
||||
for metric, mode in [(self.metric, "min"), (self.metric, "max")]:
|
||||
logdir = analysis.get_best_logdir(metric, mode=mode)
|
||||
best_config = analysis.get_best_config(metric, mode=mode)
|
||||
self.assertEquals(analysis.get_all_configs()[logdir], best_config)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
import sys
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -2,10 +2,7 @@ from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import ray
|
||||
@@ -15,39 +12,10 @@ from ray import tune
|
||||
from ray.tune import TuneError, register_trainable
|
||||
from ray.tune.ray_trial_executor import RayTrialExecutor
|
||||
from ray.tune.schedulers import TrialScheduler, FIFOScheduler
|
||||
from ray.tune.result import DONE
|
||||
from ray.tune.registry import _global_registry, TRAINABLE_CLASS
|
||||
from ray.tune.experiment import Experiment
|
||||
from ray.tune.trial import Trial
|
||||
from ray.tune.trial_runner import TrialRunner
|
||||
from ray.tune.resources import Resources, json_to_resources, resources_to_json
|
||||
from ray.tune.resources import Resources
|
||||
from ray.tune.suggest import BasicVariantGenerator
|
||||
from ray.tune.suggest.suggestion import (_MockSuggestionAlgorithm,
|
||||
SuggestionAlgorithm)
|
||||
|
||||
if sys.version_info >= (3, 3):
|
||||
from unittest.mock import patch
|
||||
else:
|
||||
from mock import patch
|
||||
|
||||
|
||||
def create_mock_components():
|
||||
class _MockScheduler(FIFOScheduler):
|
||||
errored_trials = []
|
||||
|
||||
def on_trial_error(self, trial_runner, trial):
|
||||
self.errored_trials += [trial]
|
||||
|
||||
class _MockSearchAlg(BasicVariantGenerator):
|
||||
errored_trials = []
|
||||
|
||||
def on_trial_complete(self, trial_id, error=False, **kwargs):
|
||||
if error:
|
||||
self.errored_trials += [trial_id]
|
||||
|
||||
searchalg = _MockSearchAlg()
|
||||
scheduler = _MockScheduler()
|
||||
return searchalg, scheduler
|
||||
|
||||
|
||||
class TrialRunnerTest(unittest.TestCase):
|
||||
@@ -317,794 +285,7 @@ class TrialRunnerTest(unittest.TestCase):
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(runner.trial_executor._committed_resources.cpu, 2)
|
||||
|
||||
def testErrorHandling(self):
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 1
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
_global_registry.register(TRAINABLE_CLASS, "asdf", None)
|
||||
trials = [Trial("asdf", **kwargs), Trial("__fake", **kwargs)]
|
||||
for t in trials:
|
||||
runner.add_trial(t)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
|
||||
def testThrowOnOverstep(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
runner.step()
|
||||
self.assertRaises(TuneError, runner.step)
|
||||
|
||||
def testFailureRecoveryDisabled(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
searchalg, scheduler = create_mock_components()
|
||||
|
||||
runner = TrialRunner(searchalg, scheduler=scheduler)
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
"checkpoint_freq": 1,
|
||||
"max_failures": 0,
|
||||
"config": {
|
||||
"mock_error": True,
|
||||
},
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[0].num_failures, 1)
|
||||
self.assertEqual(len(searchalg.errored_trials), 1)
|
||||
self.assertEqual(len(scheduler.errored_trials), 1)
|
||||
|
||||
def testFailureRecoveryEnabled(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
searchalg, scheduler = create_mock_components()
|
||||
|
||||
runner = TrialRunner(searchalg, scheduler=scheduler)
|
||||
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
"checkpoint_freq": 1,
|
||||
"max_failures": 1,
|
||||
"config": {
|
||||
"mock_error": True,
|
||||
},
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[0].num_failures, 1)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(len(searchalg.errored_trials), 0)
|
||||
self.assertEqual(len(scheduler.errored_trials), 0)
|
||||
|
||||
def testFailureRecoveryNodeRemoval(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
searchalg, scheduler = create_mock_components()
|
||||
|
||||
runner = TrialRunner(searchalg, scheduler=scheduler)
|
||||
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
"checkpoint_freq": 1,
|
||||
"max_failures": 1,
|
||||
"config": {
|
||||
"mock_error": True,
|
||||
},
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
with patch("ray.cluster_resources") as resource_mock:
|
||||
resource_mock.return_value = {"CPU": 1, "GPU": 1}
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
# Mimic a node failure
|
||||
resource_mock.return_value = {"CPU": 0, "GPU": 0}
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.PENDING)
|
||||
self.assertEqual(trials[0].num_failures, 1)
|
||||
self.assertEqual(len(searchalg.errored_trials), 0)
|
||||
self.assertEqual(len(scheduler.errored_trials), 1)
|
||||
|
||||
def testFailureRecoveryMaxFailures(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
"checkpoint_freq": 1,
|
||||
"max_failures": 2,
|
||||
"config": {
|
||||
"mock_error": True,
|
||||
"persistent_error": True,
|
||||
},
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[0].num_failures, 1)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[0].num_failures, 2)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[0].num_failures, 3)
|
||||
|
||||
def testCheckpointing(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 1
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.set_info.remote(1)), 1)
|
||||
path = runner.trial_executor.save(trials[0])
|
||||
kwargs["restore_path"] = path
|
||||
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[1].runner.get_info.remote()), 1)
|
||||
self.addCleanup(os.remove, path)
|
||||
|
||||
def testRestoreMetricsAfterCheckpointing(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.set_info.remote(1)), 1)
|
||||
path = runner.trial_executor.save(trials[0])
|
||||
runner.trial_executor.stop_trial(trials[0])
|
||||
kwargs["restore_path"] = path
|
||||
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[1].last_result["timesteps_since_restore"], 10)
|
||||
self.assertEqual(trials[1].last_result["iterations_since_restore"], 1)
|
||||
self.assertGreater(trials[1].last_result["time_since_restore"], 0)
|
||||
runner.step()
|
||||
self.assertEqual(trials[1].last_result["timesteps_since_restore"], 20)
|
||||
self.assertEqual(trials[1].last_result["iterations_since_restore"], 2)
|
||||
self.assertGreater(trials[1].last_result["time_since_restore"], 0)
|
||||
self.addCleanup(os.remove, path)
|
||||
|
||||
def testCheckpointingAtEnd(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 2
|
||||
},
|
||||
"checkpoint_at_end": True,
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].last_result[DONE], True)
|
||||
self.assertEqual(trials[0].has_checkpoint(), True)
|
||||
|
||||
def testResultDone(self):
|
||||
"""Tests that last_result is marked `done` after trial is complete."""
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 2
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertNotEqual(trials[0].last_result[DONE], True)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].last_result[DONE], True)
|
||||
|
||||
def testPauseThenResume(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 2
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.get_info.remote()), None)
|
||||
|
||||
self.assertEqual(ray.get(trials[0].runner.set_info.remote(1)), 1)
|
||||
|
||||
runner.trial_executor.pause_trial(trials[0])
|
||||
self.assertEqual(trials[0].status, Trial.PAUSED)
|
||||
|
||||
runner.trial_executor.resume_trial(trials[0])
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.get_info.remote()), 1)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
|
||||
def testStepHook(self):
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
runner = TrialRunner()
|
||||
|
||||
def on_step_begin(self, trialrunner):
|
||||
self._update_avail_resources()
|
||||
cnt = self.pre_step if hasattr(self, "pre_step") else 0
|
||||
setattr(self, "pre_step", cnt + 1)
|
||||
|
||||
def on_step_end(self, trialrunner):
|
||||
cnt = self.pre_step if hasattr(self, "post_step") else 0
|
||||
setattr(self, "post_step", 1 + cnt)
|
||||
|
||||
import types
|
||||
runner.trial_executor.on_step_begin = types.MethodType(
|
||||
on_step_begin, runner.trial_executor)
|
||||
runner.trial_executor.on_step_end = types.MethodType(
|
||||
on_step_end, runner.trial_executor)
|
||||
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 5
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
runner.step()
|
||||
self.assertEqual(runner.trial_executor.pre_step, 1)
|
||||
self.assertEqual(runner.trial_executor.post_step, 1)
|
||||
|
||||
def testStopTrial(self):
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 5
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
trials = [
|
||||
Trial("__fake", **kwargs),
|
||||
Trial("__fake", **kwargs),
|
||||
Trial("__fake", **kwargs),
|
||||
Trial("__fake", **kwargs)
|
||||
]
|
||||
for t in trials:
|
||||
runner.add_trial(t)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
# Stop trial while running
|
||||
runner.stop_trial(trials[0])
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[-1].status, Trial.PENDING)
|
||||
|
||||
# Stop trial while pending
|
||||
runner.stop_trial(trials[-1])
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[-1].status, Trial.TERMINATED)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[2].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[-1].status, Trial.TERMINATED)
|
||||
|
||||
def testSearchAlgNotification(self):
|
||||
"""Checks notification of trial to the Search Algorithm."""
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 2}}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(max_concurrent=10)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher)
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
|
||||
self.assertEqual(searcher.counter["result"], 1)
|
||||
self.assertEqual(searcher.counter["complete"], 1)
|
||||
|
||||
def testSearchAlgFinished(self):
|
||||
"""Checks that SearchAlg is Finished before all trials are done."""
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 1}}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(max_concurrent=10)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher)
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(len(searcher.live_trials), 0)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertTrue(runner.is_finished())
|
||||
|
||||
def testSearchAlgSchedulerInteraction(self):
|
||||
"""Checks that TrialScheduler killing trial will notify SearchAlg."""
|
||||
|
||||
class _MockScheduler(FIFOScheduler):
|
||||
def on_trial_result(self, *args, **kwargs):
|
||||
return TrialScheduler.STOP
|
||||
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 2}}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(max_concurrent=10)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher, scheduler=_MockScheduler())
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(len(searcher.live_trials), 0)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertTrue(runner.is_finished())
|
||||
|
||||
def testSearchAlgSchedulerEarlyStop(self):
|
||||
"""Early termination notif to Searcher can be turned off."""
|
||||
|
||||
class _MockScheduler(FIFOScheduler):
|
||||
def on_trial_result(self, *args, **kwargs):
|
||||
return TrialScheduler.STOP
|
||||
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 2}}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(use_early_stopped_trials=True)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher, scheduler=_MockScheduler())
|
||||
runner.step()
|
||||
runner.step()
|
||||
self.assertEqual(len(searcher.final_results), 1)
|
||||
|
||||
searcher = _MockSuggestionAlgorithm(use_early_stopped_trials=False)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher, scheduler=_MockScheduler())
|
||||
runner.step()
|
||||
runner.step()
|
||||
self.assertEqual(len(searcher.final_results), 0)
|
||||
|
||||
def testSearchAlgStalled(self):
|
||||
"""Checks that runner and searcher state is maintained when stalled."""
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {
|
||||
"run": "__fake",
|
||||
"num_samples": 3,
|
||||
"stop": {
|
||||
"training_iteration": 1
|
||||
}
|
||||
}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(max_concurrent=1)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher)
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
|
||||
trials = runner.get_trials()
|
||||
runner.step()
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(len(searcher.live_trials), 1)
|
||||
|
||||
searcher.stall = True
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[1].status, Trial.TERMINATED)
|
||||
self.assertEqual(len(searcher.live_trials), 0)
|
||||
|
||||
self.assertTrue(all(trial.is_finished() for trial in trials))
|
||||
self.assertFalse(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
|
||||
searcher.stall = False
|
||||
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[2].status, Trial.RUNNING)
|
||||
self.assertEqual(len(searcher.live_trials), 1)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[2].status, Trial.TERMINATED)
|
||||
self.assertEqual(len(searcher.live_trials), 0)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertTrue(runner.is_finished())
|
||||
|
||||
def testSearchAlgFinishes(self):
|
||||
"""Empty SearchAlg changing state in `next_trials` does not crash."""
|
||||
|
||||
class FinishFastAlg(SuggestionAlgorithm):
|
||||
_index = 0
|
||||
|
||||
def next_trials(self):
|
||||
trials = []
|
||||
self._index += 1
|
||||
|
||||
for trial in self._trial_generator:
|
||||
trials += [trial]
|
||||
break
|
||||
|
||||
if self._index > 4:
|
||||
self._finished = True
|
||||
return trials
|
||||
|
||||
def _suggest(self, trial_id):
|
||||
return {}
|
||||
|
||||
ray.init(num_cpus=2)
|
||||
experiment_spec = {
|
||||
"run": "__fake",
|
||||
"num_samples": 2,
|
||||
"stop": {
|
||||
"training_iteration": 1
|
||||
}
|
||||
}
|
||||
searcher = FinishFastAlg()
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher.add_configurations(experiments)
|
||||
|
||||
runner = TrialRunner(search_alg=searcher)
|
||||
self.assertFalse(runner.is_finished())
|
||||
runner.step() # This launches a new run
|
||||
runner.step() # This launches a 2nd run
|
||||
self.assertFalse(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
runner.step() # This kills the first run
|
||||
self.assertFalse(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
runner.step() # This kills the 2nd run
|
||||
self.assertFalse(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
runner.step() # this converts self._finished to True
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertRaises(TuneError, runner.step)
|
||||
|
||||
def testTrialSaveRestore(self):
|
||||
"""Creates different trials to test runner.checkpoint/restore."""
|
||||
ray.init(num_cpus=3)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
trials = [
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="trial_terminate",
|
||||
stopping_criterion={"training_iteration": 1},
|
||||
checkpoint_freq=1)
|
||||
]
|
||||
runner.add_trial(trials[0])
|
||||
runner.step() # start
|
||||
runner.step()
|
||||
self.assertEquals(trials[0].status, Trial.TERMINATED)
|
||||
|
||||
trials += [
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="trial_fail",
|
||||
stopping_criterion={"training_iteration": 3},
|
||||
checkpoint_freq=1,
|
||||
config={"mock_error": True})
|
||||
]
|
||||
runner.add_trial(trials[1])
|
||||
runner.step()
|
||||
runner.step()
|
||||
runner.step()
|
||||
self.assertEquals(trials[1].status, Trial.ERROR)
|
||||
|
||||
trials += [
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="trial_succ",
|
||||
stopping_criterion={"training_iteration": 2},
|
||||
checkpoint_freq=1)
|
||||
]
|
||||
runner.add_trial(trials[2])
|
||||
runner.step()
|
||||
self.assertEquals(len(runner.trial_executor.get_checkpoints()), 3)
|
||||
self.assertEquals(trials[2].status, Trial.RUNNING)
|
||||
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
for tid in ["trial_terminate", "trial_fail"]:
|
||||
original_trial = runner.get_trial(tid)
|
||||
restored_trial = runner2.get_trial(tid)
|
||||
self.assertEqual(original_trial.status, restored_trial.status)
|
||||
|
||||
restored_trial = runner2.get_trial("trial_succ")
|
||||
self.assertEqual(Trial.PENDING, restored_trial.status)
|
||||
|
||||
runner2.step()
|
||||
runner2.step()
|
||||
runner2.step()
|
||||
self.assertRaises(TuneError, runner2.step)
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def testTrialNoSave(self):
|
||||
"""Check that non-checkpointing trials are not saved."""
|
||||
ray.init(num_cpus=3)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
runner.add_trial(
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="non_checkpoint",
|
||||
stopping_criterion={"training_iteration": 2}))
|
||||
|
||||
while not all(t.status == Trial.TERMINATED
|
||||
for t in runner.get_trials()):
|
||||
runner.step()
|
||||
|
||||
runner.add_trial(
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="checkpoint",
|
||||
checkpoint_at_end=True,
|
||||
stopping_criterion={"training_iteration": 2}))
|
||||
|
||||
while not all(t.status == Trial.TERMINATED
|
||||
for t in runner.get_trials()):
|
||||
runner.step()
|
||||
|
||||
runner.add_trial(
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="pending",
|
||||
stopping_criterion={"training_iteration": 2}))
|
||||
|
||||
runner.step()
|
||||
runner.step()
|
||||
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
new_trials = runner2.get_trials()
|
||||
self.assertEquals(len(new_trials), 3)
|
||||
self.assertTrue(
|
||||
runner2.get_trial("non_checkpoint").status == Trial.TERMINATED)
|
||||
self.assertTrue(
|
||||
runner2.get_trial("checkpoint").status == Trial.TERMINATED)
|
||||
self.assertTrue(runner2.get_trial("pending").status == Trial.PENDING)
|
||||
self.assertTrue(not runner2.get_trial("pending").last_result)
|
||||
runner2.step()
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def testCheckpointWithFunction(self):
|
||||
ray.init()
|
||||
trial = Trial(
|
||||
"__fake",
|
||||
config={"callbacks": {
|
||||
"on_episode_start": lambda i: i,
|
||||
}},
|
||||
checkpoint_freq=1)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
runner.add_trial(trial)
|
||||
for i in range(5):
|
||||
runner.step()
|
||||
# force checkpoint
|
||||
runner.checkpoint()
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
new_trial = runner2.get_trials()[0]
|
||||
self.assertTrue("callbacks" in new_trial.config)
|
||||
self.assertTrue("on_episode_start" in new_trial.config["callbacks"])
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def testCheckpointOverwrite(self):
|
||||
def count_checkpoints(cdir):
|
||||
return sum((fname.startswith("experiment_state")
|
||||
and fname.endswith(".json"))
|
||||
for fname in os.listdir(cdir))
|
||||
|
||||
ray.init()
|
||||
trial = Trial("__fake", checkpoint_freq=1)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
runner.add_trial(trial)
|
||||
for i in range(5):
|
||||
runner.step()
|
||||
# force checkpoint
|
||||
runner.checkpoint()
|
||||
self.assertEquals(count_checkpoints(tmpdir), 1)
|
||||
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
for i in range(5):
|
||||
runner2.step()
|
||||
self.assertEquals(count_checkpoints(tmpdir), 2)
|
||||
|
||||
runner2.checkpoint()
|
||||
self.assertEquals(count_checkpoints(tmpdir), 2)
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def testUserCheckpoint(self):
|
||||
ray.init(num_cpus=3)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
runner.add_trial(Trial("__fake", config={"user_checkpoint_freq": 2}))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.set_info.remote(1)), 1)
|
||||
runner.step() # 0
|
||||
self.assertFalse(trials[0].has_checkpoint())
|
||||
runner.step() # 1
|
||||
self.assertFalse(trials[0].has_checkpoint())
|
||||
runner.step() # 2
|
||||
self.assertTrue(trials[0].has_checkpoint())
|
||||
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
runner2.step()
|
||||
trials2 = runner2.get_trials()
|
||||
self.assertEqual(ray.get(trials2[0].runner.get_info.remote()), 1)
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
class SearchAlgorithmTest(unittest.TestCase):
|
||||
def testNestedSuggestion(self):
|
||||
class TestSuggestion(SuggestionAlgorithm):
|
||||
def _suggest(self, trial_id):
|
||||
return {"a": {"b": {"c": {"d": 4, "e": 5}}}}
|
||||
|
||||
alg = TestSuggestion()
|
||||
alg.add_configurations({"test": {"run": "__fake"}})
|
||||
trial = alg.next_trials()[0]
|
||||
self.assertTrue("e=5" in trial.experiment_tag)
|
||||
self.assertTrue("d=4" in trial.experiment_tag)
|
||||
|
||||
|
||||
class ResourcesTest(unittest.TestCase):
|
||||
def testSubtraction(self):
|
||||
resource_1 = Resources(
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
custom_resources={
|
||||
"a": 1,
|
||||
"b": 2
|
||||
},
|
||||
extra_custom_resources={
|
||||
"a": 1,
|
||||
"b": 1
|
||||
})
|
||||
resource_2 = Resources(
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
custom_resources={
|
||||
"a": 1,
|
||||
"b": 2
|
||||
},
|
||||
extra_custom_resources={
|
||||
"a": 1,
|
||||
"b": 1
|
||||
})
|
||||
new_res = Resources.subtract(resource_1, resource_2)
|
||||
self.assertTrue(new_res.cpu == 0)
|
||||
self.assertTrue(new_res.gpu == 0)
|
||||
self.assertTrue(new_res.extra_cpu == 0)
|
||||
self.assertTrue(new_res.extra_gpu == 0)
|
||||
self.assertTrue(all(k == 0 for k in new_res.custom_resources.values()))
|
||||
self.assertTrue(
|
||||
all(k == 0 for k in new_res.extra_custom_resources.values()))
|
||||
|
||||
def testDifferentResources(self):
|
||||
resource_1 = Resources(1, 0, 0, 1, custom_resources={"a": 1, "b": 2})
|
||||
resource_2 = Resources(1, 0, 0, 1, custom_resources={"a": 1, "c": 2})
|
||||
new_res = Resources.subtract(resource_1, resource_2)
|
||||
assert "c" in new_res.custom_resources
|
||||
assert "b" in new_res.custom_resources
|
||||
self.assertTrue(new_res.cpu == 0)
|
||||
self.assertTrue(new_res.gpu == 0)
|
||||
self.assertTrue(new_res.extra_cpu == 0)
|
||||
self.assertTrue(new_res.extra_gpu == 0)
|
||||
self.assertTrue(new_res.get("a") == 0)
|
||||
|
||||
def testSerialization(self):
|
||||
original = Resources(1, 0, 0, 1, custom_resources={"a": 1, "b": 2})
|
||||
jsoned = resources_to_json(original)
|
||||
new_resource = json_to_resources(jsoned)
|
||||
self.assertEquals(original, new_resource)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
import sys
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
import ray
|
||||
from ray.rllib import _register_all
|
||||
|
||||
from ray.tune import TuneError
|
||||
from ray.tune.schedulers import FIFOScheduler
|
||||
from ray.tune.result import DONE
|
||||
from ray.tune.registry import _global_registry, TRAINABLE_CLASS
|
||||
from ray.tune.trial import Trial
|
||||
from ray.tune.trial_runner import TrialRunner
|
||||
from ray.tune.resources import Resources
|
||||
from ray.tune.suggest import BasicVariantGenerator
|
||||
|
||||
if sys.version_info >= (3, 3):
|
||||
from unittest.mock import patch
|
||||
else:
|
||||
from mock import patch
|
||||
|
||||
|
||||
def create_mock_components():
|
||||
class _MockScheduler(FIFOScheduler):
|
||||
errored_trials = []
|
||||
|
||||
def on_trial_error(self, trial_runner, trial):
|
||||
self.errored_trials += [trial]
|
||||
|
||||
class _MockSearchAlg(BasicVariantGenerator):
|
||||
errored_trials = []
|
||||
|
||||
def on_trial_complete(self, trial_id, error=False, **kwargs):
|
||||
if error:
|
||||
self.errored_trials += [trial_id]
|
||||
|
||||
searchalg = _MockSearchAlg()
|
||||
scheduler = _MockScheduler()
|
||||
return searchalg, scheduler
|
||||
|
||||
|
||||
class TrialRunnerTest2(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
ray.shutdown()
|
||||
_register_all() # re-register the evicted objects
|
||||
|
||||
def testErrorHandling(self):
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 1
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
_global_registry.register(TRAINABLE_CLASS, "asdf", None)
|
||||
trials = [Trial("asdf", **kwargs), Trial("__fake", **kwargs)]
|
||||
for t in trials:
|
||||
runner.add_trial(t)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
|
||||
def testThrowOnOverstep(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
runner.step()
|
||||
self.assertRaises(TuneError, runner.step)
|
||||
|
||||
def testFailureRecoveryDisabled(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
searchalg, scheduler = create_mock_components()
|
||||
|
||||
runner = TrialRunner(searchalg, scheduler=scheduler)
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
"checkpoint_freq": 1,
|
||||
"max_failures": 0,
|
||||
"config": {
|
||||
"mock_error": True,
|
||||
},
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[0].num_failures, 1)
|
||||
self.assertEqual(len(searchalg.errored_trials), 1)
|
||||
self.assertEqual(len(scheduler.errored_trials), 1)
|
||||
|
||||
def testFailureRecoveryEnabled(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
searchalg, scheduler = create_mock_components()
|
||||
|
||||
runner = TrialRunner(searchalg, scheduler=scheduler)
|
||||
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
"checkpoint_freq": 1,
|
||||
"max_failures": 1,
|
||||
"config": {
|
||||
"mock_error": True,
|
||||
},
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[0].num_failures, 1)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(len(searchalg.errored_trials), 0)
|
||||
self.assertEqual(len(scheduler.errored_trials), 0)
|
||||
|
||||
def testFailureRecoveryNodeRemoval(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
searchalg, scheduler = create_mock_components()
|
||||
|
||||
runner = TrialRunner(searchalg, scheduler=scheduler)
|
||||
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
"checkpoint_freq": 1,
|
||||
"max_failures": 1,
|
||||
"config": {
|
||||
"mock_error": True,
|
||||
},
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
with patch("ray.cluster_resources") as resource_mock:
|
||||
resource_mock.return_value = {"CPU": 1, "GPU": 1}
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
# Mimic a node failure
|
||||
resource_mock.return_value = {"CPU": 0, "GPU": 0}
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.PENDING)
|
||||
self.assertEqual(trials[0].num_failures, 1)
|
||||
self.assertEqual(len(searchalg.errored_trials), 0)
|
||||
self.assertEqual(len(scheduler.errored_trials), 1)
|
||||
|
||||
def testFailureRecoveryMaxFailures(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
"checkpoint_freq": 1,
|
||||
"max_failures": 2,
|
||||
"config": {
|
||||
"mock_error": True,
|
||||
"persistent_error": True,
|
||||
},
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[0].num_failures, 1)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[0].num_failures, 2)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[0].num_failures, 3)
|
||||
|
||||
def testCheckpointing(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 1
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.set_info.remote(1)), 1)
|
||||
path = runner.trial_executor.save(trials[0])
|
||||
kwargs["restore_path"] = path
|
||||
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[1].runner.get_info.remote()), 1)
|
||||
self.addCleanup(os.remove, path)
|
||||
|
||||
def testRestoreMetricsAfterCheckpointing(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.set_info.remote(1)), 1)
|
||||
path = runner.trial_executor.save(trials[0])
|
||||
runner.trial_executor.stop_trial(trials[0])
|
||||
kwargs["restore_path"] = path
|
||||
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertEqual(trials[1].last_result["timesteps_since_restore"], 10)
|
||||
self.assertEqual(trials[1].last_result["iterations_since_restore"], 1)
|
||||
self.assertGreater(trials[1].last_result["time_since_restore"], 0)
|
||||
runner.step()
|
||||
self.assertEqual(trials[1].last_result["timesteps_since_restore"], 20)
|
||||
self.assertEqual(trials[1].last_result["iterations_since_restore"], 2)
|
||||
self.assertGreater(trials[1].last_result["time_since_restore"], 0)
|
||||
self.addCleanup(os.remove, path)
|
||||
|
||||
def testCheckpointingAtEnd(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 2
|
||||
},
|
||||
"checkpoint_at_end": True,
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].last_result[DONE], True)
|
||||
self.assertEqual(trials[0].has_checkpoint(), True)
|
||||
|
||||
def testResultDone(self):
|
||||
"""Tests that last_result is marked `done` after trial is complete."""
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 2
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
runner.step()
|
||||
self.assertNotEqual(trials[0].last_result[DONE], True)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].last_result[DONE], True)
|
||||
|
||||
def testPauseThenResume(self):
|
||||
ray.init(num_cpus=1, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 2
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.get_info.remote()), None)
|
||||
|
||||
self.assertEqual(ray.get(trials[0].runner.set_info.remote(1)), 1)
|
||||
|
||||
runner.trial_executor.pause_trial(trials[0])
|
||||
self.assertEqual(trials[0].status, Trial.PAUSED)
|
||||
|
||||
runner.trial_executor.resume_trial(trials[0])
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.get_info.remote()), 1)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
import sys
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
@@ -0,0 +1,539 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import ray
|
||||
from ray.rllib import _register_all
|
||||
|
||||
from ray.tune import TuneError
|
||||
from ray.tune.schedulers import TrialScheduler, FIFOScheduler
|
||||
from ray.tune.experiment import Experiment
|
||||
from ray.tune.trial import Trial
|
||||
from ray.tune.trial_runner import TrialRunner
|
||||
from ray.tune.resources import Resources, json_to_resources, resources_to_json
|
||||
from ray.tune.suggest.suggestion import (_MockSuggestionAlgorithm,
|
||||
SuggestionAlgorithm)
|
||||
|
||||
|
||||
class TrialRunnerTest3(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
ray.shutdown()
|
||||
_register_all() # re-register the evicted objects
|
||||
|
||||
def testStepHook(self):
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
runner = TrialRunner()
|
||||
|
||||
def on_step_begin(self, trialrunner):
|
||||
self._update_avail_resources()
|
||||
cnt = self.pre_step if hasattr(self, "pre_step") else 0
|
||||
setattr(self, "pre_step", cnt + 1)
|
||||
|
||||
def on_step_end(self, trialrunner):
|
||||
cnt = self.pre_step if hasattr(self, "post_step") else 0
|
||||
setattr(self, "post_step", 1 + cnt)
|
||||
|
||||
import types
|
||||
runner.trial_executor.on_step_begin = types.MethodType(
|
||||
on_step_begin, runner.trial_executor)
|
||||
runner.trial_executor.on_step_end = types.MethodType(
|
||||
on_step_end, runner.trial_executor)
|
||||
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 5
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
runner.add_trial(Trial("__fake", **kwargs))
|
||||
runner.step()
|
||||
self.assertEqual(runner.trial_executor.pre_step, 1)
|
||||
self.assertEqual(runner.trial_executor.post_step, 1)
|
||||
|
||||
def testStopTrial(self):
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {
|
||||
"training_iteration": 5
|
||||
},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
trials = [
|
||||
Trial("__fake", **kwargs),
|
||||
Trial("__fake", **kwargs),
|
||||
Trial("__fake", **kwargs),
|
||||
Trial("__fake", **kwargs)
|
||||
]
|
||||
for t in trials:
|
||||
runner.add_trial(t)
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
# Stop trial while running
|
||||
runner.stop_trial(trials[0])
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[-1].status, Trial.PENDING)
|
||||
|
||||
# Stop trial while pending
|
||||
runner.stop_trial(trials[-1])
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[-1].status, Trial.TERMINATED)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[2].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[-1].status, Trial.TERMINATED)
|
||||
|
||||
def testSearchAlgNotification(self):
|
||||
"""Checks notification of trial to the Search Algorithm."""
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 2}}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(max_concurrent=10)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher)
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
|
||||
self.assertEqual(searcher.counter["result"], 1)
|
||||
self.assertEqual(searcher.counter["complete"], 1)
|
||||
|
||||
def testSearchAlgFinished(self):
|
||||
"""Checks that SearchAlg is Finished before all trials are done."""
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 1}}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(max_concurrent=10)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher)
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(len(searcher.live_trials), 0)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertTrue(runner.is_finished())
|
||||
|
||||
def testSearchAlgSchedulerInteraction(self):
|
||||
"""Checks that TrialScheduler killing trial will notify SearchAlg."""
|
||||
|
||||
class _MockScheduler(FIFOScheduler):
|
||||
def on_trial_result(self, *args, **kwargs):
|
||||
return TrialScheduler.STOP
|
||||
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 2}}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(max_concurrent=10)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher, scheduler=_MockScheduler())
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(len(searcher.live_trials), 0)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertTrue(runner.is_finished())
|
||||
|
||||
def testSearchAlgSchedulerEarlyStop(self):
|
||||
"""Early termination notif to Searcher can be turned off."""
|
||||
|
||||
class _MockScheduler(FIFOScheduler):
|
||||
def on_trial_result(self, *args, **kwargs):
|
||||
return TrialScheduler.STOP
|
||||
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {"run": "__fake", "stop": {"training_iteration": 2}}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(use_early_stopped_trials=True)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher, scheduler=_MockScheduler())
|
||||
runner.step()
|
||||
runner.step()
|
||||
self.assertEqual(len(searcher.final_results), 1)
|
||||
|
||||
searcher = _MockSuggestionAlgorithm(use_early_stopped_trials=False)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher, scheduler=_MockScheduler())
|
||||
runner.step()
|
||||
runner.step()
|
||||
self.assertEqual(len(searcher.final_results), 0)
|
||||
|
||||
def testSearchAlgStalled(self):
|
||||
"""Checks that runner and searcher state is maintained when stalled."""
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
experiment_spec = {
|
||||
"run": "__fake",
|
||||
"num_samples": 3,
|
||||
"stop": {
|
||||
"training_iteration": 1
|
||||
}
|
||||
}
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher = _MockSuggestionAlgorithm(max_concurrent=1)
|
||||
searcher.add_configurations(experiments)
|
||||
runner = TrialRunner(search_alg=searcher)
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
|
||||
trials = runner.get_trials()
|
||||
runner.step()
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
self.assertEqual(len(searcher.live_trials), 1)
|
||||
|
||||
searcher.stall = True
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[1].status, Trial.TERMINATED)
|
||||
self.assertEqual(len(searcher.live_trials), 0)
|
||||
|
||||
self.assertTrue(all(trial.is_finished() for trial in trials))
|
||||
self.assertFalse(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
|
||||
searcher.stall = False
|
||||
|
||||
runner.step()
|
||||
trials = runner.get_trials()
|
||||
self.assertEqual(trials[2].status, Trial.RUNNING)
|
||||
self.assertEqual(len(searcher.live_trials), 1)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[2].status, Trial.TERMINATED)
|
||||
self.assertEqual(len(searcher.live_trials), 0)
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertTrue(runner.is_finished())
|
||||
|
||||
def testSearchAlgFinishes(self):
|
||||
"""Empty SearchAlg changing state in `next_trials` does not crash."""
|
||||
|
||||
class FinishFastAlg(SuggestionAlgorithm):
|
||||
_index = 0
|
||||
|
||||
def next_trials(self):
|
||||
trials = []
|
||||
self._index += 1
|
||||
|
||||
for trial in self._trial_generator:
|
||||
trials += [trial]
|
||||
break
|
||||
|
||||
if self._index > 4:
|
||||
self._finished = True
|
||||
return trials
|
||||
|
||||
def _suggest(self, trial_id):
|
||||
return {}
|
||||
|
||||
ray.init(num_cpus=2)
|
||||
experiment_spec = {
|
||||
"run": "__fake",
|
||||
"num_samples": 2,
|
||||
"stop": {
|
||||
"training_iteration": 1
|
||||
}
|
||||
}
|
||||
searcher = FinishFastAlg()
|
||||
experiments = [Experiment.from_json("test", experiment_spec)]
|
||||
searcher.add_configurations(experiments)
|
||||
|
||||
runner = TrialRunner(search_alg=searcher)
|
||||
self.assertFalse(runner.is_finished())
|
||||
runner.step() # This launches a new run
|
||||
runner.step() # This launches a 2nd run
|
||||
self.assertFalse(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
runner.step() # This kills the first run
|
||||
self.assertFalse(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
runner.step() # This kills the 2nd run
|
||||
self.assertFalse(searcher.is_finished())
|
||||
self.assertFalse(runner.is_finished())
|
||||
runner.step() # this converts self._finished to True
|
||||
self.assertTrue(searcher.is_finished())
|
||||
self.assertRaises(TuneError, runner.step)
|
||||
|
||||
def testTrialSaveRestore(self):
|
||||
"""Creates different trials to test runner.checkpoint/restore."""
|
||||
ray.init(num_cpus=3)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
trials = [
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="trial_terminate",
|
||||
stopping_criterion={"training_iteration": 1},
|
||||
checkpoint_freq=1)
|
||||
]
|
||||
runner.add_trial(trials[0])
|
||||
runner.step() # start
|
||||
runner.step()
|
||||
self.assertEquals(trials[0].status, Trial.TERMINATED)
|
||||
|
||||
trials += [
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="trial_fail",
|
||||
stopping_criterion={"training_iteration": 3},
|
||||
checkpoint_freq=1,
|
||||
config={"mock_error": True})
|
||||
]
|
||||
runner.add_trial(trials[1])
|
||||
runner.step()
|
||||
runner.step()
|
||||
runner.step()
|
||||
self.assertEquals(trials[1].status, Trial.ERROR)
|
||||
|
||||
trials += [
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="trial_succ",
|
||||
stopping_criterion={"training_iteration": 2},
|
||||
checkpoint_freq=1)
|
||||
]
|
||||
runner.add_trial(trials[2])
|
||||
runner.step()
|
||||
self.assertEquals(len(runner.trial_executor.get_checkpoints()), 3)
|
||||
self.assertEquals(trials[2].status, Trial.RUNNING)
|
||||
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
for tid in ["trial_terminate", "trial_fail"]:
|
||||
original_trial = runner.get_trial(tid)
|
||||
restored_trial = runner2.get_trial(tid)
|
||||
self.assertEqual(original_trial.status, restored_trial.status)
|
||||
|
||||
restored_trial = runner2.get_trial("trial_succ")
|
||||
self.assertEqual(Trial.PENDING, restored_trial.status)
|
||||
|
||||
runner2.step()
|
||||
runner2.step()
|
||||
runner2.step()
|
||||
self.assertRaises(TuneError, runner2.step)
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def testTrialNoSave(self):
|
||||
"""Check that non-checkpointing trials are not saved."""
|
||||
ray.init(num_cpus=3)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
runner.add_trial(
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="non_checkpoint",
|
||||
stopping_criterion={"training_iteration": 2}))
|
||||
|
||||
while not all(t.status == Trial.TERMINATED
|
||||
for t in runner.get_trials()):
|
||||
runner.step()
|
||||
|
||||
runner.add_trial(
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="checkpoint",
|
||||
checkpoint_at_end=True,
|
||||
stopping_criterion={"training_iteration": 2}))
|
||||
|
||||
while not all(t.status == Trial.TERMINATED
|
||||
for t in runner.get_trials()):
|
||||
runner.step()
|
||||
|
||||
runner.add_trial(
|
||||
Trial(
|
||||
"__fake",
|
||||
trial_id="pending",
|
||||
stopping_criterion={"training_iteration": 2}))
|
||||
|
||||
runner.step()
|
||||
runner.step()
|
||||
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
new_trials = runner2.get_trials()
|
||||
self.assertEquals(len(new_trials), 3)
|
||||
self.assertTrue(
|
||||
runner2.get_trial("non_checkpoint").status == Trial.TERMINATED)
|
||||
self.assertTrue(
|
||||
runner2.get_trial("checkpoint").status == Trial.TERMINATED)
|
||||
self.assertTrue(runner2.get_trial("pending").status == Trial.PENDING)
|
||||
self.assertTrue(not runner2.get_trial("pending").last_result)
|
||||
runner2.step()
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def testCheckpointWithFunction(self):
|
||||
ray.init()
|
||||
trial = Trial(
|
||||
"__fake",
|
||||
config={"callbacks": {
|
||||
"on_episode_start": lambda i: i,
|
||||
}},
|
||||
checkpoint_freq=1)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
runner.add_trial(trial)
|
||||
for i in range(5):
|
||||
runner.step()
|
||||
# force checkpoint
|
||||
runner.checkpoint()
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
new_trial = runner2.get_trials()[0]
|
||||
self.assertTrue("callbacks" in new_trial.config)
|
||||
self.assertTrue("on_episode_start" in new_trial.config["callbacks"])
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def testCheckpointOverwrite(self):
|
||||
def count_checkpoints(cdir):
|
||||
return sum((fname.startswith("experiment_state")
|
||||
and fname.endswith(".json"))
|
||||
for fname in os.listdir(cdir))
|
||||
|
||||
ray.init()
|
||||
trial = Trial("__fake", checkpoint_freq=1)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
runner.add_trial(trial)
|
||||
for i in range(5):
|
||||
runner.step()
|
||||
# force checkpoint
|
||||
runner.checkpoint()
|
||||
self.assertEquals(count_checkpoints(tmpdir), 1)
|
||||
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
for i in range(5):
|
||||
runner2.step()
|
||||
self.assertEquals(count_checkpoints(tmpdir), 2)
|
||||
|
||||
runner2.checkpoint()
|
||||
self.assertEquals(count_checkpoints(tmpdir), 2)
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def testUserCheckpoint(self):
|
||||
ray.init(num_cpus=3)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
runner = TrialRunner(local_checkpoint_dir=tmpdir, checkpoint_period=0)
|
||||
runner.add_trial(Trial("__fake", config={"user_checkpoint_freq": 2}))
|
||||
trials = runner.get_trials()
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(ray.get(trials[0].runner.set_info.remote(1)), 1)
|
||||
runner.step() # 0
|
||||
self.assertFalse(trials[0].has_checkpoint())
|
||||
runner.step() # 1
|
||||
self.assertFalse(trials[0].has_checkpoint())
|
||||
runner.step() # 2
|
||||
self.assertTrue(trials[0].has_checkpoint())
|
||||
|
||||
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
|
||||
runner2.step()
|
||||
trials2 = runner2.get_trials()
|
||||
self.assertEqual(ray.get(trials2[0].runner.get_info.remote()), 1)
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
class SearchAlgorithmTest(unittest.TestCase):
|
||||
def testNestedSuggestion(self):
|
||||
class TestSuggestion(SuggestionAlgorithm):
|
||||
def _suggest(self, trial_id):
|
||||
return {"a": {"b": {"c": {"d": 4, "e": 5}}}}
|
||||
|
||||
alg = TestSuggestion()
|
||||
alg.add_configurations({"test": {"run": "__fake"}})
|
||||
trial = alg.next_trials()[0]
|
||||
self.assertTrue("e=5" in trial.experiment_tag)
|
||||
self.assertTrue("d=4" in trial.experiment_tag)
|
||||
|
||||
|
||||
class ResourcesTest(unittest.TestCase):
|
||||
def testSubtraction(self):
|
||||
resource_1 = Resources(
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
custom_resources={
|
||||
"a": 1,
|
||||
"b": 2
|
||||
},
|
||||
extra_custom_resources={
|
||||
"a": 1,
|
||||
"b": 1
|
||||
})
|
||||
resource_2 = Resources(
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
custom_resources={
|
||||
"a": 1,
|
||||
"b": 2
|
||||
},
|
||||
extra_custom_resources={
|
||||
"a": 1,
|
||||
"b": 1
|
||||
})
|
||||
new_res = Resources.subtract(resource_1, resource_2)
|
||||
self.assertTrue(new_res.cpu == 0)
|
||||
self.assertTrue(new_res.gpu == 0)
|
||||
self.assertTrue(new_res.extra_cpu == 0)
|
||||
self.assertTrue(new_res.extra_gpu == 0)
|
||||
self.assertTrue(all(k == 0 for k in new_res.custom_resources.values()))
|
||||
self.assertTrue(
|
||||
all(k == 0 for k in new_res.extra_custom_resources.values()))
|
||||
|
||||
def testDifferentResources(self):
|
||||
resource_1 = Resources(1, 0, 0, 1, custom_resources={"a": 1, "b": 2})
|
||||
resource_2 = Resources(1, 0, 0, 1, custom_resources={"a": 1, "c": 2})
|
||||
new_res = Resources.subtract(resource_1, resource_2)
|
||||
assert "c" in new_res.custom_resources
|
||||
assert "b" in new_res.custom_resources
|
||||
self.assertTrue(new_res.cpu == 0)
|
||||
self.assertTrue(new_res.gpu == 0)
|
||||
self.assertTrue(new_res.extra_cpu == 0)
|
||||
self.assertTrue(new_res.extra_gpu == 0)
|
||||
self.assertTrue(new_res.get("a") == 0)
|
||||
|
||||
def testSerialization(self):
|
||||
original = Resources(1, 0, 0, 1, custom_resources={"a": 1, "b": 2})
|
||||
jsoned = resources_to_json(original)
|
||||
new_resource = json_to_resources(jsoned)
|
||||
self.assertEquals(original, new_resource)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
sys.exit(pytest.main(["-v", __file__]))
|
||||
Reference in New Issue
Block a user