wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-16 11:27:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] API revamp fix (#10518)

Browse Source
This commit is contained in:
Richard Liaw
2020-09-05 15:34:53 -07:00
committed by GitHub
parent 8a891b3c30
commit 551c597312
26 changed files with 349 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/tests/_test_cluster_interrupt_searcher.py
+2 -1
View File
@@ -1,3 +1,4 @@
import os
import argparse
from ray.tune import run
@@ -44,9 +45,9 @@ if __name__ == "__main__":
algo = ConcurrencyLimiter(algo, max_concurrent=1)
from ray.tune import register_trainable
register_trainable("trainable", MyTrainableClass)
os.environ["TUNE_GLOBAL_CHECKPOINT_S"] = "0"
run("trainable",
search_alg=algo,
global_checkpoint_period=0,
resume=args.resume,
verbose=0,
num_samples=20,
python/ray/tune/tests/test_api.py
+14 -13
View File
@@ -245,17 +245,19 @@ class TrainableFunctionApiTest(unittest.TestCase):
register_trainable("B", B)
def f(cpus, gpus, queue_trials):
return run_experiments(
{
"foo": {
"run": "B",
"config": {
"cpu": cpus,
"gpu": gpus,
},
}
},
queue_trials=queue_trials)[0]
if not queue_trials:
os.environ["TUNE_DISABLE_QUEUE_TRIALS"] = "1"
else:
os.environ.pop("TUNE_DISABLE_QUEUE_TRIALS", None)
return run_experiments({
"foo": {
"run": "B",
"config": {
"cpu": cpus,
"gpu": gpus,
},
}
})[0]
# Should all succeed
self.assertEqual(f(0, 0, False).status, Trial.TERMINATED)
@@ -639,8 +641,7 @@ class TrainableFunctionApiTest(unittest.TestCase):
loggers=None)
trials = tune.run(test, raise_on_failed_trial=False, **config).trials
self.assertEqual(Counter(t.status for t in trials)["ERROR"], 5)
new_trials = tune.run(
test, resume=True, run_errored_only=True, **config).trials
new_trials = tune.run(test, resume="ERRORED_ONLY", **config).trials
self.assertEqual(Counter(t.status for t in new_trials)["ERROR"], 0)
self.assertTrue(
all(t.last_result.get("hello") == 123 for t in new_trials))
python/ray/tune/tests/test_cluster.py
+3 -1
View File
@@ -642,10 +642,13 @@ def test_cluster_interrupt(start_connected_cluster, tmpdir):
for line in inspect.getsource(_Mock).split("\n"))
script = """
import os
import time
import ray
from ray import tune
os.environ["TUNE_GLOBAL_CHECKPOINT_S"] = "0"
ray.init(address="{address}")
{fail_class_code}
@@ -656,7 +659,6 @@ tune.run(
stop=dict(training_iteration=5),
local_dir="{checkpoint_dir}",
checkpoint_freq=1,
global_checkpoint_period=0,
max_failures=1,
raise_on_failed_trial=False)
""".format(
python/ray/tune/tests/test_experiment_analysis.py
-1
View File
@@ -147,7 +147,6 @@ class ExperimentAnalysisSuite(unittest.TestCase):
MyTrainableClass,
name="test_example",
local_dir=self.test_dir,
return_trials=False,
stop={"training_iteration": 1},
num_samples=1,
config={
python/ray/tune/tests/test_experiment_analysis_mem.py
-1
View File
@@ -135,7 +135,6 @@ class AnalysisSuite(unittest.TestCase):
run(MyTrainableClass,
name=test_name,
local_dir=self.test_dir,
return_trials=False,
stop={"training_iteration": 1},
num_samples=self.num_samples,
config={
python/ray/tune/tests/test_ray_trial_executor.py
+4 -4
View File
@@ -16,7 +16,7 @@ from ray.cluster_utils import Cluster
class RayTrialExecutorTest(unittest.TestCase):
def setUp(self):
self.trial_executor = RayTrialExecutor(queue_trials=False)
ray.init()
ray.init(ignore_reinit_error=True)
_register_all() # Needed for flaky tests
def tearDown(self):
@@ -182,8 +182,6 @@ class RayTrialExecutorTest(unittest.TestCase):
class RayExecutorQueueTest(unittest.TestCase):
def setUp(self):
self.trial_executor = RayTrialExecutor(
queue_trials=True, refresh_period=0)
self.cluster = Cluster(
initialize_head=True,
connect=True,
@@ -193,6 +191,8 @@ class RayExecutorQueueTest(unittest.TestCase):
"num_heartbeats_timeout": 10
}
})
self.trial_executor = RayTrialExecutor(
queue_trials=True, refresh_period=0)
# Pytest doesn't play nicely with imports
_register_all()
@@ -247,8 +247,8 @@ class RayExecutorQueueTest(unittest.TestCase):
class LocalModeExecutorTest(RayTrialExecutorTest):
def setUp(self):
self.trial_executor = RayTrialExecutor(queue_trials=False)
ray.init(local_mode=True)
self.trial_executor = RayTrialExecutor(queue_trials=False)
def tearDown(self):
ray.shutdown()
python/ray/tune/tests/test_sync.py
+60 -53
View File
@@ -31,12 +31,11 @@ class TestSyncFunctionality(unittest.TestCase):
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_cloud": "echo {source} {target}"
}).trials
stop={
"training_iteration": 1
},
sync_config=tune.SyncConfig(
**{"sync_to_cloud": "echo {source} {target}"})).trials
@patch("ray.tune.sync_client.S3_PREFIX", "test")
def testCloudProperString(self):
@@ -45,26 +44,26 @@ class TestSyncFunctionality(unittest.TestCase):
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
stop={
"training_iteration": 1
},
sync_config=tune.SyncConfig(**{
"upload_dir": "test",
"sync_to_cloud": "ls {target}"
}).trials
})).trials
with self.assertRaises(ValueError):
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
stop={
"training_iteration": 1
},
sync_config=tune.SyncConfig(**{
"upload_dir": "test",
"sync_to_cloud": "ls {source}"
}).trials
})).trials
tmpdir = tempfile.mkdtemp()
logfile = os.path.join(tmpdir, "test.log")
@@ -73,13 +72,14 @@ class TestSyncFunctionality(unittest.TestCase):
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"upload_dir": "test",
"sync_to_cloud": "echo {source} {target} > " + logfile
}).trials
stop={
"training_iteration": 1
},
sync_config=tune.SyncConfig(
**{
"upload_dir": "test",
"sync_to_cloud": "echo {source} {target} > " + logfile
})).trials
with open(logfile) as f:
lines = f.read()
self.assertTrue("test" in lines)
@@ -89,42 +89,41 @@ class TestSyncFunctionality(unittest.TestCase):
"""Tests that invalid commands throw.."""
with self.assertRaises(TuneError):
# This raises TuneError because logger is init in safe zone.
sync_config = tune.SyncConfig(sync_to_driver="ls {target}")
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_driver": "ls {target}"
}).trials
stop={
"training_iteration": 1
},
sync_config=sync_config,
).trials
with self.assertRaises(TuneError):
# This raises TuneError because logger is init in safe zone.
sync_config = tune.SyncConfig(sync_to_driver="ls {source}")
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_driver": "ls {source}"
sync_config=sync_config,
stop={
"training_iteration": 1
}).trials
with patch.object(CommandBasedClient, "_execute") as mock_fn:
with patch("ray.services.get_node_ip_address") as mock_sync:
sync_config = tune.SyncConfig(
sync_to_driver="echo {source} {target}")
mock_sync.return_value = "0.0.0.0"
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_driver": "echo {source} {target}"
sync_config=sync_config,
stop={
"training_iteration": 1
}).trials
self.assertGreater(mock_fn.call_count, 0)
@@ -137,6 +136,8 @@ class TestSyncFunctionality(unittest.TestCase):
for filename in glob.glob(os.path.join(local, "*.json")):
shutil.copy(filename, remote)
sync_config = tune.SyncConfig(
upload_dir=tmpdir2, sync_to_cloud=sync_func)
[trial] = tune.run(
"__fake",
name="foo",
@@ -145,8 +146,7 @@ class TestSyncFunctionality(unittest.TestCase):
stop={
"training_iteration": 1
},
upload_dir=tmpdir2,
sync_to_cloud=sync_func).trials
sync_config=sync_config).trials
test_file_path = glob.glob(os.path.join(tmpdir2, "foo", "*.json"))
self.assertTrue(test_file_path)
shutil.rmtree(tmpdir)
@@ -167,18 +167,21 @@ class TestSyncFunctionality(unittest.TestCase):
def counter(local, remote):
mock()
tune.syncer.CLOUD_SYNC_PERIOD = 1
sync_config = tune.SyncConfig(
upload_dir="test", sync_to_cloud=counter, cloud_sync_period=1)
# This was originally set to 0.5
os.environ["TUNE_GLOBAL_CHECKPOINT_S"] = "0"
self.addCleanup(
lambda: os.environ.pop("TUNE_GLOBAL_CHECKPOINT_S", None))
[trial] = tune.run(
trainable,
name="foo",
max_failures=0,
local_dir=tmpdir,
upload_dir="test",
sync_to_cloud=counter,
stop={
"training_iteration": 10
},
global_checkpoint_period=0.5,
sync_config=sync_config,
).trials
self.assertEqual(mock.call_count, 12)
@@ -192,6 +195,9 @@ class TestSyncFunctionality(unittest.TestCase):
print("writing to", f.name)
f.write(source)
sync_config = tune.SyncConfig(
sync_to_driver=sync_func_driver, node_sync_period=5)
[trial] = tune.run(
"__fake",
name="foo",
@@ -199,12 +205,13 @@ class TestSyncFunctionality(unittest.TestCase):
stop={
"training_iteration": 1
},
sync_to_driver=sync_func_driver).trials
sync_config=sync_config).trials
test_file_path = os.path.join(trial.logdir, "test.log2")
self.assertFalse(os.path.exists(test_file_path))
with patch("ray.services.get_node_ip_address") as mock_sync:
mock_sync.return_value = "0.0.0.0"
sync_config = tune.SyncConfig(sync_to_driver=sync_func_driver)
[trial] = tune.run(
"__fake",
name="foo",
@@ -212,7 +219,7 @@ class TestSyncFunctionality(unittest.TestCase):
stop={
"training_iteration": 1
},
sync_to_driver=sync_func_driver).trials
sync_config=sync_config).trials
test_file_path = os.path.join(trial.logdir, "test.log2")
self.assertTrue(os.path.exists(test_file_path))
os.remove(test_file_path)
@@ -223,17 +230,17 @@ class TestSyncFunctionality(unittest.TestCase):
def sync_func(source, target):
pass
sync_config = tune.SyncConfig(sync_to_driver=sync_func)
with patch.object(CommandBasedClient, "_execute") as mock_sync:
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_driver": sync_func
}).trials
stop={
"training_iteration": 1
},
sync_config=sync_config).trials
self.assertEqual(mock_sync.call_count, 0)
python/ray/tune/tests/test_trial_runner_3.py
+2 -7
View File
@@ -385,10 +385,7 @@ class TrialRunnerTest3(unittest.TestCase):
assert trials[0].status == Trial.ERROR
del runner
new_runner = TrialRunner(
run_errored_only=False,
resume=True,
local_checkpoint_dir=self.tmpdir)
new_runner = TrialRunner(resume=True, local_checkpoint_dir=self.tmpdir)
assert len(new_runner.get_trials()) == 3
assert Trial.ERROR in (t.status for t in new_runner.get_trials())
@@ -418,9 +415,7 @@ class TrialRunnerTest3(unittest.TestCase):
del runner
new_runner = TrialRunner(
run_errored_only=True,
resume=True,
local_checkpoint_dir=self.tmpdir)
resume="ERRORED_ONLY", local_checkpoint_dir=self.tmpdir)
assert len(new_runner.get_trials()) == 3
assert Trial.ERROR not in (t.status for t in new_runner.get_trials())
# The below is just a check for standard behavior.
python/ray/tune/tests/test_tune_server.py
+1 -1
View File
@@ -29,7 +29,7 @@ class TuneServerSuite(unittest.TestCase):
def basicSetup(self):
ray.init(num_cpus=4, num_gpus=1)
port = get_valid_port()
self.runner = TrialRunner(launch_web_server=True, server_port=port)
self.runner = TrialRunner(server_port=port)
runner = self.runner
kwargs = {
"stopping_criterion": {