wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-12 12:51:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] Change the log syncing behavior (#4450)

Browse Source 
* Change the log syncing behavior

* fix up abstractions for syncer

* Finished checkpoint syncing

* Code

* Set of changes to get things running

* Fixes for log syncing

* Fix parts

* Lint and other fixes

* fix some test

* Remove extra parsing functionality

* some test fixes

* Fix up cloud syncing

* Another thing to do

* Fix up tests and local sync

Changes LogSync into a mixin, and adds tests for different
functionalities.

* Fix up tests, start on local migration

* fix distributed migrations

* comments

* formatting

* Better checkpoint directory handling

* fix tests

* fix tests

* fix click

* comments

* formatting comments

* formatting and comments

* sync function deprecations

* syncfunction

* Add documentation for Syncing and Uploading

* nit

* BaseSyncer as base for Mixin in edge case

* more docs

* clean up assertions

* validate

* nit

* Update test_cluster.py

* betterdoc

* Update tune-usage.rst

* cleanup

* nit
This commit is contained in:
Kristian Hartikainen
2019-07-02 20:46:00 -07:00
committed by Richard Liaw
parent 71d4637b75
commit 9e0192bc0b
14 changed files with 718 additions and 466 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/tests/test_trial_runner.py
+178 -52
View File
@@ -3,6 +3,7 @@ from __future__ import division
from __future__ import print_function
import copy
import glob
import os
import shutil
import sys
@@ -315,21 +316,6 @@ class TrainableFunctionApiTest(unittest.TestCase):
}
})
def testUploadDirNone(self):
def train(config, reporter):
reporter(timesteps_total=1)
[trial] = run_experiments({
"foo": {
"run": train,
"upload_dir": None,
"config": {
"a": "b"
},
}
})
self.assertFalse(trial.upload_dir)
def testLogdirStartingWithTilde(self):
local_dir = "~/ray_results/local_dir"
@@ -930,50 +916,190 @@ class RunExperimentTest(unittest.TestCase):
str(trial), "{}_{}_321".format(trial.trainable_name,
trial.trial_id))
def testSyncFunction(self):
def fail_sync_local():
[trial] = run_experiments({
"foo": {
"run": "__fake",
class TestSyncFunctionality(unittest.TestCase):
def setUp(self):
ray.init()
def tearDown(self):
ray.shutdown()
_register_all() # re-register the evicted objects
@patch("ray.tune.syncer.S3_PREFIX", "test")
def testNoUploadDir(self):
"""No Upload Dir is given."""
with self.assertRaises(AssertionError):
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_cloud": "echo {source} {target}"
})
@patch("ray.tune.syncer.S3_PREFIX", "test")
def testCloudProperString(self):
with self.assertRaises(ValueError):
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"upload_dir": "test",
"sync_function": "ls {remote_dir}"
}
})
"sync_to_cloud": "ls {target}"
})
self.assertRaises(AssertionError, fail_sync_local)
def fail_sync_remote():
[trial] = run_experiments({
"foo": {
"run": "__fake",
with self.assertRaises(ValueError):
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"upload_dir": "test",
"sync_function": "ls {local_dir}"
}
})
"sync_to_cloud": "ls {source}"
})
self.assertRaises(AssertionError, fail_sync_remote)
tmpdir = tempfile.mkdtemp()
logfile = os.path.join(tmpdir, "test.log")
def sync_func(local, remote):
with open(os.path.join(local, "test.log"), "w") as f:
f.write(remote)
[trial] = run_experiments({
"foo": {
"run": "__fake",
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"upload_dir": "test",
"sync_function": tune.function(sync_func)
}
})
self.assertTrue(os.path.exists(os.path.join(trial.logdir, "test.log")))
"sync_to_cloud": "echo {source} {target} > " + logfile
})
with open(logfile) as f:
lines = f.read()
self.assertTrue("test" in lines)
shutil.rmtree(tmpdir)
def testClusterProperString(self):
"""Tests that invalid commands throw.."""
with self.assertRaises(TuneError):
# This raises TuneError because logger is init in safe zone.
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_driver": "ls {target}"
})
with self.assertRaises(TuneError):
# This raises TuneError because logger is init in safe zone.
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_driver": "ls {source}"
})
with patch("ray.tune.syncer.CommandSyncer.sync_function"
) as mock_fn, patch(
"ray.services.get_node_ip_address") as mock_sync:
mock_sync.return_value = "0.0.0.0"
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"sync_to_driver": "echo {source} {target}"
})
self.assertGreater(mock_fn.call_count, 0)
def testCloudFunctions(self):
tmpdir = tempfile.mkdtemp()
tmpdir2 = tempfile.mkdtemp()
os.mkdir(os.path.join(tmpdir2, "foo"))
def sync_func(local, remote):
for filename in glob.glob(os.path.join(local, "*.json")):
shutil.copy(filename, remote)
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
local_dir=tmpdir,
stop={"training_iteration": 1},
upload_dir=tmpdir2,
sync_to_cloud=tune.function(sync_func))
test_file_path = glob.glob(os.path.join(tmpdir2, "foo", "*.json"))
self.assertTrue(test_file_path)
shutil.rmtree(tmpdir)
shutil.rmtree(tmpdir2)
def testClusterSyncFunction(self):
def sync_func_driver(source, target):
assert ":" in source, "Source not a remote path."
assert ":" not in target, "Target is supposed to be local."
with open(os.path.join(target, "test.log2"), "w") as f:
print("writing to", f.name)
f.write(source)
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
stop={"training_iteration": 1},
sync_to_driver=tune.function(sync_func_driver))
test_file_path = os.path.join(trial.logdir, "test.log2")
self.assertFalse(os.path.exists(test_file_path))
with patch("ray.services.get_node_ip_address") as mock_sync:
mock_sync.return_value = "0.0.0.0"
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
stop={"training_iteration": 1},
sync_to_driver=tune.function(sync_func_driver))
test_file_path = os.path.join(trial.logdir, "test.log2")
self.assertTrue(os.path.exists(test_file_path))
os.remove(test_file_path)
def testNoSync(self):
def sync_func(source, target):
pass
with patch("ray.tune.syncer.CommandSyncer.sync_function") as mock_sync:
[trial] = tune.run(
"__fake",
name="foo",
max_failures=0,
**{
"stop": {
"training_iteration": 1
},
"upload_dir": "test",
"sync_to_driver": tune.function(sync_func),
"sync_to_cloud": tune.function(sync_func)
})
self.assertEqual(mock_sync.call_count, 0)
class VariantGeneratorTest(unittest.TestCase):
@@ -1960,7 +2086,7 @@ class TrialRunnerTest(unittest.TestCase):
ray.init(num_cpus=3)
tmpdir = tempfile.mkdtemp()
runner = TrialRunner(metadata_checkpoint_dir=tmpdir)
runner = TrialRunner(local_checkpoint_dir=tmpdir)
trials = [
Trial(
"__fake",
@@ -1999,7 +2125,7 @@ class TrialRunnerTest(unittest.TestCase):
self.assertEquals(len(runner.trial_executor.get_checkpoints()), 3)
self.assertEquals(trials[2].status, Trial.RUNNING)
runner2 = TrialRunner.restore(tmpdir)
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
for tid in ["trial_terminate", "trial_fail"]:
original_trial = runner.get_trial(tid)
restored_trial = runner2.get_trial(tid)
@@ -2019,7 +2145,7 @@ class TrialRunnerTest(unittest.TestCase):
ray.init(num_cpus=3)
tmpdir = tempfile.mkdtemp()
runner = TrialRunner(metadata_checkpoint_dir=tmpdir)
runner = TrialRunner(local_checkpoint_dir=tmpdir)
runner.add_trial(
Trial(
@@ -2051,7 +2177,7 @@ class TrialRunnerTest(unittest.TestCase):
runner.step()
runner.step()
runner2 = TrialRunner.restore(tmpdir)
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
new_trials = runner2.get_trials()
self.assertEquals(len(new_trials), 3)
self.assertTrue(
@@ -2074,13 +2200,13 @@ class TrialRunnerTest(unittest.TestCase):
},
checkpoint_freq=1)
tmpdir = tempfile.mkdtemp()
runner = TrialRunner(metadata_checkpoint_dir=tmpdir)
runner = TrialRunner(local_checkpoint_dir=tmpdir)
runner.add_trial(trial)
for i in range(5):
runner.step()
# force checkpoint
runner.checkpoint()
runner2 = TrialRunner.restore(tmpdir)
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
new_trial = runner2.get_trials()[0]
self.assertTrue("callbacks" in new_trial.config)
self.assertTrue("on_episode_start" in new_trial.config["callbacks"])
@@ -2095,7 +2221,7 @@ class TrialRunnerTest(unittest.TestCase):
ray.init()
trial = Trial("__fake", checkpoint_freq=1)
tmpdir = tempfile.mkdtemp()
runner = TrialRunner(metadata_checkpoint_dir=tmpdir)
runner = TrialRunner(local_checkpoint_dir=tmpdir)
runner.add_trial(trial)
for i in range(5):
runner.step()
@@ -2103,7 +2229,7 @@ class TrialRunnerTest(unittest.TestCase):
runner.checkpoint()
self.assertEquals(count_checkpoints(tmpdir), 1)
runner2 = TrialRunner.restore(tmpdir)
runner2 = TrialRunner(resume="LOCAL", local_checkpoint_dir=tmpdir)
for i in range(5):
runner2.step()
self.assertEquals(count_checkpoints(tmpdir), 2)