mirror of
https://github.com/wassname/ray.git
synced 2026-08-08 11:25:28 +08:00
[rllib] Initial work on integrating hyperparameter search tool (#1107)
* clean up train * update * update train script * add tuned examples * add agent catalog * add tune lib * update * fix * testS * remove * train docs * comments * todo * fix resource parsing * fix cr test * add test * try to fix travis test
This commit is contained in:
Eric Liang
@@ -60,56 +60,56 @@ docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
python /ray/python/ray/rllib/train.py \
|
||||
--env PongDeterministic-v0 \
|
||||
--alg A3C \
|
||||
--num-iterations 2 \
|
||||
--stop '{"training_iteration": 2}' \
|
||||
--config '{"num_workers": 16}'
|
||||
|
||||
docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
python /ray/python/ray/rllib/train.py \
|
||||
--env CartPole-v1 \
|
||||
--alg PPO \
|
||||
--num-iterations 2 \
|
||||
--stop '{"training_iteration": 2}' \
|
||||
--config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "sgd_stepsize": 1e-4, "sgd_batchsize": 64, "timesteps_per_batch": 2000, "num_workers": 1, "model": {"free_log_std": true}}'
|
||||
|
||||
docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
python /ray/python/ray/rllib/train.py \
|
||||
--env CartPole-v1 \
|
||||
--alg PPO \
|
||||
--num-iterations 2 \
|
||||
--stop '{"training_iteration": 2}' \
|
||||
--config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "sgd_stepsize": 1e-4, "sgd_batchsize": 64, "timesteps_per_batch": 2000, "num_workers": 1, "use_gae": false}'
|
||||
|
||||
docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
python /ray/python/ray/rllib/train.py \
|
||||
--env Pendulum-v0 \
|
||||
--alg ES \
|
||||
--num-iterations 2 \
|
||||
--stop '{"training_iteration": 2}' \
|
||||
--config '{"stepsize": 0.01}'
|
||||
|
||||
docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
python /ray/python/ray/rllib/train.py \
|
||||
--env CartPole-v0 \
|
||||
--alg A3C \
|
||||
--num-iterations 2 \
|
||||
--stop '{"training_iteration": 2}' \
|
||||
--config '{"use_lstm": false}'
|
||||
|
||||
docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
python /ray/python/ray/rllib/train.py \
|
||||
--env CartPole-v0 \
|
||||
--alg DQN \
|
||||
--num-iterations 2 \
|
||||
--stop '{"training_iteration": 2}' \
|
||||
--config '{"lr": 1e-3, "schedule_max_timesteps": 100000, "exploration_fraction": 0.1, "exploration_final_eps": 0.02, "dueling": false, "hiddens": [], "model": {"fcnet_hiddens": [64], "fcnet_activation": "relu"}}'
|
||||
|
||||
docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
python /ray/python/ray/rllib/train.py \
|
||||
--env PongNoFrameskip-v4 \
|
||||
--alg DQN \
|
||||
--num-iterations 2 \
|
||||
--stop '{"training_iteration": 2}' \
|
||||
--config '{"lr": 1e-4, "schedule_max_timesteps": 2000000, "buffer_size": 10000, "exploration_fraction": 0.1, "exploration_final_eps": 0.01, "sample_batch_size": 4, "learning_starts": 10000, "target_network_update_freq": 1000, "gamma": 0.99, "prioritized_replay": true}'
|
||||
|
||||
docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
python /ray/python/ray/rllib/train.py \
|
||||
--env MontezumaRevenge-v0 \
|
||||
--alg PPO \
|
||||
--num-iterations 2 \
|
||||
--stop '{"training_iteration": 2}' \
|
||||
--config '{"kl_coeff": 1.0, "num_sgd_iter": 10, "sgd_stepsize": 1e-4, "sgd_batchsize": 64, "timesteps_per_batch": 2000, "num_workers": 1, "model": {"dim": 40, "conv_filters": [[16, [8, 8], 4], [32, [4, 4], 2], [512, [5, 5], 1]]}, "extra_frameskip": 4}'
|
||||
|
||||
docker run --shm-size=10G --memory=10G $DOCKER_SHA \
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import unittest
|
||||
|
||||
import ray
|
||||
from ray.tune.trial import Trial, Resources
|
||||
from ray.tune.trial_runner import TrialRunner
|
||||
from ray.tune.config_parser import parse_to_trials
|
||||
|
||||
|
||||
class ConfigParserTest(unittest.TestCase):
|
||||
def testParseToTrials(self):
|
||||
trials = parse_to_trials({
|
||||
"tune-pong": {
|
||||
"env": "Pong-v0",
|
||||
"alg": "PPO",
|
||||
"num_trials": 2,
|
||||
"config": {
|
||||
"foo": "bar"
|
||||
},
|
||||
},
|
||||
})
|
||||
self.assertEqual(len(trials), 2)
|
||||
self.assertEqual(trials[0].env_name, "Pong-v0")
|
||||
self.assertEqual(trials[0].config, {"foo": "bar"})
|
||||
self.assertEqual(trials[0].alg, "PPO")
|
||||
self.assertEqual(trials[0].agent_id, "0")
|
||||
self.assertEqual(trials[0].local_dir, "/tmp/ray/tune-pong")
|
||||
self.assertEqual(trials[1].agent_id, "1")
|
||||
|
||||
def testEval(self):
|
||||
trials = parse_to_trials({
|
||||
"tune-pong": {
|
||||
"env": "Pong-v0",
|
||||
"config": {
|
||||
"foo": {
|
||||
"eval": "2 + 2"
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
self.assertEqual(len(trials), 1)
|
||||
self.assertEqual(trials[0].config, {"foo": 4})
|
||||
self.assertEqual(trials[0].agent_id, "0_foo=4")
|
||||
|
||||
def testGridSearch(self):
|
||||
trials = parse_to_trials({
|
||||
"tune-pong": {
|
||||
"env": "Pong-v0",
|
||||
"num_trials": 6,
|
||||
"config": {
|
||||
"bar": {
|
||||
"grid_search": [True, False]
|
||||
},
|
||||
"foo": {
|
||||
"grid_search": [1, 2, 3]
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
self.assertEqual(len(trials), 6)
|
||||
self.assertEqual(trials[0].config, {"bar": True, "foo": 1})
|
||||
self.assertEqual(trials[0].agent_id, "0_bar=True_foo=1")
|
||||
self.assertEqual(trials[1].config, {"bar": False, "foo": 1})
|
||||
self.assertEqual(trials[1].agent_id, "1_bar=False_foo=1")
|
||||
self.assertEqual(trials[2].config, {"bar": True, "foo": 2})
|
||||
self.assertEqual(trials[3].config, {"bar": False, "foo": 2})
|
||||
self.assertEqual(trials[4].config, {"bar": True, "foo": 3})
|
||||
self.assertEqual(trials[5].config, {"bar": False, "foo": 3})
|
||||
|
||||
def testGridSearchAndEval(self):
|
||||
trials = parse_to_trials({
|
||||
"tune-pong": {
|
||||
"env": "Pong-v0",
|
||||
"num_trials": 1,
|
||||
"config": {
|
||||
"qux": {
|
||||
"eval": "2 + 2"
|
||||
},
|
||||
"bar": {
|
||||
"grid_search": [True, False]
|
||||
},
|
||||
"foo": {
|
||||
"grid_search": [1, 2, 3]
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
self.assertEqual(len(trials), 1)
|
||||
self.assertEqual(trials[0].config, {"bar": True, "foo": 1, "qux": 4})
|
||||
self.assertEqual(trials[0].agent_id, "0_bar=True_foo=1_qux=4")
|
||||
|
||||
|
||||
class TrialRunnerTest(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
ray.worker.cleanup()
|
||||
|
||||
def testTrialStatus(self):
|
||||
ray.init()
|
||||
trial = Trial("CartPole-v0", "__fake")
|
||||
self.assertEqual(trial.status, Trial.PENDING)
|
||||
trial.start()
|
||||
self.assertEqual(trial.status, Trial.RUNNING)
|
||||
trial.stop()
|
||||
self.assertEqual(trial.status, Trial.TERMINATED)
|
||||
trial.stop(error=True)
|
||||
self.assertEqual(trial.status, Trial.ERROR)
|
||||
|
||||
def testTrialErrorOnStart(self):
|
||||
ray.init()
|
||||
trial = Trial("CartPole-v0", "asdf")
|
||||
try:
|
||||
trial.start()
|
||||
except Exception as e:
|
||||
self.assertIn("Unknown algorithm", str(e))
|
||||
|
||||
def testResourceScheduler(self):
|
||||
ray.init(num_cpus=4, num_gpus=1)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {"training_iteration": 1},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
trials = [
|
||||
Trial("CartPole-v0", "__fake", **kwargs),
|
||||
Trial("CartPole-v0", "__fake", **kwargs)]
|
||||
for t in trials:
|
||||
runner.add_trial(t)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.TERMINATED)
|
||||
self.assertEqual(trials[1].status, Trial.TERMINATED)
|
||||
|
||||
def testMultiStepRun(self):
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {"training_iteration": 5},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
trials = [
|
||||
Trial("CartPole-v0", "__fake", **kwargs),
|
||||
Trial("CartPole-v0", "__fake", **kwargs)]
|
||||
for t in trials:
|
||||
runner.add_trial(t)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.RUNNING)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
|
||||
def testErrorHandling(self):
|
||||
ray.init(num_cpus=4, num_gpus=2)
|
||||
runner = TrialRunner()
|
||||
kwargs = {
|
||||
"stopping_criterion": {"training_iteration": 1},
|
||||
"resources": Resources(cpu=1, gpu=1),
|
||||
}
|
||||
trials = [
|
||||
Trial("CartPole-v0", "asdf", **kwargs),
|
||||
Trial("CartPole-v0", "__fake", **kwargs)]
|
||||
for t in trials:
|
||||
runner.add_trial(t)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[1].status, Trial.PENDING)
|
||||
|
||||
runner.step()
|
||||
self.assertEqual(trials[0].status, Trial.ERROR)
|
||||
self.assertEqual(trials[1].status, Trial.RUNNING)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user