wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-18 12:20:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

[rllib] clarify train batch size for PPO (#2793)

Browse Source 
It's possible to configure PPO in a way that ends up discarding most of the samples (they are treated as "stragglers"). Add a warning when this happens, and raise an exception if the waste is particularly egregious.
This commit is contained in:
Eric Liang
2018-09-05 12:06:13 -07:00
committed by GitHub
parent c87a9114cd
commit 995ac24a2c
24 changed files with 83 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/examples/pbt_ppo_example.py
+8 -8
View File
@@ -21,8 +21,8 @@ if __name__ == "__main__":
# Postprocess the perturbed config to ensure it's still valid
def explore(config):
# ensure we collect enough timesteps to do sgd
if config["timesteps_per_batch"] < config["sgd_batchsize"] * 2:
config["timesteps_per_batch"] = config["sgd_batchsize"] * 2
if config["train_batch_size"] < config["sgd_minibatch_size"] * 2:
config["train_batch_size"] = config["sgd_minibatch_size"] * 2
# ensure we run at least one sgd iter
if config["num_sgd_iter"] < 1:
config["num_sgd_iter"] = 1
@@ -37,10 +37,10 @@ if __name__ == "__main__":
hyperparam_mutations={
"lambda": lambda: random.uniform(0.9, 1.0),
"clip_param": lambda: random.uniform(0.01, 0.5),
"sgd_stepsize": [1e-3, 5e-4, 1e-4, 5e-5, 1e-5],
"lr": [1e-3, 5e-4, 1e-4, 5e-5, 1e-5],
"num_sgd_iter": lambda: random.randint(1, 30),
"sgd_batchsize": lambda: random.randint(128, 16384),
"timesteps_per_batch": lambda: random.randint(2000, 160000),
"sgd_minibatch_size": lambda: random.randint(128, 16384),
"train_batch_size": lambda: random.randint(2000, 160000),
},
custom_explore_fn=explore)
@@ -61,13 +61,13 @@ if __name__ == "__main__":
# These params are tuned from a fixed starting value.
"lambda": 0.95,
"clip_param": 0.2,
"sgd_stepsize": 1e-4,
"lr": 1e-4,
# These params start off randomly drawn from a set.
"num_sgd_iter":
lambda spec: random.choice([10, 20, 30]),
"sgd_batchsize":
"sgd_minibatch_size":
lambda spec: random.choice([128, 512, 2048]),
"timesteps_per_batch":
"train_batch_size":
lambda spec: random.choice([10000, 20000, 40000])
},
},