wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-08 00:34:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Removed iteritems and xrange for python3 in rl_pong (#182)

Browse Source 
* Removed iteritems and xrange for python3

* Remove unused variable.
This commit is contained in:
Wapaul1
2017-01-05 20:37:00 -08:00
committed by Robert Nishihara
parent cac473b557
commit 417c04bac8
1 changed files with 7 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/rl_pong/driver.py
+7 -13
View File
@@ -6,7 +6,6 @@ from __future__ import division
from __future__ import print_function
import numpy as np
import cPickle as pickle
import ray
import gym
@@ -17,7 +16,6 @@ batch_size = 10 # every how many episodes to do a param update?
learning_rate = 1e-4
gamma = 0.99 # discount factor for reward
decay_rate = 0.99 # decay factor for RMSProp leaky sum of grad^2
resume = False # resume from previous checkpoint?
D = 80 * 80 # input dimensionality: 80x80 grid
@@ -50,7 +48,7 @@ def discount_rewards(r):
"""take 1D float array of rewards and compute discounted reward"""
discounted_r = np.zeros_like(r)
running_add = 0
for t in reversed(xrange(0, r.size)):
for t in reversed(range(0, r.size)):
if r[t] != 0: running_add = 0 # reset the sum, since this was a game boundary (pong specific!)
running_add = running_add * gamma + r[t]
discounted_r[t] = running_add
@@ -117,14 +115,11 @@ if __name__ == "__main__":
# Run the reinforcement learning
running_reward = None
batch_num = 1
if resume:
model = pickle.load(open("save.p", "rb"))
else:
model = {}
model["W1"] = np.random.randn(H, D) / np.sqrt(D) # "Xavier" initialization
model["W2"] = np.random.randn(H) / np.sqrt(H)
grad_buffer = {k: np.zeros_like(v) for k, v in model.iteritems()} # update buffers that add up gradients over a batch
rmsprop_cache = {k: np.zeros_like(v) for k, v in model.iteritems()} # rmsprop memory
model = {}
model["W1"] = np.random.randn(H, D) / np.sqrt(D) # "Xavier" initialization
model["W2"] = np.random.randn(H) / np.sqrt(H)
grad_buffer = {k: np.zeros_like(v) for k, v in model.items()} # update buffers that add up gradients over a batch
rmsprop_cache = {k: np.zeros_like(v) for k, v in model.items()} # rmsprop memory
while True:
model_id = ray.put(model)
@@ -140,10 +135,9 @@ if __name__ == "__main__":
for k in model: grad_buffer[k] += grad[k] # accumulate grad over batch
running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01
print("Batch {}. episode reward total was {}. running mean: {}".format(batch_num, reward_sum, running_reward))
for k, v in model.iteritems():
for k, v in model.items():
g = grad_buffer[k] # gradient
rmsprop_cache[k] = decay_rate * rmsprop_cache[k] + (1 - decay_rate) * g ** 2
model[k] += learning_rate * g / (np.sqrt(rmsprop_cache[k]) + 1e-5)
grad_buffer[k] = np.zeros_like(v) # reset batch gradient buffer
batch_num += 1
if batch_num % 10 == 0: pickle.dump(model, open("save.p", "wb"))