wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-11 11:53:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update Readme

Browse Source
This commit is contained in:
Shangtong Zhang
2017-05-11 21:30:04 -06:00
parent 4ce59dc419
commit 4786bb8990
5 changed files with 69 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
async_agent.py
+19 -9
View File
@@ -13,7 +13,7 @@ from network import *
class AsyncAgent:
def __init__(self, task_fn, network_fn, optimizer_fn, policy_fn, discount, step_limit,
target_network_update_freq, n_workers, batch_size, test_interval):
target_network_update_freq, n_workers, batch_size, test_interval, test_repeats):
self.network_fn = network_fn
self.learning_network = network_fn()
self.learning_network.share_memory()
@@ -35,14 +35,14 @@ class AsyncAgent:
self.n_workers = n_workers
self.batch_size = batch_size
self.test_interval = test_interval
self.test_repeats = test_repeats
def deterministic_episode(self, task):
def deterministic_episode(self, task, network):
state = np.asarray([task.reset()])
total_rewards = 0
steps = 0
while True and steps < self.step_limit:
with self.network_lock:
action_values = self.learning_network.predict(state)
action_values = network.predict(state)
steps += 1
action = np.argmax(action_values.flatten())
state, reward, terminal, _ = task.step(action)
@@ -68,10 +68,14 @@ class AsyncAgent:
terminal = True
episode = 0
episode_steps = 0
episode_return = 0
while True and not self.stop_signal.value:
batch_states, batch_actions, batch_rewards = [], [], []
if terminal:
if id == 0:
print 'worker %d, episode %d, return %f' % (id, episode, episode_return)
episode_steps = 0
episode_return = 0
episode += 1
policy.update_epsilon()
terminal = False
@@ -86,6 +90,7 @@ class AsyncAgent:
action = policy.sample(value.flatten())
batch_actions.append(action)
state, reward, terminal, _ = task.step(action)
episode_return += reward
state = state.reshape([1, -1])
if not terminal:
with self.network_lock:
@@ -93,6 +98,9 @@ class AsyncAgent:
reward += self.discount * q_next
batch_rewards.append(reward)
if episode_steps > self.step_limit:
terminal = True
worker_network.zero_grad()
worker_network.gradient(np.vstack(batch_states), batch_actions, batch_rewards)
self.async_update(worker_network, optimizer)
@@ -106,13 +114,15 @@ class AsyncAgent:
procs = [mp.Process(target=self.worker, args=(i, )) for i in range(self.n_workers)]
for p in procs: p.start()
task = self.task_fn()
test_network = self.network_fn()
while True:
if self.total_steps.value % self.test_interval == 0:
test_repeats = 5
rewards = np.zeros(test_repeats)
for i in range(test_repeats):
rewards[i] = self.deterministic_episode(task)
print 'total stpes: %d, test process epsidoe reward: %f' %\
with self.network_lock:
test_network.load_state_dict(self.learning_network.state_dict())
rewards = np.zeros(self.test_repeats)
for i in range(self.test_repeats):
rewards[i] = self.deterministic_episode(task, test_network)
print 'total steps: %d, averaged return per episode: %f' %\
(self.total_steps.value, np.mean(rewards))
if np.mean(rewards) > task.success_threshold:
self.stop_signal.value = True