wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update for latest gym

Browse Source
This commit is contained in:
Shangtong Zhang
2018-02-09 22:02:58 -07:00
parent e7f036bc6b
commit d4aa8d4a22
6 changed files with 36 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/A2C_agent.py
+6 -6
View File
@@ -12,7 +12,6 @@ from component import *
import pickle
import os
import time
import gym.monitoring
class A2CAgent:
def __init__(self, config):
@@ -53,8 +52,9 @@ class A2CAgent:
config = self.config
for _ in range(config.iteration_log_interval):
self.iteration(deterministic)
config.logger.info('max/min reward %f/%f' %
(np.max(self.last_episode_rewards), np.min(self.last_episode_rewards)))
config.logger.info('max/min reward %f/%f, policy loss %f, entropy loss %f, value loss %f' %
(np.max(self.last_episode_rewards), np.min(self.last_episode_rewards),
self.policy_loss, self.entropy_loss, self.value_loss))
return self.last_episode_rewards.mean(), config.rollout_length * config.num_workers * \
config.iteration_log_interval
@@ -107,9 +107,9 @@ class A2CAgent:
entropy_loss = torch.sum(prob * log_prob, dim=1, keepdim=True)
value_loss = 0.5 * (Variable(returns) - value).pow(2)
self.config.logger.scalar_summary('policy_loss', np.mean(policy_loss.data.cpu().numpy()))
self.config.logger.scalar_summary('entropy_loss', np.mean(entropy_loss.data.cpu().numpy()))
self.config.logger.scalar_summary('value_loss', np.mean(value_loss.data.cpu().numpy()))
self.policy_loss = np.mean(policy_loss.data.cpu().numpy())
self.entropy_loss = np.mean(entropy_loss.data.cpu().numpy())
self.value_loss = np.mean(value_loss.data.cpu().numpy())
self.optimizer.zero_grad()
(policy_loss + config.entropy_weight * entropy_loss +
agent/DDPG_agent.py
-1
View File
@@ -12,7 +12,6 @@ from component import *
import pickle
import os
import time
import gym.monitoring
class DDPGAgent:
def __init__(self, config):
component/atari_wrapper.py
+24 -18
View File
@@ -13,7 +13,7 @@ class NoopResetEnv(gym.Wrapper):
self.noop_max = noop_max
assert env.unwrapped.get_action_meanings()[0] == 'NOOP'
def _reset(self):
def reset(self):
""" Do no-op action for a number of steps in [1, noop_max]."""
self.env.reset()
noops = np.random.randint(1, self.noop_max + 1)
@@ -21,6 +21,9 @@ class NoopResetEnv(gym.Wrapper):
obs, _, _, _ = self.env.step(0)
return obs
def step(self, action):
return self.env.step(action)
class FireResetEnv(gym.Wrapper):
def __init__(self, env=None):
"""Take action on reset for environments that are fixed until firing."""
@@ -28,12 +31,15 @@ class FireResetEnv(gym.Wrapper):
assert env.unwrapped.get_action_meanings()[1] == 'FIRE'
assert len(env.unwrapped.get_action_meanings()) >= 3
def _reset(self):
def reset(self):
self.env.reset()
obs, _, _, _ = self.env.step(1)
obs, _, _, _ = self.env.step(2)
return obs
def step(self, action):
return self.env.step(action)
class EpisodicLifeEnv(gym.Wrapper):
def __init__(self, env=None):
"""Make end-of-life == end-of-episode, but only reset on true game over.
@@ -42,9 +48,9 @@ class EpisodicLifeEnv(gym.Wrapper):
super(EpisodicLifeEnv, self).__init__(env)
self.lives = 0
self.was_real_done = True
self.was_real_reset = False
self.was_realreset = False
def _step(self, action):
def step(self, action):
obs, reward, done, info = self.env.step(action)
self.was_real_done = done
# check current lives, make loss of life terminal,
@@ -58,18 +64,18 @@ class EpisodicLifeEnv(gym.Wrapper):
self.lives = lives
return obs, reward, done, info
def _reset(self):
def reset(self):
"""Reset only when lives are exhausted.
This way all states are still reachable even though lives are episodic,
and the learner need not know about any of this behind-the-scenes.
"""
if self.was_real_done:
obs = self.env.reset()
self.was_real_reset = True
self.was_realreset = True
else:
# no-op step to advance from terminal/lost life state
obs, _, _, _ = self.env.step(0)
self.was_real_reset = False
self.was_realreset = False
self.lives = self.env.unwrapped.ale.lives()
return obs
@@ -81,7 +87,7 @@ class MaxAndSkipEnv(gym.Wrapper):
self._obs_buffer = deque(maxlen=2)
self._skip = skip
def _step(self, action):
def step(self, action):
total_reward = 0.0
done = None
for _ in range(self._skip):
@@ -95,7 +101,7 @@ class MaxAndSkipEnv(gym.Wrapper):
return max_frame, total_reward, done, info
def _reset(self):
def reset(self):
"""Clear past frame buffer and init. to first obs. from inner env."""
self._obs_buffer.clear()
obs = self.env.reset()
@@ -115,13 +121,13 @@ class DatasetEnv(gym.Wrapper):
self.saved_obs = []
self.saved_actions = []
def _step(self, action):
def step(self, action):
obs, reward, done, info = self.env.step(action)
self.saved_actions.append(action)
self.saved_obs.append(obs)
return obs, reward, done, info
def _reset(self):
def reset(self):
obs = self.env.reset()
self.saved_obs.append(obs)
return obs
@@ -130,7 +136,7 @@ class ProcessFrame(gym.Wrapper):
def __init__(self, env=None, frame_size=84):
super(ProcessFrame, self).__init__(env)
self.frame_size = frame_size
self.observation_space = spaces.Box(low=0, high=255, shape=(1, frame_size, frame_size))
self.observation_space = spaces.Box(low=0, high=255, shape=(1, frame_size, frame_size), dtype=np.uint8)
def process(self, obs):
obs = color.rgb2gray(obs)
@@ -138,11 +144,11 @@ class ProcessFrame(gym.Wrapper):
obs = (255 * obs).astype(np.uint8).reshape((1, ) + obs.shape)
return obs
def _step(self, action):
def step(self, action):
obs, reward, done, info = self.env.step(action)
return self.process(obs), reward, done, info
def _reset(self):
def reset(self):
return self.process(self.env.reset())
class NormalizeFrame(gym.Wrapper):
@@ -152,11 +158,11 @@ class NormalizeFrame(gym.Wrapper):
def _normalize(self, obs):
return np.asarray(obs, dtype=np.float32) / 255.0
def _step(self, action):
def step(self, action):
obs, reward, done, info = self.env.step(action)
return self._normalize(obs), reward, done, info
def _reset(self):
def reset(self):
return self._normalize(self.env.reset())
class StackFrame(gym.Wrapper):
@@ -165,12 +171,12 @@ class StackFrame(gym.Wrapper):
self.history_length = history_length
self.buffer = None
def _reset(self):
def reset(self):
state = self.env.reset()
self.buffer = [state] * self.history_length
return np.vstack(self.buffer)
def _step(self, action):
def step(self, action):
state, reward, done, info = self.env.step(action)
self.buffer.pop(0)
self.buffer.append(state)
component/task.py
+1
View File
@@ -127,6 +127,7 @@ class Roboschool(BasicTask):
def sub_task(parent_pipe, pipe, task_fn):
parent_pipe.close()
task = task_fn()
task.env.seed(np.random.randint(0, sys.maxsize))
while True:
op, data = pipe.recv()
if op == 'step':
main.py
+5 -5
View File
@@ -168,11 +168,11 @@ def a2c_pixel_atari(name):
history_length=config.history_length)
config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers)
task = config.task_fn()
# config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.0007)
config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.0001)
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.0007)
# config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.0001)
# config.network_fn = lambda: OpenAIActorCriticConvNet(
config.network_fn = lambda: NatureActorCriticConvNet(
config.history_length, task.task.env.action_space.n, gpu=2)
config.history_length, task.task.env.action_space.n, gpu=0)
config.reward_shift_fn = lambda r: np.sign(r)
config.policy_fn = SamplePolicy
config.discount = 0.99
@@ -333,8 +333,8 @@ if __name__ == '__main__':
# dqn_pixel_atari('PongNoFrameskip-v4')
# async_pixel_atari('PongNoFrameskip-v4')
a3c_pixel_atari('PongNoFrameskip-v4')
# a2c_pixel_atari('PongNoFrameskip-v4')
# a3c_pixel_atari('PongNoFrameskip-v4')
a2c_pixel_atari('PongNoFrameskip-v4')
# dqn_pixel_atari('BreakoutNoFrameskip-v4')
# async_pixel_atari('BreakoutNoFrameskip-v4')
utils/misc.py
-7
View File
@@ -7,7 +7,6 @@
import numpy as np
import pickle
import os
import gym.monitoring
def run_episodes(agent):
config = agent.config
@@ -31,12 +30,6 @@ def run_episodes(agent):
agent_type, config.tag, agent.task.name), 'wb') as f:
pickle.dump([steps, rewards], f)
if config.render_episode_freq and ep % config.render_episode_freq == 0:
video_recoder = gym.monitoring.VideoRecorder(
env=agent.task.env, base_path='./data/video/%s-%s-%s-%d' % (agent_type, config.tag, agent.task.name, ep))
agent.episode(True, video_recoder)
video_recoder.close()
if config.episode_limit and ep > config.episode_limit:
break