mirror of
https://github.com/wassname/DeepRL.git
synced 2026-09-04 16:14:05 +08:00
Major refactor
This commit is contained in:
Shangtong Zhang
@@ -0,0 +1,5 @@
|
||||
from atari_wrapper import *
|
||||
from policy import *
|
||||
from replay import *
|
||||
from task import *
|
||||
from random_process import *
|
||||
@@ -0,0 +1,149 @@
|
||||
# This file is copied/apdated from
|
||||
# https://raw.githubusercontent.com/transedward/pytorch-dqn/master/utils/atari_wrapper.py
|
||||
|
||||
import numpy as np
|
||||
from collections import deque
|
||||
import gym
|
||||
from gym import spaces
|
||||
from PIL import Image
|
||||
|
||||
class NoopResetEnv(gym.Wrapper):
|
||||
def __init__(self, env=None, noop_max=30):
|
||||
"""Sample initial states by taking random number of no-ops on reset.
|
||||
No-op is assumed to be action 0.
|
||||
"""
|
||||
super(NoopResetEnv, self).__init__(env)
|
||||
self.noop_max = noop_max
|
||||
assert env.unwrapped.get_action_meanings()[0] == 'NOOP'
|
||||
|
||||
def _reset(self):
|
||||
""" Do no-op action for a number of steps in [1, noop_max]."""
|
||||
self.env.reset()
|
||||
noops = np.random.randint(1, self.noop_max + 1)
|
||||
for _ in range(noops):
|
||||
obs, _, _, _ = self.env.step(0)
|
||||
return obs
|
||||
|
||||
class FireResetEnv(gym.Wrapper):
|
||||
def __init__(self, env=None):
|
||||
"""Take action on reset for environments that are fixed until firing."""
|
||||
super(FireResetEnv, self).__init__(env)
|
||||
assert env.unwrapped.get_action_meanings()[1] == 'FIRE'
|
||||
assert len(env.unwrapped.get_action_meanings()) >= 3
|
||||
|
||||
def _reset(self):
|
||||
self.env.reset()
|
||||
obs, _, _, _ = self.env.step(1)
|
||||
obs, _, _, _ = self.env.step(2)
|
||||
return obs
|
||||
|
||||
class EpisodicLifeEnv(gym.Wrapper):
|
||||
def __init__(self, env=None):
|
||||
"""Make end-of-life == end-of-episode, but only reset on true game over.
|
||||
Done by DeepMind for the DQN and co. since it helps value estimation.
|
||||
"""
|
||||
super(EpisodicLifeEnv, self).__init__(env)
|
||||
self.lives = 0
|
||||
self.was_real_done = True
|
||||
self.was_real_reset = False
|
||||
|
||||
def _step(self, action):
|
||||
obs, reward, done, info = self.env.step(action)
|
||||
self.was_real_done = done
|
||||
# check current lives, make loss of life terminal,
|
||||
# then update lives to handle bonus lives
|
||||
lives = self.env.unwrapped.ale.lives()
|
||||
if lives < self.lives and lives > 0:
|
||||
# for Qbert somtimes we stay in lives == 0 condtion for a few frames
|
||||
# so its important to keep lives > 0, so that we only reset once
|
||||
# the environment advertises done.
|
||||
done = True
|
||||
self.lives = lives
|
||||
return obs, reward, done, info
|
||||
|
||||
def _reset(self):
|
||||
"""Reset only when lives are exhausted.
|
||||
This way all states are still reachable even though lives are episodic,
|
||||
and the learner need not know about any of this behind-the-scenes.
|
||||
"""
|
||||
if self.was_real_done:
|
||||
obs = self.env.reset()
|
||||
self.was_real_reset = True
|
||||
else:
|
||||
# no-op step to advance from terminal/lost life state
|
||||
obs, _, _, _ = self.env.step(0)
|
||||
self.was_real_reset = False
|
||||
self.lives = self.env.unwrapped.ale.lives()
|
||||
return obs
|
||||
|
||||
class MaxAndSkipEnv(gym.Wrapper):
|
||||
def __init__(self, env=None, skip=4):
|
||||
"""Return only every `skip`-th frame"""
|
||||
super(MaxAndSkipEnv, self).__init__(env)
|
||||
# most recent raw observations (for max pooling across time steps)
|
||||
self._obs_buffer = deque(maxlen=2)
|
||||
self._skip = skip
|
||||
|
||||
def _step(self, action):
|
||||
total_reward = 0.0
|
||||
done = None
|
||||
for _ in range(self._skip):
|
||||
obs, reward, done, info = self.env.step(action)
|
||||
self._obs_buffer.append(obs)
|
||||
total_reward += reward
|
||||
if done:
|
||||
break
|
||||
|
||||
max_frame = np.max(np.stack(self._obs_buffer), axis=0)
|
||||
|
||||
return max_frame, total_reward, done, info
|
||||
|
||||
def _reset(self):
|
||||
"""Clear past frame buffer and init. to first obs. from inner env."""
|
||||
self._obs_buffer.clear()
|
||||
obs = self.env.reset()
|
||||
self._obs_buffer.append(obs)
|
||||
return obs
|
||||
|
||||
def _process_frame84(frame):
|
||||
img = np.reshape(frame, [210, 160, 3]).astype(np.float32)
|
||||
img = img[:, :, 0] * 0.299 + img[:, :, 1] * 0.587 + img[:, :, 2] * 0.114
|
||||
img = Image.fromarray(img)
|
||||
resized_screen = img.resize((84, 110), Image.BILINEAR)
|
||||
resized_screen = np.array(resized_screen)
|
||||
x_t = resized_screen[18:102, :]
|
||||
x_t = np.reshape(x_t, [1, 84, 84])
|
||||
return x_t.astype(np.uint8)
|
||||
|
||||
def _process_frame42(frame):
|
||||
img = np.reshape(frame, [210, 160, 3]).astype(np.float32)
|
||||
img = img[:, :, 0] * 0.299 + img[:, :, 1] * 0.587 + img[:, :, 2] * 0.114
|
||||
img = img[34:34 + 160, :160]
|
||||
img = Image.fromarray(img)
|
||||
img = img.resize((80, 80), Image.BILINEAR)
|
||||
img = img.resize((42, 42), Image.BILINEAR)
|
||||
resized_screen = np.array(img).reshape(1, 42, 42)
|
||||
return resized_screen.astype(np.uint8)
|
||||
|
||||
class ProcessFrame(gym.Wrapper):
|
||||
def __init__(self, env=None, frame_size=84):
|
||||
super(ProcessFrame, self).__init__(env)
|
||||
self.observation_space = spaces.Box(low=0, high=255, shape=(1, frame_size, frame_size))
|
||||
if frame_size == 84:
|
||||
self.process_fn = _process_frame84
|
||||
elif frame_size == 42:
|
||||
self.process_fn = _process_frame42
|
||||
else:
|
||||
assert False, "Unknown frame size"
|
||||
|
||||
def _step(self, action):
|
||||
obs, reward, done, info = self.env.step(action)
|
||||
return self.process_fn(obs), reward, done, info
|
||||
|
||||
def _reset(self):
|
||||
return self.process_fn(self.env.reset())
|
||||
|
||||
class ClippedRewardsWrapper(gym.Wrapper):
|
||||
def _step(self, action):
|
||||
obs, reward, done, info = self.env.step(action)
|
||||
return obs, np.sign(reward), done, info
|
||||
@@ -0,0 +1,57 @@
|
||||
#######################################################################
|
||||
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) #
|
||||
# Permission given to modify the code as long as you keep this #
|
||||
# declaration at the top #
|
||||
#######################################################################
|
||||
|
||||
import numpy as np
|
||||
|
||||
class GreedyPolicy:
|
||||
def __init__(self, epsilon, final_step, min_epsilon):
|
||||
self.init_epsilon = self.epsilon = epsilon
|
||||
self.current_steps = 0
|
||||
self.min_epsilon = min_epsilon
|
||||
self.final_step = final_step
|
||||
|
||||
def sample(self, action_value, deterministic=False):
|
||||
if deterministic:
|
||||
return np.argmax(action_value)
|
||||
if np.random.rand() < self.epsilon:
|
||||
return np.random.randint(0, len(action_value))
|
||||
return np.argmax(action_value)
|
||||
|
||||
def update_epsilon(self):
|
||||
self.epsilon = self.init_epsilon - float(self.current_steps) / self.final_step * (self.init_epsilon - self.min_epsilon)
|
||||
self.epsilon = max(self.epsilon, self.min_epsilon)
|
||||
self.current_steps += 1
|
||||
|
||||
class StochasticGreedyPolicy:
|
||||
def __init__(self, epsilons, final_step, min_epsilons, probs):
|
||||
self.policies = []
|
||||
self.probs = probs
|
||||
for epsilon, min_epsilon in zip(epsilons, min_epsilons):
|
||||
self.policies.append(GreedyPolicy(epsilon, final_step, min_epsilon))
|
||||
|
||||
def sample(self, action_value, deterministic=False):
|
||||
return np.random.choice(self.policies, p=self.probs).sample(action_value, deterministic)
|
||||
|
||||
def update_epsilon(self):
|
||||
for policy in self.policies:
|
||||
policy.update_epsilon()
|
||||
|
||||
class SamplePolicy:
|
||||
def sample(self, action_value, deterministic=False):
|
||||
if deterministic:
|
||||
return np.argmax(action_value)
|
||||
return np.random.choice(np.arange(len(action_value)), p=action_value)
|
||||
def update_epsilon(self):
|
||||
pass
|
||||
|
||||
class GaussianPolicy:
|
||||
def sample(self, mean, var, deterministic=False):
|
||||
if deterministic:
|
||||
return mean
|
||||
return mean + np.sqrt(var) * np.random.randn(*mean.shape)
|
||||
|
||||
def update_epsilon(self):
|
||||
pass
|
||||
@@ -0,0 +1,49 @@
|
||||
# copy from https://github.com/ghliu/pytorch-ddpg/blob/master/random_process.py
|
||||
import numpy as np
|
||||
|
||||
# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/random.py
|
||||
|
||||
class RandomProcess(object):
|
||||
def reset_states(self):
|
||||
pass
|
||||
|
||||
class AnnealedGaussianProcess(RandomProcess):
|
||||
def __init__(self, mu, sigma, sigma_min, n_steps_annealing):
|
||||
self.mu = mu
|
||||
self.sigma = sigma
|
||||
self.n_steps = 0
|
||||
|
||||
if sigma_min is not None:
|
||||
self.m = -float(sigma - sigma_min) / float(n_steps_annealing)
|
||||
self.c = sigma
|
||||
self.sigma_min = sigma_min
|
||||
else:
|
||||
self.m = 0.
|
||||
self.c = sigma
|
||||
self.sigma_min = sigma
|
||||
|
||||
@property
|
||||
def current_sigma(self):
|
||||
sigma = max(self.sigma_min, self.m * float(self.n_steps) + self.c)
|
||||
return sigma
|
||||
|
||||
|
||||
# Based on http://math.stackexchange.com/questions/1287634/implementing-ornstein-uhlenbeck-in-matlab
|
||||
class OrnsteinUhlenbeckProcess(AnnealedGaussianProcess):
|
||||
def __init__(self, theta, mu=0., sigma=1., dt=1e-2, x0=None, size=1, sigma_min=None, n_steps_annealing=1000):
|
||||
super(OrnsteinUhlenbeckProcess, self).__init__(mu=mu, sigma=sigma, sigma_min=sigma_min, n_steps_annealing=n_steps_annealing)
|
||||
self.theta = theta
|
||||
self.mu = mu
|
||||
self.dt = dt
|
||||
self.x0 = x0
|
||||
self.size = size
|
||||
self.reset_states()
|
||||
|
||||
def sample(self):
|
||||
x = self.x_prev + self.theta * (self.mu - self.x_prev) * self.dt + self.current_sigma * np.sqrt(self.dt) * np.random.normal(size=self.size)
|
||||
self.x_prev = x
|
||||
self.n_steps += 1
|
||||
return x
|
||||
|
||||
def reset_states(self):
|
||||
self.x_prev = self.x0 if self.x0 is not None else np.zeros(self.size)
|
||||
@@ -0,0 +1,94 @@
|
||||
#######################################################################
|
||||
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) #
|
||||
# Permission given to modify the code as long as you keep this #
|
||||
# declaration at the top #
|
||||
#######################################################################
|
||||
|
||||
import numpy as np
|
||||
|
||||
class Replay:
|
||||
def __init__(self, memory_size, batch_size, dtype=np.float32):
|
||||
self.memory_size = memory_size
|
||||
self.batch_size = batch_size
|
||||
self.dtype = dtype
|
||||
|
||||
self.states = None
|
||||
self.actions = np.empty(self.memory_size, dtype=np.int8)
|
||||
self.rewards = np.empty(self.memory_size)
|
||||
self.next_states = None
|
||||
self.terminals = np.empty(self.memory_size, dtype=np.int8)
|
||||
|
||||
self.pos = 0
|
||||
self.full = False
|
||||
|
||||
|
||||
def feed(self, experience):
|
||||
state, action, reward, next_state, done = experience
|
||||
|
||||
if self.states is None:
|
||||
self.states = np.empty((self.memory_size, ) + state.shape, dtype=self.dtype)
|
||||
self.next_states = np.empty((self.memory_size, ) + state.shape, dtype=self.dtype)
|
||||
|
||||
self.states[self.pos][:] = state
|
||||
self.actions[self.pos] = action
|
||||
self.rewards[self.pos] = reward
|
||||
self.next_states[self.pos][:] = next_state
|
||||
self.terminals[self.pos] = done
|
||||
|
||||
self.pos += 1
|
||||
if self.pos == self.memory_size:
|
||||
self.full = True
|
||||
self.pos = 0
|
||||
|
||||
def sample(self):
|
||||
upper_bound = self.memory_size if self.full else self.pos
|
||||
sampled_indices = np.random.randint(0, upper_bound, size=self.batch_size)
|
||||
return [self.states[sampled_indices],
|
||||
self.actions[sampled_indices],
|
||||
self.rewards[sampled_indices],
|
||||
self.next_states[sampled_indices],
|
||||
self.terminals[sampled_indices]]
|
||||
|
||||
class HighDimActionReplay:
|
||||
def __init__(self, memory_size, batch_size, dtype=np.float32):
|
||||
self.memory_size = memory_size
|
||||
self.batch_size = batch_size
|
||||
self.dtype = dtype
|
||||
|
||||
self.states = None
|
||||
self.actions = None
|
||||
self.rewards = np.empty(self.memory_size)
|
||||
self.next_states = None
|
||||
self.terminals = np.empty(self.memory_size, dtype=np.int8)
|
||||
|
||||
self.pos = 0
|
||||
self.full = False
|
||||
|
||||
|
||||
def feed(self, experience):
|
||||
state, action, reward, next_state, done = experience
|
||||
|
||||
if self.states is None:
|
||||
self.states = np.empty((self.memory_size, ) + state.shape, dtype=self.dtype)
|
||||
self.actions = np.empty((self.memory_size, ) + action.shape)
|
||||
self.next_states = np.empty((self.memory_size, ) + state.shape, dtype=self.dtype)
|
||||
|
||||
self.states[self.pos][:] = state
|
||||
self.actions[self.pos][:] = action
|
||||
self.rewards[self.pos] = reward
|
||||
self.next_states[self.pos][:] = next_state
|
||||
self.terminals[self.pos] = done
|
||||
|
||||
self.pos += 1
|
||||
if self.pos == self.memory_size:
|
||||
self.full = True
|
||||
self.pos = 0
|
||||
|
||||
def sample(self):
|
||||
upper_bound = self.memory_size if self.full else self.pos
|
||||
sampled_indices = np.random.randint(0, upper_bound, size=self.batch_size)
|
||||
return [self.states[sampled_indices],
|
||||
self.actions[sampled_indices],
|
||||
self.rewards[sampled_indices],
|
||||
self.next_states[sampled_indices],
|
||||
self.terminals[sampled_indices]]
|
||||
@@ -0,0 +1,110 @@
|
||||
#######################################################################
|
||||
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) #
|
||||
# Permission given to modify the code as long as you keep this #
|
||||
# declaration at the top #
|
||||
#######################################################################
|
||||
import gym
|
||||
import sys
|
||||
import numpy as np
|
||||
from atari_wrapper import *
|
||||
|
||||
class BasicTask:
|
||||
def __init__(self):
|
||||
self.normalized_state = True
|
||||
|
||||
def normalize_state(self, state):
|
||||
return state
|
||||
|
||||
def reset(self):
|
||||
state = self.env.reset()
|
||||
if self.normalized_state:
|
||||
return self.normalize_state(state)
|
||||
return state
|
||||
|
||||
def step(self, action):
|
||||
next_state, reward, done, info = self.env.step(action)
|
||||
if self.normalized_state:
|
||||
next_state = self.normalize_state(next_state)
|
||||
return next_state, np.sign(reward), done, info
|
||||
|
||||
def random_action(self):
|
||||
return self.env.action_space.sample()
|
||||
|
||||
class MountainCar(BasicTask):
|
||||
name = 'MountainCar-v0'
|
||||
success_threshold = -110
|
||||
|
||||
def __init__(self):
|
||||
BasicTask.__init__(self)
|
||||
self.env = gym.make(self.name)
|
||||
self.env._max_episode_steps = sys.maxsize
|
||||
|
||||
class CartPole(BasicTask):
|
||||
name = 'CartPole-v0'
|
||||
success_threshold = 195
|
||||
|
||||
def __init__(self):
|
||||
BasicTask.__init__(self)
|
||||
self.env = gym.make(self.name)
|
||||
self.env._max_episode_steps = sys.maxsize
|
||||
|
||||
class LunarLander(BasicTask):
|
||||
name = 'LunarLander-v2'
|
||||
success_threshold = 200
|
||||
|
||||
def __init__(self):
|
||||
BasicTask.__init__(self)
|
||||
self.env = gym.make(self.name)
|
||||
|
||||
class PixelAtari(BasicTask):
|
||||
def __init__(self, name, no_op, frame_skip, normalized_state=True,
|
||||
frame_size=84, success_threshold=1000):
|
||||
BasicTask.__init__(self)
|
||||
self.normalized_state = normalized_state
|
||||
self.name = name
|
||||
self.success_threshold = success_threshold
|
||||
env = gym.make(name)
|
||||
assert 'NoFrameskip' in env.spec.id
|
||||
env = EpisodicLifeEnv(env)
|
||||
env = NoopResetEnv(env, noop_max=no_op)
|
||||
env = MaxAndSkipEnv(env, skip=frame_skip)
|
||||
if 'FIRE' in env.unwrapped.get_action_meanings():
|
||||
env = FireResetEnv(env)
|
||||
env = ProcessFrame(env, frame_size)
|
||||
self.env = ClippedRewardsWrapper(env)
|
||||
|
||||
def normalize_state(self, state):
|
||||
return np.asarray(state, dtype=np.float32) / 255.0
|
||||
|
||||
class Pendulum(BasicTask):
|
||||
name = 'Pendulum-v0'
|
||||
success_threshold = 200
|
||||
|
||||
def __init__(self):
|
||||
BasicTask.__init__(self)
|
||||
self.env = gym.make(self.name)
|
||||
self.env._max_episode_steps = sys.maxsize
|
||||
self.action_dim = self.env.action_space.shape[0]
|
||||
self.state_dim = self.env.observation_space.shape[0]
|
||||
|
||||
def step(self, action):
|
||||
# action = 2 * np.clip(action, -1, 1)
|
||||
action = np.clip(action, -2, 2)
|
||||
next_state, reward, done, info = self.env.step(action)
|
||||
return next_state, reward, done, info
|
||||
|
||||
class BipedalWalker(BasicTask):
|
||||
name = 'BipedalWalker-v2'
|
||||
success_threshold = 2000
|
||||
|
||||
def __init__(self):
|
||||
BasicTask.__init__(self)
|
||||
self.env = gym.make(self.name)
|
||||
self.env._max_episode_steps = sys.maxsize
|
||||
self.action_dim = self.env.action_space.shape[0]
|
||||
self.state_dim = self.env.observation_space.shape[0]
|
||||
|
||||
def step(self, action):
|
||||
action = np.clip(action, -1, 1)
|
||||
next_state, reward, done, info = self.env.step(action)
|
||||
return next_state, reward, done, info
|
||||
Reference in New Issue
Block a user