wassname/kair_algorithms_draft
1
0
Fork 0
mirror of https://github.com/wassname/kair_algorithms_draft.git synced 2026-09-09 11:25:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add random initial action in ddpg (#13)

Browse Source 
* Add random initial actions in ddpg

* Add reacher-v2 example of ddpg
This commit is contained in:
Jinwoo Park (Curt)
2019-02-18 08:57:36 +09:00
committed by GitHub
parent ecb42d30d2
commit d2b670015c
9 changed files with 272 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/algorithms/common/buffer/priortized_replay_buffer.py
+2 -10
View File
@@ -36,18 +36,17 @@ class PrioritizedReplayBuffer(ReplayBuffer):
"""
def __init__(
self, buffer_size: int, batch_size: int, demo: list = None, alpha: float = 0.6
self, buffer_size: int, batch_size: int, alpha: float = 0.6
):
"""Initialization.
Args:
buffer_size (int): size of replay buffer for experience
batch_size (int): size of a batched sampled from replay buffer for training
demo (list): demonstration
alpha (float): alpha parameter for prioritized replay buffer
"""
super(PrioritizedReplayBuffer, self).__init__(buffer_size, batch_size, demo)
super(PrioritizedReplayBuffer, self).__init__(buffer_size, batch_size)
assert alpha >= 0
self.buffer_size = buffer_size
self.alpha = alpha
@@ -62,13 +61,6 @@ class PrioritizedReplayBuffer(ReplayBuffer):
self.min_tree = MinSegmentTree(tree_capacity)
self.init_priority = 1.0
# for init priority of demo
if demo:
for _ in range(len(demo)):
self.sum_tree[self.tree_idx] = self.init_priority ** self.alpha
self.min_tree[self.tree_idx] = self.init_priority ** self.alpha
self.tree_idx += 1
def add(
self,
state: np.ndarray,
scripts/algorithms/common/buffer/replay_buffer.py
+5 -4
View File
@@ -22,7 +22,7 @@ class ReplayBuffer:
"""
def __init__(self, buffer_size: int, batch_size: int, demo: list = None):
def __init__(self, buffer_size: int, batch_size: int):
"""Initialize a ReplayBuffer object.
Args:
@@ -31,7 +31,7 @@ class ReplayBuffer:
demo (list) : demonstration list
"""
self.buffer = list() if not demo else demo
self.buffer: list = list()
self.buffer_size = buffer_size
self.batch_size = batch_size
self.idx = 0
@@ -55,13 +55,14 @@ class ReplayBuffer:
def extend(self, transitions: list):
"""Add experiences to memory."""
self.buffer.extend(transitions)
for transition in transitions:
self.add(*transition)
def sample(
self
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Randomly sample a batch of experiences from memory."""
idxs = np.random.randint(0, len(self.buffer), size=self.batch_size)
idxs = np.random.choice(len(self.buffer), size=self.batch_size, replace=False)
states, actions, rewards, next_states, dones = [], [], [], [], []
scripts/algorithms/common/noise.py
+30 -3
View File
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
"""Noise classes for baselines."""
"""Noise classes for algorithms."""
import copy
import random
@@ -7,6 +7,31 @@ import random
import numpy as np
class GaussianNoise:
"""Gaussian Noise.
Taken from https://github.com/vitchyr/rlkit
"""
def __init__(
self,
min_sigma: float = 1.0,
max_sigma: float = 1.0,
decay_period: int = 1000000,
):
"""Initialization."""
self.max_sigma = max_sigma
self.min_sigma = min_sigma
self.decay_period = decay_period
def sample(self, action_size: int, t: int = 0) -> float:
"""Get an action with gaussian noise."""
sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(
1.0, t / self.decay_period
)
return np.random.normal(0, sigma, size=action_size)
class OUNoise:
"""Ornstein-Uhlenbeck process.
@@ -15,7 +40,9 @@ class OUNoise:
ddpg-pendulum/ddpg_agent.py
"""
def __init__(self, size, mu=0.0, theta=0.15, sigma=0.2):
def __init__(
self, size: int, mu: float = 0.0, theta: float = 0.15, sigma: float = 0.2
):
"""Initialize parameters and noise process."""
self.state = np.float64(0.0)
self.mu = mu * np.ones(size)
@@ -27,7 +54,7 @@ class OUNoise:
"""Reset the internal state (= noise) to mean (mu)."""
self.state = copy.copy(self.mu)
def sample(self):
def sample(self) -> float:
"""Update internal state and return it as a noise sample."""
x = self.state
dx = self.theta * (self.mu - x) + self.sigma * np.array(
scripts/algorithms/ddpg/agent.py
+40 -12
View File
@@ -38,6 +38,8 @@ class Agent(AbstractAgent):
actor_optimizer (Optimizer): optimizer for training actor
critic_optimizer (Optimizer): optimizer for training critic
curr_state (np.ndarray): temporary storage of the current state
total_step (int): total step numbers
episode_step (int): step number of the current episode
"""
@@ -68,20 +70,30 @@ class Agent(AbstractAgent):
self.hyper_params = hyper_params
self.curr_state = np.zeros((1,))
self.noise = noise
self.total_step = 0
self.episode_step = 0
# load the optimizer and model parameters
if args.load_from is not None and os.path.exists(args.load_from):
self.load_params(args.load_from)
# replay memory
self.memory = ReplayBuffer(
hyper_params["BUFFER_SIZE"], hyper_params["BATCH_SIZE"]
)
if not self.args.test:
# replay memory
self.memory = ReplayBuffer(
hyper_params["BUFFER_SIZE"], hyper_params["BATCH_SIZE"]
)
def select_action(self, state: np.ndarray) -> torch.Tensor:
def select_action(self, state: np.ndarray) -> np.ndarray:
"""Select an action from the input space."""
self.curr_state = state
# if initial random action should be conducted
if (
self.total_step < self.hyper_params["INITIAL_RANDOM_ACTION"]
and not self.args.test
):
return self.env.action_space.sample()
state = torch.FloatTensor(state).to(device)
selected_action = self.actor(state)
@@ -89,14 +101,21 @@ class Agent(AbstractAgent):
selected_action += torch.FloatTensor(self.noise.sample()).to(device)
selected_action = torch.clamp(selected_action, -1.0, 1.0)
return selected_action
return selected_action.detach().cpu().numpy()
def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]:
def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:
"""Take an action and return the response of the env."""
action = action.detach().cpu().numpy()
self.total_step += 1
self.episode_step += 1
next_state, reward, done, _ = self.env.step(action)
self.memory.add(self.curr_state, action, reward, next_state, done)
if not self.args.test:
# if the last state is not a terminal state, store done as false
done_bool = (
False if self.episode_step == self.args.max_episode_steps else done
)
self.memory.add(self.curr_state, action, reward, next_state, done_bool)
return next_state, reward, done
@@ -171,9 +190,17 @@ class Agent(AbstractAgent):
total_loss = loss.sum()
print(
"[INFO] episode %d total score: %d, total loss: %f\n"
"actor_loss: %.3f critic_loss: %.3f\n"
% (i, score, total_loss, loss[0], loss[1]) # actor loss # critic loss
"[INFO] episode %d, episode step: %d, total step: %d, total score: %d\n"
"total loss: %f actor_loss: %.3f critic_loss: %.3f\n"
% (
i,
self.episode_step,
self.total_step,
score,
total_loss,
loss[0],
loss[1],
) # actor loss # critic loss
)
if self.args.log:
@@ -198,6 +225,7 @@ class Agent(AbstractAgent):
state = self.env.reset()
done = False
score = 0
self.episode_step = 0
loss_episode = list()
while not done:
scripts/algorithms/per/ddpg_agent.py
+11 -10
View File
@@ -75,24 +75,24 @@ class Agent(AbstractAgent):
self.load_params(args.load_from)
# replay memory
self.beta = self.hyper_params["PER_BETA"]
self.memory = PrioritizedReplayBuffer(
self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"],
alpha=self.hyper_params["PER_ALPHA"],
)
if not self.args.test:
self.beta = self.hyper_params["PER_BETA"]
self.memory = PrioritizedReplayBuffer(
self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"],
alpha=self.hyper_params["PER_ALPHA"],
)
def select_action(self, state: np.ndarray) -> torch.Tensor:
"""Select an action from the input space."""
self.curr_state = state
state = torch.FloatTensor(state).to(device)
selected_action = self.actor(state)
if not self.args.test:
selected_action = self.actor(state)
selected_action += torch.FloatTensor(self.noise.sample()).to(device)
selected_action = torch.clamp(selected_action, -1.0, 1.0)
selected_action = torch.clamp(selected_action, -1.0, 1.0)
return selected_action
@@ -101,7 +101,8 @@ class Agent(AbstractAgent):
action = action.detach().cpu().numpy()
next_state, reward, done, _ = self.env.step(action)
self.memory.add(self.curr_state, action, reward, next_state, done)
if not self.args.test:
self.memory.add(self.curr_state, action, reward, next_state, done)
return next_state, reward, done