wassname/kair_algorithms_draft
1
0
Fork 0
mirror of https://github.com/wassname/kair_algorithms_draft.git synced 2026-09-09 11:25:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Convert code to python 2.7 (#35)

Browse Source 
* Convert code format to python2.7 (SAC)

* Convert code format python2.7 (TD3, all fD)

* Remove no use import and black setting

* Change SAC param

* Change env name Reacher-v2 to v1

* Remove old version reacher training script

* Convert code format python2.7

* Modify .travis.yml

* Add install command python3.6 & black on Makefile

* Fix seperator to tab on Makefile

* Modify Makefile

* Fix little error

* Change td3 gamma parameter
This commit is contained in:
Kyunghwan Kim
2019-03-25 19:07:19 +09:00
committed by Whi Kwon
parent 81a9d861b6
commit d2769dfa9d
39 changed files with 154 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/algorithms/__init__.py
View File
scripts/algorithms/common/__init__.py
View File
scripts/algorithms/common/abstract/__init__.py
View File
scripts/algorithms/common/abstract/agent.py
+12 -12
View File
@@ -5,18 +5,16 @@
- Contact: curt.park@medipixel.io
"""
import argparse
import os
import subprocess
from abc import ABC, abstractmethod
from typing import Tuple
from abc import ABCMeta, abstractmethod
import gym
import numpy as np
import torch
class AbstractAgent(ABC):
class AbstractAgent:
"""Abstract Agent used for all agents.
Attributes:
@@ -27,7 +25,9 @@ class AbstractAgent(ABC):
"""
def __init__(self, env: gym.Env, args: argparse.Namespace):
__metaclass__ = ABCMeta
def __init__(self, env, args):
"""Initialization.
Args:
@@ -52,11 +52,11 @@ class AbstractAgent(ABC):
)
@abstractmethod
def select_action(self, state: np.ndarray):
def select_action(self, state):
pass
@abstractmethod
def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]:
def step(self, action):
pass
@abstractmethod
@@ -68,7 +68,7 @@ class AbstractAgent(ABC):
pass
@abstractmethod
def save_params(self, params: dict, n_episode: int):
def save_params(self, params, n_episode):
if not os.path.exists("./save"):
os.mkdir("./save")
@@ -77,7 +77,7 @@ class AbstractAgent(ABC):
path = os.path.join("./save/" + save_name + "_ep_" + str(n_episode) + ".pt")
torch.save(params, path)
print("[INFO] Saved the model and optimizer to", path)
print ("[INFO] Saved the model and optimizer to", path)
@abstractmethod
def write_log(self, *args):
@@ -106,7 +106,7 @@ class AbstractAgent(ABC):
score += reward
step += 1
print(
print (
"[INFO] episode %d\tstep: %d\ttotal score: %d"
% (i_episode, step, score)
)
@@ -118,7 +118,7 @@ class AbstractAgent(ABC):
class NormalizedActions(gym.ActionWrapper):
"""Rescale and relocate the actions."""
def action(self, action: np.ndarray) -> np.ndarray:
def action(self, action):
"""Change the range (-1, 1) to (low, high)."""
low = self.action_space.low
high = self.action_space.high
@@ -131,7 +131,7 @@ class NormalizedActions(gym.ActionWrapper):
return action
def reverse_action(self, action: np.ndarray) -> np.ndarray:
def reverse_action(self, action):
"""Change the range (low, high) to (-1, 1)."""
low = self.action_space.low
high = self.action_space.high
scripts/algorithms/common/buffer/__init__.py
View File
scripts/algorithms/common/buffer/priortized_replay_buffer.py
+16 -34
View File
@@ -8,7 +8,6 @@
"""
import random
from typing import Tuple
import numpy as np
import torch
@@ -35,7 +34,7 @@ class PrioritizedReplayBuffer(ReplayBuffer):
"""
def __init__(self, buffer_size: int, batch_size: int, alpha: float = 0.6):
def __init__(self, buffer_size, batch_size, alpha=0.6):
"""Initialization.
Args:
@@ -59,27 +58,22 @@ class PrioritizedReplayBuffer(ReplayBuffer):
self.min_tree = MinSegmentTree(tree_capacity)
self._max_priority = 1.0
def add(
self,
state: np.ndarray,
action: np.ndarray,
reward: np.float64,
next_state: np.ndarray,
done: bool,
):
def add(self, state, action, reward, next_state, done):
"""Add experience and priority."""
idx = self.tree_idx
self.tree_idx = (self.tree_idx + 1) % self.buffer_size
super().add(state, action, reward, next_state, done)
super(PrioritizedReplayBuffer, self).add(
state, action, reward, next_state, done
)
self.sum_tree[idx] = self._max_priority ** self.alpha
self.min_tree[idx] = self._max_priority ** self.alpha
def extend(self, transitions: list):
def extend(self, transitions):
"""Add experiences to memory."""
raise NotImplementedError
def _sample_proportional(self, batch_size: int) -> list:
def _sample_proportional(self, batch_size):
"""Sample indices based on proportional."""
indices = []
p_total = self.sum_tree.sum(0, len(self.buffer) - 1)
@@ -92,7 +86,7 @@ class PrioritizedReplayBuffer(ReplayBuffer):
indices.append(idx)
return indices
def sample(self, beta: float = 0.4) -> Tuple[torch.Tensor, ...]:
def sample(self, beta=0.4):
"""Sample a batch of experiences."""
assert beta > 0
@@ -127,7 +121,7 @@ class PrioritizedReplayBuffer(ReplayBuffer):
return experiences
def update_priorities(self, indices: list, priorities: np.ndarray):
def update_priorities(self, indices, priorities):
"""Update priorities of sampled transitions."""
assert len(indices) == len(priorities)
@@ -153,14 +147,7 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
epsilon_d (float) : epsilon_d parameter to update priority using demo
"""
def __init__(
self,
buffer_size: int,
batch_size: int,
demo: list,
alpha: float = 0.6,
epsilon_d: float = 1.0,
):
def __init__(self, buffer_size, batch_size, demo, alpha=0.6, epsilon_d=1.0):
"""Initialization.
Args:
buffer_size (int): size of replay buffer for experience
@@ -181,14 +168,7 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
self.min_tree[self.tree_idx] = self._max_priority ** self.alpha
self.tree_idx += 1
def add(
self,
state: np.ndarray,
action: np.ndarray,
reward: np.float64,
next_state: np.ndarray,
done: bool,
):
def add(self, state, action, reward, next_state, done):
"""Add experience and priority."""
idx = self.tree_idx
# buffer is full
@@ -196,7 +176,9 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
self.tree_idx = self.demo_size
else:
self.tree_idx = self.tree_idx + 1
super().add(state, action, reward, next_state, done)
super(PrioritizedReplayBuffer, self).add(
state, action, reward, next_state, done
)
self.sum_tree[idx] = self._max_priority ** self.alpha
self.min_tree[idx] = self._max_priority ** self.alpha
@@ -204,7 +186,7 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
# update current total size
self.total_size = self.demo_size + len(self.buffer)
def sample(self, beta: float = 0.4) -> Tuple[torch.Tensor, ...]:
def sample(self, beta=0.4):
"""Sample a batch of experiences."""
assert beta > 0
@@ -266,7 +248,7 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
return experiences
def update_priorities(self, indices: list, priorities: np.ndarray):
def update_priorities(self, indices, priorities):
"""Update priorities of sampled transitions."""
assert len(indices) == len(priorities)
scripts/algorithms/common/buffer/replay_buffer.py
+11 -21
View File
@@ -2,7 +2,6 @@
"""Replay buffer for baselines."""
from collections import deque
from typing import Any, Deque, List, Tuple
import numpy as np
import torch
@@ -25,7 +24,7 @@ class ReplayBuffer:
"""
def __init__(self, buffer_size: int, batch_size: int):
def __init__(self, buffer_size, batch_size):
"""Initialize a ReplayBuffer object.
Args:
@@ -33,19 +32,12 @@ class ReplayBuffer:
batch_size (int): size of a batched sampled from replay buffer for training
"""
self.buffer: list = list()
self.buffer = list()
self.buffer_size = buffer_size
self.batch_size = batch_size
self.idx = 0
def add(
self,
state: np.ndarray,
action: np.ndarray,
reward: np.float64,
next_state: np.ndarray,
done: bool,
):
def add(self, state, action, reward, next_state, done):
"""Add a new experience to memory."""
data = (state, action, reward, next_state, done)
@@ -55,14 +47,12 @@ class ReplayBuffer:
else:
self.buffer.append(data)
def extend(self, transitions: list):
def extend(self, transitions):
"""Add experiences to memory."""
for transition in transitions:
self.add(*transition)
def sample(
self
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
def sample(self):
"""Randomly sample a batch of experiences from memory."""
idxs = np.random.choice(len(self.buffer), size=self.batch_size, replace=False)
@@ -84,7 +74,7 @@ class ReplayBuffer:
return states, actions, rewards, next_states, dones
def __len__(self) -> int:
def __len__(self):
"""Return the current size of internal memory."""
return len(self.buffer)
@@ -100,7 +90,7 @@ class NStepTransitionBuffer:
"""
def __init__(self, buffer_size: int, n_step: int, gamma: float, demo: list = None):
def __init__(self, buffer_size, n_step, gamma, demo=None):
"""Initialize a ReplayBuffer object.
Args:
@@ -110,9 +100,9 @@ class NStepTransitionBuffer:
"""
assert buffer_size > 0
self.n_step_buffer: Deque = deque(maxlen=n_step)
self.n_step_buffer = deque(maxlen=n_step)
self.buffer_size = buffer_size
self.buffer: list = list()
self.buffer = list()
self.n_step = n_step
self.gamma = gamma
self.demo_size = 0
@@ -125,7 +115,7 @@ class NStepTransitionBuffer:
self.buffer.extend([None] * self.buffer_size)
def add(self, transition: Tuple[np.ndarray, ...]) -> Tuple[Any, ...]:
def add(self, transition):
"""Add a new transition to memory."""
self.n_step_buffer.append(transition)
@@ -146,7 +136,7 @@ class NStepTransitionBuffer:
# return a single step transition to insert to replay buffer
return self.n_step_buffer[0]
def sample(self, indices: List[int]) -> Tuple[torch.Tensor, ...]:
def sample(self, indices):
"""Randomly sample a batch of experiences from memory."""
states, actions, rewards, next_states, dones = [], [], [], [], []
scripts/algorithms/common/buffer/segment_tree.py
+10 -13
View File
@@ -2,7 +2,6 @@
"""Segment tree for Proirtized Replay Buffer."""
import operator
from typing import Callable
class SegmentTree:
@@ -18,7 +17,7 @@ class SegmentTree:
"""
def __init__(self, capacity: int, operation: Callable, init_value: float):
def __init__(self, capacity, operation, init_value):
"""Initialization.
Args:
@@ -34,9 +33,7 @@ class SegmentTree:
self.tree = [init_value for _ in range(2 * capacity)]
self.operation = operation
def _operate_helper(
self, start: int, end: int, node: int, node_start: int, node_end: int
) -> float:
def _operate_helper(self, start, end, node, node_start, node_end):
"""Returns result of operation in segment."""
if start == node_start and end == node_end:
return self.tree[node]
@@ -52,7 +49,7 @@ class SegmentTree:
self._operate_helper(mid + 1, end, 2 * node + 1, mid + 1, node_end),
)
def operate(self, start: int = 0, end: int = 0) -> float:
def operate(self, start=0, end=0):
"""Returns result of applying `self.operation`."""
if end <= 0:
end += self.capacity
@@ -60,7 +57,7 @@ class SegmentTree:
return self._operate_helper(start, end, 1, 0, self.capacity - 1)
def __setitem__(self, idx: int, val: float):
def __setitem__(self, idx, val):
"""Set value in tree."""
idx += self.capacity
self.tree[idx] = val
@@ -70,7 +67,7 @@ class SegmentTree:
self.tree[idx] = self.operation(self.tree[2 * idx], self.tree[2 * idx + 1])
idx //= 2
def __getitem__(self, idx: int) -> float:
def __getitem__(self, idx):
"""Get real value in leaf node of tree."""
assert 0 <= idx < self.capacity
@@ -85,7 +82,7 @@ class SumSegmentTree(SegmentTree):
"""
def __init__(self, capacity: int):
def __init__(self, capacity):
"""Initialization.
Args:
@@ -96,11 +93,11 @@ class SumSegmentTree(SegmentTree):
capacity=capacity, operation=operator.add, init_value=0.0
)
def sum(self, start: int = 0, end: int = 0) -> float:
def sum(self, start=0, end=0):
"""Returns arr[start] + ... + arr[end]."""
return super(SumSegmentTree, self).operate(start, end)
def retrieve(self, upperbound: float) -> int:
def retrieve(self, upperbound):
"""Find the highest index `i` about upper bound in the tree"""
assert 0 <= upperbound <= self.sum() + 1e-5
@@ -125,7 +122,7 @@ class MinSegmentTree(SegmentTree):
"""
def __init__(self, capacity: int):
def __init__(self, capacity):
"""Initialization.
Args:
@@ -136,6 +133,6 @@ class MinSegmentTree(SegmentTree):
capacity=capacity, operation=min, init_value=float("inf")
)
def min(self, start: int = 0, end: int = 0) -> float:
def min(self, start=0, end=0):
"""Returns min(arr[start], ..., arr[end])."""
return super(MinSegmentTree, self).operate(start, end)
scripts/algorithms/common/helper_functions.py
+6 -13
View File
@@ -7,28 +7,25 @@
import random
from collections import deque
from typing import Deque, List, Tuple
import gym
import numpy as np
import torch
import torch.nn as nn
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
def identity(x: torch.Tensor) -> torch.Tensor:
def identity(x):
"""Return input without any change."""
return x
def soft_update(local: nn.Module, target: nn.Module, tau: float):
def soft_update(local, target, tau):
"""Soft-update: target = tau*local + (1-tau)*target."""
for t_param, l_param in zip(target.parameters(), local.parameters()):
t_param.data.copy_(tau * l_param.data + (1.0 - tau) * t_param.data)
def set_random_seed(seed: int, env: gym.Env):
def set_random_seed(seed, env):
"""Set random seed"""
env.seed(seed)
torch.manual_seed(seed)
@@ -36,16 +33,14 @@ def set_random_seed(seed: int, env: gym.Env):
random.seed(seed)
def get_n_step_info_from_demo(
demo: List, n_step: int, gamma: float
) -> Tuple[List, List]:
def get_n_step_info_from_demo(demo, n_step, gamma):
"""Return 1 step and n step demos."""
assert demo
assert n_step > 1
demos_1_step = list()
demos_n_step = list()
n_step_buffer: Deque = deque(maxlen=n_step)
n_step_buffer = deque(maxlen=n_step)
for transition in demo:
n_step_buffer.append(transition)
@@ -63,9 +58,7 @@ def get_n_step_info_from_demo(
return demos_1_step, demos_n_step
def get_n_step_info(
n_step_buffer: Deque, gamma: float
) -> Tuple[np.int64, np.ndarray, bool]:
def get_n_step_info(n_step_buffer, gamma):
"""Return n step reward, next state, and done."""
# info of the last transition
reward, next_state, done = n_step_buffer[-1][-3:]
scripts/algorithms/common/networks/__init__.py
View File
scripts/algorithms/common/networks/lstm.py
+10 -13
View File
@@ -5,9 +5,6 @@
- Contact: whikwon@gmail.com
"""
from typing import Callable
import torch
import torch.nn as nn
import torch.nn.functional as F
@@ -30,13 +27,13 @@ class LSTM(nn.Module):
def __init__(
self,
input_size: int,
output_size: int,
hidden_sizes: list,
hidden_activation: Callable = F.relu,
output_activation: Callable = identity,
use_output_layer: bool = True,
init_w: float = 3e-3,
input_size,
output_size,
hidden_sizes,
hidden_activation=F.relu,
output_activation=identity,
use_output_layer=True,
init_w=3e-3,
):
"""Initialization.
@@ -59,7 +56,7 @@ class LSTM(nn.Module):
self.output_activation = output_activation
self.use_output_layer = use_output_layer
self.hidden_layers: list = []
self.hidden_layers = []
in_size = self.input_size
for i, next_size in enumerate(hidden_sizes):
lstm = nn.LSTM(in_size, next_size, batch_first=True)
@@ -73,14 +70,14 @@ class LSTM(nn.Module):
self.output_layer.weight.data.uniform_(-init_w, init_w)
self.output_layer.bias.data.uniform_(-init_w, init_w)
def get_last_activation(self, x: torch.Tensor) -> torch.Tensor:
def get_last_activation(self, x):
"""Get the activation of the last hidden layer."""
for hidden_layer in self.hidden_layers:
x, _ = hidden_layer(x)
x = self.hidden_activation(x)
return x
def forward(self, x: torch.Tensor) -> torch.Tensor:
def forward(self, x):
"""Forward method implementation."""
assert self.use_output_layer
scripts/algorithms/common/networks/mlp.py
+23 -27
View File
@@ -5,8 +5,6 @@
- Contact: kh.kim@medipixel.io
"""
from typing import Callable, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
@@ -31,13 +29,13 @@ class MLP(nn.Module):
def __init__(
self,
input_size: int,
output_size: int,
hidden_sizes: list,
hidden_activation: Callable = F.relu,
output_activation: Callable = identity,
use_output_layer: bool = True,
init_w: float = 3e-3,
input_size,
output_size,
hidden_sizes,
hidden_activation=F.relu,
output_activation=identity,
use_output_layer=True,
init_w=3e-3,
):
"""Initialization.
@@ -61,7 +59,7 @@ class MLP(nn.Module):
self.use_output_layer = use_output_layer
# set hidden layers
self.hidden_layers: list = []
self.hidden_layers = []
in_size = self.input_size
for i, next_size in enumerate(hidden_sizes):
fc = nn.Linear(in_size, next_size)
@@ -75,13 +73,13 @@ class MLP(nn.Module):
self.output_layer.weight.data.uniform_(-init_w, init_w)
self.output_layer.bias.data.uniform_(-init_w, init_w)
def get_last_activation(self, x: torch.Tensor) -> torch.Tensor:
def get_last_activation(self, x):
"""Get the activation of the last hidden layer."""
for hidden_layer in self.hidden_layers:
x = self.hidden_activation(hidden_layer(x))
return x
def forward(self, x: torch.Tensor) -> torch.Tensor:
def forward(self, x):
"""Forward method implementation."""
assert self.use_output_layer
@@ -96,7 +94,7 @@ class MLP(nn.Module):
class FlattenMLP(MLP):
"""Baseline of Multilayer perceptron for Flatten input."""
def forward(self, *args: torch.Tensor) -> torch.Tensor:
def forward(self, *args):
"""Forward method implementation."""
states, actions = args
flat_inputs = torch.cat((states, actions), dim=-1)
@@ -116,14 +114,14 @@ class GaussianDist(MLP):
def __init__(
self,
input_size: int,
output_size: int,
hidden_sizes: list,
hidden_activation: Callable = F.relu,
mu_activation: Callable = torch.tanh,
log_std_min: float = -20,
log_std_max: float = 2,
init_w: float = 3e-3,
input_size,
output_size,
hidden_sizes,
hidden_activation=F.relu,
mu_activation=torch.tanh,
log_std_min=-20,
log_std_max=2,
init_w=3e-3,
):
"""Initialization."""
super(GaussianDist, self).__init__(
@@ -149,7 +147,7 @@ class GaussianDist(MLP):
self.mu_layer.weight.data.uniform_(-init_w, init_w)
self.mu_layer.bias.data.uniform_(-init_w, init_w)
def get_dist_params(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]:
def get_dist_params(self, x):
"""Return gausian distribution parameters."""
hidden = super(GaussianDist, self).get_last_activation(x)
@@ -165,7 +163,7 @@ class GaussianDist(MLP):
return mu, log_std, std
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]:
def forward(self, x):
"""Forward method implementation."""
mu, _, std = self.get_dist_params(x)
@@ -181,11 +179,9 @@ class TanhGaussianDistParams(GaussianDist):
def __init__(self, **kwargs):
"""Initialization."""
super(TanhGaussianDistParams, self).__init__(**kwargs, mu_activation=identity)
super(TanhGaussianDistParams, self).__init__(mu_activation=identity, **kwargs)
def forward(
self, x: torch.Tensor, epsilon: float = 1e-6
) -> Tuple[torch.Tensor, ...]:
def forward(self, x, epsilon=1e-6):
"""Forward method implementation."""
mu, _, std = super(TanhGaussianDistParams, self).get_dist_params(x)
scripts/algorithms/common/noise.py
+4 -12
View File
@@ -13,20 +13,14 @@ class GaussianNoise:
Taken from https://github.com/vitchyr/rlkit
"""
def __init__(
self,
action_dim: int,
min_sigma: float = 1.0,
max_sigma: float = 1.0,
decay_period: int = 1000000,
):
def __init__(self, action_dim, min_sigma=1.0, max_sigma=1.0, decay_period=1000000):
"""Initialization."""
self.action_dim = action_dim
self.min_sigma = min_sigma
self.max_sigma = max_sigma
self.decay_period = decay_period
def sample(self, t: int = 0) -> float:
def sample(self, t=0):
"""Get an action with gaussian noise."""
sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * min(
1.0, t / self.decay_period
@@ -42,9 +36,7 @@ class OUNoise:
ddpg-pendulum/ddpg_agent.py
"""
def __init__(
self, size: int, mu: float = 0.0, theta: float = 0.15, sigma: float = 0.2
):
def __init__(self, size, mu=0.0, theta=0.15, sigma=0.2):
"""Initialize parameters and noise process."""
self.state = np.float64(0.0)
self.mu = mu * np.ones(size)
@@ -56,7 +48,7 @@ class OUNoise:
"""Reset the internal state (= noise) to mean (mu)."""
self.state = copy.copy(self.mu)
def sample(self) -> float:
def sample(self):
"""Update internal state and return it as a noise sample."""
x = self.state
dx = self.theta * (self.mu - x) + self.sigma * np.array(
scripts/algorithms/fd/__init__.py
View File
scripts/algorithms/fd/sac_agent.py
+5 -16
View File
@@ -10,7 +10,6 @@
"""
import pickle
from typing import List, Tuple
import numpy as np
import torch
@@ -39,6 +38,8 @@ class Agent(SACAgent):
if not self.args.test:
# load demo replay memory
# TODO: should make new demo to set protocol 2
# e.g. pickle.dump(your_object, your_file, protocol=2)
with open(self.args.demo_path, "rb") as f:
demos = pickle.load(f)
@@ -65,7 +66,7 @@ class Agent(SACAgent):
epsilon_d=self.hyper_params["PER_EPS_DEMO"],
)
def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]):
def _add_transition_to_memory(self, transition):
"""Add 1 step and n step transitions to memory."""
# add n-step transition
if self.use_n_step:
@@ -77,19 +78,7 @@ class Agent(SACAgent):
self.memory.add(*transition)
# pylint: disable=too-many-statements
def update_model(
self,
experiences: Tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
List[int],
],
) -> Tuple[torch.Tensor, torch.Tensor]:
def update_model(self, experiences):
"""Train the model after each episode."""
states, actions, rewards, next_states, dones, weights, indices, eps_d = (
experiences
@@ -212,7 +201,7 @@ class Agent(SACAgent):
def pretrain(self):
"""Pretraining steps."""
pretrain_loss = list()
print("[INFO] Pre-Train %d steps." % self.hyper_params["PRETRAIN_STEP"])
print ("[INFO] Pre-Train %d steps." % self.hyper_params["PRETRAIN_STEP"])
for i_step in range(1, self.hyper_params["PRETRAIN_STEP"] + 1):
loss = self.update_model()
pretrain_loss.append(loss) # for logging
scripts/algorithms/fd/td3_agent.py
+7 -22
View File
@@ -9,7 +9,6 @@
"""
import pickle
from typing import List, Tuple
import numpy as np
import torch
@@ -38,6 +37,8 @@ class Agent(TD3Agent):
if not self.args.test:
# load demo replay memory
# TODO: should make new demo to set protocol 2
# e.g. pickle.dump(your_object, your_file, protocol=2)
with open(self.args.demo_path, "rb") as f:
demos = pickle.load(f)
@@ -64,7 +65,7 @@ class Agent(TD3Agent):
epsilon_d=self.hyper_params["PER_EPS_DEMO"],
)
def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]):
def _add_transition_to_memory(self, transition):
"""Add 1 step and n step transitions to memory."""
# add n-step transition
if self.use_n_step:
@@ -75,9 +76,7 @@ class Agent(TD3Agent):
if transition:
self.memory.add(*transition)
def _get_critic_loss(
self, experiences: Tuple[torch.Tensor, ...], gamma: float
) -> torch.Tensor:
def _get_critic_loss(self, experiences, gamma):
"""Return element-wise critic loss."""
states, actions, rewards, next_states, dones = experiences[:5]
@@ -99,9 +98,7 @@ class Agent(TD3Agent):
torch.cat((next_states, next_actions), dim=-1)
)
target_values = torch.min(target_values1, target_values2)
target_values = (
rewards + (self.hyper_params["GAMMA"] * target_values * masks).detach()
)
target_values = rewards + (gamma * target_values * masks).detach()
# train critic
values1 = self.critic1(torch.cat((states, actions), dim=-1))
@@ -113,19 +110,7 @@ class Agent(TD3Agent):
return critic1_loss_element_wise, critic2_loss_element_wise
# pylint: disable=too-many-statements
def update_model(
self,
experiences: Tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
List[int],
],
) -> Tuple[torch.Tensor, torch.Tensor]:
def update_model(self, experiences):
"""Train the model after each episode."""
states, actions, rewards, next_states, dones, weights, indices, eps_d = (
experiences
@@ -195,7 +180,7 @@ class Agent(TD3Agent):
def pretrain(self):
"""Pretraining steps."""
pretrain_loss = list()
print("[INFO] Pre-Train %d steps." % self.hyper_params["PRETRAIN_STEP"])
print ("[INFO] Pre-Train %d steps." % self.hyper_params["PRETRAIN_STEP"])
for i_step in range(1, self.hyper_params["PRETRAIN_STEP"] + 1):
loss = self.update_model()
pretrain_loss.append(loss) # for logging
scripts/algorithms/sac/__init__.py
View File
scripts/algorithms/sac/agent.py
+12 -30
View File
@@ -7,11 +7,8 @@
https://arxiv.org/pdf/1812.05905.pdf
"""
import argparse
import os
from typing import Tuple
import gym
import numpy as np
import torch
import torch.nn.functional as F
@@ -50,15 +47,7 @@ class Agent(AbstractAgent):
"""
def __init__(
self,
env: gym.Env,
args: argparse.Namespace,
hyper_params: dict,
models: tuple,
optims: tuple,
target_entropy: float,
):
def __init__(self, env, args, hyper_params, models, optims, target_entropy):
"""Initialization.
Args:
@@ -103,7 +92,7 @@ class Agent(AbstractAgent):
self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"]
)
def select_action(self, state: np.ndarray) -> np.ndarray:
def select_action(self, state):
"""Select an action from the input space."""
self.curr_state = state
state = self._preprocess_state(state)
@@ -122,12 +111,12 @@ class Agent(AbstractAgent):
return selected_action.detach().cpu().numpy()
def _preprocess_state(self, state: np.ndarray) -> torch.Tensor:
def _preprocess_state(self, state):
"""Preprocess state so that actor selects an action."""
state = torch.FloatTensor(state).to(device)
return state
def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:
def step(self, action):
"""Take an action and return the response of the env."""
self.total_step += 1
self.episode_step += 1
@@ -144,16 +133,11 @@ class Agent(AbstractAgent):
return next_state, reward, done
def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]):
def _add_transition_to_memory(self, transition):
"""Add 1 step and n step transitions to memory."""
self.memory.add(*transition)
def update_model(
self,
experiences: Tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
],
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
def update_model(self, experiences):
"""Train the model after each episode."""
states, actions, rewards, next_states, dones = experiences
new_actions, log_prob, pre_tanh_value, mu, std = self.actor(states)
@@ -238,10 +222,10 @@ class Agent(AbstractAgent):
alpha_loss.data,
)
def load_params(self, path: str):
def load_params(self, path):
"""Load model and optimizer parameters."""
if not os.path.exists(path):
print("[ERROR] the input path does not exist. ->", path)
print ("[ERROR] the input path does not exist. ->", path)
return
params = torch.load(path)
@@ -258,9 +242,9 @@ class Agent(AbstractAgent):
if self.hyper_params["AUTO_ENTROPY_TUNING"]:
self.alpha_optimizer.load_state_dict(params["alpha_optim"])
print("[INFO] loaded the model and optimizer from", path)
print ("[INFO] loaded the model and optimizer from", path)
def save_params(self, n_episode: int):
def save_params(self, n_episode):
"""Save model and optimizer parameters."""
params = {
"actor": self.actor.state_dict(),
@@ -279,13 +263,11 @@ class Agent(AbstractAgent):
AbstractAgent.save_params(self, params, n_episode)
def write_log(
self, i: int, loss: np.ndarray, score: float = 0.0, delayed_update: int = 1
):
def write_log(self, i, loss, score=0.0, delayed_update=1):
"""Write log about loss and score"""
total_loss = loss.sum()
print(
print (
"[INFO] episode %d, episode_step %d, total step %d, total score: %d\n"
"total loss: %.3f actor_loss: %.3f qf_1_loss: %.3f qf_2_loss: %.3f "
"vf_loss: %.3f alpha_loss: %.3f\n"
scripts/algorithms/td3/__init__.py
View File
scripts/algorithms/td3/agent.py
+11 -27
View File
@@ -6,11 +6,8 @@
- Paper: https://arxiv.org/pdf/1802.09477.pdf
"""
import argparse
import os
from typing import Tuple
import gym
import numpy as np
import torch
import torch.nn.functional as F
@@ -44,15 +41,7 @@ class Agent(AbstractAgent):
"""
def __init__(
self,
env: gym.Env,
args: argparse.Namespace,
hyper_params: dict,
models: tuple,
optims: tuple,
noises: tuple,
):
def __init__(self, env, args, hyper_params, models, optims, noises):
"""Initialization.
Args:
@@ -90,7 +79,7 @@ class Agent(AbstractAgent):
self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"]
)
def select_action(self, state: np.ndarray) -> np.ndarray:
def select_action(self, state):
"""Select an action from the input space."""
# initial training step, try random action for exploration
random_action_count = self.hyper_params["INITIAL_RANDOM_ACTIONS"]
@@ -109,7 +98,7 @@ class Agent(AbstractAgent):
return selected_action.detach().cpu().numpy()
def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]:
def step(self, action):
"""Take an action and return the response of the env."""
self.total_steps += 1
self.episode_steps += 1
@@ -126,16 +115,11 @@ class Agent(AbstractAgent):
return next_state, reward, done
def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]):
def _add_transition_to_memory(self, transition):
"""Add 1 step and n step transitions to memory."""
self.memory.add(*transition)
def update_model(
self,
experiences: Tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
],
) -> Tuple[torch.Tensor, torch.Tensor]:
def update_model(self, experiences):
"""Train the model after each episode."""
states, actions, rewards, next_states, dones = experiences
@@ -190,10 +174,10 @@ class Agent(AbstractAgent):
return actor_loss.data, critic1_loss.data, critic2_loss.data
def load_params(self, path: str):
def load_params(self, path):
"""Load model and optimizer parameters."""
if not os.path.exists(path):
print("[ERROR] the input path does not exist. ->", path)
print ("[ERROR] the input path does not exist. ->", path)
return
params = torch.load(path)
@@ -205,9 +189,9 @@ class Agent(AbstractAgent):
self.critic2_target.load_state_dict(params["critic2_target_state_dict"])
self.actor_optim.load_state_dict(params["actor_optim_state_dict"])
self.critic_optim.load_state_dict(params["critic_optim_state_dict"])
print("[INFO] loaded the model and optimizer from", path)
print ("[INFO] loaded the model and optimizer from", path)
def save_params(self, n_episode: int):
def save_params(self, n_episode):
"""Save model and optimizer parameters."""
params = {
"actor_state_dict": self.actor.state_dict(),
@@ -222,11 +206,11 @@ class Agent(AbstractAgent):
AbstractAgent.save_params(self, params, n_episode)
def write_log(self, i: int, loss: np.ndarray, score: int):
def write_log(self, i, loss, score):
"""Write log about loss and score"""
total_loss = loss.sum()
print(
print (
"[INFO] total_steps: %d episode: %d total score: %d, total loss: %f\n"
"actor_loss: %.3f critic1_loss: %.3f critic2_loss: %.3f\n"
% (self.total_steps, i, score, total_loss, loss[0], loss[1], loss[2])