wassname/kair_algorithms_draft
1
0
Fork 0
mirror of https://github.com/wassname/kair_algorithms_draft.git synced 2026-09-09 11:25:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add overall setting and ddpg baseline (#1)

Browse Source 
* Add overall CI settings

* Add specific build dir to travis

* Add before install/script condition to travis

* Add ddpg baseline

* Add wandb, remove algorithms except ddpg

* Remove init file in script

* Separate config file for ddpg

* Remove unnecessary examples

* Remove unnecessary args opt

* Add pre-commit setting

* Change pre-commit settings

* Change travis-ci setting

* Fix travis-ci issue

* Modify argparse arguments, fix requirements

* Change arguments order
This commit is contained in:
Whi Kwon
2019-02-05 20:07:46 +09:00
committed by GitHub
parent c7362ee828
commit 7f4756a1d4
17 changed files with 931 additions and 578 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/.flake8
-5
View File
@@ -1,5 +0,0 @@
[flake8]
ignore = E203, E266, E501, W503
max-line-length = 88
max-complexity = 18
select = B,C,E,F,W,T4,B9
scripts/.pre-commit-config.yaml
+13
View File
@@ -0,0 +1,13 @@
repos:
- repo: local
hooks:
- id: format
name: format
language: system
entry: make format
types: [python]
- id: test
name: test
language: system
entry: make test
types: [python]
scripts/.pylintrc
-573
View File
@@ -1,573 +0,0 @@
[MASTER]
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code.
extension-pkg-whitelist=
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
# Add files or directories matching the regex patterns to the blacklist. The
# regex matches against base names, not paths.
ignore-patterns=
# Python code to execute, usually for sys.path manipulation such as
# pygtk.require().
#init-hook=
# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
# number of processors available to use.
jobs=1
# Control the amount of potential inferred values when inferring a single
# object. This can help the performance when dealing with large functions or
# complex, nested conditions.
limit-inference-results=100
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
# Pickle collected data for later comparisons.
persistent=yes
# Specify a configuration file.
#rcfile=
# When enabled, pylint would attempt to guess common misconfiguration and emit
# user-friendly hints instead of false-positive error messages.
suggestion-mode=yes
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.
confidence=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once). You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use "--disable=all --enable=classes
# --disable=W".
disable=print-statement,
bad-continuation,
no-else-return,
broad-except,
missing-docstring,
invalid-name,
parameter-unpacking,
unpacking-in-except,
old-raise-syntax,
backtick,
long-suffix,
old-ne-operator,
old-octal-literal,
import-star-module-level,
non-ascii-bytes-literal,
raw-checker-failed,
bad-inline-option,
locally-disabled,
locally-enabled,
file-ignored,
suppressed-message,
useless-suppression,
deprecated-pragma,
use-symbolic-message-instead,
apply-builtin,
basestring-builtin,
buffer-builtin,
cmp-builtin,
coerce-builtin,
execfile-builtin,
file-builtin,
long-builtin,
raw_input-builtin,
reduce-builtin,
standarderror-builtin,
unicode-builtin,
xrange-builtin,
coerce-method,
delslice-method,
getslice-method,
setslice-method,
no-absolute-import,
old-division,
dict-iter-method,
dict-view-method,
next-method-called,
metaclass-assignment,
indexing-exception,
raising-string,
reload-builtin,
oct-method,
hex-method,
nonzero-method,
cmp-method,
input-builtin,
round-builtin,
intern-builtin,
unichr-builtin,
map-builtin-not-iterating,
zip-builtin-not-iterating,
range-builtin-not-iterating,
filter-builtin-not-iterating,
using-cmp-argument,
eq-without-hash,
div-method,
idiv-method,
rdiv-method,
exception-message-attribute,
invalid-str-codec,
sys-max-int,
bad-python3-import,
deprecated-string-function,
deprecated-str-translate-call,
deprecated-itertools-function,
deprecated-types-field,
next-method-defined,
dict-items-not-iterating,
dict-keys-not-iterating,
dict-values-not-iterating,
deprecated-operator-function,
deprecated-urllib-function,
xreadlines-attribute,
deprecated-sys-function,
exception-escape,
comprehension-escape,
no-member,
useless-import-alias,
too-many-locals,
too-few-public-methods,
arguments-differ,
duplicate-code,
protected-access,
too-many-instance-attributes,
fixme
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once). See also the "--disable" option for examples.
enable=c-extension-no-member
[REPORTS]
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details.
#msg-template=
# Set the output format. Available formats are text, parseable, colorized, json
# and msvs (visual studio). You can also give a reporter class, e.g.
# mypackage.mymodule.MyReporterClass.
output-format=text
# Tells whether to display a full report or only the messages.
reports=no
# Activate the evaluation score.
score=yes
[REFACTORING]
# Maximum number of nested blocks for function / method body
max-nested-blocks=5
# Complete name of functions that never returns. When checking for
# inconsistent-return-statements if a never returning function is called then
# it will be considered as an explicit return statement and no message will be
# printed.
never-returning-functions=sys.exit
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format.
logging-modules=logging
[SPELLING]
# Limits count of emitted suggestions for spelling mistakes.
max-spelling-suggestions=4
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package..
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,
XXX,
TODO
[TYPECHECK]
# List of decorators that produce context managers, such as
# contextlib.contextmanager. Add to this list to register other decorators that
# produce valid context managers.
contextmanager-decorators=contextlib.contextmanager
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# Tells whether to warn about missing members when the owner of the attribute
# is inferred to be None.
ignore-none=yes
# This flag controls whether pylint should warn about no-member and similar
# checks whenever an opaque object is returned when inferring. The inference
# can return multiple potential results while evaluating a Python object, but
# some branches might not be evaluated, which results in partial inference. In
# that case, it might be useful to still emit no-member and other checks for
# the rest of the inferred objects.
ignore-on-opaque-inference=yes
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=optparse.Values,thread._local,_thread._local
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=
# Show a hint with possible names when a member name was not found. The aspect
# of finding the hint is based on edit distance.
missing-member-hint=yes
# The minimum edit distance a name should have in order to be considered a
# similar match for a missing member name.
missing-member-hint-distance=1
# The total number of similar names that should be taken in consideration when
# showing a hint for a missing member.
missing-member-max-choices=1
[VARIABLES]
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# Tells whether unused global variables should be treated as a violation.
allow-global-unused-variables=yes
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,
_cb
# A regular expression matching the name of dummy variables (i.e. expected to
# not be used).
dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
# Argument names that match this expression will be ignored. Default to name
# with leading underscore.
ignored-argument-names=_.*|^ignored_|^unused_
# Tells whether we should check for unused import in __init__ files.
init-import=no
# List of qualified module names which can have objects that can redefine
# builtins.
redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io
[FORMAT]
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
# Maximum number of characters on a single line.
max-line-length=100
# Maximum number of lines in a module.
max-module-lines=1000
# List of optional constructs for which whitespace checking is disabled. `dict-
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
# `empty-line` allows space-only lines.
no-space-check=trailing-comma,
dict-separator
# Allow the body of a class to be on the same line as the declaration if body
# contains single statement.
single-line-class-stmt=no
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
[SIMILARITIES]
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
# Minimum lines number of a similarity.
min-similarity-lines=4
[BASIC]
# Naming style matching correct argument names.
argument-naming-style=snake_case
# Regular expression matching correct argument names. Overrides argument-
# naming-style.
#argument-rgx=
# Naming style matching correct attribute names.
attr-naming-style=snake_case
# Regular expression matching correct attribute names. Overrides attr-naming-
# style.
#attr-rgx=
# Bad variable names which should always be refused, separated by a comma.
bad-names=foo,
bar,
baz,
toto,
tutu,
tata
# Naming style matching correct class attribute names.
class-attribute-naming-style=any
# Regular expression matching correct class attribute names. Overrides class-
# attribute-naming-style.
#class-attribute-rgx=
# Naming style matching correct class names.
class-naming-style=PascalCase
# Regular expression matching correct class names. Overrides class-naming-
# style.
#class-rgx=
# Naming style matching correct constant names.
const-naming-style=UPPER_CASE
# Regular expression matching correct constant names. Overrides const-naming-
# style.
#const-rgx=
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=-1
# Naming style matching correct function names.
function-naming-style=snake_case
# Regular expression matching correct function names. Overrides function-
# naming-style.
#function-rgx=
# Good variable names which should always be accepted, separated by a comma.
good-names=i,
j,
k,
ex,
Run,
ch
_
# Include a hint for the correct naming format with invalid-name.
include-naming-hint=no
# Naming style matching correct inline iteration names.
inlinevar-naming-style=any
# Regular expression matching correct inline iteration names. Overrides
# inlinevar-naming-style.
#inlinevar-rgx=
# Naming style matching correct method names.
method-naming-style=snake_case
# Regular expression matching correct method names. Overrides method-naming-
# style.
#method-rgx=
# Naming style matching correct module names.
module-naming-style=snake_case
# Regular expression matching correct module names. Overrides module-naming-
# style.
#module-rgx=
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
# List of decorators that produce properties, such as abc.abstractproperty. Add
# to this list to register other decorators that produce valid properties.
# These decorators are taken in consideration only for invalid-name.
property-classes=abc.abstractproperty
# Naming style matching correct variable names.
variable-naming-style=snake_case
# Regular expression matching correct variable names. Overrides variable-
# naming-style.
#variable-rgx=
[IMPORTS]
# Allow wildcard imports from modules that define __all__.
allow-wildcard-with-all=no
# Analyse import fallback blocks. This can be used to support both Python 2 and
# 3 compatible code, which means that the block might have code that exists
# only in one or another interpreter, leading to false positives when analysed.
analyse-fallback-blocks=no
# Deprecated modules which should not be used, separated by a comma.
deprecated-modules=optparse,tkinter.tix
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled).
ext-import-graph=
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled).
import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled).
int-import-graph=
# Force import order to recognize a module as part of the standard
# compatibility libraries.
known-standard-library=
# Force import order to recognize a module as part of a third party library.
known-third-party=enchant
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,
__new__,
setUp
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,
_fields,
_replace,
_source,
_make
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=cls
[DESIGN]
# Maximum number of arguments for function / method.
max-args=10
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Maximum number of boolean expressions in an if statement.
max-bool-expr=5
# Maximum number of branch for function / method body.
max-branches=12
# Maximum number of locals for function / method body.
max-locals=15
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
# Maximum number of return / yield for function / method body.
max-returns=6
# Maximum number of statements in function / method body.
max-statements=55
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception".
overgeneral-exceptions=Exception
scripts/__init__.py
View File
scripts/algorithms/common/abstract/agent.py
+139
View File
@@ -0,0 +1,139 @@
# -*- coding: utf-8 -*-
"""Abstract Agent used for all agents.
- Author: Curt Park
- Contact: curt.park@medipixel.io
"""
import argparse
import os
import subprocess
from abc import ABC, abstractmethod
from typing import Tuple
import gym
import numpy as np
import torch
class AbstractAgent(ABC):
"""Abstract Agent used for all agents.
Attributes:
env (gym.Env): openAI Gym environment with discrete action space
args (argparse.Namespace): arguments including hyperparameters and training settings
state_dim (int): dimension of state space
action_dim (int): dimension of action space
sha (str): sha code of current git commit
"""
def __init__(self, env: gym.Env, args: argparse.Namespace):
"""Initialization.
Args:
env (gym.Env): openAI Gym environment with discrete action space
args (argparse.Namespace): arguments including hyperparameters and training settings
"""
self.args = args
self.env = NormalizedActions(env)
if self.args.max_episode_steps > 0:
env._max_episode_steps = self.args.max_episode_steps
else:
self.args.max_episode_steps = env._max_episode_steps
# for logging
self.sha = (
subprocess.check_output(["git", "rev-parse", "--short", "HEAD"])[:-1]
.decode("ascii")
.strip()
)
@abstractmethod
def select_action(self, state: np.ndarray):
pass
@abstractmethod
def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]:
pass
@abstractmethod
def update_model(self, *args):
pass
@abstractmethod
def load_params(self, *args):
pass
@abstractmethod
def save_params(self, name: str, params: dict, n_episode: int):
if not os.path.exists("./save"):
os.mkdir("./save")
path = os.path.join(
"./save/" + name + "_" + self.sha + "_ep_" + str(n_episode) + ".pt"
)
torch.save(params, path)
print("[INFO] Saved the model and optimizer to", path)
@abstractmethod
def write_log(self, *args):
pass
@abstractmethod
def train(self):
pass
def test(self):
"""Test the agent."""
for i_episode in range(self.args.episode_num):
state = self.env.reset()
done = False
score = 0
while not done:
if self.args.render and i_episode >= self.args.render_after:
self.env.render()
action = self.select_action(state)
next_state, reward, done = self.step(action)
state = next_state
score += reward
print("[INFO] episode %d\ttotal score: %d" % (i_episode, score))
# termination
self.env.close()
class NormalizedActions(gym.ActionWrapper):
"""Rescale and relocate the actions."""
def action(self, action: np.ndarray) -> np.ndarray:
"""Change the range (-1, 1) to (low, high)."""
low = self.action_space.low
high = self.action_space.high
scale_factor = (high - low) / 2
reloc_factor = high - scale_factor
action = action * scale_factor + reloc_factor
action = np.clip(action, low, high)
return action
def reverse_action(self, action: np.ndarray) -> np.ndarray:
"""Change the range (low, high) to (-1, 1)."""
low = self.action_space.low
high = self.action_space.high
scale_factor = (high - low) / 2
reloc_factor = high - scale_factor
action = (action - reloc_factor) / scale_factor
action = np.clip(action, -1.0, 1.0)
return action
scripts/algorithms/common/buffer/replay_buffer.py
+72
View File
@@ -0,0 +1,72 @@
# -*- coding: utf-8 -*-
"""Replay buffer for baselines."""
import random
from collections import deque
import numpy as np
import torch
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
class ReplayBuffer:
"""Fixed-size buffer to store experience tuples.
Taken from Udacity deep-reinforcement-learning github repository:
https://github.com/udacity/deep-reinforcement-learning/blob/master/
ddpg-pendulum/ddpg_agent.py
Attributes:
buffer (deque): deque of replay buffer
batch_size (int): size of a batched sampled from replay buffer for training
"""
def __init__(self, buffer_size, batch_size, seed, demo=None):
"""Initialize a ReplayBuffer object.
Args:
buffer_size (int): size of replay buffer for experience
batch_size (int): size of a batched sampled from replay buffer for training
seed (int): random seed
demo (deque) : demonstration deque
"""
self.buffer = deque(maxlen=buffer_size) if not demo else demo
self.batch_size = batch_size
random.seed(seed)
def add(self, state, action, reward, next_state, done):
"""Add a new experience to memory."""
self.buffer.append((state, action, reward, next_state, done))
def extend(self, transitions):
"""Add experiences to memory."""
self.buffer.extend(transitions)
def sample(self):
"""Randomly sample a batch of experiences from memory."""
experiences = random.sample(self.buffer, k=self.batch_size)
states, actions, rewards, next_states, dones = [], [], [], [], []
for e in experiences:
states.append(np.expand_dims(e[0], axis=0))
actions.append(e[1])
rewards.append(e[2])
next_states.append(np.expand_dims(e[3], axis=0))
dones.append(e[4])
states = torch.from_numpy(np.vstack(states)).float().to(device)
actions = torch.from_numpy(np.vstack(actions)).float().to(device)
rewards = torch.from_numpy(np.vstack(rewards)).float().to(device)
next_states = torch.from_numpy(np.vstack(next_states)).float().to(device)
dones = torch.from_numpy(np.vstack(dones).astype(np.uint8)).float().to(device)
return (states, actions, rewards, next_states, dones)
def __len__(self):
"""Return the current size of internal memory."""
return len(self.buffer)
scripts/algorithms/common/helper_functions.py
+20
View File
@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
"""Common util functions for all algorithms.
- Author: Curt Park
- Contact: curt.park@medipixel.io
"""
import torch
import torch.nn as nn
def identity(x: torch.Tensor) -> torch.Tensor:
"""Return input without any change."""
return x
def soft_update(local: nn.Module, target: nn.Module, tau: float):
"""Soft-update: target = tau*local + (1-tau)*target."""
for t_param, l_param in zip(target.parameters(), local.parameters()):
t_param.data.copy_(tau * l_param.data + (1.0 - tau) * t_param.data)
scripts/algorithms/common/networks/mlp.py
+204
View File
@@ -0,0 +1,204 @@
# -*- coding: utf-8 -*-
"""MLP module for model of algorithms
- Author: Kh Kim
- Contact: kh.kim@medipixel.io
"""
from typing import Callable, Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.distributions import Normal
from algorithms.common.helper_functions import identity
class MLP(nn.Module):
"""Baseline of Multilayer perceptron.
Attributes:
input_size (int): size of input
output_size (int): size of output layer
hidden_sizes (list): sizes of hidden layers
hidden_activation (function): activation function of hidden layers
output_activation (function): activation function of output layer
hidden_layers (list): list containing linear layers
use_output_layer (bool): whether or not to use the last layer
"""
def __init__(
self,
input_size: int,
output_size: int,
hidden_sizes: list,
hidden_activation: Callable = F.relu,
output_activation: Callable = identity,
use_output_layer: bool = True,
init_w: float = 3e-3,
):
"""Initialization.
Args:
input_size (int): size of input
output_size (int): size of output layer
hidden_sizes (list): number of hidden layers
hidden_activation (function): activation function of hidden layers
output_activation (function): activation function of output layer
use_output_layer (bool): whether or not to use the last layer
init_w (float): weight initialization bound for the last layer
"""
super(MLP, self).__init__()
self.hidden_sizes = hidden_sizes
self.input_size = input_size
self.output_size = output_size
self.hidden_activation = hidden_activation
self.output_activation = output_activation
self.use_output_layer = use_output_layer
# set hidden layers
self.hidden_layers: list = []
in_size = self.input_size
for i, next_size in enumerate(hidden_sizes):
fc = nn.Linear(in_size, next_size)
in_size = next_size
self.__setattr__("hidden_fc{}".format(i), fc)
self.hidden_layers.append(fc)
# set output layers
if self.use_output_layer:
self.output_layer = nn.Linear(in_size, output_size)
self.output_layer.weight.data.uniform_(-init_w, init_w)
self.output_layer.bias.data.uniform_(-init_w, init_w)
def get_last_activation(self, x: torch.Tensor) -> torch.Tensor:
"""Get the activation of the last hidden layer."""
for hidden_layer in self.hidden_layers:
x = self.hidden_activation(hidden_layer(x))
return x
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""Forward method implementation."""
assert self.use_output_layer
x = self.get_last_activation(x)
output = self.output_layer(x)
output = self.output_activation(output)
return output
class GaussianDist(MLP):
"""Multilayer perceptron with Gaussian distribution output.
Attributes:
mu_activation (function): bounding function for mean
log_std_clamping (bool): whether or not to clamp log std
log_std_min (float): lower bound of log std
log_std_max (float): upper bound of log std
mu_layer (nn.Linear): output layer for mean
log_std_layer (nn.Linear): output layer for log std
"""
def __init__(
self,
input_size: int,
output_size: int,
hidden_sizes: list,
hidden_activation: Callable = F.relu,
mu_activation: Callable = torch.tanh,
log_std_min: float = -20,
log_std_max: float = 2,
init_w: float = 3e-3,
):
"""Initialization.
"""
super(GaussianDist, self).__init__(
input_size=input_size,
output_size=output_size,
hidden_sizes=hidden_sizes,
hidden_activation=hidden_activation,
use_output_layer=False,
)
self.mu_activation = mu_activation
self.log_std_min = log_std_min
self.log_std_max = log_std_max
in_size = hidden_sizes[-1]
# set log_std layer
self.log_std_layer = nn.Linear(in_size, output_size)
self.log_std_layer.weight.data.uniform_(-init_w, init_w)
self.log_std_layer.bias.data.uniform_(-init_w, init_w)
# set mean layer
self.mu_layer = nn.Linear(in_size, output_size)
self.mu_layer.weight.data.uniform_(-init_w, init_w)
self.mu_layer.bias.data.uniform_(-init_w, init_w)
def get_dist_params(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]:
"""Return gausian distribution parameters."""
hidden = super(GaussianDist, self).get_last_activation(x)
# get mean
mu = self.mu_activation(self.mu_layer(hidden))
# get std
log_std = torch.clamp(
self.log_std_layer(hidden), self.log_std_min, self.log_std_max
)
std = torch.exp(log_std)
return mu, log_std, std
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]:
"""Forward method implementation."""
mu, _, std = self.get_dist_params(x)
# get normal distribution and action
dist = Normal(mu, std)
action = dist.sample()
return action, dist
class GaussianDistParams(GaussianDist):
"""Multilayer perceptron with Gaussian distribution params output."""
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]:
"""Forward method implementation."""
mu, log_std, std = super(GaussianDistParams, self).get_dist_params(x)
return mu, log_std, std
class TanhGaussianDistParams(GaussianDist):
"""Multilayer perceptron with Gaussian distribution output."""
def __init__(self, **kwargs):
"""Initialization."""
super(TanhGaussianDistParams, self).__init__(**kwargs, mu_activation=identity)
def forward(
self, x: torch.Tensor, epsilon: float = 1e-6
) -> Tuple[torch.Tensor, ...]:
"""Forward method implementation."""
mu, _, std = super(TanhGaussianDistParams, self).get_dist_params(x)
# sampling actions
dist = Normal(mu, std)
z = dist.rsample()
# normalize action and log_prob
# see appendix C of 'https://arxiv.org/pdf/1812.05905.pdf'
action = torch.tanh(z)
log_prob = dist.log_prob(z) - torch.log(1 - action.pow(2) + epsilon)
log_prob = log_prob.sum(-1, keepdim=True)
return action, log_prob, z, mu, std
scripts/algorithms/common/noise.py
+39
View File
@@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
"""Noise classes for baselines."""
import copy
import random
import numpy as np
class OUNoise:
"""Ornstein-Uhlenbeck process.
Taken from Udacity deep-reinforcement-learning github repository:
https://github.com/udacity/deep-reinforcement-learning/blob/master/
ddpg-pendulum/ddpg_agent.py
"""
def __init__(self, size, seed, mu=0.0, theta=0.15, sigma=0.2):
"""Initialize parameters and noise process."""
self.state = np.float64(0.0)
self.mu = mu * np.ones(size)
self.theta = theta
self.sigma = sigma
self.reset()
random.seed(seed)
def reset(self):
"""Reset the internal state (= noise) to mean (mu)."""
self.state = copy.copy(self.mu)
def sample(self):
"""Update internal state and return it as a noise sample."""
x = self.state
dx = self.theta * (self.mu - x) + self.sigma * np.array(
[random.random() for _ in range(len(x))]
)
self.state = x + dx
return self.state
scripts/algorithms/ddpg/agent.py
+226
View File
@@ -0,0 +1,226 @@
# -*- coding: utf-8 -*-
"""DDPG agent for episodic tasks in OpenAI Gym.
- Author: Curt Park
- Contact: curt.park@medipixel.io
- Paper: https://arxiv.org/pdf/1509.02971.pdf
"""
import argparse
import os
from typing import Tuple
import gym
import numpy as np
import torch
import torch.nn.functional as F
import wandb
import algorithms.common.helper_functions as common_utils
from algorithms.common.abstract.agent import AbstractAgent
from algorithms.common.buffer.replay_buffer import ReplayBuffer
from algorithms.common.noise import OUNoise
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
class Agent(AbstractAgent):
"""ActorCritic interacting with environment.
Attributes:
memory (ReplayBuffer): replay memory
noise (OUNoise): random noise for exploration
hyper_params (dict): hyper-parameters
actor (nn.Module): actor model to select actions
actor_target (nn.Module): target actor model to select actions
critic (nn.Module): critic model to predict state values
critic_target (nn.Module): target critic model to predict state values
actor_optimizer (Optimizer): optimizer for training actor
critic_optimizer (Optimizer): optimizer for training critic
curr_state (np.ndarray): temporary storage of the current state
"""
def __init__(
self,
env: gym.Env,
args: argparse.Namespace,
hyper_params: dict,
models: tuple,
optims: tuple,
noise: OUNoise,
):
"""Initialization.
Args:
env (gym.Env): openAI Gym environment with discrete action space
args (argparse.Namespace): arguments including hyperparameters and training settings
hyper_params (dict): hyper-parameters
models (tuple): models including actor and critic
optims (tuple): optimizers for actor and critic
noise (OUNoise): random noise for exploration
"""
AbstractAgent.__init__(self, env, args)
self.actor, self.actor_target, self.critic, self.critic_target = models
self.actor_optimizer, self.critic_optimizer = optims
self.hyper_params = hyper_params
self.curr_state = np.zeros((1,))
self.noise = noise
# load the optimizer and model parameters
if args.load_from is not None and os.path.exists(args.load_from):
self.load_params(args.load_from)
# replay memory
self.memory = ReplayBuffer(
hyper_params["BUFFER_SIZE"], hyper_params["BATCH_SIZE"], self.args.seed
)
def select_action(self, state: np.ndarray) -> torch.Tensor:
"""Select an action from the input space."""
self.curr_state = state
state = torch.FloatTensor(state).to(device)
selected_action = self.actor(state)
selected_action += torch.FloatTensor(self.noise.sample()).to(device)
selected_action = torch.clamp(selected_action, -1.0, 1.0)
return selected_action
def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]:
"""Take an action and return the response of the env."""
action = action.detach().cpu().numpy()
next_state, reward, done, _ = self.env.step(action)
self.memory.add(self.curr_state, action, reward, next_state, done)
return next_state, reward, done
def update_model(
self,
experiences: Tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
],
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Train the model after each episode."""
states, actions, rewards, next_states, dones = experiences
# G_t = r + gamma * v(s_{t+1}) if state != Terminal
# = r otherwise
masks = 1 - dones
next_actions = self.actor_target(next_states)
next_values = self.critic_target(torch.cat((next_states, next_actions), dim=-1))
curr_returns = rewards + self.hyper_params["GAMMA"] * next_values * masks
curr_returns = curr_returns.to(device)
# train critic
values = self.critic(torch.cat((states, actions), dim=-1))
critic_loss = F.mse_loss(values, curr_returns)
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# train actor
actions = self.actor(states)
actor_loss = -self.critic(torch.cat((states, actions), dim=-1)).mean()
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# update target networks
tau = self.hyper_params["TAU"]
common_utils.soft_update(self.actor, self.actor_target, tau)
common_utils.soft_update(self.critic, self.critic_target, tau)
return actor_loss.data, critic_loss.data
def load_params(self, path: str):
"""Load model and optimizer parameters."""
if not os.path.exists(path):
print("[ERROR] the input path does not exist. ->", path)
return
params = torch.load(path)
self.actor.load_state_dict(params["actor_state_dict"])
self.actor_target.load_state_dict(params["actor_target_state_dict"])
self.critic.load_state_dict(params["critic_state_dict"])
self.critic_target.load_state_dict(params["critic_target_state_dict"])
self.actor_optimizer.load_state_dict(params["actor_optim_state_dict"])
self.critic_optimizer.load_state_dict(params["critic_optim_state_dict"])
print("[INFO] loaded the model and optimizer from", path)
def save_params(self, n_episode: int):
"""Save model and optimizer parameters."""
params = {
"actor_state_dict": self.actor.state_dict(),
"actor_target_state_dict": self.actor_target.state_dict(),
"critic_state_dict": self.critic.state_dict(),
"critic_target_state_dict": self.critic_target.state_dict(),
"actor_optim_state_dict": self.actor_optimizer.state_dict(),
"critic_optim_state_dict": self.critic_optimizer.state_dict(),
}
AbstractAgent.save_params(self, self.args.algo, params, n_episode)
def write_log(self, i: int, loss: np.ndarray, score: int):
"""Write log about loss and score"""
total_loss = loss.sum()
print(
"[INFO] episode %d total score: %d, total loss: %f\n"
"actor_loss: %.3f critic_loss: %.3f\n"
% (i, score, total_loss, loss[0], loss[1]) # actor loss # critic loss
)
if self.args.log:
wandb.log(
{
"score": score,
"total loss": total_loss,
"actor loss": loss[0],
"critic loss": loss[1],
}
)
def train(self):
"""Train the agent."""
# logger
if self.args.log:
wandb.init()
wandb.config.update(self.hyper_params)
wandb.watch([self.actor, self.critic], log="parameters")
for i_episode in range(1, self.args.episode_num + 1):
state = self.env.reset()
done = False
score = 0
loss_episode = list()
while not done:
if self.args.render and i_episode >= self.args.render_after:
self.env.render()
action = self.select_action(state)
next_state, reward, done = self.step(action)
if len(self.memory) >= self.hyper_params["BATCH_SIZE"]:
experiences = self.memory.sample()
loss = self.update_model(experiences)
loss_episode.append(loss) # for logging
state = next_state
score += reward
# logging
if loss_episode:
avg_loss = np.vstack(loss_episode).mean(axis=0)
self.write_log(i_episode, avg_loss, score)
if i_episode % self.args.save_period == 0:
self.save_params(i_episode)
# termination
self.env.close()
scripts/examples/lunarlander_continuous_v2/ddpg.py
+104
View File
@@ -0,0 +1,104 @@
# -*- coding: utf-8 -*-
"""Run module for DDPG on LunarLanderContinuous-v2.
- Author: Curt Park
- Contact: curt.park@medipixel.io
"""
import argparse
import gym
import torch
import torch.optim as optim
from algorithms.common.networks.mlp import MLP
from algorithms.common.noise import OUNoise
from algorithms.ddpg.agent import Agent
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# hyper parameters
hyper_params = {
"GAMMA": 0.99,
"TAU": 1e-3,
"BUFFER_SIZE": int(1e5),
"BATCH_SIZE": 128,
"LR_ACTOR": 1e-3,
"LR_CRITIC": 1e-3,
"OU_NOISE_THETA": 0.0,
"OU_NOISE_SIGMA": 0.0,
"WEIGHT_DECAY": 1e-6,
}
def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int):
"""Run training or test.
Args:
env (gym.Env): openAI Gym environment with continuous action space
args (argparse.Namespace): arguments including training settings
state_dim (int): dimension of states
action_dim (int): dimension of actions
"""
hidden_sizes = [256, 256]
# create actor
actor = MLP(
input_size=state_dim,
output_size=action_dim,
hidden_sizes=hidden_sizes,
output_activation=torch.tanh,
).to(device)
actor_target = MLP(
input_size=state_dim,
output_size=action_dim,
hidden_sizes=hidden_sizes,
output_activation=torch.tanh,
).to(device)
actor_target.load_state_dict(actor.state_dict())
# create critic
critic = MLP(
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes
).to(device)
critic_target = MLP(
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes
).to(device)
critic_target.load_state_dict(critic.state_dict())
# create optimizer
actor_optim = optim.Adam(
actor.parameters(),
lr=hyper_params["LR_ACTOR"],
weight_decay=hyper_params["WEIGHT_DECAY"],
)
critic_optim = optim.Adam(
critic.parameters(),
lr=hyper_params["LR_CRITIC"],
weight_decay=hyper_params["WEIGHT_DECAY"],
)
# noise
noise = OUNoise(
action_dim,
args.seed,
theta=hyper_params["OU_NOISE_THETA"],
sigma=hyper_params["OU_NOISE_SIGMA"],
)
# make tuples to create an agent
models = (actor, actor_target, critic, critic_target)
optims = (actor_optim, critic_optim)
# create an agent
agent = Agent(env, args, hyper_params, models, optims, noise)
# run
if args.test:
agent.test()
else:
agent.train()
scripts/mypy.ini
-5
View File
@@ -1,5 +0,0 @@
# Global options:
[mypy]
python_version = 3.6
ignore_missing_imports = True
scripts/requirements-dev.txt
+16
View File
@@ -0,0 +1,16 @@
pre-commit
# formatting
black
isort
# testing
flake8==3.6.0
flake8-bugbear
flake8-docstrings
pytest>=4.0.0
pytest-flake8
pytest-cov
# generating requirements
pipreqs
scripts/requirements.txt
+5
View File
@@ -0,0 +1,5 @@
gym
numpy
torch==1.0.0
typing
wandb
scripts/run_lunarlander_continuous.py
+67
View File
@@ -0,0 +1,67 @@
# -*- coding: utf-8 -*-
"""Train or test baselines on LunarLanderContinuous-v2.
- Author: Curt Park
- Contact: curt.park@medipixel.io
"""
import argparse
import importlib
import gym
import numpy as np
import torch
# configurations
parser = argparse.ArgumentParser(description="Pytorch RL baselines")
parser.add_argument(
"--seed", type=int, default=777, help="random seed for reproducibility"
)
parser.add_argument("--algo", type=str, default="ddpg", help="choose an algorithm")
parser.add_argument(
"--load-from",
type=str,
default=None,
help="load the saved model and optimizer at the beginning",
)
parser.add_argument("--episode-num", type=int, default=1500, help="total episode num")
parser.add_argument(
"--max-episode-steps", type=int, default=300, help="max episode step"
)
parser.add_argument(
"--off-render", dest="render", action="store_false", help="turn off rendering"
)
parser.add_argument(
"--render-after",
type=int,
default=0,
help="start rendering after the input number of episode",
)
parser.add_argument("--save-period", type=int, default=100, help="save model period")
parser.add_argument("--log", action="store_true", help="turn on logging")
parser.add_argument("--test", action="store_true", help="test mode (no training)")
parser.set_defaults(render=True)
args = parser.parse_args()
def main():
"""Main."""
# env initialization
env = gym.make("LunarLanderContinuous-v2")
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.shape[0]
# set a random seed
env.seed(args.seed)
torch.manual_seed(args.seed)
np.random.seed(args.seed)
# run
module_path = "examples.lunarlander_continuous_v2." + args.algo
example = importlib.import_module(module_path)
example.run(env, args, state_dim, action_dim)
if __name__ == "__main__":
main()