From 7f4756a1d466110b739bb93b46851ba505481c43 Mon Sep 17 00:00:00 2001 From: Whi Kwon <30768596+whikwon@users.noreply.github.com> Date: Tue, 5 Feb 2019 20:07:46 +0900 Subject: [PATCH] Add overall setting and ddpg baseline (#1) * Add overall CI settings * Add specific build dir to travis * Add before install/script condition to travis * Add ddpg baseline * Add wandb, remove algorithms except ddpg * Remove init file in script * Separate config file for ddpg * Remove unnecessary examples * Remove unnecessary args opt * Add pre-commit setting * Change pre-commit settings * Change travis-ci setting * Fix travis-ci issue * Modify argparse arguments, fix requirements * Change arguments order --- scripts/.flake8 => .flake8 | 0 .travis.yml | 13 + Makefile | 13 + scripts/.pre-commit-config.yaml | 13 + scripts/.pylintrc | 573 ------------------ scripts/__init__.py | 0 scripts/algorithms/common/abstract/agent.py | 139 +++++ .../algorithms/common/buffer/replay_buffer.py | 72 +++ scripts/algorithms/common/helper_functions.py | 20 + scripts/algorithms/common/networks/mlp.py | 204 +++++++ scripts/algorithms/common/noise.py | 39 ++ scripts/algorithms/ddpg/agent.py | 226 +++++++ .../lunarlander_continuous_v2/ddpg.py | 104 ++++ scripts/mypy.ini | 5 - scripts/requirements-dev.txt | 16 + scripts/requirements.txt | 5 + scripts/run_lunarlander_continuous.py | 67 ++ 17 files changed, 931 insertions(+), 578 deletions(-) rename scripts/.flake8 => .flake8 (100%) create mode 100644 .travis.yml create mode 100644 Makefile create mode 100644 scripts/.pre-commit-config.yaml delete mode 100644 scripts/.pylintrc delete mode 100644 scripts/__init__.py create mode 100644 scripts/algorithms/common/abstract/agent.py create mode 100644 scripts/algorithms/common/buffer/replay_buffer.py create mode 100644 scripts/algorithms/common/helper_functions.py create mode 100644 scripts/algorithms/common/networks/mlp.py create mode 100644 scripts/algorithms/common/noise.py create mode 100644 scripts/algorithms/ddpg/agent.py create mode 100644 scripts/examples/lunarlander_continuous_v2/ddpg.py delete mode 100644 scripts/mypy.ini create mode 100644 scripts/requirements-dev.txt create mode 100644 scripts/requirements.txt create mode 100644 scripts/run_lunarlander_continuous.py diff --git a/scripts/.flake8 b/.flake8 similarity index 100% rename from scripts/.flake8 rename to .flake8 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..8903b57 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +dist: xenial + +language: python +python: + - "3.6" + +install: + - make dep + - make dev + +script: + # run static analysis and format check + - make test diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..654a3a4 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +test: + pytest --flake8 # --cov=algorithms + +format: + black . + isort -y + +dev: + pip install -r scripts/requirements-dev.txt + pre-commit install + +dep: + pip install -r scripts/requirements.txt diff --git a/scripts/.pre-commit-config.yaml b/scripts/.pre-commit-config.yaml new file mode 100644 index 0000000..c6df5a2 --- /dev/null +++ b/scripts/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: +- repo: local + hooks: + - id: format + name: format + language: system + entry: make format + types: [python] + - id: test + name: test + language: system + entry: make test + types: [python] diff --git a/scripts/.pylintrc b/scripts/.pylintrc deleted file mode 100644 index 760e642..0000000 --- a/scripts/.pylintrc +++ /dev/null @@ -1,573 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=print-statement, - bad-continuation, - no-else-return, - broad-except, - missing-docstring, - invalid-name, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - locally-enabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - no-member, - useless-import-alias, - too-many-locals, - too-few-public-methods, - arguments-differ, - duplicate-code, - protected-access, - too-many-instance-attributes, - fixme - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package.. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. -#class-attribute-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - ch - _ - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. -#variable-rgx= - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=10 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement. -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=55 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception". -overgeneral-exceptions=Exception diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/algorithms/common/abstract/agent.py b/scripts/algorithms/common/abstract/agent.py new file mode 100644 index 0000000..c52b5ba --- /dev/null +++ b/scripts/algorithms/common/abstract/agent.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +"""Abstract Agent used for all agents. + +- Author: Curt Park +- Contact: curt.park@medipixel.io +""" + +import argparse +import os +import subprocess +from abc import ABC, abstractmethod +from typing import Tuple + +import gym +import numpy as np +import torch + + +class AbstractAgent(ABC): + """Abstract Agent used for all agents. + + Attributes: + env (gym.Env): openAI Gym environment with discrete action space + args (argparse.Namespace): arguments including hyperparameters and training settings + state_dim (int): dimension of state space + action_dim (int): dimension of action space + sha (str): sha code of current git commit + + """ + + def __init__(self, env: gym.Env, args: argparse.Namespace): + """Initialization. + + Args: + env (gym.Env): openAI Gym environment with discrete action space + args (argparse.Namespace): arguments including hyperparameters and training settings + + """ + self.args = args + self.env = NormalizedActions(env) + if self.args.max_episode_steps > 0: + env._max_episode_steps = self.args.max_episode_steps + else: + self.args.max_episode_steps = env._max_episode_steps + + # for logging + self.sha = ( + subprocess.check_output(["git", "rev-parse", "--short", "HEAD"])[:-1] + .decode("ascii") + .strip() + ) + + @abstractmethod + def select_action(self, state: np.ndarray): + pass + + @abstractmethod + def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]: + pass + + @abstractmethod + def update_model(self, *args): + pass + + @abstractmethod + def load_params(self, *args): + pass + + @abstractmethod + def save_params(self, name: str, params: dict, n_episode: int): + if not os.path.exists("./save"): + os.mkdir("./save") + + path = os.path.join( + "./save/" + name + "_" + self.sha + "_ep_" + str(n_episode) + ".pt" + ) + torch.save(params, path) + + print("[INFO] Saved the model and optimizer to", path) + + @abstractmethod + def write_log(self, *args): + pass + + @abstractmethod + def train(self): + pass + + def test(self): + """Test the agent.""" + for i_episode in range(self.args.episode_num): + state = self.env.reset() + done = False + score = 0 + + while not done: + if self.args.render and i_episode >= self.args.render_after: + self.env.render() + + action = self.select_action(state) + next_state, reward, done = self.step(action) + + state = next_state + score += reward + + print("[INFO] episode %d\ttotal score: %d" % (i_episode, score)) + + # termination + self.env.close() + + +class NormalizedActions(gym.ActionWrapper): + """Rescale and relocate the actions.""" + + def action(self, action: np.ndarray) -> np.ndarray: + """Change the range (-1, 1) to (low, high).""" + low = self.action_space.low + high = self.action_space.high + + scale_factor = (high - low) / 2 + reloc_factor = high - scale_factor + + action = action * scale_factor + reloc_factor + action = np.clip(action, low, high) + + return action + + def reverse_action(self, action: np.ndarray) -> np.ndarray: + """Change the range (low, high) to (-1, 1).""" + low = self.action_space.low + high = self.action_space.high + + scale_factor = (high - low) / 2 + reloc_factor = high - scale_factor + + action = (action - reloc_factor) / scale_factor + action = np.clip(action, -1.0, 1.0) + + return action diff --git a/scripts/algorithms/common/buffer/replay_buffer.py b/scripts/algorithms/common/buffer/replay_buffer.py new file mode 100644 index 0000000..a37ef45 --- /dev/null +++ b/scripts/algorithms/common/buffer/replay_buffer.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +"""Replay buffer for baselines.""" + +import random +from collections import deque + +import numpy as np +import torch + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + +class ReplayBuffer: + """Fixed-size buffer to store experience tuples. + + Taken from Udacity deep-reinforcement-learning github repository: + https://github.com/udacity/deep-reinforcement-learning/blob/master/ + ddpg-pendulum/ddpg_agent.py + + Attributes: + buffer (deque): deque of replay buffer + batch_size (int): size of a batched sampled from replay buffer for training + + """ + + def __init__(self, buffer_size, batch_size, seed, demo=None): + """Initialize a ReplayBuffer object. + + Args: + buffer_size (int): size of replay buffer for experience + batch_size (int): size of a batched sampled from replay buffer for training + seed (int): random seed + demo (deque) : demonstration deque + + """ + self.buffer = deque(maxlen=buffer_size) if not demo else demo + + self.batch_size = batch_size + random.seed(seed) + + def add(self, state, action, reward, next_state, done): + """Add a new experience to memory.""" + self.buffer.append((state, action, reward, next_state, done)) + + def extend(self, transitions): + """Add experiences to memory.""" + self.buffer.extend(transitions) + + def sample(self): + """Randomly sample a batch of experiences from memory.""" + experiences = random.sample(self.buffer, k=self.batch_size) + + states, actions, rewards, next_states, dones = [], [], [], [], [] + + for e in experiences: + states.append(np.expand_dims(e[0], axis=0)) + actions.append(e[1]) + rewards.append(e[2]) + next_states.append(np.expand_dims(e[3], axis=0)) + dones.append(e[4]) + + states = torch.from_numpy(np.vstack(states)).float().to(device) + actions = torch.from_numpy(np.vstack(actions)).float().to(device) + rewards = torch.from_numpy(np.vstack(rewards)).float().to(device) + next_states = torch.from_numpy(np.vstack(next_states)).float().to(device) + dones = torch.from_numpy(np.vstack(dones).astype(np.uint8)).float().to(device) + + return (states, actions, rewards, next_states, dones) + + def __len__(self): + """Return the current size of internal memory.""" + return len(self.buffer) diff --git a/scripts/algorithms/common/helper_functions.py b/scripts/algorithms/common/helper_functions.py new file mode 100644 index 0000000..7a1e22f --- /dev/null +++ b/scripts/algorithms/common/helper_functions.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +"""Common util functions for all algorithms. + +- Author: Curt Park +- Contact: curt.park@medipixel.io +""" + +import torch +import torch.nn as nn + + +def identity(x: torch.Tensor) -> torch.Tensor: + """Return input without any change.""" + return x + + +def soft_update(local: nn.Module, target: nn.Module, tau: float): + """Soft-update: target = tau*local + (1-tau)*target.""" + for t_param, l_param in zip(target.parameters(), local.parameters()): + t_param.data.copy_(tau * l_param.data + (1.0 - tau) * t_param.data) diff --git a/scripts/algorithms/common/networks/mlp.py b/scripts/algorithms/common/networks/mlp.py new file mode 100644 index 0000000..449daad --- /dev/null +++ b/scripts/algorithms/common/networks/mlp.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +"""MLP module for model of algorithms + +- Author: Kh Kim +- Contact: kh.kim@medipixel.io +""" + +from typing import Callable, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.distributions import Normal + +from algorithms.common.helper_functions import identity + + +class MLP(nn.Module): + """Baseline of Multilayer perceptron. + + Attributes: + input_size (int): size of input + output_size (int): size of output layer + hidden_sizes (list): sizes of hidden layers + hidden_activation (function): activation function of hidden layers + output_activation (function): activation function of output layer + hidden_layers (list): list containing linear layers + use_output_layer (bool): whether or not to use the last layer + + """ + + def __init__( + self, + input_size: int, + output_size: int, + hidden_sizes: list, + hidden_activation: Callable = F.relu, + output_activation: Callable = identity, + use_output_layer: bool = True, + init_w: float = 3e-3, + ): + """Initialization. + + Args: + input_size (int): size of input + output_size (int): size of output layer + hidden_sizes (list): number of hidden layers + hidden_activation (function): activation function of hidden layers + output_activation (function): activation function of output layer + use_output_layer (bool): whether or not to use the last layer + init_w (float): weight initialization bound for the last layer + + """ + super(MLP, self).__init__() + + self.hidden_sizes = hidden_sizes + self.input_size = input_size + self.output_size = output_size + self.hidden_activation = hidden_activation + self.output_activation = output_activation + self.use_output_layer = use_output_layer + + # set hidden layers + self.hidden_layers: list = [] + in_size = self.input_size + for i, next_size in enumerate(hidden_sizes): + fc = nn.Linear(in_size, next_size) + in_size = next_size + self.__setattr__("hidden_fc{}".format(i), fc) + self.hidden_layers.append(fc) + + # set output layers + if self.use_output_layer: + self.output_layer = nn.Linear(in_size, output_size) + self.output_layer.weight.data.uniform_(-init_w, init_w) + self.output_layer.bias.data.uniform_(-init_w, init_w) + + def get_last_activation(self, x: torch.Tensor) -> torch.Tensor: + """Get the activation of the last hidden layer.""" + for hidden_layer in self.hidden_layers: + x = self.hidden_activation(hidden_layer(x)) + return x + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward method implementation.""" + assert self.use_output_layer + + x = self.get_last_activation(x) + + output = self.output_layer(x) + output = self.output_activation(output) + + return output + + +class GaussianDist(MLP): + """Multilayer perceptron with Gaussian distribution output. + + Attributes: + mu_activation (function): bounding function for mean + log_std_clamping (bool): whether or not to clamp log std + log_std_min (float): lower bound of log std + log_std_max (float): upper bound of log std + mu_layer (nn.Linear): output layer for mean + log_std_layer (nn.Linear): output layer for log std + """ + + def __init__( + self, + input_size: int, + output_size: int, + hidden_sizes: list, + hidden_activation: Callable = F.relu, + mu_activation: Callable = torch.tanh, + log_std_min: float = -20, + log_std_max: float = 2, + init_w: float = 3e-3, + ): + """Initialization. + + """ + super(GaussianDist, self).__init__( + input_size=input_size, + output_size=output_size, + hidden_sizes=hidden_sizes, + hidden_activation=hidden_activation, + use_output_layer=False, + ) + + self.mu_activation = mu_activation + self.log_std_min = log_std_min + self.log_std_max = log_std_max + in_size = hidden_sizes[-1] + + # set log_std layer + self.log_std_layer = nn.Linear(in_size, output_size) + self.log_std_layer.weight.data.uniform_(-init_w, init_w) + self.log_std_layer.bias.data.uniform_(-init_w, init_w) + + # set mean layer + self.mu_layer = nn.Linear(in_size, output_size) + self.mu_layer.weight.data.uniform_(-init_w, init_w) + self.mu_layer.bias.data.uniform_(-init_w, init_w) + + def get_dist_params(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]: + """Return gausian distribution parameters.""" + hidden = super(GaussianDist, self).get_last_activation(x) + + # get mean + mu = self.mu_activation(self.mu_layer(hidden)) + + # get std + log_std = torch.clamp( + self.log_std_layer(hidden), self.log_std_min, self.log_std_max + ) + std = torch.exp(log_std) + + return mu, log_std, std + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]: + """Forward method implementation.""" + mu, _, std = self.get_dist_params(x) + + # get normal distribution and action + dist = Normal(mu, std) + action = dist.sample() + + return action, dist + + +class GaussianDistParams(GaussianDist): + """Multilayer perceptron with Gaussian distribution params output.""" + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, ...]: + """Forward method implementation.""" + mu, log_std, std = super(GaussianDistParams, self).get_dist_params(x) + + return mu, log_std, std + + +class TanhGaussianDistParams(GaussianDist): + """Multilayer perceptron with Gaussian distribution output.""" + + def __init__(self, **kwargs): + """Initialization.""" + super(TanhGaussianDistParams, self).__init__(**kwargs, mu_activation=identity) + + def forward( + self, x: torch.Tensor, epsilon: float = 1e-6 + ) -> Tuple[torch.Tensor, ...]: + """Forward method implementation.""" + mu, _, std = super(TanhGaussianDistParams, self).get_dist_params(x) + + # sampling actions + dist = Normal(mu, std) + z = dist.rsample() + + # normalize action and log_prob + # see appendix C of 'https://arxiv.org/pdf/1812.05905.pdf' + action = torch.tanh(z) + log_prob = dist.log_prob(z) - torch.log(1 - action.pow(2) + epsilon) + log_prob = log_prob.sum(-1, keepdim=True) + + return action, log_prob, z, mu, std diff --git a/scripts/algorithms/common/noise.py b/scripts/algorithms/common/noise.py new file mode 100644 index 0000000..d99202e --- /dev/null +++ b/scripts/algorithms/common/noise.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +"""Noise classes for baselines.""" + +import copy +import random + +import numpy as np + + +class OUNoise: + """Ornstein-Uhlenbeck process. + + Taken from Udacity deep-reinforcement-learning github repository: + https://github.com/udacity/deep-reinforcement-learning/blob/master/ + ddpg-pendulum/ddpg_agent.py + """ + + def __init__(self, size, seed, mu=0.0, theta=0.15, sigma=0.2): + """Initialize parameters and noise process.""" + self.state = np.float64(0.0) + self.mu = mu * np.ones(size) + self.theta = theta + self.sigma = sigma + self.reset() + + random.seed(seed) + + def reset(self): + """Reset the internal state (= noise) to mean (mu).""" + self.state = copy.copy(self.mu) + + def sample(self): + """Update internal state and return it as a noise sample.""" + x = self.state + dx = self.theta * (self.mu - x) + self.sigma * np.array( + [random.random() for _ in range(len(x))] + ) + self.state = x + dx + return self.state diff --git a/scripts/algorithms/ddpg/agent.py b/scripts/algorithms/ddpg/agent.py new file mode 100644 index 0000000..3fa65c2 --- /dev/null +++ b/scripts/algorithms/ddpg/agent.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +"""DDPG agent for episodic tasks in OpenAI Gym. + +- Author: Curt Park +- Contact: curt.park@medipixel.io +- Paper: https://arxiv.org/pdf/1509.02971.pdf +""" + +import argparse +import os +from typing import Tuple + +import gym +import numpy as np +import torch +import torch.nn.functional as F +import wandb + +import algorithms.common.helper_functions as common_utils +from algorithms.common.abstract.agent import AbstractAgent +from algorithms.common.buffer.replay_buffer import ReplayBuffer +from algorithms.common.noise import OUNoise + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + +class Agent(AbstractAgent): + """ActorCritic interacting with environment. + + Attributes: + memory (ReplayBuffer): replay memory + noise (OUNoise): random noise for exploration + hyper_params (dict): hyper-parameters + actor (nn.Module): actor model to select actions + actor_target (nn.Module): target actor model to select actions + critic (nn.Module): critic model to predict state values + critic_target (nn.Module): target critic model to predict state values + actor_optimizer (Optimizer): optimizer for training actor + critic_optimizer (Optimizer): optimizer for training critic + curr_state (np.ndarray): temporary storage of the current state + + """ + + def __init__( + self, + env: gym.Env, + args: argparse.Namespace, + hyper_params: dict, + models: tuple, + optims: tuple, + noise: OUNoise, + ): + """Initialization. + + Args: + env (gym.Env): openAI Gym environment with discrete action space + args (argparse.Namespace): arguments including hyperparameters and training settings + hyper_params (dict): hyper-parameters + models (tuple): models including actor and critic + optims (tuple): optimizers for actor and critic + noise (OUNoise): random noise for exploration + + """ + AbstractAgent.__init__(self, env, args) + + self.actor, self.actor_target, self.critic, self.critic_target = models + self.actor_optimizer, self.critic_optimizer = optims + self.hyper_params = hyper_params + self.curr_state = np.zeros((1,)) + self.noise = noise + + # load the optimizer and model parameters + if args.load_from is not None and os.path.exists(args.load_from): + self.load_params(args.load_from) + + # replay memory + self.memory = ReplayBuffer( + hyper_params["BUFFER_SIZE"], hyper_params["BATCH_SIZE"], self.args.seed + ) + + def select_action(self, state: np.ndarray) -> torch.Tensor: + """Select an action from the input space.""" + self.curr_state = state + + state = torch.FloatTensor(state).to(device) + selected_action = self.actor(state) + selected_action += torch.FloatTensor(self.noise.sample()).to(device) + + selected_action = torch.clamp(selected_action, -1.0, 1.0) + + return selected_action + + def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]: + """Take an action and return the response of the env.""" + action = action.detach().cpu().numpy() + next_state, reward, done, _ = self.env.step(action) + + self.memory.add(self.curr_state, action, reward, next_state, done) + + return next_state, reward, done + + def update_model( + self, + experiences: Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor + ], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Train the model after each episode.""" + states, actions, rewards, next_states, dones = experiences + + # G_t = r + gamma * v(s_{t+1}) if state != Terminal + # = r otherwise + masks = 1 - dones + next_actions = self.actor_target(next_states) + next_values = self.critic_target(torch.cat((next_states, next_actions), dim=-1)) + curr_returns = rewards + self.hyper_params["GAMMA"] * next_values * masks + curr_returns = curr_returns.to(device) + + # train critic + values = self.critic(torch.cat((states, actions), dim=-1)) + critic_loss = F.mse_loss(values, curr_returns) + self.critic_optimizer.zero_grad() + critic_loss.backward() + self.critic_optimizer.step() + + # train actor + actions = self.actor(states) + actor_loss = -self.critic(torch.cat((states, actions), dim=-1)).mean() + self.actor_optimizer.zero_grad() + actor_loss.backward() + self.actor_optimizer.step() + + # update target networks + tau = self.hyper_params["TAU"] + common_utils.soft_update(self.actor, self.actor_target, tau) + common_utils.soft_update(self.critic, self.critic_target, tau) + + return actor_loss.data, critic_loss.data + + def load_params(self, path: str): + """Load model and optimizer parameters.""" + if not os.path.exists(path): + print("[ERROR] the input path does not exist. ->", path) + return + + params = torch.load(path) + self.actor.load_state_dict(params["actor_state_dict"]) + self.actor_target.load_state_dict(params["actor_target_state_dict"]) + self.critic.load_state_dict(params["critic_state_dict"]) + self.critic_target.load_state_dict(params["critic_target_state_dict"]) + self.actor_optimizer.load_state_dict(params["actor_optim_state_dict"]) + self.critic_optimizer.load_state_dict(params["critic_optim_state_dict"]) + print("[INFO] loaded the model and optimizer from", path) + + def save_params(self, n_episode: int): + """Save model and optimizer parameters.""" + params = { + "actor_state_dict": self.actor.state_dict(), + "actor_target_state_dict": self.actor_target.state_dict(), + "critic_state_dict": self.critic.state_dict(), + "critic_target_state_dict": self.critic_target.state_dict(), + "actor_optim_state_dict": self.actor_optimizer.state_dict(), + "critic_optim_state_dict": self.critic_optimizer.state_dict(), + } + + AbstractAgent.save_params(self, self.args.algo, params, n_episode) + + def write_log(self, i: int, loss: np.ndarray, score: int): + """Write log about loss and score""" + total_loss = loss.sum() + + print( + "[INFO] episode %d total score: %d, total loss: %f\n" + "actor_loss: %.3f critic_loss: %.3f\n" + % (i, score, total_loss, loss[0], loss[1]) # actor loss # critic loss + ) + + if self.args.log: + wandb.log( + { + "score": score, + "total loss": total_loss, + "actor loss": loss[0], + "critic loss": loss[1], + } + ) + + def train(self): + """Train the agent.""" + # logger + if self.args.log: + wandb.init() + wandb.config.update(self.hyper_params) + wandb.watch([self.actor, self.critic], log="parameters") + + for i_episode in range(1, self.args.episode_num + 1): + state = self.env.reset() + done = False + score = 0 + loss_episode = list() + + while not done: + if self.args.render and i_episode >= self.args.render_after: + self.env.render() + + action = self.select_action(state) + next_state, reward, done = self.step(action) + + if len(self.memory) >= self.hyper_params["BATCH_SIZE"]: + experiences = self.memory.sample() + loss = self.update_model(experiences) + loss_episode.append(loss) # for logging + + state = next_state + score += reward + + # logging + if loss_episode: + avg_loss = np.vstack(loss_episode).mean(axis=0) + self.write_log(i_episode, avg_loss, score) + + if i_episode % self.args.save_period == 0: + self.save_params(i_episode) + + # termination + self.env.close() diff --git a/scripts/examples/lunarlander_continuous_v2/ddpg.py b/scripts/examples/lunarlander_continuous_v2/ddpg.py new file mode 100644 index 0000000..988597e --- /dev/null +++ b/scripts/examples/lunarlander_continuous_v2/ddpg.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +"""Run module for DDPG on LunarLanderContinuous-v2. + +- Author: Curt Park +- Contact: curt.park@medipixel.io +""" + +import argparse + +import gym +import torch +import torch.optim as optim + +from algorithms.common.networks.mlp import MLP +from algorithms.common.noise import OUNoise +from algorithms.ddpg.agent import Agent + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +# hyper parameters +hyper_params = { + "GAMMA": 0.99, + "TAU": 1e-3, + "BUFFER_SIZE": int(1e5), + "BATCH_SIZE": 128, + "LR_ACTOR": 1e-3, + "LR_CRITIC": 1e-3, + "OU_NOISE_THETA": 0.0, + "OU_NOISE_SIGMA": 0.0, + "WEIGHT_DECAY": 1e-6, +} + + +def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int): + """Run training or test. + + Args: + env (gym.Env): openAI Gym environment with continuous action space + args (argparse.Namespace): arguments including training settings + state_dim (int): dimension of states + action_dim (int): dimension of actions + + """ + hidden_sizes = [256, 256] + + # create actor + actor = MLP( + input_size=state_dim, + output_size=action_dim, + hidden_sizes=hidden_sizes, + output_activation=torch.tanh, + ).to(device) + + actor_target = MLP( + input_size=state_dim, + output_size=action_dim, + hidden_sizes=hidden_sizes, + output_activation=torch.tanh, + ).to(device) + actor_target.load_state_dict(actor.state_dict()) + + # create critic + critic = MLP( + input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes + ).to(device) + + critic_target = MLP( + input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes + ).to(device) + critic_target.load_state_dict(critic.state_dict()) + + # create optimizer + actor_optim = optim.Adam( + actor.parameters(), + lr=hyper_params["LR_ACTOR"], + weight_decay=hyper_params["WEIGHT_DECAY"], + ) + + critic_optim = optim.Adam( + critic.parameters(), + lr=hyper_params["LR_CRITIC"], + weight_decay=hyper_params["WEIGHT_DECAY"], + ) + + # noise + noise = OUNoise( + action_dim, + args.seed, + theta=hyper_params["OU_NOISE_THETA"], + sigma=hyper_params["OU_NOISE_SIGMA"], + ) + + # make tuples to create an agent + models = (actor, actor_target, critic, critic_target) + optims = (actor_optim, critic_optim) + + # create an agent + agent = Agent(env, args, hyper_params, models, optims, noise) + + # run + if args.test: + agent.test() + else: + agent.train() diff --git a/scripts/mypy.ini b/scripts/mypy.ini deleted file mode 100644 index 74e9b94..0000000 --- a/scripts/mypy.ini +++ /dev/null @@ -1,5 +0,0 @@ -# Global options: - -[mypy] -python_version = 3.6 -ignore_missing_imports = True diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt new file mode 100644 index 0000000..691420a --- /dev/null +++ b/scripts/requirements-dev.txt @@ -0,0 +1,16 @@ +pre-commit + +# formatting +black +isort + +# testing +flake8==3.6.0 +flake8-bugbear +flake8-docstrings +pytest>=4.0.0 +pytest-flake8 +pytest-cov + +# generating requirements +pipreqs diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..188dcd9 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,5 @@ +gym +numpy +torch==1.0.0 +typing +wandb diff --git a/scripts/run_lunarlander_continuous.py b/scripts/run_lunarlander_continuous.py new file mode 100644 index 0000000..5db4a01 --- /dev/null +++ b/scripts/run_lunarlander_continuous.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +"""Train or test baselines on LunarLanderContinuous-v2. + +- Author: Curt Park +- Contact: curt.park@medipixel.io +""" + +import argparse +import importlib + +import gym +import numpy as np +import torch + +# configurations +parser = argparse.ArgumentParser(description="Pytorch RL baselines") +parser.add_argument( + "--seed", type=int, default=777, help="random seed for reproducibility" +) +parser.add_argument("--algo", type=str, default="ddpg", help="choose an algorithm") +parser.add_argument( + "--load-from", + type=str, + default=None, + help="load the saved model and optimizer at the beginning", +) +parser.add_argument("--episode-num", type=int, default=1500, help="total episode num") +parser.add_argument( + "--max-episode-steps", type=int, default=300, help="max episode step" +) +parser.add_argument( + "--off-render", dest="render", action="store_false", help="turn off rendering" +) +parser.add_argument( + "--render-after", + type=int, + default=0, + help="start rendering after the input number of episode", +) +parser.add_argument("--save-period", type=int, default=100, help="save model period") +parser.add_argument("--log", action="store_true", help="turn on logging") +parser.add_argument("--test", action="store_true", help="test mode (no training)") +parser.set_defaults(render=True) + +args = parser.parse_args() + + +def main(): + """Main.""" + # env initialization + env = gym.make("LunarLanderContinuous-v2") + state_dim = env.observation_space.shape[0] + action_dim = env.action_space.shape[0] + + # set a random seed + env.seed(args.seed) + torch.manual_seed(args.seed) + np.random.seed(args.seed) + + # run + module_path = "examples.lunarlander_continuous_v2." + args.algo + example = importlib.import_module(module_path) + example.run(env, args, state_dim, action_dim) + + +if __name__ == "__main__": + main()