Convert code to python 2.7 (#35)

* Convert code format to python2.7 (SAC)

* Convert code format python2.7 (TD3, all fD)

* Remove no use import and black setting

* Change SAC param

* Change env name Reacher-v2 to v1

* Remove old version reacher training script

* Convert code format python2.7

* Modify .travis.yml

* Add install command python3.6 & black on Makefile

* Fix seperator to tab on Makefile

* Modify Makefile

* Fix little error

* Change td3 gamma parameter

scripts/algorithms/common/buffer/priortized_replay_buffer.py

@@ -8,7 +8,6 @@
 """
 
 import random
-from typing import Tuple
 
 import numpy as np
 import torch
@@ -35,7 +34,7 @@ class PrioritizedReplayBuffer(ReplayBuffer):
 
         """
 
-    def __init__(self, buffer_size: int, batch_size: int, alpha: float = 0.6):
+    def __init__(self, buffer_size, batch_size, alpha=0.6):
         """Initialization.
 
         Args:
@@ -59,27 +58,22 @@ class PrioritizedReplayBuffer(ReplayBuffer):
         self.min_tree = MinSegmentTree(tree_capacity)
         self._max_priority = 1.0
 
-    def add(
-        self,
-        state: np.ndarray,
-        action: np.ndarray,
-        reward: np.float64,
-        next_state: np.ndarray,
-        done: bool,
-    ):
+    def add(self, state, action, reward, next_state, done):
         """Add experience and priority."""
         idx = self.tree_idx
         self.tree_idx = (self.tree_idx + 1) % self.buffer_size
-        super().add(state, action, reward, next_state, done)
+        super(PrioritizedReplayBuffer, self).add(
+            state, action, reward, next_state, done
+        )
 
         self.sum_tree[idx] = self._max_priority ** self.alpha
         self.min_tree[idx] = self._max_priority ** self.alpha
 
-    def extend(self, transitions: list):
+    def extend(self, transitions):
         """Add experiences to memory."""
         raise NotImplementedError
 
-    def _sample_proportional(self, batch_size: int) -> list:
+    def _sample_proportional(self, batch_size):
         """Sample indices based on proportional."""
         indices = []
         p_total = self.sum_tree.sum(0, len(self.buffer) - 1)
@@ -92,7 +86,7 @@ class PrioritizedReplayBuffer(ReplayBuffer):
             indices.append(idx)
         return indices
 
-    def sample(self, beta: float = 0.4) -> Tuple[torch.Tensor, ...]:
+    def sample(self, beta=0.4):
         """Sample a batch of experiences."""
         assert beta > 0
 
@@ -127,7 +121,7 @@ class PrioritizedReplayBuffer(ReplayBuffer):
 
         return experiences
 
-    def update_priorities(self, indices: list, priorities: np.ndarray):
+    def update_priorities(self, indices, priorities):
         """Update priorities of sampled transitions."""
         assert len(indices) == len(priorities)
 
@@ -153,14 +147,7 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
         epsilon_d (float) : epsilon_d parameter to update priority using demo
         """
 
-    def __init__(
-        self,
-        buffer_size: int,
-        batch_size: int,
-        demo: list,
-        alpha: float = 0.6,
-        epsilon_d: float = 1.0,
-    ):
+    def __init__(self, buffer_size, batch_size, demo, alpha=0.6, epsilon_d=1.0):
         """Initialization.
         Args:
             buffer_size (int): size of replay buffer for experience
@@ -181,14 +168,7 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
             self.min_tree[self.tree_idx] = self._max_priority ** self.alpha
             self.tree_idx += 1
 
-    def add(
-        self,
-        state: np.ndarray,
-        action: np.ndarray,
-        reward: np.float64,
-        next_state: np.ndarray,
-        done: bool,
-    ):
+    def add(self, state, action, reward, next_state, done):
         """Add experience and priority."""
         idx = self.tree_idx
         # buffer is full
@@ -196,7 +176,9 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
             self.tree_idx = self.demo_size
         else:
             self.tree_idx = self.tree_idx + 1
-        super().add(state, action, reward, next_state, done)
+        super(PrioritizedReplayBuffer, self).add(
+            state, action, reward, next_state, done
+        )
 
         self.sum_tree[idx] = self._max_priority ** self.alpha
         self.min_tree[idx] = self._max_priority ** self.alpha
@@ -204,7 +186,7 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
         # update current total size
         self.total_size = self.demo_size + len(self.buffer)
 
-    def sample(self, beta: float = 0.4) -> Tuple[torch.Tensor, ...]:
+    def sample(self, beta=0.4):
         """Sample a batch of experiences."""
         assert beta > 0
 
@@ -266,7 +248,7 @@ class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
 
         return experiences
 
-    def update_priorities(self, indices: list, priorities: np.ndarray):
+    def update_priorities(self, indices, priorities):
         """Update priorities of sampled transitions."""
         assert len(indices) == len(priorities)
 

scripts/algorithms/common/buffer/replay_buffer.py

@@ -2,7 +2,6 @@
 """Replay buffer for baselines."""
 
 from collections import deque
-from typing import Any, Deque, List, Tuple
 
 import numpy as np
 import torch
@@ -25,7 +24,7 @@ class ReplayBuffer:
 
     """
 
-    def __init__(self, buffer_size: int, batch_size: int):
+    def __init__(self, buffer_size, batch_size):
         """Initialize a ReplayBuffer object.
 
         Args:
@@ -33,19 +32,12 @@ class ReplayBuffer:
             batch_size (int): size of a batched sampled from replay buffer for training
 
         """
-        self.buffer: list = list()
+        self.buffer = list()
         self.buffer_size = buffer_size
         self.batch_size = batch_size
         self.idx = 0
 
-    def add(
-        self,
-        state: np.ndarray,
-        action: np.ndarray,
-        reward: np.float64,
-        next_state: np.ndarray,
-        done: bool,
-    ):
+    def add(self, state, action, reward, next_state, done):
         """Add a new experience to memory."""
         data = (state, action, reward, next_state, done)
 
@@ -55,14 +47,12 @@ class ReplayBuffer:
         else:
             self.buffer.append(data)
 
-    def extend(self, transitions: list):
+    def extend(self, transitions):
         """Add experiences to memory."""
         for transition in transitions:
             self.add(*transition)
 
-    def sample(
-        self
-    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+    def sample(self):
         """Randomly sample a batch of experiences from memory."""
         idxs = np.random.choice(len(self.buffer), size=self.batch_size, replace=False)
 
@@ -84,7 +74,7 @@ class ReplayBuffer:
 
         return states, actions, rewards, next_states, dones
 
-    def __len__(self) -> int:
+    def __len__(self):
         """Return the current size of internal memory."""
         return len(self.buffer)
 
@@ -100,7 +90,7 @@ class NStepTransitionBuffer:
 
     """
 
-    def __init__(self, buffer_size: int, n_step: int, gamma: float, demo: list = None):
+    def __init__(self, buffer_size, n_step, gamma, demo=None):
         """Initialize a ReplayBuffer object.
 
         Args:
@@ -110,9 +100,9 @@ class NStepTransitionBuffer:
         """
         assert buffer_size > 0
 
-        self.n_step_buffer: Deque = deque(maxlen=n_step)
+        self.n_step_buffer = deque(maxlen=n_step)
         self.buffer_size = buffer_size
-        self.buffer: list = list()
+        self.buffer = list()
         self.n_step = n_step
         self.gamma = gamma
         self.demo_size = 0
@@ -125,7 +115,7 @@ class NStepTransitionBuffer:
 
         self.buffer.extend([None] * self.buffer_size)
 
-    def add(self, transition: Tuple[np.ndarray, ...]) -> Tuple[Any, ...]:
+    def add(self, transition):
         """Add a new transition to memory."""
         self.n_step_buffer.append(transition)
 
@@ -146,7 +136,7 @@ class NStepTransitionBuffer:
         # return a single step transition to insert to replay buffer
         return self.n_step_buffer[0]
 
-    def sample(self, indices: List[int]) -> Tuple[torch.Tensor, ...]:
+    def sample(self, indices):
         """Randomly sample a batch of experiences from memory."""
         states, actions, rewards, next_states, dones = [], [], [], [], []
 

scripts/algorithms/common/buffer/segment_tree.py

@@ -2,7 +2,6 @@
 """Segment tree for Proirtized Replay Buffer."""
 
 import operator
-from typing import Callable
 
 
 class SegmentTree:
@@ -18,7 +17,7 @@ class SegmentTree:
 
     """
 
-    def __init__(self, capacity: int, operation: Callable, init_value: float):
+    def __init__(self, capacity, operation, init_value):
         """Initialization.
 
         Args:
@@ -34,9 +33,7 @@ class SegmentTree:
         self.tree = [init_value for _ in range(2 * capacity)]
         self.operation = operation
 
-    def _operate_helper(
-        self, start: int, end: int, node: int, node_start: int, node_end: int
-    ) -> float:
+    def _operate_helper(self, start, end, node, node_start, node_end):
         """Returns result of operation in segment."""
         if start == node_start and end == node_end:
             return self.tree[node]
@@ -52,7 +49,7 @@ class SegmentTree:
                     self._operate_helper(mid + 1, end, 2 * node + 1, mid + 1, node_end),
                 )
 
-    def operate(self, start: int = 0, end: int = 0) -> float:
+    def operate(self, start=0, end=0):
         """Returns result of applying `self.operation`."""
         if end <= 0:
             end += self.capacity
@@ -60,7 +57,7 @@ class SegmentTree:
 
         return self._operate_helper(start, end, 1, 0, self.capacity - 1)
 
-    def __setitem__(self, idx: int, val: float):
+    def __setitem__(self, idx, val):
         """Set value in tree."""
         idx += self.capacity
         self.tree[idx] = val
@@ -70,7 +67,7 @@ class SegmentTree:
             self.tree[idx] = self.operation(self.tree[2 * idx], self.tree[2 * idx + 1])
             idx //= 2
 
-    def __getitem__(self, idx: int) -> float:
+    def __getitem__(self, idx):
         """Get real value in leaf node of tree."""
         assert 0 <= idx < self.capacity
 
@@ -85,7 +82,7 @@ class SumSegmentTree(SegmentTree):
 
     """
 
-    def __init__(self, capacity: int):
+    def __init__(self, capacity):
         """Initialization.
 
         Args:
@@ -96,11 +93,11 @@ class SumSegmentTree(SegmentTree):
             capacity=capacity, operation=operator.add, init_value=0.0
         )
 
-    def sum(self, start: int = 0, end: int = 0) -> float:
+    def sum(self, start=0, end=0):
         """Returns arr[start] + ... + arr[end]."""
         return super(SumSegmentTree, self).operate(start, end)
 
-    def retrieve(self, upperbound: float) -> int:
+    def retrieve(self, upperbound):
         """Find the highest index `i` about upper bound in the tree"""
         assert 0 <= upperbound <= self.sum() + 1e-5
 
@@ -125,7 +122,7 @@ class MinSegmentTree(SegmentTree):
 
     """
 
-    def __init__(self, capacity: int):
+    def __init__(self, capacity):
         """Initialization.
 
         Args:
@@ -136,6 +133,6 @@ class MinSegmentTree(SegmentTree):
             capacity=capacity, operation=min, init_value=float("inf")
         )
 
-    def min(self, start: int = 0, end: int = 0) -> float:
+    def min(self, start=0, end=0):
         """Returns min(arr[start], ...,  arr[end])."""
         return super(MinSegmentTree, self).operate(start, end)