Convert code to python 2.7 (#35)

* Convert code format to python2.7 (SAC)

* Convert code format python2.7 (TD3, all fD)

* Remove no use import and black setting

* Change SAC param

* Change env name Reacher-v2 to v1

* Remove old version reacher training script

* Convert code format python2.7

* Modify .travis.yml

* Add install command python3.6 & black on Makefile

* Fix seperator to tab on Makefile

* Modify Makefile

* Fix little error

* Change td3 gamma parameter

scripts/algorithms/sac/agent.py

@@ -7,11 +7,8 @@
          https://arxiv.org/pdf/1812.05905.pdf
 """
 
-import argparse
 import os
-from typing import Tuple
 
-import gym
 import numpy as np
 import torch
 import torch.nn.functional as F
@@ -50,15 +47,7 @@ class Agent(AbstractAgent):
 
     """
 
-    def __init__(
-        self,
-        env: gym.Env,
-        args: argparse.Namespace,
-        hyper_params: dict,
-        models: tuple,
-        optims: tuple,
-        target_entropy: float,
-    ):
+    def __init__(self, env, args, hyper_params, models, optims, target_entropy):
         """Initialization.
 
         Args:
@@ -103,7 +92,7 @@ class Agent(AbstractAgent):
                 self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"]
             )
 
-    def select_action(self, state: np.ndarray) -> np.ndarray:
+    def select_action(self, state):
         """Select an action from the input space."""
         self.curr_state = state
         state = self._preprocess_state(state)
@@ -122,12 +111,12 @@ class Agent(AbstractAgent):
 
         return selected_action.detach().cpu().numpy()
 
-    def _preprocess_state(self, state: np.ndarray) -> torch.Tensor:
+    def _preprocess_state(self, state):
         """Preprocess state so that actor selects an action."""
         state = torch.FloatTensor(state).to(device)
         return state
 
-    def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:
+    def step(self, action):
         """Take an action and return the response of the env."""
         self.total_step += 1
         self.episode_step += 1
@@ -144,16 +133,11 @@ class Agent(AbstractAgent):
 
         return next_state, reward, done
 
-    def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]):
+    def _add_transition_to_memory(self, transition):
         """Add 1 step and n step transitions to memory."""
         self.memory.add(*transition)
 
-    def update_model(
-        self,
-        experiences: Tuple[
-            torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
-        ],
-    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+    def update_model(self, experiences):
         """Train the model after each episode."""
         states, actions, rewards, next_states, dones = experiences
         new_actions, log_prob, pre_tanh_value, mu, std = self.actor(states)
@@ -238,10 +222,10 @@ class Agent(AbstractAgent):
             alpha_loss.data,
         )
 
-    def load_params(self, path: str):
+    def load_params(self, path):
         """Load model and optimizer parameters."""
         if not os.path.exists(path):
-            print("[ERROR] the input path does not exist. ->", path)
+            print ("[ERROR] the input path does not exist. ->", path)
             return
 
         params = torch.load(path)
@@ -258,9 +242,9 @@ class Agent(AbstractAgent):
         if self.hyper_params["AUTO_ENTROPY_TUNING"]:
             self.alpha_optimizer.load_state_dict(params["alpha_optim"])
 
-        print("[INFO] loaded the model and optimizer from", path)
+        print ("[INFO] loaded the model and optimizer from", path)
 
-    def save_params(self, n_episode: int):
+    def save_params(self, n_episode):
         """Save model and optimizer parameters."""
         params = {
             "actor": self.actor.state_dict(),
@@ -279,13 +263,11 @@ class Agent(AbstractAgent):
 
         AbstractAgent.save_params(self, params, n_episode)
 
-    def write_log(
-        self, i: int, loss: np.ndarray, score: float = 0.0, delayed_update: int = 1
-    ):
+    def write_log(self, i, loss, score=0.0, delayed_update=1):
         """Write log about loss and score"""
         total_loss = loss.sum()
 
-        print(
+        print (
             "[INFO] episode %d, episode_step %d, total step %d, total score: %d\n"
             "total loss: %.3f actor_loss: %.3f qf_1_loss: %.3f qf_2_loss: %.3f "
             "vf_loss: %.3f alpha_loss: %.3f\n"