Add DDPGfD, TD3fD and SACfD (#22)

* Format repository

* Clone files from medipixel repo

* Fix DDPGfDAgent.update_model()

* Fix bug on _initialize()

* Add demo-path parameter and demo data

* Rename init_priority to _max_priority for PER

This makes PER and PERfD consistent.

* Make i_episode attribute of DDPGAgent

* Clone SAC code from medipixel repo

* Fix update_model() for SACfD

* Fix _initialize() for SACfD

* Add is_discrete attribute to AbstractAgent for SACfD

* Add i_episode attribute to SACAgent for SACfD

* Modularize DDPGAgent and SACAgent

* Modify hyperparameters for DDPGfD and SACfD

* Add NStepBuffer

* Add n-step to DDPGfD

* Add n-step to SACfD

* Add TD3fD without n-step

* Attempt to tune hyperparameters

* Remove discrete environment check in SAC

* Implement n-step on TD3fD

* Fix step function of TD3

No done check, and _add_transition_to_memory was not called.

* Fix actor loss calculation for TD3fD

* Attempt to tune hyperparameters

* Print both critic losses

* Fix typo bug

* Attempt to tune hyperparameters

* Fix bug in n-step demo retrieval

* Fix bug in n-step transition addition

scripts/algorithms/sac/agent.py

@@ -46,6 +46,7 @@ class Agent(AbstractAgent):
         hyper_params (dict): hyper-parameters
         total_step (int): total step numbers
         episode_step (int): step number of the current episode
+        i_episode (int): current episode number
 
     """
 
@@ -78,6 +79,7 @@ class Agent(AbstractAgent):
         self.curr_state = np.zeros((1,))
         self.total_step = 0
         self.episode_step = 0
+        self.i_episode = 0
 
         # automatic entropy tuning
         if self.hyper_params["AUTO_ENTROPY_TUNING"]:
@@ -91,15 +93,20 @@ class Agent(AbstractAgent):
         if args.load_from is not None and os.path.exists(args.load_from):
             self.load_params(args.load_from)
 
+        self._initialize()
+
+    def _initialize(self):
+        """Initialize non-common things."""
         if not self.args.test:
             # replay memory
             self.memory = ReplayBuffer(
-                hyper_params["BUFFER_SIZE"], hyper_params["BATCH_SIZE"]
+                self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"]
             )
 
     def select_action(self, state: np.ndarray) -> np.ndarray:
         """Select an action from the input space."""
         self.curr_state = state
+        state = self._preprocess_state(state)
 
         # if initial random action should be conducted
         if (
@@ -108,7 +115,6 @@ class Agent(AbstractAgent):
         ):
             return self.env.action_space.sample()
 
-        state = torch.FloatTensor(state).to(device)
         if self.args.test:
             _, _, _, selected_action, _ = self.actor(state)
         else:
@@ -116,6 +122,11 @@ class Agent(AbstractAgent):
 
         return selected_action.detach().cpu().numpy()
 
+    def _preprocess_state(self, state: np.ndarray) -> torch.Tensor:
+        """Preprocess state so that actor selects an action."""
+        state = torch.FloatTensor(state).to(device)
+        return state
+
     def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:
         """Take an action and return the response of the env."""
         self.total_step += 1
@@ -128,10 +139,15 @@ class Agent(AbstractAgent):
             done_bool = (
                 False if self.episode_step == self.args.max_episode_steps else done
             )
-            self.memory.add(self.curr_state, action, reward, next_state, done_bool)
+            transition = (self.curr_state, action, reward, next_state, done_bool)
+            self._add_transition_to_memory(transition)
 
         return next_state, reward, done
 
+    def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]):
+        """Add 1 step and n step transitions to memory."""
+        self.memory.add(*transition)
+
     def update_model(
         self,
         experiences: Tuple[
@@ -308,7 +324,7 @@ class Agent(AbstractAgent):
             wandb.config.update(self.hyper_params)
             wandb.watch([self.actor, self.vf, self.qf_1, self.qf_2], log="parameters")
 
-        for i_episode in range(1, self.args.episode_num + 1):
+        for self.i_episode in range(1, self.args.episode_num + 1):
             state = self.env.reset()
             done = False
             score = 0
@@ -316,7 +332,7 @@ class Agent(AbstractAgent):
             loss_episode = list()
 
             while not done:
-                if self.args.render and i_episode >= self.args.render_after:
+                if self.args.render and self.i_episode >= self.args.render_after:
                     self.env.render()
 
                 action = self.select_action(state)
@@ -335,11 +351,11 @@ class Agent(AbstractAgent):
             if loss_episode:
                 avg_loss = np.vstack(loss_episode).mean(axis=0)
                 self.write_log(
-                    i_episode, avg_loss, score, self.hyper_params["DELAYED_UPDATE"]
+                    self.i_episode, avg_loss, score, self.hyper_params["DELAYED_UPDATE"]
                 )
 
-            if i_episode % self.args.save_period == 0:
-                self.save_params(i_episode)
+            if self.i_episode % self.args.save_period == 0:
+                self.save_params(self.i_episode)
 
         # termination
         self.env.close()