Add DDPGfD, TD3fD and SACfD (#22)

* Format repository * Clone files from medipixel repo * Fix DDPGfDAgent.update_model() * Fix bug on _initialize() * Add demo-path parameter and demo data * Rename init_priority to _max_priority for PER This makes PER and PERfD consistent. * Make i_episode attribute of DDPGAgent * Clone SAC code from medipixel repo * Fix update_model() for SACfD * Fix _initialize() for SACfD * Add is_discrete attribute to AbstractAgent for SACfD * Add i_episode attribute to SACAgent for SACfD * Modularize DDPGAgent and SACAgent * Modify hyperparameters for DDPGfD and SACfD * Add NStepBuffer * Add n-step to DDPGfD * Add n-step to SACfD * Add TD3fD without n-step * Attempt to tune hyperparameters * Remove discrete environment check in SAC * Implement n-step on TD3fD * Fix step function of TD3 No done check, and _add_transition_to_memory was not called. * Fix actor loss calculation for TD3fD * Attempt to tune hyperparameters * Print both critic losses * Fix typo bug * Attempt to tune hyperparameters * Fix bug in n-step demo retrieval * Fix bug in n-step transition addition
2026-08-25 11:19:30 +08:00 · 2019-03-14 11:06:54 +09:00
parent ee014e5a93
commit ca5c99bc41
17 changed files with 1377 additions and 49 deletions
@@ -31,13 +31,11 @@ class PrioritizedReplayBuffer(ReplayBuffer):
        tree_idx (int): next index of tree
        sum_tree (SumSegmentTree): sum tree for prior
        min_tree (MinSegmentTree): min tree for min prior to get max weight
-        init_priority (float): lower bound of priority
+        _max_priority (float): max priority

        """

-    def __init__(
-        self, buffer_size: int, batch_size: int, alpha: float = 0.6
-    ):
+    def __init__(self, buffer_size: int, batch_size: int, alpha: float = 0.6):
        """Initialization.

        Args:
@@ -59,7 +57,7 @@ class PrioritizedReplayBuffer(ReplayBuffer):

        self.sum_tree = SumSegmentTree(tree_capacity)
        self.min_tree = MinSegmentTree(tree_capacity)
-        self.init_priority = 1.0
+        self._max_priority = 1.0

    def add(
        self,
@@ -74,8 +72,8 @@ class PrioritizedReplayBuffer(ReplayBuffer):
        self.tree_idx = (self.tree_idx + 1) % self.buffer_size
        super().add(state, action, reward, next_state, done)

-        self.sum_tree[idx] = self.init_priority ** self.alpha
-        self.min_tree[idx] = self.init_priority ** self.alpha
+        self.sum_tree[idx] = self._max_priority ** self.alpha
+        self.min_tree[idx] = self._max_priority ** self.alpha

    def extend(self, transitions: list):
        """Add experiences to memory."""
@@ -140,4 +138,143 @@ class PrioritizedReplayBuffer(ReplayBuffer):
            self.sum_tree[idx] = priority ** self.alpha
            self.min_tree[idx] = priority ** self.alpha

-            self.init_priority = max(self.init_priority, priority)
+            self._max_priority = max(self._max_priority, priority)
+
+
+class PrioritizedReplayBufferfD(PrioritizedReplayBuffer):
+    """Create Prioritized Replay buffer with demo.
+    Taken from OpenAI baselines github repository:
+    https://github.com/openai/baselines/blob/master/baselines/deepq/replay_buffer.py
+    Attributes:
+        demo (list): list of demo replay buffer
+        buffer_size (int): size of replay buffer for experience
+        demo_size (int): size of replay buffer for demonstration
+        total_size (int): sum of demo size and number of samples of experience
+        epsilon_d (float) : epsilon_d parameter to update priority using demo
+        """
+
+    def __init__(
+        self,
+        buffer_size: int,
+        batch_size: int,
+        demo: list,
+        alpha: float = 0.6,
+        epsilon_d: float = 1.0,
+    ):
+        """Initialization.
+        Args:
+            buffer_size (int): size of replay buffer for experience
+            batch_size (int): size of a batched sampled from replay buffer for training
+            demo (list): demonstration
+            alpha (float): alpha parameter for prioritized replay buffer
+            epsilon_d (float) : epsilon_d parameter to update priority using demo
+        """
+        super(PrioritizedReplayBufferfD, self).__init__(buffer_size, batch_size, alpha)
+        self.demo = demo
+        self.demo_size = len(demo)
+        self.total_size = self.demo_size + len(self.buffer)
+        self.epsilon_d = epsilon_d
+
+        # for init priority of demo
+        for _ in range(self.demo_size):
+            self.sum_tree[self.tree_idx] = self._max_priority ** self.alpha
+            self.min_tree[self.tree_idx] = self._max_priority ** self.alpha
+            self.tree_idx += 1
+
+    def add(
+        self,
+        state: np.ndarray,
+        action: np.ndarray,
+        reward: np.float64,
+        next_state: np.ndarray,
+        done: bool,
+    ):
+        """Add experience and priority."""
+        idx = self.tree_idx
+        # buffer is full
+        if (self.tree_idx + 1) % (self.buffer_size + self.demo_size) == 0:
+            self.tree_idx = self.demo_size
+        else:
+            self.tree_idx = self.tree_idx + 1
+        super().add(state, action, reward, next_state, done)
+
+        self.sum_tree[idx] = self._max_priority ** self.alpha
+        self.min_tree[idx] = self._max_priority ** self.alpha
+
+        # update current total size
+        self.total_size = self.demo_size + len(self.buffer)
+
+    def sample(self, beta: float = 0.4) -> Tuple[torch.Tensor, ...]:
+        """Sample a batch of experiences."""
+        assert beta > 0
+
+        indices = self._sample_proportional(self.batch_size)
+        states, actions, rewards, next_states, dones = [], [], [], [], []
+        weights, eps_d = [], []
+
+        # get max weight
+        p_min = self.min_tree.min() / self.sum_tree.sum()
+        max_weight = (p_min * self.total_size) ** (-beta)
+
+        for i in indices:
+            # sample from buffer
+            if i < self.demo_size:
+                s, a, r, n_s, d = self.demo[i]
+                eps_d.append(self.epsilon_d)
+            else:
+                s, a, r, n_s, d = self.buffer[i - self.demo_size]
+                eps_d.append(0.0)
+
+            # append transition info
+            states.append(np.array(s, copy=False))
+            actions.append(np.array(a, copy=False))
+            rewards.append(np.array(r, copy=False))
+            next_states.append(np.array(n_s, copy=False))
+            dones.append(np.array(float(d), copy=False))
+
+            # calculate weights
+            p_sample = self.sum_tree[i] / self.sum_tree.sum()
+            weight = (p_sample * self.total_size) ** (-beta)
+            weights.append(weight / max_weight)
+
+        states_ = torch.FloatTensor(np.array(states)).to(device)
+        actions_ = torch.FloatTensor(np.array(actions)).to(device)
+        rewards_ = torch.FloatTensor(np.array(rewards).reshape(-1, 1)).to(device)
+        next_states_ = torch.FloatTensor(np.array(next_states)).to(device)
+        dones_ = torch.FloatTensor(np.array(dones).reshape(-1, 1)).to(device)
+        weights_ = torch.FloatTensor(np.array(weights).reshape(-1, 1)).to(device)
+        eps_d = np.array(eps_d)
+
+        if torch.cuda.is_available():
+            states_ = states_.cuda(non_blocking=True)
+            actions_ = actions_.cuda(non_blocking=True)
+            rewards_ = rewards_.cuda(non_blocking=True)
+            next_states_ = next_states_.cuda(non_blocking=True)
+            dones_ = dones_.cuda(non_blocking=True)
+            weights_ = weights_.cuda(non_blocking=True)
+
+        experiences = (
+            states_,
+            actions_,
+            rewards_,
+            next_states_,
+            dones_,
+            weights_,
+            indices,
+            eps_d,
+        )
+
+        return experiences
+
+    def update_priorities(self, indices: list, priorities: np.ndarray):
+        """Update priorities of sampled transitions."""
+        assert len(indices) == len(priorities)
+
+        for idx, priority in zip(indices, priorities):
+            assert priority > 0
+            assert 0 <= idx < self.total_size
+
+            self.sum_tree[idx] = priority ** self.alpha
+            self.min_tree[idx] = priority ** self.alpha
+
+            self._max_priority = max(self._max_priority, priority)
@@ -1,11 +1,14 @@
 # -*- coding: utf-8 -*-
 """Replay buffer for baselines."""

-from typing import Tuple
+from collections import deque
+from typing import Any, Deque, List, Tuple

 import numpy as np
 import torch

+from algorithms.common.helper_functions import get_n_step_info
+
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")


@@ -84,3 +87,88 @@ class ReplayBuffer:
    def __len__(self) -> int:
        """Return the current size of internal memory."""
        return len(self.buffer)
+
+
+class NStepTransitionBuffer:
+    """Fixed-size buffer to store experience tuples.
+
+    Attributes:
+        buffer (list): list of replay buffer
+        buffer_size (int): buffer size not storing demos
+        demo_size (int): size of a demo to permanently store in the buffer
+        cursor (int): position to store next transition coming in
+
+    """
+
+    def __init__(self, buffer_size: int, n_step: int, gamma: float, demo: list = None):
+        """Initialize a ReplayBuffer object.
+
+        Args:
+            buffer_size (int): size of replay buffer for experience
+            demo (list): demonstration transitions
+
+        """
+        assert buffer_size > 0
+
+        self.n_step_buffer: Deque = deque(maxlen=n_step)
+        self.buffer_size = buffer_size
+        self.buffer: list = list()
+        self.n_step = n_step
+        self.gamma = gamma
+        self.demo_size = 0
+        self.cursor = 0
+
+        # if demo exists
+        if demo:
+            self.demo_size = len(demo)
+            self.buffer.extend(demo)
+
+        self.buffer.extend([None] * self.buffer_size)
+
+    def add(self, transition: Tuple[np.ndarray, ...]) -> Tuple[Any, ...]:
+        """Add a new transition to memory."""
+        self.n_step_buffer.append(transition)
+
+        # single step transition is not ready
+        if len(self.n_step_buffer) < self.n_step:
+            return ()
+
+        # add a multi step transition
+        reward, next_state, done = get_n_step_info(self.n_step_buffer, self.gamma)
+        curr_state, action = self.n_step_buffer[0][:2]
+        new_transition = (curr_state, action, reward, next_state, done)
+
+        # insert the new transition to buffer
+        idx = self.demo_size + self.cursor
+        self.buffer[idx] = new_transition
+        self.cursor = (self.cursor + 1) % self.buffer_size
+
+        # return a single step transition to insert to replay buffer
+        return self.n_step_buffer[0]
+
+    def sample(self, indices: List[int]) -> Tuple[torch.Tensor, ...]:
+        """Randomly sample a batch of experiences from memory."""
+        states, actions, rewards, next_states, dones = [], [], [], [], []
+
+        for i in indices:
+            s, a, r, n_s, d = self.buffer[i]
+            states.append(np.array(s, copy=False))
+            actions.append(np.array(a, copy=False))
+            rewards.append(np.array(r, copy=False))
+            next_states.append(np.array(n_s, copy=False))
+            dones.append(np.array(float(d), copy=False))
+
+        states_ = torch.FloatTensor(np.array(states)).to(device)
+        actions_ = torch.FloatTensor(np.array(actions)).to(device)
+        rewards_ = torch.FloatTensor(np.array(rewards).reshape(-1, 1)).to(device)
+        next_states_ = torch.FloatTensor(np.array(next_states)).to(device)
+        dones_ = torch.FloatTensor(np.array(dones).reshape(-1, 1)).to(device)
+
+        if torch.cuda.is_available():
+            states_ = states_.cuda(non_blocking=True)
+            actions_ = actions_.cuda(non_blocking=True)
+            rewards_ = rewards_.cuda(non_blocking=True)
+            next_states_ = next_states_.cuda(non_blocking=True)
+            dones_ = dones_.cuda(non_blocking=True)
+
+        return states_, actions_, rewards_, next_states_, dones_