commit 2ad8b9f4d20c1f680f30f0627300ee65167291cd Author: chenyingyinglalala <18759377116@163.com> Date: Sun Nov 3 10:39:13 2019 +0800 add diff --git a/Char00 Conventional Algorithms/Q-learning.py b/Char00 Conventional Algorithms/Q-learning.py new file mode 100644 index 0000000..14047b2 --- /dev/null +++ b/Char00 Conventional Algorithms/Q-learning.py @@ -0,0 +1,98 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import time + +ALPHA = 0.1 +GAMMA = 0.95 +EPSILION = 0.9 +N_STATE = 20 +ACTIONS = ['left', 'right'] +MAX_EPISODES = 200 +FRESH_TIME = 0.1 + +def build_q_table(n_state, actions): + q_table = pd.DataFrame( + np.zeros((n_state, len(actions))), + np.arange(n_state), + actions + ) + return q_table + +def choose_action(state, q_table): + #epslion - greedy policy + state_action = q_table.loc[state,:] + if np.random.uniform()>EPSILION or (state_action==0).all(): + action_name = np.random.choice(ACTIONS) + else: + action_name = state_action.idxmax() + return action_name + +def get_env_feedback(state, action): + if action=='right': + if state == N_STATE-2: + next_state = 'terminal' + reward = 1 + else: + next_state = state+1 + reward = -0.5 + else: + if state == 0: + next_state = 0 + + else: + next_state = state-1 + reward = -0.5 + return next_state, reward + +def update_env(state,episode, step_counter): + env = ['-'] *(N_STATE-1)+['T'] + if state =='terminal': + print("Episode {}, the total step is {}".format(episode+1, step_counter)) + final_env = ['-'] *(N_STATE-1)+['T'] + return True, step_counter + else: + env[state]='*' + env = ''.join(env) + print(env) + time.sleep(FRESH_TIME) + return False, step_counter + + +def q_learning(): + q_table = build_q_table(N_STATE, ACTIONS) + step_counter_times = [] + for episode in range(MAX_EPISODES): + state = 0 + is_terminal = False + step_counter = 0 + update_env(state, episode, step_counter) + while not is_terminal: + action = choose_action(state,q_table) + next_state, reward = get_env_feedback(state, action) + next_q = q_table.loc[state, action] + if next_state == 'terminal': + is_terminal = True + q_target = reward + else: + delta = reward + GAMMA*q_table.iloc[next_state,:].max()-q_table.loc[state, action] + q_table.loc[state, action] += ALPHA*delta + state = next_state + is_terminal,steps = update_env(state, episode, step_counter+1) + step_counter+=1 + if is_terminal: + step_counter_times.append(steps) + + return q_table, step_counter_times + +def main(): + q_table, step_counter_times= q_learning() + print("Q table\n{}\n".format(q_table)) + print('end') + + plt.plot(step_counter_times,'g-') + plt.ylabel("steps") + plt.show() + print("The step_counter_times is {}".format(step_counter_times)) + +main() diff --git a/Char00 Conventional Algorithms/Sarsa.py b/Char00 Conventional Algorithms/Sarsa.py new file mode 100644 index 0000000..b96dbf7 --- /dev/null +++ b/Char00 Conventional Algorithms/Sarsa.py @@ -0,0 +1,104 @@ + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import time + +ALPHA = 0.1 +GAMMA = 0.95 +EPSILION = 0.9 +N_STATE = 6 +ACTIONS = ['left', 'right'] +MAX_EPISODES = 200 +FRESH_TIME = 0.1 + +def build_q_table(n_state, actions): + q_table = pd.DataFrame( + np.zeros((n_state, len(actions))), + np.arange(n_state), + actions + ) + return q_table + +def choose_action(state, q_table): + #epslion - greedy policy + state_action = q_table.loc[state,:] + if np.random.uniform()>EPSILION or (state_action==0).all(): + action_name = np.random.choice(ACTIONS) + else: + action_name = state_action.idxmax() + return action_name + +def get_env_feedback(state, action): + if action=='right': + if state == N_STATE-2: + next_state = 'terminal' + reward = 1 + else: + next_state = state+1 + reward = -0.5 + else: + if state == 0: + next_state = 0 + + else: + next_state = state-1 + reward = -0.5 + return next_state, reward + +def update_env(state,episode, step_counter): + env = ['-'] *(N_STATE-1)+['T'] + if state =='terminal': + print("Episode {}, the total step is {}".format(episode+1, step_counter)) + final_env = ['-'] *(N_STATE-1)+['T'] + return True, step_counter + else: + env[state]='*' + env = ''.join(env) + print(env) + time.sleep(FRESH_TIME) + return False, step_counter + + +def sarsa_learning(): + q_table = build_q_table(N_STATE, ACTIONS) + step_counter_times = [] + for episode in range(MAX_EPISODES): + state = 0 + is_terminal = False + step_counter = 0 + update_env(state, episode, step_counter) + while not is_terminal: + action = choose_action(state,q_table) + next_state, reward = get_env_feedback(state, action) + if next_state != 'terminal': + next_action = choose_action(next_state, q_table) #sarsa update method + else: + next_action = action + next_q = q_table.loc[state, action] + + if next_state == 'terminal': + is_terminal = True + q_target = reward + else: + delta = reward + GAMMA*q_table.loc[next_state,next_action]-q_table.loc[state, action] + q_table.loc[state, action] += ALPHA*delta + state = next_state + is_terminal,steps = update_env(state, episode, step_counter+1) + step_counter+=1 + if is_terminal: + step_counter_times.append(steps) + + return q_table, step_counter_times + +def main(): + q_table, step_counter_times= sarsa_learning() + print("Q table\n{}\n".format(q_table)) + print('end') + + plt.plot(step_counter_times,'g-') + plt.ylabel("steps") + plt.show() + print("The step_counter_times is {}".format(step_counter_times)) + +main() diff --git a/Char00 Conventional Algorithms/gridworld.py b/Char00 Conventional Algorithms/gridworld.py new file mode 100644 index 0000000..4b01c0a --- /dev/null +++ b/Char00 Conventional Algorithms/gridworld.py @@ -0,0 +1,152 @@ + +import numpy as np + +class GridWorld: + + def __init__(self, tot_row, tot_col): + self.action_space_size = 4 + self.world_row = tot_row + self.world_col = tot_col + #The world is a matrix of size row x col x 2 + #The first layer contains the obstacles + #The second layer contains the rewards + #self.world_matrix = np.zeros((tot_row, tot_col, 2)) + self.transition_matrix = np.ones((self.action_space_size, self.action_space_size))/ self.action_space_size + #self.transition_array = np.ones(self.action_space_size) / self.action_space_size + self.reward_matrix = np.zeros((tot_row, tot_col)) + self.state_matrix = np.zeros((tot_row, tot_col)) + self.position = [np.random.randint(tot_row), np.random.randint(tot_col)] + + #def setTransitionArray(self, transition_array): + #if(transition_array.shape != self.transition_array): + #raise ValueError('The shape of the two matrices must be the same.') + #self.transition_array = transition_array + + def setTransitionMatrix(self, transition_matrix): + '''Set the reward matrix. + + The transition matrix here is intended as a matrix which has a line + for each action and the element of the row are the probabilities to + executes each action when a command is given. For example: + [[0.55, 0.25, 0.10, 0.10] + [0.25, 0.25, 0.25, 0.25] + [0.30, 0.20, 0.40, 0.10] + [0.10, 0.20, 0.10, 0.60]] + + This matrix defines the transition rules for all the 4 possible actions. + The first row corresponds to the probabilities of executing each one of + the 4 actions when the policy orders to the robot to go UP. In this case + the transition model says that with a probability of 0.55 the robot will + go UP, with a probaiblity of 0.25 RIGHT, 0.10 DOWN and 0.10 LEFT. + ''' + if(transition_matrix.shape != self.transition_matrix.shape): + raise ValueError('The shape of the two matrices must be the same.') + self.transition_matrix = transition_matrix + + def setRewardMatrix(self, reward_matrix): + '''Set the reward matrix. + + ''' + if(reward_matrix.shape != self.reward_matrix.shape): + raise ValueError('The shape of the matrix does not match with the shape of the world.') + self.reward_matrix = reward_matrix + + def setStateMatrix(self, state_matrix): + '''Set the obstacles in the world. + + The input to the function is a matrix with the + same size of the world + -1 for states which are not walkable. + +1 for terminal states + 0 for all the walkable states (non terminal) + The following matrix represents the 4x3 world + used in the series "dissecting reinforcement learning" + [[0, 0, 0, +1] + [0, -1, 0, +1] + [0, 0, 0, 0]] + ''' + if(state_matrix.shape != self.state_matrix.shape): + raise ValueError('The shape of the matrix does not match with the shape of the world.') + self.state_matrix = state_matrix + + def setPosition(self, index_row=None, index_col=None): + ''' Set the position of the robot in a specific state. + + ''' + if(index_row is None or index_col is None): self.position = [np.random.randint(tot_row), np.random.randint(tot_col)] + else: self.position = [index_row, index_col] + + def render(self): + ''' Print the current world in the terminal. + + O represents the robot position + - respresent empty states. + # represents obstacles + * represents terminal states + ''' + graph = "" + for row in range(self.world_row): + row_string = "" + for col in range(self.world_col): + if(self.position == [row, col]): row_string += u" \u25CB " # u" \u25CC " + else: + if(self.state_matrix[row, col] == 0): row_string += ' - ' + elif(self.state_matrix[row, col] == -1): row_string += ' # ' + elif(self.state_matrix[row, col] == +1): row_string += ' * ' + row_string += '\n' + graph += row_string + print(graph) + + def reset(self, exploring_starts=False): + ''' Set the position of the robot in the bottom left corner. + + It returns the first observation + ''' + if exploring_starts: + while(True): + row = np.random.randint(0, self.world_row) + col = np.random.randint(0, self.world_col) + if(self.state_matrix[row, col] == 0): break + self.position = [row, col] + else: + self.position = [self.world_row-1, 0] + #reward = self.reward_matrix[self.position[0], self.position[1]] + return self.position + + def step(self, action): + ''' One step in the world. + + [observation, reward, done = env.step(action)] + The robot moves one step in the world based on the action given. + The action can be 0=UP, 1=RIGHT, 2=DOWN, 3=LEFT + @return observation the position of the robot after the step + @return reward the reward associated with the next state + @return done True if the state is terminal + ''' + if(action >= self.action_space_size): + raise ValueError('The action is not included in the action space.') + + #Based on the current action and the probability derived + #from the trasition model it chooses a new actio to perform + action = np.random.choice(4, 1, p=self.transition_matrix[int(action),:]) + #action = self.transition_model(action) + + #Generating a new position based on the current position and action + if(action == 0): new_position = [self.position[0]-1, self.position[1]] #UP + elif(action == 1): new_position = [self.position[0], self.position[1]+1] #RIGHT + elif(action == 2): new_position = [self.position[0]+1, self.position[1]] #DOWN + elif(action == 3): new_position = [self.position[0], self.position[1]-1] #LEFT + else: raise ValueError('The action is not included in the action space.') + + #Check if the new position is a valid position + #print(self.state_matrix) + if (new_position[0]>=0 and new_position[0]=0 and new_position[1]= MEMORY_CAPACITY: + dqn.learn() + if done: + print("episode: {} , the episode reward is {}".format(i, round(ep_reward, 3))) + if done: + break + state = next_state + r = copy.copy(reward) + reward_list.append(r) + ax.set_xlim(0,300) + #ax.cla() + ax.plot(reward_list, 'g-', label='total_loss') + plt.pause(0.001) + + +if __name__ == '__main__': + main() diff --git a/Char01 DQN/DQN/pic/finish_episode.jpg b/Char01 DQN/DQN/pic/finish_episode.jpg new file mode 100644 index 0000000..b66475d Binary files /dev/null and b/Char01 DQN/DQN/pic/finish_episode.jpg differ diff --git a/Char01 DQN/DQN/pic/readme.md b/Char01 DQN/DQN/pic/readme.md new file mode 100644 index 0000000..8178c76 --- /dev/null +++ b/Char01 DQN/DQN/pic/readme.md @@ -0,0 +1 @@ +readme diff --git a/Char01 DQN/DQN/pic/value_loss.jpg b/Char01 DQN/DQN/pic/value_loss.jpg new file mode 100644 index 0000000..5f72838 Binary files /dev/null and b/Char01 DQN/DQN/pic/value_loss.jpg differ diff --git a/Char01 DQN/DQN_CartPole-v0.py b/Char01 DQN/DQN_CartPole-v0.py new file mode 100644 index 0000000..c7ac767 --- /dev/null +++ b/Char01 DQN/DQN_CartPole-v0.py @@ -0,0 +1,120 @@ +import argparse +import pickle +from collections import namedtuple +from itertools import count + +import os, time +import numpy as np +import matplotlib.pyplot as plt + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal, Categorical +from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler +from tensorboardX import SummaryWriter + +# Hyper-parameters +seed = 1 +render = False +num_episodes = 2000 +env = gym.make('CartPole-v0').unwrapped +num_state = env.observation_space.shape[0] +num_action = env.action_space.n +torch.manual_seed(seed) +env.seed(seed) + +Transition = namedtuple('Transition', ['state', 'action', 'reward', 'next_state']) + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = nn.Linear(num_state, 100) + self.fc2 = nn.Linear(100, num_action) + + def forward(self, x): + x = F.relu(self.fc1(x)) + action_value = self.fc2(x) + return action_value + +class DQN(): + + capacity = 8000 + learning_rate = 1e-3 + memory_count = 0 + batch_size = 256 + gamma = 0.995 + update_count = 0 + + def __init__(self): + super(DQN, self).__init__() + self.target_net, self.act_net = Net(), Net() + self.memory = [None]*self.capacity + self.optimizer = optim.Adam(self.act_net.parameters(), self.learning_rate) + self.loss_func = nn.MSELoss() + self.writer = SummaryWriter('./DQN/logs') + + + def select_action(self,state): + state = torch.tensor(state, dtype=torch.float).unsqueeze(0) + value = self.act_net(state) + action_max_value, index = torch.max(value, 1) + action = index.item() + if np.random.rand(1) >= 0.9: # epslion greedy + action = np.random.choice(range(num_action), 1).item() + return action + + def store_transition(self,transition): + index = self.memory_count % self.capacity + self.memory[index] = transition + self.memory_count += 1 + return self.memory_count >= self.capacity + + def update(self): + if self.memory_count >= self.capacity: + state = torch.tensor([t.state for t in self.memory]).float() + action = torch.LongTensor([t.action for t in self.memory]).view(-1,1).long() + reward = torch.tensor([t.reward for t in self.memory]).float() + next_state = torch.tensor([t.next_state for t in self.memory]).float() + + reward = (reward - reward.mean()) / (reward.std() + 1e-7) + with torch.no_grad(): + target_v = reward + self.gamma * self.target_net(next_state).max(1)[0] + + #Update... + for index in BatchSampler(SubsetRandomSampler(range(len(self.memory))), batch_size=self.batch_size, drop_last=False): + v = (self.act_net(state).gather(1, action))[index] + loss = self.loss_func(target_v[index].unsqueeze(1), (self.act_net(state).gather(1, action))[index]) + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + self.writer.add_scalar('loss/value_loss', loss, self.update_count) + self.update_count +=1 + if self.update_count % 100 ==0: + self.target_net.load_state_dict(self.act_net.state_dict()) + else: + print("Memory Buff is too less") +def main(): + + agent = DQN() + for i_ep in range(num_episodes): + state = env.reset() + if render: env.render() + for t in range(10000): + action = agent.select_action(state) + next_state, reward, done, info = env.step(action) + if render: env.render() + transition = Transition(state, action, reward, next_state) + agent.store_transition(transition) + state = next_state + if done or t >=9999: + agent.writer.add_scalar('live/finish_step', t+1, global_step=i_ep) + agent.update() + if i_ep % 10 == 0: + print("episodes {}, step is {} ".format(i_ep, t)) + break + +if __name__ == '__main__': + main() diff --git a/Char01 DQN/DQN_MountainCar-v0.py b/Char01 DQN/DQN_MountainCar-v0.py new file mode 100644 index 0000000..7f8b246 --- /dev/null +++ b/Char01 DQN/DQN_MountainCar-v0.py @@ -0,0 +1,120 @@ +import argparse +import pickle +from collections import namedtuple +from itertools import count + +import os, time +import numpy as np +import matplotlib.pyplot as plt + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal, Categorical +from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler +from tensorboardX import SummaryWriter + +# Hyper-parameters +seed = 1 +render = False +num_episodes = 400000 +env = gym.make('MountainCar-v0').unwrapped +num_state = env.observation_space.shape[0] +num_action = env.action_space.n +torch.manual_seed(seed) +env.seed(seed) + +Transition = namedtuple('Transition', ['state', 'action', 'reward', 'next_state']) + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = nn.Linear(num_state, 100) + self.fc2 = nn.Linear(100, num_action) + + def forward(self, x): + x = F.relu(self.fc1(x)) + action_prob = self.fc2(x) + return action_prob + +class DQN(): + + capacity = 8000 + learning_rate = 1e-3 + memory_count = 0 + batch_size = 256 + gamma = 0.995 + update_count = 0 + + def __init__(self): + super(DQN, self).__init__() + self.target_net, self.act_net = Net(), Net() + self.memory = [None]*self.capacity + self.optimizer = optim.Adam(self.act_net.parameters(), self.learning_rate) + self.loss_func = nn.MSELoss() + self.writer = SummaryWriter('./DQN/logs') + + + def select_action(self,state): + state = torch.tensor(state, dtype=torch.float).unsqueeze(0) + value = self.act_net(state) + action_max_value, index = torch.max(value, 1) + action = index.item() + if np.random.rand(1) >= 0.9: # epslion greedy + action = np.random.choice(range(num_action), 1).item() + return action + + def store_transition(self,transition): + index = self.memory_count % self.capacity + self.memory[index] = transition + self.memory_count += 1 + return self.memory_count >= self.capacity + + def update(self): + if self.memory_count >= self.capacity: + state = torch.tensor([t.state for t in self.memory]).float() + action = torch.LongTensor([t.action for t in self.memory]).view(-1,1).long() + reward = torch.tensor([t.reward for t in self.memory]).float() + next_state = torch.tensor([t.next_state for t in self.memory]).float() + + reward = (reward - reward.mean()) / (reward.std() + 1e-7) + with torch.no_grad(): + target_v = reward + self.gamma * self.target_net(next_state).max(1)[0] + + #Update... + for index in BatchSampler(SubsetRandomSampler(range(len(self.memory))), batch_size=self.batch_size, drop_last=False): + v = (self.act_net(state).gather(1, action))[index] + loss = self.loss_func(target_v[index].unsqueeze(1), (self.act_net(state).gather(1, action))[index]) + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + self.writer.add_scalar('loss/value_loss', loss, self.update_count) + self.update_count +=1 + if self.update_count % 100 ==0: + self.target_net.load_state_dict(self.act_net.state_dict()) + else: + print("Memory Buff is too less") +def main(): + + agent = DQN() + for i_ep in range(num_episodes): + state = env.reset() + if render: env.render() + for t in range(10000): + action = agent.select_action(state) + next_state, reward, done, info = env.step(action) + if render: env.render() + transition = Transition(state, action, reward, next_state) + agent.store_transition(transition) + state = next_state + if done or t >=9999: + agent.writer.add_scalar('live/finish_step', t+1, global_step=i_ep) + agent.update() + if i_ep % 10 == 0: + print("episodes {}, step is {} ".format(i_ep, t)) + break + +if __name__ == '__main__': + main() diff --git a/Char01 DQN/DQN_mountain_car_v1.py b/Char01 DQN/DQN_mountain_car_v1.py new file mode 100644 index 0000000..45b6a56 --- /dev/null +++ b/Char01 DQN/DQN_mountain_car_v1.py @@ -0,0 +1,133 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import optim +import matplotlib.pyplot as plt +import gym + + +#hyper parameters +EPSILON = 0.9 +GAMMA = 0.9 +LR = 0.01 +MEMORY_CAPACITY = 2000 +Q_NETWORK_ITERATION = 100 +BATCH_SIZE = 32 + +EPISODES = 400 +env = gym.make('MountainCar-v0') +env = env.unwrapped +NUM_STATES = env.observation_space.shape[0] # 2 +NUM_ACTIONS = env.action_space.n + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + + self.fc1 = nn.Linear(NUM_STATES, 30) + self.fc1.weight.data.normal_(0, 0.1) + self.fc2 = nn.Linear(30, NUM_ACTIONS) + self.fc2.weight.data.normal_(0, 0.1) + + + def forward(self, x): + x = self.fc1(x) + x = F.relu(x) + x = self.fc2(x) + + return x + +class Dqn(): + def __init__(self): + self.eval_net, self.target_net = Net(), Net() + self.memory = np.zeros((MEMORY_CAPACITY, NUM_STATES *2 +2)) + # state, action ,reward and next state + self.memory_counter = 0 + self.learn_counter = 0 + self.optimizer = optim.Adam(self.eval_net.parameters(), LR) + self.loss = nn.MSELoss() + + self.fig, self.ax = plt.subplots() + + def store_trans(self, state, action, reward, next_state): + if self.memory_counter % 500 ==0: + print("The experience pool collects {} time experience".format(self.memory_counter)) + index = self.memory_counter % MEMORY_CAPACITY + trans = np.hstack((state, [action], [reward], next_state)) + self.memory[index,] = trans + self.memory_counter += 1 + + def choose_action(self, state): + # notation that the function return the action's index nor the real action + # EPSILON + state = torch.unsqueeze(torch.FloatTensor(state) ,0) + if np.random.randn() <= EPSILON: + action_value = self.eval_net.forward(state) + action = torch.max(action_value, 1)[1].data.numpy() # get action whose q is max + action = action[0] #get the action index + else: + action = np.random.randint(0,NUM_ACTIONS) + return action + + def plot(self, ax, x): + ax.cla() + ax.set_xlabel("episode") + ax.set_ylabel("total reward") + ax.plot(x, 'b-') + plt.pause(0.000000000000001) + + def learn(self): + # learn 100 times then the target network update + if self.learn_counter % Q_NETWORK_ITERATION ==0: + self.target_net.load_state_dict(self.eval_net.state_dict()) + self.learn_counter+=1 + + sample_index = np.random.choice(MEMORY_CAPACITY, BATCH_SIZE) + batch_memory = self.memory[sample_index, :] + batch_state = torch.FloatTensor(batch_memory[:, :NUM_STATES]) + #note that the action must be a int + batch_action = torch.LongTensor(batch_memory[:, NUM_STATES:NUM_STATES+1].astype(int)) + batch_reward = torch.FloatTensor(batch_memory[:, NUM_STATES+1: NUM_STATES+2]) + batch_next_state = torch.FloatTensor(batch_memory[:, -NUM_STATES:]) + + q_eval = self.eval_net(batch_state).gather(1, batch_action) + q_next = self.target_net(batch_next_state).detach() + q_target = batch_reward + GAMMA*q_next.max(1)[0].view(BATCH_SIZE, 1) + + loss = self.loss(q_eval, q_target) + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + + +def main(): + net = Dqn() + print("The DQN is collecting experience...") + step_counter_list = [] + for episode in range(EPISODES): + state = env.reset() + step_counter = 0 + while True: + step_counter +=1 + env.render() + action = net.choose_action(state) + next_state, reward, done, info = env.step(action) + reward = reward * 100 if reward >0 else reward * 5 + net.store_trans(state, action, reward, next_state) + + if net.memory_counter >= MEMORY_CAPACITY: + net.learn() + if done: + print("episode {}, the reward is {}".format(episode, round(reward, 3))) + if done: + step_counter_list.append(step_counter) + net.plot(net.ax, step_counter_list) + break + + state = next_state + +if __name__ == '__main__': + main() diff --git a/Char01 DQN/naiveDQN.py b/Char01 DQN/naiveDQN.py new file mode 100644 index 0000000..3ebdc7c --- /dev/null +++ b/Char01 DQN/naiveDQN.py @@ -0,0 +1,124 @@ +# Nota that this network won't work because the reward is always 1 + +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import gym + +# hyper-parameters +BATCH_SIZE = 128 +LR = 0.01 +GAMMA = 0.90 +EPISILO = 0.9 +MEMORY_CAPACITY = 20000 +Q_NETWORK_ITERATION = 100 + +env = gym.make("CartPole-v0") +env = env.unwrapped +NUM_ACTIONS = env.action_space.n +NUM_STATES = env.observation_space.shape[0] +ENV_A_SHAPE = 0 if isinstance(env.action_space.sample(), int) else env.action_space.sample.shape +class Net(nn.Module): + """docstring for Net""" + def __init__(self): + super(Net, self).__init__() + self.fc1 = nn.Linear(NUM_STATES, 50) + self.fc1.weight.data.normal_(0,0.1) + self.fc2 = nn.Linear(50,30) + self.fc2.weight.data.normal_(0,0.1) + self.out = nn.Linear(30,NUM_ACTIONS) + self.out.weight.data.normal_(0,0.1) + + def forward(self,x): + x = self.fc1(x) + x = F.relu(x) + x = self.fc2(x) + x = F.relu(x) + action_prob = self.out(x) + return action_prob + +class DQN(): + """docstring for DQN""" + def __init__(self): + super(DQN, self).__init__() + self.eval_net, self.target_net = Net(), Net() + + self.learn_step_counter = 0 + self.memory_counter = 0 + self.memory = np.zeros((MEMORY_CAPACITY, NUM_STATES * 2 + 2)) + # why the NUM_STATE*2 +2 + # When we store the memory, we put the state, action, reward and next_state in the memory + # here reward and action is a number, state is a ndarray + self.optimizer = torch.optim.Adam(self.eval_net.parameters(), lr=LR) + self.loss_func = nn.MSELoss() + + def choose_action(self, state): + state = torch.unsqueeze(torch.FloatTensor(state), 0) # get a 1D array + if np.random.randn() <= EPISILO:# greedy policy + action_value = self.eval_net.forward(state) + action = torch.max(action_value, 1)[1].data.numpy() + action = action[0] if ENV_A_SHAPE == 0 else action.reshape(ENV_A_SHAPE) + else: # random policy + action = np.random.randint(0,NUM_ACTIONS) + action = action if ENV_A_SHAPE ==0 else action.reshape(ENV_A_SHAPE) + return action + + + def store_transition(self, state, action, reward, next_state): + transition = np.hstack((state, [action, reward], next_state)) + index = self.memory_counter % MEMORY_CAPACITY + self.memory[index, :] = transition + self.memory_counter += 1 + + + def learn(self): + + #update the parameters + if self.learn_step_counter % Q_NETWORK_ITERATION ==0: + self.target_net.load_state_dict(self.eval_net.state_dict()) + self.learn_step_counter+=1 + + #sample batch from memory + sample_index = np.random.choice(MEMORY_CAPACITY, BATCH_SIZE) + batch_memory = self.memory[sample_index, :] + batch_state = torch.FloatTensor(batch_memory[:, :NUM_STATES]) + batch_action = torch.LongTensor(batch_memory[:, NUM_STATES:NUM_STATES+1].astype(int)) + batch_reward = torch.FloatTensor(batch_memory[:, NUM_STATES+1:NUM_STATES+2]) + batch_next_state = torch.FloatTensor(batch_memory[:,-NUM_STATES:]) + + #q_eval + q_eval = self.eval_net(batch_state).gather(1, batch_action) + q_next = self.target_net(batch_next_state).detach() + q_target = batch_reward + GAMMA * q_next.max(1)[0].view(BATCH_SIZE, 1) + loss = self.loss_func(q_eval, q_target) + + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + +def main(): + dqn = DQN() + episodes = 400 + print("Collecting Experience....") + for i in range(episodes): + state = env.reset() + ep_reward = 0 + while True: + env.render() + action = dqn.choose_action(state) + next_state, reward, done, info = env.step(action) + + dqn.store_transition(state, action, reward, next_state) + ep_reward += reward + + if dqn.memory_counter >= MEMORY_CAPACITY: + dqn.learn() + if done: + print("episode: {} , the episode reward is {}".format(i, round(ep_reward, 3))) + if done: + break + state = next_state + +if __name__ == '__main__': + main() diff --git a/Char01 DQN/readme.md b/Char01 DQN/readme.md new file mode 100644 index 0000000..fafd5a5 --- /dev/null +++ b/Char01 DQN/readme.md @@ -0,0 +1,14 @@ +# Requirment： + +- tensorflow 1.10 +- pytorch 4.1 +- tensorboardX +- gym + +## Tips for MountainCar-v0 env: + +This is very sparse for MountainCar-v0, it is 0 at the beginning, only when the top of the mountain is 1, there is a reward. This leads to the fact that if the sample to the top of the mountain is not taken during training, basically the train will not come out. So you can change the reward, for example, to change to the current position of the Car is positively related. Of course, there is a more advanced approach to inverse reinforcement learning (using GAN). + +![value_loss](DQN/pic/value_loss.jpg) +![step](DQN/pic/finish_episode.jpg) +This is value loss for DQN, We can see that the loss increaded to 1e13 however, the network work well. This is because the training is going on, the target_net and act_net are very different, so the calculated loss becomes very large. The previous loss was small because the reward was very sparse, resulting in a small update of the two networks. diff --git a/Char02 Policy Gradient/PolicyGradient.py b/Char02 Policy Gradient/PolicyGradient.py new file mode 100644 index 0000000..b4159d5 --- /dev/null +++ b/Char02 Policy Gradient/PolicyGradient.py @@ -0,0 +1,106 @@ + +import argparse +import gym +import numpy as np +from itertools import count + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import matplotlib.pyplot as plt +from torch.distributions import Categorical + + + +parser = argparse.ArgumentParser(description='PyTorch REINFORCE example') +parser.add_argument('--gamma', type=float, default=0.99, metavar='G', + help='discount factor (default: 0.99)') +parser.add_argument('--seed', type=int, default=543, metavar='N', + help='random seed (default: 543)') +parser.add_argument('--render', action='store_true', + help='render the environment') +parser.add_argument('--log-interval', type=int, default=10, metavar='N', + help='interval between training status logs (default: 10)') +args = parser.parse_args() + + +env = gym.make('CartPole-v0') +env.seed(args.seed) +torch.manual_seed(args.seed) + + +class Policy(nn.Module): + def __init__(self): + super(Policy, self).__init__() + self.affine1 = nn.Linear(4, 128) + self.affine2 = nn.Linear(128, 2) + + self.saved_log_probs = [] + self.rewards = [] + + def forward(self, x): + x = F.relu(self.affine1(x)) + action_scores = self.affine2(x) + return F.softmax(action_scores, dim=1) + + +policy = Policy() +optimizer = optim.Adam(policy.parameters(), lr=1e-2) +eps = np.finfo(np.float32).eps.item() + + +def select_action(state): + state = torch.from_numpy(state).float().unsqueeze(0) + probs = policy(state) + m = Categorical(probs) + action = m.sample() + policy.saved_log_probs.append(m.log_prob(action)) + return action.item() + + +def finish_episode(): + R = 0 + policy_loss = [] + rewards = [] + for r in policy.rewards[::-1]: + R = r + args.gamma * R + rewards.insert(0, R) + rewards = torch.tensor(rewards) + rewards = (rewards - rewards.mean()) / (rewards.std() + eps) + for log_prob, reward in zip(policy.saved_log_probs, rewards): + policy_loss.append(-log_prob * reward) + optimizer.zero_grad() + policy_loss = torch.cat(policy_loss).sum() + policy_loss.backward() + optimizer.step() + del policy.rewards[:] + del policy.saved_log_probs[:] + + +def main(): + running_reward = 10 + for i_episode in count(1): + state = env.reset() + for t in range(10000): # Don't infinite loop while learning + action = select_action(state) + state, reward, done, _ = env.step(action) + if args.render: + env.render() + policy.rewards.append(reward) + if done: + break + + running_reward = running_reward * 0.99 + t * 0.01 + finish_episode() + if i_episode % args.log_interval == 0: + print('Episode {}\tLast length: {:5d}\tAverage length: {:.2f}'.format( + i_episode, t, running_reward)) + if running_reward > env.spec.reward_threshold: + print("Solved! Running reward is now {} and " + "the last episode runs to {} time steps!".format(running_reward, t)) + break + + +if __name__ == '__main__': + main() diff --git a/Char02 Policy Gradient/REINFORCE.py b/Char02 Policy Gradient/REINFORCE.py new file mode 100644 index 0000000..7982545 --- /dev/null +++ b/Char02 Policy Gradient/REINFORCE.py @@ -0,0 +1,103 @@ +import argparse +import gym +import numpy as np +from itertools import count + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Categorical + + +parser = argparse.ArgumentParser(description='PyTorch REINFORCE example') +parser.add_argument('--gamma', type=float, default=0.99, metavar='G', + help='discount factor (default: 0.99)') +parser.add_argument('--seed', type=int, default=543, metavar='N', + help='random seed (default: 543)') +parser.add_argument('--render', action='store_true', + help='render the environment') +parser.add_argument('--log-interval', type=int, default=10, metavar='N', + help='interval between training status logs (default: 10)') +args = parser.parse_args() + + +env = gym.make('CartPole-v0') +env.seed(args.seed) +torch.manual_seed(args.seed) + + +class Policy(nn.Module): + def __init__(self): + super(Policy, self).__init__() + self.affine1 = nn.Linear(4, 128) + self.affine2 = nn.Linear(128, 2) + + self.saved_log_probs = [] + self.rewards = [] + + def forward(self, x): + x = F.relu(self.affine1(x)) + action_scores = self.affine2(x) + return F.softmax(action_scores, dim=1) + + +policy = Policy() +optimizer = optim.Adam(policy.parameters(), lr=1e-2) +eps = np.finfo(np.float32).eps.item() + + +def select_action(state): + state = torch.from_numpy(state).float().unsqueeze(0) + probs = policy(state) + m = Categorical(probs) + action = m.sample() + policy.saved_log_probs.append(m.log_prob(action)) + return action.item() + + +def finish_episode(): + R = 0 + policy_loss = [] + rewards = [] + for r in policy.rewards[::-1]: + R = r + args.gamma * R + rewards.insert(0, R) + rewards = torch.tensor(rewards) + rewards = (rewards - rewards.mean()) / (rewards.std() + eps) + for log_prob, reward in zip(policy.saved_log_probs, rewards): + policy_loss.append(-log_prob * reward) + optimizer.zero_grad() + policy_loss = torch.cat(policy_loss).sum() + policy_loss.backward() + optimizer.step() + del policy.rewards[:] + del policy.saved_log_probs[:] + + +def main(): + running_reward = 10 + for i_episode in count(1): + state = env.reset() + for t in range(10000): # Don't infinite loop while learning + action = select_action(state) + state, reward, done, _ = env.step(action) + if args.render: + env.render() + policy.rewards.append(reward) + if done: + break + + running_reward = running_reward * 0.99 + t * 0.01 + finish_episode() + if i_episode % args.log_interval == 0: + print('Episode {}\tLast length: {:5d}\tAverage length: {:.2f}'.format( + i_episode, t, running_reward)) + if running_reward > env.spec.reward_threshold: + print("Solved! Running reward is now {} and " + "the last episode runs to {} time steps!".format(running_reward, t)) + break + + +if __name__ == '__main__': + main() diff --git a/Char02 Policy Gradient/REINFORCE_with_Baseline.py b/Char02 Policy Gradient/REINFORCE_with_Baseline.py new file mode 100644 index 0000000..7cd346e --- /dev/null +++ b/Char02 Policy Gradient/REINFORCE_with_Baseline.py @@ -0,0 +1,154 @@ + + + +import torch.nn as nn +import torch.nn.functional as F +import gym +import torch +from torch.distributions import Categorical +import torch.optim as optim +from copy import deepcopy +import argparse +import matplotlib.pyplot as plt +from tensorboardX import SummaryWriter +from torch.nn.utils import clip_grad_norm_ +render = False + +#parser = argparse.ArgumentParser(description='PyTorch REINFORCE example with baseline') +# parser.add_argument('--render', action='store_true', default=True, +# help='render the environment') +#args = parser.parse_args() + +#神经网络的输出： Net1输入state，输出action_prob; Net2 输入state,输出各个action_reward +class Policy(nn.Module): + def __init__(self,n_states, n_hidden, n_output): + super(Policy, self).__init__() + self.linear1 = nn.Linear(n_states, n_hidden) + self.linear2 = nn.Linear(n_hidden, n_output) + + #这是policy的参数 + self.reward = [] + self.log_act_probs = [] + self.Gt = [] + self.sigma = [] +#这是state_action_func的参数 + # self.Reward = [] + # self.s_value = [] + + def forward(self, x): + x = F.relu(self.linear1(x)) + output = F.softmax(self.linear2(x), dim= 1) + # self.act_probs.append(action_probs) + return output + + + +env = gym.make('CartPole-v0') +# writer = SummaryWriter('./yingyingying') +# state = env.reset() +n_states = env.observation_space.shape[0] +n_actions = env.action_space.n + +policy = Policy(n_states, 128, n_actions) +s_value_func = Policy(n_states, 128, 1) + + +alpha_theta = 1e-3 +optimizer_theta = optim.Adam(policy.parameters(), lr=alpha_theta) +# alpha_w = 1e-3 #初始化 +# optimizer_w = optim.Adam(policy.parameters(), lr=alpha_w) +gamma = 0.99 + + + +seed = 1 +env.seed(seed) +torch.manual_seed(seed) +live_time = [] + +def loop_episode(): + + state = env.reset() + if render: env.render() + policy_loss = [] + s_value = [] + state_sequence = [] + log_act_prob = [] + for t in range(1000): + state = torch.from_numpy(state).unsqueeze(0).float() # 在第0维增加一个维度，将数据组织成[N , .....] 形式 + state_sequence.append(deepcopy(state)) + action_probs = policy(state) + m = Categorical(action_probs) + action = m.sample() + m_log_prob = m.log_prob(action) + log_act_prob.append(m_log_prob) + # policy.log_act_probs.append(m_log_prob) + action = action.item() + next_state, re, done, _ = env.step(action) + if render: env.render() + policy.reward.append(re) + if done: + live_time.append(t) + break + state = next_state + + R = 0 + Gt = [] + + # get Gt value + for r in policy.reward[::-1]: + R = r + gamma * R + Gt.insert(0, R) + # s_value_func.sigma.insert(0,sigma) + # policy.Gt.insert(0,R) + + + # update step by step + for i in range(len(Gt)): + + + + G = Gt[i] + V = s_value_func(state_sequence[i]) + delta = G - V + + # update value network + alpha_w = 1e-3 # 初始化 + + optimizer_w = optim.Adam(policy.parameters(), lr=alpha_w) + optimizer_w.zero_grad() + policy_loss_w =-delta + policy_loss_w.backward(retain_graph = True) + clip_grad_norm_(policy_loss_w, 0.1) + optimizer_w.step() + + # update policy network + optimizer_theta.zero_grad() + policy_loss_theta = - log_act_prob[i] * delta + policy_loss_theta.backward(retain_graph = True) + clip_grad_norm_(policy_loss_theta, 0.1) + optimizer_theta.step() + + del policy.log_act_probs[:] + del policy.reward[:] + + +def plot(live_time): + plt.ion() + plt.grid() + plt.plot(live_time, 'g-') + plt.xlabel('running step') + plt.ylabel('live time') + plt.pause(0.000001) + + + +if __name__ == '__main__': + + #生成若干episode + # graph_data = torch.autograd.Variable(torch.ones(1,4)) + # writer.add_graph(policy, (graph_data, )) + for i_episode in range(1000): + loop_episode() + plot(live_time) + #policy.plot(live_time) diff --git a/Char02 Policy Gradient/Run_Model.py b/Char02 Policy Gradient/Run_Model.py new file mode 100644 index 0000000..57cb27a --- /dev/null +++ b/Char02 Policy Gradient/Run_Model.py @@ -0,0 +1,104 @@ +# MountainCar V0 +# +import numpy as np +import gym +import matplotlib.pyplot as plt +from itertools import count + +import torch +import torch.nn as nn +from torch.nn import functional as F +from torch.optim import adam +from torch.distributions import Categorical + +env = gym.make('MountainCar-v0') +env = env.unwrapped +env.seed(1) + +torch.manual_seed(1) +plt.ion() + + +#Hyperparameters +learning_rate = 0.02 +gamma = 0.995 +episodes = 1000 + +eps = np.finfo(np.float32).eps.item() + +action_space = env.action_space.n +state_space = env.observation_space.shape[0] + +class Policy(nn.Module): + def __init__(self): + super(Policy, self).__init__() + + self.fc1 = nn.Linear(state_space, 20) + #self.fc2 = nn.Linear(128,64) + self.fc3 = nn.Linear(20, action_space) + + self.gamma = gamma + self.saved_log_probs = [] + self.rewards = [] + + def forward(self, x): + + x = F.relu(self.fc1(x)) + #x = F.relu(self.fc2(x)) + x = F.softmax(self.fc3(x), dim=1) + + return x + +policy = torch.load('policyNet.pkl') + +def plot(steps): + ax = plt.subplot(111) + ax.cla() + ax.set_title('Training') + ax.set_xlabel('Episode') + ax.set_ylabel('Run Time') + ax.plot(steps) + RunTime = len(steps) + path = './PG_MountainCar-v0/'+'RunTime'+str(RunTime)+'.jpg' + if len(steps) % 100 == 0: + #plt.savefig(path) + pass + plt.pause(0.0000001) + +def selct_action(state): + state = torch.from_numpy(state).float().unsqueeze(0) + probs = policy(state) + c = Categorical(probs) + action = c.sample() + + + #policy.saved_log_probs.append(c.log_prob(action)) + action = action.item() + return action + +def run_Model(): + running_reward = 0 + steps = [] + for episode in count(60000): + state = env.reset() + + for t in range(10000): + action = selct_action(state) + state, reward ,done, info = env.step(action) + env.render() + #policy.rewards.append(reward) + + if done: + print("Episode {}, live time = {}".format(episode, t)) + steps.append(t) + plot(steps) + break + if episode % 50 == 0: + pass + #torch.save(policy, 'policyNet.pkl') + + running_reward = running_reward * policy.gamma - t*0.01 + #finish_episode() + +if __name__ == '__main__': + run_Model() \ No newline at end of file diff --git a/Char02 Policy Gradient/naive-policy-gradient.py b/Char02 Policy Gradient/naive-policy-gradient.py new file mode 100644 index 0000000..bb76c98 --- /dev/null +++ b/Char02 Policy Gradient/naive-policy-gradient.py @@ -0,0 +1,153 @@ +import numpy as np +import gym +import matplotlib.pyplot as plt +from itertools import count + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +from torch.distributions import Categorical + + +#Hyperparameters +learning_rate = 0.01 +gamma = 0.98 + +num_episode = 5000 +batch_size = 32 + + +env = gym.make('CartPole-v0') +state_space = env.observation_space.shape[0] +action_space = env.action_space.n + +def plot_durations(episode_durations): + plt.ion() + plt.figure(2) + plt.clf() + duration_t = torch.FloatTensor(episode_durations) + plt.title('Training') + plt.xlabel('Episodes') + plt.ylabel('Duration') + plt.plot(duration_t.numpy()) + + if len(duration_t) >= 100: + means = duration_t.unfold(0,100,1).mean(1).view(-1) + means = torch.cat((torch.zeros(99), means)) + plt.plot(means.numpy()) + + plt.pause(0.00001) + +class Policy(nn.Module): + + def __init__(self): + super(Policy, self).__init__() + + self.state_space = state_space + self.action_space = action_space + + self.fc1 = nn.Linear(self.state_space, 128) + self.fc2 = nn.Linear(128, self.action_space) + + def forward(self, x): + x = self.fc1(x) + #x = F.dropout(x, 0.5) + x = F.relu(x) + x = F.softmax(self.fc2(x), dim=-1) + + return x + +policy = Policy() +optimizer = torch.optim.Adam(policy.parameters(), lr=learning_rate) + + + +def train(): + + episode_durations = [] + #Batch_history + state_pool = [] + action_pool = [] + reward_pool = [] + steps = 0 + + for episode in range(num_episode): + state = env.reset() + state = torch.from_numpy(state).float() + state = Variable(state) + + env.render() + + for t in count(): + probs = policy(state) + c = Categorical(probs) + action = c.sample() + + action = action.data.numpy().astype('int32') + next_state, reward, done, info = env.step(action) + reward = 0 if done else reward # correct the reward + env.render() + + state_pool.append(state) + action_pool.append(float(action)) + reward_pool.append(reward) + + state = next_state + state = torch.from_numpy(state).float() + state = Variable(state) + + steps += 1 + + if done: + episode_durations.append(t+1) + plot_durations(episode_durations) + break + + # update policy + if episode >0 and episode % batch_size == 0: + + r = 0 + ''' + for i in reversed(range(steps)): + if reward_pool[i] == 0: + running_add = 0 + else: + running_add = running_add * gamma +reward_pool[i] + reward_pool[i] = running_add + ''' + for i in reversed(range(steps)): + if reward_pool[i] == 0: + r = 0 + else: + r = r * gamma + reward_pool[i] + reward_pool[i] = r + + #Normalize reward + reward_mean = np.mean(reward_pool) + reward_std = np.std(reward_pool) + reward_pool = (reward_pool-reward_mean)/reward_std + + #gradiend desent + optimizer.zero_grad() + + for i in range(steps): + state = state_pool[i] + action = Variable(torch.FloatTensor([action_pool[i]])) + reward = reward_pool[i] + + probs = policy(state) + c = Categorical(probs) + + loss = -c.log_prob(action) * reward + loss.backward() + + optimizer.step() + + # clear the batch pool + state_pool = [] + action_pool = [] + reward_pool = [] + steps = 0 + +train() diff --git a/Char02 Policy Gradient/pytorch_MountainCar-v0.py b/Char02 Policy Gradient/pytorch_MountainCar-v0.py new file mode 100644 index 0000000..e1e83ec --- /dev/null +++ b/Char02 Policy Gradient/pytorch_MountainCar-v0.py @@ -0,0 +1,134 @@ +# MountainCar V0 + +import numpy as np +import gym +import matplotlib.pyplot as plt +from itertools import count + +import torch +import torch.nn as nn +from torch.nn import functional as F +from torch.optim import adam +from torch.distributions import Categorical + +env = gym.make('MountainCar-v0') +env = env.unwrapped +env.seed(1) + +torch.manual_seed(1) +plt.ion() + + +#Hyperparameters +learning_rate = 0.02 +gamma = 0.995 +episodes = 1000 + +eps = np.finfo(np.float32).eps.item() + +action_space = env.action_space.n +state_space = env.observation_space.shape[0] + + +class Policy(nn.Module): + def __init__(self): + super(Policy, self).__init__() + + self.fc1 = nn.Linear(state_space, 20) + #self.fc2 = nn.Linear(128,64) + self.fc3 = nn.Linear(20, action_space) + + self.gamma = gamma + self.saved_log_probs = [] + self.rewards = [] + + def forward(self, x): + + x = F.relu(self.fc1(x)) + #x = F.relu(self.fc2(x)) + x = F.softmax(self.fc3(x), dim=1) + + return x + +policy = Policy() +optimizer = adam.Adam(policy.parameters(), lr=learning_rate) + +def selct_action(state): + state = torch.from_numpy(state).float().unsqueeze(0) + probs = policy(state) + c = Categorical(probs) + action = c.sample() + + + policy.saved_log_probs.append(c.log_prob(action)) + action = action.item() + return action + +def finish_episode(): + R = 0 + policy_loss = [] + rewards = [] + + for r in policy.rewards[::-1]: + R = r + policy.gamma * R + rewards.insert(0, R) + + # Formalize reward + rewards = torch.tensor(rewards) + rewards = (rewards - rewards.mean())/(rewards.std() + eps) + + # get loss + for reward, log_prob in zip(rewards, policy.saved_log_probs): + policy_loss.append(-log_prob * reward) + + optimizer.zero_grad() + policy_loss = torch.cat(policy_loss).sum() + policy_loss.backward() + optimizer.step() + + + + del policy.rewards[:] + del policy.saved_log_probs[:] + +def plot(steps): + ax = plt.subplot(111) + ax.cla() + ax.set_title('Training') + ax.set_xlabel('Episode') + ax.set_ylabel('Run Time') + ax.plot(steps) + RunTime = len(steps) + path = './PG_MountainCar-v0/'+'RunTime'+str(RunTime)+'.jpg' + if len(steps) % 100 == 0: + plt.savefig(path) + plt.pause(0.0000001) + + + +def main(): + + running_reward = 0 + steps = [] + for episode in count(60000): + state = env.reset() + + for t in range(10000): + action = selct_action(state) + state, reward ,done, info = env.step(action) + env.render() + policy.rewards.append(reward) + + if done: + print("Episode {}, live time = {}".format(episode, t)) + steps.append(t) + plot(steps) + break + if episode % 50 == 0: + torch.save(policy, 'policyNet.pkl') + + running_reward = running_reward * policy.gamma - t*0.01 + finish_episode() + +if __name__ == '__main__': + main() diff --git a/Char03 Actor-Critic/AC_CartPole-v0.py b/Char03 Actor-Critic/AC_CartPole-v0.py new file mode 100644 index 0000000..6f4040f --- /dev/null +++ b/Char03 Actor-Critic/AC_CartPole-v0.py @@ -0,0 +1,128 @@ +import gym, os +import numpy as np +import matplotlib.pyplot as plt +from itertools import count +from collections import namedtuple + +import torch +import torch.nn as nn +import torch.optim as optim +import torch.nn.functional as F +from torch.distributions import Categorical + +#Parameters +env = gym.make('CartPole-v0') +env = env.unwrapped + +env.seed(1) +torch.manual_seed(1) + +state_space = env.observation_space.shape[0] +action_space = env.action_space.n + + +#Hyperparameters +learning_rate = 0.01 +gamma = 0.99 +episodes = 20000 +render = False +eps = np.finfo(np.float32).eps.item() +SavedAction = namedtuple('SavedAction', ['log_prob', 'value']) + +class Policy(nn.Module): + def __init__(self): + super(Policy, self).__init__() + self.fc1 = nn.Linear(state_space, 32) + + self.action_head = nn.Linear(32, action_space) + self.value_head = nn.Linear(32, 1) # Scalar Value + + self.save_actions = [] + self.rewards = [] + os.makedirs('./AC_CartPole-v0', exist_ok=True) + + def forward(self, x): + x = F.relu(self.fc1(x)) + action_score = self.action_head(x) + state_value = self.value_head(x) + + return F.softmax(action_score, dim=-1), state_value + +model = Policy() +optimizer = optim.Adam(model.parameters(), lr=learning_rate) + +def plot(steps): + ax = plt.subplot(111) + ax.cla() + ax.grid() + ax.set_title('Training') + ax.set_xlabel('Episode') + ax.set_ylabel('Run Time') + ax.plot(steps) + RunTime = len(steps) + + path = './AC_CartPole-v0/' + 'RunTime' + str(RunTime) + '.jpg' + if len(steps) % 200 == 0: + plt.savefig(path) + plt.pause(0.0000001) + +def select_action(state): + state = torch.from_numpy(state).float() + probs, state_value = model(state) + m = Categorical(probs) + action = m.sample() + model.save_actions.append(SavedAction(m.log_prob(action), state_value)) + + return action.item() + + +def finish_episode(): + R = 0 + save_actions = model.save_actions + policy_loss = [] + value_loss = [] + rewards = [] + + for r in model.rewards[::-1]: + R = r + gamma * R + rewards.insert(0, R) + + rewards = torch.tensor(rewards) + rewards = (rewards - rewards.mean()) / (rewards.std() + eps) + + for (log_prob , value), r in zip(save_actions, rewards): + reward = r - value.item() + policy_loss.append(-log_prob * reward) + value_loss.append(F.smooth_l1_loss(value, torch.tensor([r]))) + + optimizer.zero_grad() + loss = torch.stack(policy_loss).sum() + torch.stack(value_loss).sum() + loss.backward() + optimizer.step() + + del model.rewards[:] + del model.save_actions[:] + +def main(): + running_reward = 10 + live_time = [] + for i_episode in count(episodes): + state = env.reset() + for t in count(): + action = select_action(state) + state, reward, done, info = env.step(action) + if render: env.render() + model.rewards.append(reward) + + if done or t >= 1000: + break + running_reward = running_reward * 0.99 + t * 0.01 + live_time.append(t) + plot(live_time) + if i_episode % 100 == 0: + modelPath = './AC_CartPole_Model/ModelTraing'+str(i_episode)+'Times.pkl' + torch.save(model, modelPath) + finish_episode() + +if __name__ == '__main__': + main() diff --git a/Char03 Actor-Critic/AC_MountainCar-v0.py b/Char03 Actor-Critic/AC_MountainCar-v0.py new file mode 100644 index 0000000..68c4a94 --- /dev/null +++ b/Char03 Actor-Critic/AC_MountainCar-v0.py @@ -0,0 +1,145 @@ +import gym, os +import numpy as np +import matplotlib.pyplot as plt +from itertools import count +from collections import namedtuple + +import torch +import torch.nn as nn +from torch.nn import functional as F +from torch.optim import Adam +from torch.distributions import Categorical +from torch.nn.functional import smooth_l1_loss + +#Hyperparameters +LEARNING_RATE = 0.01 +GAMMA = 0.995 +NUM_EPISODES = 50000 +RENDER = False +#env info +env = gym.make('MountainCar-v0') +env = env.unwrapped + +env.seed(1) +torch.manual_seed(1) + +num_state = env.observation_space.shape[0] +num_action = env.action_space.n +eps = np.finfo(np.float32).eps.item() +plt.ion() +saveAction = namedtuple('SavedActions',['probs', 'action_values']) + +class Module(nn.Module): + def __init__(self): + super(Module, self).__init__() + self.fc1 = nn.Linear(num_state, 128) + #self.fc2 = nn.Linear(64, 128) + + self.action_head = nn.Linear(128, num_action) + self.value_head = nn.Linear(128, 1) + self.policy_action_value = [] + self.rewards = [] + + self.gamma = GAMMA + os.makedirs('/AC_MountainCar-v0_Model/', exist_ok=True) + + + def forward(self, x): + x = F.relu(self.fc1(x)) + #x = F.relu(self.fc2(x)) + + probs = F.softmax(self.action_head(x)) + value = self.value_head(x) + return probs, value + +policy = Module() +optimizer = Adam(policy.parameters(), lr=LEARNING_RATE) + +def plot(steps): + ax = plt.subplot(111) + ax.cla() + ax.grid() + ax.set_title('Training') + ax.set_xlabel('Episode') + ax.set_ylabel('Run Time') + ax.plot(steps) + RunTime = len(steps) + path = './AC_MountainCar-v0/' + 'RunTime' + str(RunTime) + '.jpg' + if len(steps) % 200 == 0: + plt.savefig(path) + plt.pause(0.0000001) + + +def select_action(state): + state = torch.from_numpy(state).float() + probs, value = policy(state) + c = Categorical(probs) + action = c.sample() + log_prob = c.log_prob(action) + + + policy.policy_action_value.append(saveAction(log_prob, value)) + action = action.item() + return action + + +def finish_episode(): + rewards = [] + saveActions = policy.policy_action_value + policy_loss = [] + value_loss = [] + R = 0 + + for r in policy.rewards[::-1]: + R = r + policy.gamma * R + rewards.insert(0, R) + + # Normalize the reward + rewards = torch.tensor(rewards) + rewards = (rewards - rewards.mean()) / (rewards.std() + eps) + + #Figure out loss + for (log_prob, value), r in zip(saveActions, rewards): + reward = r - value.item() + policy_loss.append(-log_prob * reward) + value_loss.append(smooth_l1_loss(value, torch.tensor([r]) )) + + optimizer.zero_grad() + loss = torch.stack(policy_loss).sum() + torch.stack(policy_loss).sum() + loss.backward() + optimizer.step() + + del policy.rewards[:] + del policy.policy_action_value[:] + + +def main(): + run_steps = [] + for i_episode in range(NUM_EPISODES): + state = env.reset() + if RENDER: env.render() + + for t in count(): + action = select_action(state) + state , reward, done, _ = env.step(action) + reward = state[0] + reward + if RENDER: env.render() + policy.rewards.append(reward) + + if done: + run_steps.append(t) + print("Epiosde {} , run step is {} ".format(i_episode+1 , t+1)) + break + + finish_episode() + plot(run_steps) + + if i_episode % 100 == 0 and i_episode !=0: + + modelPath = './AC_MountainCar-v0_Model/ModelTraing' + str(i_episode) + 'Times.pkl' + torch.save(policy, modelPath) + + + +if __name__ == '__main__': + main() diff --git a/Char04 A2C/A2C.py b/Char04 A2C/A2C.py new file mode 100644 index 0000000..b565ae1 --- /dev/null +++ b/Char04 A2C/A2C.py @@ -0,0 +1,158 @@ + +import math +import random + +import gym +import numpy as np + +import torch +import torch.nn as nn +import torch.optim as optim +import torch.nn.functional as F +from torch.distributions import Categorical + +import matplotlib.pyplot as plt + +use_cuda = torch.cuda.is_available() +device = torch.device("cuda" if use_cuda else "cpu") + +from multiprocessing_env import SubprocVecEnv + +num_envs = 8 +env_name = "CartPole-v0" + +def make_env(): + def _thunk(): + env = gym.make(env_name) + return env + return _thunk + +plt.ion() +envs = [make_env() for i in range(num_envs)] +envs = SubprocVecEnv(envs) # 8 env + +env = gym.make(env_name) # a single env + +class ActorCritic(nn.Module): + def __init__(self, num_inputs, num_outputs, hidden_size, std=0.0): + super(ActorCritic, self).__init__() + + self.critic = nn.Sequential( + nn.Linear(num_inputs, hidden_size), + nn.ReLU(), + nn.Linear(hidden_size, 1) + ) + + self.actor = nn.Sequential( + nn.Linear(num_inputs, hidden_size), + nn.ReLU(), + nn.Linear(hidden_size, num_outputs), + nn.Softmax(dim=1), + ) + + def forward(self, x): + value = self.critic(x) + probs = self.actor(x) + dist = Categorical(probs) + return dist, value + + +def test_env(vis=False): + state = env.reset() + if vis: env.render() + done = False + total_reward = 0 + while not done: + state = torch.FloatTensor(state).unsqueeze(0).to(device) + dist, _ = model(state) + next_state, reward, done, _ = env.step(dist.sample().cpu().numpy()[0]) + state = next_state + if vis: env.render() + total_reward += reward + return total_reward + + +def compute_returns(next_value, rewards, masks, gamma=0.99): + R = next_value + returns = [] + for step in reversed(range(len(rewards))): + R = rewards[step] + gamma * R * masks[step] + returns.insert(0, R) + return returns + +def plot(frame_idx, rewards): + plt.plot(rewards,'b-') + plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1])) + plt.pause(0.0001) + + +num_inputs = envs.observation_space.shape[0] +num_outputs = envs.action_space.n + +#Hyper params: +hidden_size = 256 +lr = 1e-3 +num_steps = 5 + +model = ActorCritic(num_inputs, num_outputs, hidden_size).to(device) +optimizer = optim.Adam(model.parameters()) + + +max_frames = 20000 +frame_idx = 0 +test_rewards = [] + + +state = envs.reset() + +while frame_idx < max_frames: + + log_probs = [] + values = [] + rewards = [] + masks = [] + entropy = 0 + + # rollout trajectory + for _ in range(num_steps): + state = torch.FloatTensor(state).to(device) + dist, value = model(state) + + action = dist.sample() + next_state, reward, done, _ = envs.step(action.cpu().numpy()) + + log_prob = dist.log_prob(action) + entropy += dist.entropy().mean() + + log_probs.append(log_prob) + values.append(value) + rewards.append(torch.FloatTensor(reward).unsqueeze(1).to(device)) + masks.append(torch.FloatTensor(1 - done).unsqueeze(1).to(device)) + + state = next_state + frame_idx += 1 + + if frame_idx % 100 == 0: + test_rewards.append(np.mean([test_env() for _ in range(10)])) + plot(frame_idx, test_rewards) + + next_state = torch.FloatTensor(next_state).to(device) + _, next_value = model(next_state) + returns = compute_returns(next_value, rewards, masks) + + log_probs = torch.cat(log_probs) + returns = torch.cat(returns).detach() + values = torch.cat(values) + + advantage = returns - values + + actor_loss = -(log_probs * advantage.detach()).mean() + critic_loss = advantage.pow(2).mean() + + loss = actor_loss + 0.5 * critic_loss - 0.001 * entropy + + optimizer.zero_grad() + loss.backward() + optimizer.step() + +#test_env(True) diff --git a/Char04 A2C/multiprocessing_env.py b/Char04 A2C/multiprocessing_env.py new file mode 100644 index 0000000..46bbc08 --- /dev/null +++ b/Char04 A2C/multiprocessing_env.py @@ -0,0 +1,153 @@ +#This code is from openai baseline +#https://github.com/openai/baselines/tree/master/baselines/common/vec_env + +import numpy as np +from multiprocessing import Process, Pipe + +def worker(remote, parent_remote, env_fn_wrapper): + parent_remote.close() + env = env_fn_wrapper.x() + while True: + cmd, data = remote.recv() + if cmd == 'step': + ob, reward, done, info = env.step(data) + if done: + ob = env.reset() + remote.send((ob, reward, done, info)) + elif cmd == 'reset': + ob = env.reset() + remote.send(ob) + elif cmd == 'reset_task': + ob = env.reset_task() + remote.send(ob) + elif cmd == 'close': + remote.close() + break + elif cmd == 'get_spaces': + remote.send((env.observation_space, env.action_space)) + else: + raise NotImplementedError + +class VecEnv(object): + """ + An abstract asynchronous, vectorized environment. + """ + def __init__(self, num_envs, observation_space, action_space): + self.num_envs = num_envs + self.observation_space = observation_space + self.action_space = action_space + + def reset(self): + """ + Reset all the environments and return an array of + observations, or a tuple of observation arrays. + If step_async is still doing work, that work will + be cancelled and step_wait() should not be called + until step_async() is invoked again. + """ + pass + + def step_async(self, actions): + """ + Tell all the environments to start taking a step + with the given actions. + Call step_wait() to get the results of the step. + You should not call this if a step_async run is + already pending. + """ + pass + + def step_wait(self): + """ + Wait for the step taken with step_async(). + Returns (obs, rews, dones, infos): + - obs: an array of observations, or a tuple of + arrays of observations. + - rews: an array of rewards + - dones: an array of "episode done" booleans + - infos: a sequence of info objects + """ + pass + + def close(self): + """ + Clean up the environments' resources. + """ + pass + + def step(self, actions): + self.step_async(actions) + return self.step_wait() + + +class CloudpickleWrapper(object): + """ + Uses cloudpickle to serialize contents (otherwise multiprocessing tries to use pickle) + """ + def __init__(self, x): + self.x = x + def __getstate__(self): + import cloudpickle + return cloudpickle.dumps(self.x) + def __setstate__(self, ob): + import pickle + self.x = pickle.loads(ob) + + +class SubprocVecEnv(VecEnv): + def __init__(self, env_fns, spaces=None): + """ + envs: list of gym environments to run in subprocesses + """ + self.waiting = False + self.closed = False + nenvs = len(env_fns) + self.nenvs = nenvs + self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)]) + self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn))) + for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)] + for p in self.ps: + p.daemon = True # if the main process crashes, we should not cause things to hang + p.start() + for remote in self.work_remotes: + remote.close() + + self.remotes[0].send(('get_spaces', None)) + observation_space, action_space = self.remotes[0].recv() + VecEnv.__init__(self, len(env_fns), observation_space, action_space) + + def step_async(self, actions): + for remote, action in zip(self.remotes, actions): + remote.send(('step', action)) + self.waiting = True + + def step_wait(self): + results = [remote.recv() for remote in self.remotes] + self.waiting = False + obs, rews, dones, infos = zip(*results) + return np.stack(obs), np.stack(rews), np.stack(dones), infos + + def reset(self): + for remote in self.remotes: + remote.send(('reset', None)) + return np.stack([remote.recv() for remote in self.remotes]) + + def reset_task(self): + for remote in self.remotes: + remote.send(('reset_task', None)) + return np.stack([remote.recv() for remote in self.remotes]) + + def close(self): + if self.closed: + return + if self.waiting: + for remote in self.remotes: + remote.recv() + for remote in self.remotes: + remote.send(('close', None)) + for p in self.ps: + p.join() + self.closed = True + + def __len__(self): + return self.nenvs diff --git a/Char05 DDPG/DDPG.py b/Char05 DDPG/DDPG.py new file mode 100644 index 0000000..671835c --- /dev/null +++ b/Char05 DDPG/DDPG.py @@ -0,0 +1,268 @@ +import argparse +from itertools import count + +import os, sys, random +import numpy as np + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from tensorboardX import SummaryWriter + +''' +Implementation of Deep Deterministic Policy Gradients (DDPG) with pytorch +riginal paper: https://arxiv.org/abs/1509.02971 +Not the author's implementation ! +''' + +parser = argparse.ArgumentParser() +parser.add_argument('--mode', default='train', type=str) # mode = 'train' or 'test' +# OpenAI gym environment name, # ['BipedalWalker-v2', 'Pendulum-v0'] or any continuous environment +# Note that DDPG is feasible about hyper-parameters. +# You should fine-tuning if you change to another environment. +parser.add_argument("--env_name", default="Pendulum-v0") +parser.add_argument('--tau', default=0.005, type=float) # target smoothing coefficient +parser.add_argument('--target_update_interval', default=1, type=int) +parser.add_argument('--test_iteration', default=10, type=int) + +parser.add_argument('--learning_rate', default=1e-3, type=float) +parser.add_argument('--gamma', default=0.99, type=int) # discounted factor +parser.add_argument('--capacity', default=50000, type=int) # replay buffer size +parser.add_argument('--batch_size', default=64, type=int) # mini batch size +parser.add_argument('--seed', default=False, type=bool) +parser.add_argument('--random_seed', default=9527, type=int) +# optional parameters + +parser.add_argument('--sample_frequency', default=256, type=int) +parser.add_argument('--render', default=False, type=bool) # show UI or not +parser.add_argument('--log_interval', default=50, type=int) # +parser.add_argument('--load', default=False, type=bool) # load model +parser.add_argument('--render_interval', default=100, type=int) # after render_interval, the env.render() will work +parser.add_argument('--exploration_noise', default=0.1, type=float) +parser.add_argument('--max_episode', default=100000, type=int) # num of games +parser.add_argument('--max_length_of_trajectory', default=2000, type=int) # num of games +parser.add_argument('--print_log', default=5, type=int) +parser.add_argument('--update_iteration', default=10, type=int) +args = parser.parse_args() + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +script_name = os.path.basename(__file__) +env = gym.make(args.env_name).unwrapped + +if args.seed: + env.seed(args.random_seed) + torch.manual_seed(args.random_seed) + np.random.seed(args.random_seed) + +state_dim = env.observation_space.shape[0] +action_dim = env.action_space.shape[0] +max_action = float(env.action_space.high[0]) +min_Val = torch.tensor(1e-7).float().to(device) # min value + +directory = './exp' + script_name + args.env_name +'./' + +class Replay_buffer(): + ''' + Code based on: + https://github.com/openai/baselines/blob/master/baselines/deepq/replay_buffer.py + Expects tuples of (state, next_state, action, reward, done) + ''' + def __init__(self, max_size=args.capacity): + self.storage = [] + self.max_size = max_size + self.ptr = 0 + + def push(self, data): + if len(self.storage) == self.max_size: + self.storage[int(self.ptr)] = data + self.ptr = (self.ptr + 1) % self.max_size + else: + self.storage.append(data) + + def sample(self, batch_size): + ind = np.random.randint(0, len(self.storage), size=batch_size) + x, y, u, r, d = [], [], [], [], [] + + for i in ind: + X, Y, U, R, D = self.storage[i] + x.append(np.array(X, copy=False)) + y.append(np.array(Y, copy=False)) + u.append(np.array(U, copy=False)) + r.append(np.array(R, copy=False)) + d.append(np.array(D, copy=False)) + + return np.array(x), np.array(y), np.array(u), np.array(r).reshape(-1, 1), np.array(d).reshape(-1, 1) + + +class Actor(nn.Module): + def __init__(self, state_dim, action_dim, max_action): + super(Actor, self).__init__() + + self.l1 = nn.Linear(state_dim, 400) + self.l2 = nn.Linear(400, 300) + self.l3 = nn.Linear(300, action_dim) + + self.max_action = max_action + + def forward(self, x): + x = F.relu(self.l1(x)) + x = F.relu(self.l2(x)) + x = self.max_action * torch.tanh(self.l3(x)) + return x + + +class Critic(nn.Module): + def __init__(self, state_dim, action_dim): + super(Critic, self).__init__() + + self.l1 = nn.Linear(state_dim + action_dim, 400) + self.l2 = nn.Linear(400 , 300) + self.l3 = nn.Linear(300, 1) + + def forward(self, x, u): + x = F.relu(self.l1(torch.cat([x, u], 1))) + x = F.relu(self.l2(x)) + x = self.l3(x) + return x + + +class DDPG(object): + def __init__(self, state_dim, action_dim, max_action): + self.actor = Actor(state_dim, action_dim, max_action).to(device) + self.actor_target = Actor(state_dim, action_dim, max_action).to(device) + self.actor_target.load_state_dict(self.actor.state_dict()) + self.actor_optimizer = optim.Adam(self.actor.parameters(), args.learning_rate) + + self.critic = Critic(state_dim, action_dim).to(device) + self.critic_target = Critic(state_dim, action_dim).to(device) + self.critic_target.load_state_dict(self.critic.state_dict()) + self.critic_optimizer = optim.Adam(self.critic.parameters(), args.learning_rate) + self.replay_buffer = Replay_buffer() + self.writer = SummaryWriter(directory) + self.num_critic_update_iteration = 0 + self.num_actor_update_iteration = 0 + self.num_training = 0 + + def select_action(self, state): + state = torch.FloatTensor(state.reshape(1, -1)).to(device) + return self.actor(state).cpu().data.numpy().flatten() + + def update(self): + + for it in range(args.update_iteration): + # Sample replay buffer + x, y, u, r, d = self.replay_buffer.sample(args.batch_size) + state = torch.FloatTensor(x).to(device) + action = torch.FloatTensor(u).to(device) + next_state = torch.FloatTensor(y).to(device) + done = torch.FloatTensor(d).to(device) + reward = torch.FloatTensor(r).to(device) + + # Compute the target Q value + target_Q = self.critic_target(next_state, self.actor_target(next_state)) + target_Q = reward + ((1 - done) * args.gamma * target_Q).detach() + + # Get current Q estimate + current_Q = self.critic(state, action) + + # Compute critic loss + critic_loss = F.mse_loss(current_Q, target_Q) + self.writer.add_scalar('Loss/critic_loss', critic_loss, global_step=self.num_critic_update_iteration) + # Optimize the critic + self.critic_optimizer.zero_grad() + critic_loss.backward() + self.critic_optimizer.step() + + # Compute actor loss + actor_loss = -self.critic(state, self.actor(state)).mean() + self.writer.add_scalar('Loss/actor_loss', actor_loss, global_step=self.num_actor_update_iteration) + + # Optimize the actor + self.actor_optimizer.zero_grad() + actor_loss.backward() + self.actor_optimizer.step() + + # Update the frozen target models + for param, target_param in zip(self.critic.parameters(), self.critic_target.parameters()): + target_param.data.copy_(args.tau * param.data + (1 - args.tau) * target_param.data) + + for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()): + target_param.data.copy_(args.tau * param.data + (1 - args.tau) * target_param.data) + + self.num_actor_update_iteration += 1 + self.num_critic_update_iteration += 1 + + def save(self): + torch.save(self.actor.state_dict(), directory + 'actor.pth') + torch.save(self.critic.state_dict(), directory + 'critic.pth') + # print("====================================") + # print("Model has been saved...") + # print("====================================") + + def load(self): + self.actor.load_state_dict(torch.load(directory + 'actor.pth')) + self.critic.load_state_dict(torch.load(directory + 'critic.pth')) + print("====================================") + print("model has been loaded...") + print("====================================") + +def main(): + agent = DDPG(state_dim, action_dim, max_action) + ep_r = 0 + if args.mode == 'test': + agent.load() + for i in range(args.test_iteration): + state = env.reset() + for t in count(): + action = agent.select_action(state) + next_state, reward, done, info = env.step(np.float32(action)) + ep_r += reward + env.render() + if done or t >= args.max_length_of_trajectory: + print("Ep_i \t{}, the ep_r is \t{:0.2f}, the step is \t{}".format(i, ep_r, t)) + ep_r = 0 + break + state = next_state + + elif args.mode == 'train': + print("====================================") + print("Collection Experience...") + print("====================================") + if args.load: agent.load() + for i in range(args.max_episode): + state = env.reset() + for t in count(): + action = agent.select_action(state) + + # issue 3 add noise to action + action = (action + np.random.normal(0, args.exploration_noise, size=env.action_space.shape[0])).clip( + env.action_space.low, env.action_space.high) + + next_state, reward, done, info = env.step(action) + ep_r += reward + if args.render and i >= args.render_interval : env.render() + agent.replay_buffer.push((state, next_state, action, reward, np.float(done))) + # if (i+1) % 10 == 0: + # print('Episode {}, The memory size is {} '.format(i, len(agent.replay_buffer.storage))) + + state = next_state + if done or t >= args.max_length_of_trajectory: + agent.writer.add_scalar('ep_r', ep_r, global_step=i) + if i % args.print_log == 0: + print("Ep_i \t{}, the ep_r is \t{:0.2f}, the step is \t{}".format(i, ep_r, t)) + ep_r = 0 + break + + if i % args.log_interval == 0: + agent.save() + if len(agent.replay_buffer.storage) >= args.capacity-1: + agent.update() + + else: + raise NameError("mode wrong!!!") + +if __name__ == '__main__': + main() diff --git a/Char05 DDPG/DDPG_exp.jpg b/Char05 DDPG/DDPG_exp.jpg new file mode 100644 index 0000000..4a9ecaa Binary files /dev/null and b/Char05 DDPG/DDPG_exp.jpg differ diff --git a/Char05 DDPG/README.md b/Char05 DDPG/README.md new file mode 100644 index 0000000..2a95653 --- /dev/null +++ b/Char05 DDPG/README.md @@ -0,0 +1,10 @@ +# DDPG +- Original paper: https://arxiv.org/abs/1509.02971 +- OPENAI Baselines post: https://blog.openai.com/better-exploration-with-parameter-noise/ + +**Note that DDPG is feasible about hyper-parameters. You should fine-tuning if you change to another environment.** + +Episode reward in Pendulum-v0: + +![ep_r](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/Char05%20DDPG/DDPG_exp.jpg) + diff --git a/Char07 PPO/PPO2.py b/Char07 PPO/PPO2.py new file mode 100644 index 0000000..ad3f9d5 --- /dev/null +++ b/Char07 PPO/PPO2.py @@ -0,0 +1,188 @@ + +import argparse +import pickle +from collections import namedtuple + +import os +import numpy as np +import matplotlib.pyplot as plt + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler + +# Parameters +parser = argparse.ArgumentParser(description='Solve the Pendulum-v0 with PPO') +parser.add_argument( + '--gamma', type=float, default=0.9, metavar='G', help='discount factor (default: 0.9)') +parser.add_argument('--seed', type=int, default=0, metavar='N', help='random seed (default: 0)') +parser.add_argument('--render', action='store_true', default=True, help='render the environment') +parser.add_argument( + '--log-interval', + type=int, + default=10, + metavar='N', + help='interval between training status logs (default: 10)') +args = parser.parse_args() + +env = gym.make('Pendulum-v0').unwrapped +num_state = env.observation_space.shape[0] +num_action = env.action_space.shape[0] +torch.manual_seed(args.seed) +env.seed(args.seed) + +Transition = namedtuple('Transition',['state', 'aciton', 'reward', 'a_log_prob', 'next_state']) +TrainRecord = namedtuple('TrainRecord',['episode', 'reward']) + +class Actor(nn.Module): + def __init__(self): + super(Actor, self).__init__() + self.fc1 = nn.Linear(num_state, 64) + self.fc2 = nn.Linear(64,8) + self.mu_head = nn.Linear(8, 1) + self.sigma_head = nn.Linear(8, 1) + + def forward(self, x): + x = F.leaky_relu(self.fc1(x)) + x = F.leaky_relu(self.fc2(x)) + + mu = self.mu_head(x) + sigma = self.sigma_head(x) + + return mu, sigma + +class Critic(nn.Module): + def __init__(self): + super(Critic, self).__init__() + self.fc1 = nn.Linear(num_state, 64) + self.fc2 = nn.Linear(64, 8) + self.state_value= nn.Linear(8, 1) + + def forward(self, x): + x = F.leaky_relu(self.fc1(x)) + x = F.leaky_relu(self.fc2(x)) + value = self.state_value(x) + return value + +class PPO(): + clip_param = 0.2 + max_grad_norm = 0.5 + ppo_epoch = 10 + buffer_capacity = 1000 + batch_size = 8 + + def __init__(self): + super(PPO, self).__init__() + self.actor_net = Actor().float() + self.critic_net = Critic().float() + self.buffer = [] + self.counter = 0 + self.training_step = 0 + + self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 1e-3) + self.critic_net_optimizer = optim.Adam(self.critic_net.parameters(), 4e-3) + if not os.path.exists('../param'): + os.makedirs('../param/net_param') + os.makedirs('../param/img') + + def select_action(self, state): + state = torch.from_numpy(state).float().unsqueeze(0) + with torch.no_grad(): + mu, sigma = self.actor_net(state) + dist = Normal(mu, sigma) + action = dist.sample() + action_log_prob = dist.log_prob(action) + action = action.clamp(-2, 2) + return action.item(), action_log_prob.item() + + + def get_value(self, state): + state = torch.from_numpy(state) + with torch.no_grad(): + value = self.critic_net(state) + return value.item() + + def save_param(self): + torch.save(self.actor_net.state_dict(), '../param/net_param/actor_net'+str(time.time())[:10],+'.pkl') + torch.save(self.critic_net.state_dict(), '../param/net_param/critic_net'+str(time.time())[:10],+'.pkl') + + def store_transition(self, transition): + self.buffer.append(transition) + self.counter+=1 + return counter % self.buffer_capacity == 0 + + def update(self): + self.training_step +=1 + + state = torch.tensor([t.state for t in self.buffer ], dtype=torch.float) + action = torch.tensor([t.action for t in self.buffer], dtype=torch.float).view(-1, 1) + reward = torch.tensor([t.reward for t in self.buffer], dtype=torch.float).view(-1, 1) + next_state = torch.tensor([t.next_state for t in self.buffer], dtype=torch.float) + old_action_log_prob = torch.tensor([t.a_log_prob for t in self.buffer], dtype=torch.float).view(-1, 1) + + reward = (reward - reward.mean())/(reward.std() + 1e-10) + with torch.no_grad(): + target_v = reward + args.gamma * self.critic_net(next_state) + + advantage = (target_v - self.critic_net(state)).detach() + for _ in range(self.ppo_epoch): # iteration ppo_epoch + for index in BatchSampler(SubsetRandomSampler(range(self.buffer_capacity), self.batch_size, True)): + # epoch iteration, PPO core!!! + mu, sigma = self.actor_net(state[index]) + n = Normal(mu, sigma) + action_log_prob = n.log_prob(action[index]) + ratio = torch.exp(action_log_prob - old_action_log_prob) + + L1 = ratio * advantage[index] + L2 = torch.clamp(ratio, 1-self.clip_param, 1+self.clip_param) * advantage[index] + action_loss = -torch.min(L1, L2).mean() # MAX->MIN desent + self.actor_optimizer.zero_grad() + action_loss.backward() + nn.utils.clip_grad_norm_(self.actor_net.parameters(), self.max_grad_norm) + self.actor_optimizer.step() + + value_loss = F.smooth_l1_loss(self.critic_net(state[index]), target_v[index]) + self.critic_net_optimizer.zero_grad() + value_loss.backward() + nn.utils.clip_grad_norm_(self.critic_net.parameters(), self.max_grad_norm) + self.critic_net_optimizer.step() + + del self.buffer[:] + +def main(): + + agent = PPO() + + training_records = [] + running_reward = -1000 + + for i_epoch in range(1000): + score = 0 + state = env.reset() + if args.render: env.render() + for t in range(200): + action, action_log_prob = agent.select_action(state) + next_state, reward, done, info = env.step(action) + trans = Transition(state, action, reward, action_log_prob, next_state) + if args.render: env.render() + if agent.store_transition(trans): + agent.update() + score += reward + state = next_state + + running_reward = running_reward * 0.9 + score * 0.1 + training_records.append(TrainingRecord(i_epoch, running_reward)) + if i_epoch % 10 ==0: + print("Epoch {}, Moving average score is: {:.2f} ".format(i_epoch, running_reward)) + if running_reward > -200: + print("Solved! Moving average score is now {}!".format(running_reward)) + env.close() + agent.save_param() + break + +if __name__ == '__main__': + main() diff --git a/Char07 PPO/PPO_CartPole_v0.py b/Char07 PPO/PPO_CartPole_v0.py new file mode 100644 index 0000000..597cf0a --- /dev/null +++ b/Char07 PPO/PPO_CartPole_v0.py @@ -0,0 +1,174 @@ +import argparse +import pickle +from collections import namedtuple +from itertools import count + +import os, time +import numpy as np +import matplotlib.pyplot as plt + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal, Categorical +from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler +from tensorboardX import SummaryWriter + +# Parameters +gamma = 0.99 +render = False +seed = 1 +log_interval = 10 + +env = gym.make('CartPole-v0').unwrapped +num_state = env.observation_space.shape[0] +num_action = env.action_space.n +torch.manual_seed(seed) +env.seed(seed) +Transition = namedtuple('Transition', ['state', 'action', 'a_log_prob', 'reward', 'next_state']) + +class Actor(nn.Module): + def __init__(self): + super(Actor, self).__init__() + self.fc1 = nn.Linear(num_state, 100) + self.action_head = nn.Linear(100, num_action) + + def forward(self, x): + x = F.relu(self.fc1(x)) + action_prob = F.softmax(self.action_head(x), dim=1) + return action_prob + + +class Critic(nn.Module): + def __init__(self): + super(Critic, self).__init__() + self.fc1 = nn.Linear(num_state, 100) + self.state_value = nn.Linear(100, 1) + + def forward(self, x): + x = F.relu(self.fc1(x)) + value = self.state_value(x) + return value + + +class PPO(): + clip_param = 0.2 + max_grad_norm = 0.5 + ppo_update_time = 10 + buffer_capacity = 1000 + batch_size = 32 + + def __init__(self): + super(PPO, self).__init__() + self.actor_net = Actor() + self.critic_net = Critic() + self.buffer = [] + self.counter = 0 + self.training_step = 0 + self.writer = SummaryWriter('../exp') + + self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 1e-3) + self.critic_net_optimizer = optim.Adam(self.critic_net.parameters(), 3e-3) + if not os.path.exists('../param'): + os.makedirs('../param/net_param') + os.makedirs('../param/img') + + def select_action(self, state): + state = torch.from_numpy(state).float().unsqueeze(0) + with torch.no_grad(): + action_prob = self.actor_net(state) + c = Categorical(action_prob) + action = c.sample() + return action.item(), action_prob[:,action.item()].item() + + def get_value(self, state): + state = torch.from_numpy(state) + with torch.no_grad(): + value = self.critic_net(state) + return value.item() + + def save_param(self): + torch.save(self.actor_net.state_dict(), '../param/net_param/actor_net' + str(time.time())[:10], +'.pkl') + torch.save(self.critic_net.state_dict(), '../param/net_param/critic_net' + str(time.time())[:10], +'.pkl') + + def store_transition(self, transition): + self.buffer.append(transition) + self.counter += 1 + + + def update(self, i_ep): + state = torch.tensor([t.state for t in self.buffer], dtype=torch.float) + action = torch.tensor([t.action for t in self.buffer], dtype=torch.long).view(-1, 1) + reward = [t.reward for t in self.buffer] + # update: don't need next_state + #reward = torch.tensor([t.reward for t in self.buffer], dtype=torch.float).view(-1, 1) + #next_state = torch.tensor([t.next_state for t in self.buffer], dtype=torch.float) + old_action_log_prob = torch.tensor([t.a_log_prob for t in self.buffer], dtype=torch.float).view(-1, 1) + + R = 0 + Gt = [] + for r in reward[::-1]: + R = r + gamma * R + Gt.insert(0, R) + Gt = torch.tensor(Gt, dtype=torch.float) + #print("The agent is updateing....") + for i in range(self.ppo_update_time): + for index in BatchSampler(SubsetRandomSampler(range(len(self.buffer))), self.batch_size, False): + if self.training_step % 1000 ==0: + print('I_ep {} ，train {} times'.format(i_ep,self.training_step)) + #with torch.no_grad(): + Gt_index = Gt[index].view(-1, 1) + V = self.critic_net(state[index]) + delta = Gt_index - V + advantage = delta.detach() + # epoch iteration, PPO core!!! + action_prob = self.actor_net(state[index]).gather(1, action[index]) # new policy + + ratio = (action_prob/old_action_log_prob[index]) + surr1 = ratio * advantage + surr2 = torch.clamp(ratio, 1 - self.clip_param, 1 + self.clip_param) * advantage + + # update actor network + action_loss = -torch.min(surr1, surr2).mean() # MAX->MIN desent + self.writer.add_scalar('loss/action_loss', action_loss, global_step=self.training_step) + self.actor_optimizer.zero_grad() + action_loss.backward() + nn.utils.clip_grad_norm_(self.actor_net.parameters(), self.max_grad_norm) + self.actor_optimizer.step() + + #update critic network + value_loss = F.mse_loss(Gt_index, V) + self.writer.add_scalar('loss/value_loss', value_loss, global_step=self.training_step) + self.critic_net_optimizer.zero_grad() + value_loss.backward() + nn.utils.clip_grad_norm_(self.critic_net.parameters(), self.max_grad_norm) + self.critic_net_optimizer.step() + self.training_step += 1 + + del self.buffer[:] # clear experience + + +def main(): + agent = PPO() + for i_epoch in range(1000): + state = env.reset() + if render: env.render() + + for t in count(): + action, action_prob = agent.select_action(state) + next_state, reward, done, _ = env.step(action) + trans = Transition(state, action, action_prob, reward, next_state) + if render: env.render() + agent.store_transition(trans) + state = next_state + + if done : + if len(agent.buffer) >= agent.batch_size:agent.update(i_epoch) + agent.writer.add_scalar('liveTime/livestep', t, global_step=i_epoch) + break + +if __name__ == '__main__': + main() + print("end") diff --git a/Char07 PPO/PPO_MountainCar-v0.py b/Char07 PPO/PPO_MountainCar-v0.py new file mode 100644 index 0000000..6509b1b --- /dev/null +++ b/Char07 PPO/PPO_MountainCar-v0.py @@ -0,0 +1,176 @@ +import argparse +import pickle +from collections import namedtuple +from itertools import count + +import os, time +import numpy as np +import matplotlib.pyplot as plt + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal, Categorical +from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler +from tensorboardX import SummaryWriter + +# Parameters +env_name = 'MountainCar-v0' +gamma = 0.99 +render = False +seed = 1 +log_interval = 10 + +env = gym.make(env_name).unwrapped +num_state = env.observation_space.shape[0] +num_action = env.action_space.n +torch.manual_seed(seed) +env.seed(seed) +Transition = namedtuple('Transition', ['state', 'action', 'a_log_prob', 'reward', 'next_state']) + + +class Actor(nn.Module): + def __init__(self): + super(Actor, self).__init__() + self.fc1 = nn.Linear(num_state, 128) + self.action_head = nn.Linear(128, num_action) + + def forward(self, x): + x = F.relu(self.fc1(x)) + action_prob = F.softmax(self.action_head(x), dim=1) + return action_prob + + +class Critic(nn.Module): + def __init__(self): + super(Critic, self).__init__() + self.fc1 = nn.Linear(num_state, 128) + self.state_value = nn.Linear(128, 1) + + def forward(self, x): + x = F.relu(self.fc1(x)) + value = self.state_value(x) + return value + + +class PPO(): + clip_param = 0.2 + max_grad_norm = 0.5 + ppo_update_time = 10 + buffer_capacity = 8000 + batch_size = 32 + + def __init__(self): + super(PPO, self).__init__() + self.actor_net = Actor() + self.critic_net = Critic() + self.buffer = [] + self.counter = 0 + self.training_step = 0 + self.writer = SummaryWriter('../exp') + + self.actor_optimizer = optim.Adam(self.actor_net.parameters(), 1e-3) + self.critic_net_optimizer = optim.Adam(self.critic_net.parameters(), 3e-3) + if not os.path.exists('../param'): + os.makedirs('../param/net_param') + os.makedirs('../param/img') + + def select_action(self, state): + state = torch.from_numpy(state).float().unsqueeze(0) + with torch.no_grad(): + action_prob = self.actor_net(state) + c = Categorical(action_prob) + action = c.sample() + return action.item(), action_prob[:, action.item()].item() + + def get_value(self, state): + state = torch.from_numpy(state) + with torch.no_grad(): + value = self.critic_net(state) + return value.item() + + def save_param(self): + torch.save(self.actor_net.state_dict(), '../param/net_param/actor_net' + str(time.time())[:10], +'.pkl') + torch.save(self.critic_net.state_dict(), '../param/net_param/critic_net' + str(time.time())[:10], +'.pkl') + + def store_transition(self, transition): + self.buffer.append(transition) + self.counter += 1 + + def update(self, i_ep): + state = torch.tensor([t.state for t in self.buffer], dtype=torch.float) + action = torch.tensor([t.action for t in self.buffer], dtype=torch.long).view(-1, 1) + reward = [t.reward for t in self.buffer] + # update: don't need next_state + # reward = torch.tensor([t.reward for t in self.buffer], dtype=torch.float).view(-1, 1) + # next_state = torch.tensor([t.next_state for t in self.buffer], dtype=torch.float) + old_action_log_prob = torch.tensor([t.a_log_prob for t in self.buffer], dtype=torch.float).view(-1, 1) + + R = 0 + Gt = [] + for r in reward[::-1]: + R = r + gamma * R + Gt.insert(0, R) + Gt = torch.tensor(Gt, dtype=torch.float) + # print("The agent is updateing....") + for i in range(self.ppo_update_time): + for index in BatchSampler(SubsetRandomSampler(range(len(self.buffer))), self.batch_size, False): + if self.training_step % 1000 == 0: + print('I_ep {} ，train {} times'.format(i_ep, self.training_step)) + # with torch.no_grad(): + Gt_index = Gt[index].view(-1, 1) + V = self.critic_net(state[index]) + delta = Gt_index - V + advantage = delta.detach() + # epoch iteration, PPO core!!! + action_prob = self.actor_net(state[index]).gather(1, action[index]) # new policy + + ratio = (action_prob / old_action_log_prob[index]) + surr1 = ratio * advantage + surr2 = torch.clamp(ratio, 1 - self.clip_param, 1 + self.clip_param) * advantage + + # update actor network + action_loss = -torch.min(surr1, surr2).mean() # MAX->MIN desent + self.writer.add_scalar('loss/action_loss', action_loss, global_step=self.training_step) + self.actor_optimizer.zero_grad() + action_loss.backward() + nn.utils.clip_grad_norm_(self.actor_net.parameters(), self.max_grad_norm) + self.actor_optimizer.step() + + # update critic network + value_loss = F.mse_loss(Gt_index, V) + self.writer.add_scalar('loss/value_loss', value_loss, global_step=self.training_step) + self.critic_net_optimizer.zero_grad() + value_loss.backward() + nn.utils.clip_grad_norm_(self.critic_net.parameters(), self.max_grad_norm) + self.critic_net_optimizer.step() + self.training_step += 1 + + del self.buffer[:] # clear experience + + +def main(): + agent = PPO() + for i_epoch in range(1000): + state = env.reset() + if render: env.render() + + for t in count(): + action, action_prob = agent.select_action(state) + next_state, reward, done, _ = env.step(action) + trans = Transition(state, action, action_prob, reward, next_state) + if render: env.render() + agent.store_transition(trans) + state = next_state + + if done: + if len(agent.buffer) >= agent.batch_size: agent.update(i_epoch) + agent.writer.add_scalar('Steptime/steptime', t, global_step=i_epoch) + break + + +if __name__ == '__main__': + main() + print("end") diff --git a/Char07 PPO/PPO_pendulum.py b/Char07 PPO/PPO_pendulum.py new file mode 100644 index 0000000..b9c9271 --- /dev/null +++ b/Char07 PPO/PPO_pendulum.py @@ -0,0 +1,195 @@ +import argparse +import pickle +from collections import namedtuple + +import matplotlib.pyplot as plt + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler + +parser = argparse.ArgumentParser(description='Solve the Pendulum-v0 with PPO') +parser.add_argument( + '--gamma', type=float, default=0.9, metavar='G', help='discount factor (default: 0.9)') +parser.add_argument('--seed', type=int, default=0, metavar='N', help='random seed (default: 0)') +parser.add_argument('--render', action='store_true', help='render the environment') +parser.add_argument( + '--log-interval', + type=int, + default=10, + metavar='N', + help='interval between training status logs (default: 10)') +args = parser.parse_args() + +torch.manual_seed(args.seed) + +TrainingRecord = namedtuple('TrainingRecord', ['ep', 'reward']) +Transition = namedtuple('Transition', ['s', 'a', 'a_log_p', 'r', 's_']) + + +class ActorNet(nn.Module): + + def __init__(self): + super(ActorNet, self).__init__() + self.fc = nn.Linear(3, 100) + self.mu_head = nn.Linear(100, 1) + self.sigma_head = nn.Linear(100, 1) + + def forward(self, x): + x = F.relu(self.fc(x)) + mu = 2.0 * F.tanh(self.mu_head(x)) + sigma = F.softplus(self.sigma_head(x)) + return (mu, sigma) + + +class CriticNet(nn.Module): + + def __init__(self): + super(CriticNet, self).__init__() + self.fc = nn.Linear(3, 100) + self.v_head = nn.Linear(100, 1) + + def forward(self, x): + x = F.relu(self.fc(x)) + state_value = self.v_head(x) + return state_value + + +class Agent(): + + clip_param = 0.2 + max_grad_norm = 0.5 + ppo_epoch = 10 + buffer_capacity, batch_size = 1000, 32 + + def __init__(self): + self.training_step = 0 + self.anet = ActorNet().float() + self.cnet = CriticNet().float() + self.buffer = [] + self.counter = 0 + + self.optimizer_a = optim.Adam(self.anet.parameters(), lr=1e-4) + self.optimizer_c = optim.Adam(self.cnet.parameters(), lr=3e-4) + + def select_action(self, state): + state = torch.from_numpy(state).float().unsqueeze(0) + with torch.no_grad(): + (mu, sigma) = self.anet(state) + dist = Normal(mu, sigma) + action = dist.sample() + action_log_prob = dist.log_prob(action) + action = action.clamp(-2.0, 2.0) + return action.item(), action_log_prob.item() + + def get_value(self, state): + + state = torch.from_numpy(state).float().unsqueeze(0) + with torch.no_grad(): + state_value = self.cnet(state) + return state_value.item() + + def save_param(self): + torch.save(self.anet.state_dict(), 'param/ppo_anet_params.pkl') + torch.save(self.cnet.state_dict(), 'param/ppo_cnet_params.pkl') + + def store(self, transition): + self.buffer.append(transition) + self.counter += 1 + return self.counter % self.buffer_capacity == 0 + + def update(self): + self.training_step += 1 + + s = torch.tensor([t.s for t in self.buffer], dtype=torch.float) + a = torch.tensor([t.a for t in self.buffer], dtype=torch.float).view(-1, 1) + r = torch.tensor([t.r for t in self.buffer], dtype=torch.float).view(-1, 1) + s_ = torch.tensor([t.s_ for t in self.buffer], dtype=torch.float) + + old_action_log_probs = torch.tensor( + [t.a_log_p for t in self.buffer], dtype=torch.float).view(-1, 1) + + r = (r - r.mean()) / (r.std() + 1e-5) + with torch.no_grad(): + target_v = r + args.gamma * self.cnet(s_) + + adv = (target_v - self.cnet(s)).detach() + + for _ in range(self.ppo_epoch): + for index in BatchSampler( + SubsetRandomSampler(range(self.buffer_capacity)), self.batch_size, False): + + (mu, sigma) = self.anet(s[index]) + dist = Normal(mu, sigma) + action_log_probs = dist.log_prob(a[index]) + ratio = torch.exp(action_log_probs - old_action_log_probs[index]) + + surr1 = ratio * adv[index] + surr2 = torch.clamp(ratio, 1.0 - self.clip_param, + 1.0 + self.clip_param) * adv[index] + action_loss = -torch.min(surr1, surr2).mean() + + self.optimizer_a.zero_grad() + action_loss.backward() + nn.utils.clip_grad_norm_(self.anet.parameters(), self.max_grad_norm) + self.optimizer_a.step() + + value_loss = F.smooth_l1_loss(self.cnet(s[index]), target_v[index]) + self.optimizer_c.zero_grad() + value_loss.backward() + nn.utils.clip_grad_norm_(self.cnet.parameters(), self.max_grad_norm) + self.optimizer_c.step() + + del self.buffer[:] + + +def main(): + env = gym.make('Pendulum-v0') + env.seed(args.seed) + + agent = Agent() + + training_records = [] + running_reward = -1000 + state = env.reset() + for i_ep in range(1000): + score = 0 + state = env.reset() + + for t in range(200): + action, action_log_prob = agent.select_action(state) + state_, reward, done, _ = env.step([action]) + if args.render: + env.render() + if agent.store(Transition(state, action, action_log_prob, (reward + 8) / 8, state_)): + agent.update() + score += reward + state = state_ + + running_reward = running_reward * 0.9 + score * 0.1 + training_records.append(TrainingRecord(i_ep, running_reward)) + + if i_ep % args.log_interval == 0: + print('Ep {}\tMoving average score: {:.2f}\t'.format(i_ep, running_reward)) + if running_reward > -200: + print("Solved! Moving average score is now {}!".format(running_reward)) + env.close() + agent.save_param() + with open('log/ppo_training_records.pkl', 'wb') as f: + pickle.dump(training_records, f) + break + + plt.plot([r.ep for r in training_records], [r.reward for r in training_records]) + plt.title('PPO') + plt.xlabel('Episode') + plt.ylabel('Moving averaged episode reward') + plt.savefig("img/ppo.png") + plt.show() + + +if __name__ == '__main__': + main() diff --git a/Char07 PPO/readme.md b/Char07 PPO/readme.md new file mode 100644 index 0000000..19bbdc5 --- /dev/null +++ b/Char07 PPO/readme.md @@ -0,0 +1,6 @@ +# PPO + +- Original paper: https://arxiv.org/abs/1707.06347 +- Openai Baselines blog post: https://blog.openai.com/openai-baselines-ppo/ + +Notice: This is not the author's implementation! diff --git a/Char08 ACER/readme.md b/Char08 ACER/readme.md new file mode 100644 index 0000000..13cdc46 --- /dev/null +++ b/Char08 ACER/readme.md @@ -0,0 +1,3 @@ +#ACER + +actor-critic with experience replay diff --git a/Char09 SAC/SAC.py b/Char09 SAC/SAC.py new file mode 100644 index 0000000..b04419d --- /dev/null +++ b/Char09 SAC/SAC.py @@ -0,0 +1,307 @@ +import argparse +import pickle +from collections import namedtuple +from itertools import count + +import os +import numpy as np + + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from torch.autograd import grad +from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler +from tensorboardX import SummaryWriter + + +''' +Implementation of soft actor critic +Original paper: https://arxiv.org/abs/1801.01290 +Not the author's implementation ! +''' + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +parser = argparse.ArgumentParser() + + +parser.add_argument("--env_name", default="Pendulum-v0") # OpenAI gym environment name +parser.add_argument('--tau', default=0.005, type=float) # target smoothing coefficient +parser.add_argument('--target_update_interval', default=1, type=int) +parser.add_argument('--gradient_steps', default=1, type=int) + + +parser.add_argument('--learning_rate', default=3e-4, type=int) +parser.add_argument('--gamma', default=0.99, type=int) # discount gamma +parser.add_argument('--capacity', default=10000, type=int) # replay buffer size +parser.add_argument('--iteration', default=100000, type=int) # num of games +parser.add_argument('--batch_size', default=128, type=int) # mini batch size +parser.add_argument('--seed', default=1, type=int) + +# optional parameters +parser.add_argument('--num_hidden_layers', default=2, type=int) +parser.add_argument('--num_hidden_units_per_layer', default=256, type=int) +parser.add_argument('--sample_frequency', default=256, type=int) +parser.add_argument('--activation', default='Relu', type=str) +parser.add_argument('--render', default=False, type=bool) # show UI or not +parser.add_argument('--log_interval', default=2000, type=int) # +parser.add_argument('--load', default=False, type=bool) # load model +args = parser.parse_args() + +class NormalizedActions(gym.ActionWrapper): + def _action(self, action): + low = self.action_space.low + high = self.action_space.high + + action = low + (action + 1.0) * 0.5 * (high - low) + action = np.clip(action, low, high) + + return action + + def _reverse_action(self, action): + low = self.action_space.low + high = self.action_space.high + + action = 2 * (action - low) / (high - low) - 1 + action = np.clip(action, low, high) + + return action + + +env = NormalizedActions(gym.make(args.env_name)) + +# Set seeds +env.seed(args.seed) +torch.manual_seed(args.seed) +np.random.seed(args.seed) + +state_dim = env.observation_space.shape[0] +action_dim = env.action_space.shape[0] +max_action = float(env.action_space.high[0]) +min_Val = torch.tensor(1e-7).float() +Transition = namedtuple('Transition', ['s', 'a', 'r', 's_', 'd']) + +class Actor(nn.Module): + def __init__(self, state_dim, min_log_std=-20, max_log_std=2): + super(Actor, self).__init__() + self.fc1 = nn.Linear(state_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.mu_head = nn.Linear(256, 1) + self.log_std_head = nn.Linear(256, 1) + self.max_action = max_action + + self.min_log_std = min_log_std + self.max_log_std = max_log_std + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + mu = self.mu_head(x) + log_std_head = F.relu(self.log_std_head(x)) + log_std_head = torch.clamp(log_std_head, self.min_log_std, self.max_log_std) + return mu, log_std_head + + +class Critic(nn.Module): + def __init__(self, state_dim): + super(Critic, self).__init__() + self.fc1 = nn.Linear(state_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.fc3 = nn.Linear(256, 1) + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class Q(nn.Module): + def __init__(self, state_dim, action_dim): + super(Q, self).__init__() + self.fc1 = nn.Linear(state_dim + action_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.fc3 = nn.Linear(256, 1) + + def forward(self, s, a): + s = s.reshape(-1, state_dim) + a = a.reshape(-1, action_dim) + x = torch.cat((s, a), -1) # combination s and a + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class SAC(): + def __init__(self): + super(SAC, self).__init__() + + self.policy_net = Actor(state_dim).to(device) + self.value_net = Critic(state_dim).to(device) + self.Q_net = Q(state_dim, action_dim).to(device) + self.Target_value_net = Critic(state_dim).to(device) + + self.replay_buffer = [Transition] * args.capacity + self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) + self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) + self.Q_optimizer = optim.Adam(self.Q_net.parameters(), lr=args.learning_rate) + self.num_transition = 0 # pointer of replay buffer + self.num_training = 1 + self.writer = SummaryWriter('./exp-SAC') + + self.value_criterion = nn.MSELoss() + self.Q_criterion = nn.MSELoss() + + for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): + target_param.data.copy_(param.data) + + os.makedirs('./SAC_model/', exist_ok=True) + + def select_action(self, state): + state = torch.FloatTensor(state).to(device) + mu, log_sigma = self.policy_net(state) + sigma = torch.exp(log_sigma) + dist = Normal(mu, sigma) + z = dist.sample() + action = torch.tanh(z).detach().cpu().numpy() + return action.item() # return a scalar, float32 + + def store(self, s, a, r, s_, d): + index = self.num_transition % args.capacity + transition = Transition(s, a, r, s_, d) + self.replay_buffer[index] = transition + self.num_transition += 1 + + def get_action_log_prob(self, state): + + batch_mu, batch_log_sigma = self.policy_net(state) + batch_sigma = torch.exp(batch_log_sigma) + dist = Normal(batch_mu, batch_sigma) + z = dist.sample() + action = torch.tanh(z) + log_prob = dist.log_prob(z) - torch.log(1 - action.pow(2) + min_Val) + return action, log_prob, z, batch_mu, batch_log_sigma + + + def update(self): + if self.num_training % 500 == 0: + print("Training ... {} ".format(self.num_training)) + s = torch.tensor([t.s for t in self.replay_buffer]).float().to(device) + a = torch.tensor([t.a for t in self.replay_buffer]).to(device) + r = torch.tensor([t.r for t in self.replay_buffer]).to(device) + s_ = torch.tensor([t.s_ for t in self.replay_buffer]).float().to(device) + d = torch.tensor([t.d for t in self.replay_buffer]).float().to(device) + + for _ in range(args.gradient_steps): + #for index in BatchSampler(SubsetRandomSampler(range(args.capacity)), args.batch_size, False): + index = np.random.choice(range(args.capacity), args.batch_size, replace=False) + bn_s = s[index] + bn_a = a[index].reshape(-1, 1) + bn_r = r[index].reshape(-1, 1) + bn_s_ = s_[index] + bn_d = d[index].reshape(-1, 1) + + + target_value = self.Target_value_net(bn_s_) + next_q_value = bn_r + (1 - bn_d) * args.gamma * target_value + + excepted_value = self.value_net(bn_s) + excepted_Q = self.Q_net(bn_s, bn_a) + + sample_action, log_prob, z, batch_mu, batch_log_sigma = self.get_action_log_prob(bn_s) + excepted_new_Q = self.Q_net(bn_s, sample_action) + next_value = excepted_new_Q - log_prob + + # !!!Note that the actions are sampled according to the current policy, + # instead of replay buffer. (From original paper) + + V_loss = self.value_criterion(excepted_value, next_value.detach()) # J_V + V_loss = V_loss.mean() + + # Single Q_net this is different from original paper!!! + Q_loss = self.Q_criterion(excepted_Q, next_q_value.detach()) # J_Q + Q_loss = Q_loss.mean() + + log_policy_target = excepted_new_Q - excepted_value + + pi_loss = log_prob * (log_prob- log_policy_target).detach() + pi_loss = pi_loss.mean() + + self.writer.add_scalar('Loss/V_loss', V_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/Q_loss', Q_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/pi_loss', pi_loss, global_step=self.num_training) + # mini batch gradient descent + self.value_optimizer.zero_grad() + V_loss.backward(retain_graph=True) + nn.utils.clip_grad_norm_(self.value_net.parameters(), 0.5) + self.value_optimizer.step() + + self.Q_optimizer.zero_grad() + Q_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.Q_net.parameters(), 0.5) + self.Q_optimizer.step() + + self.policy_optimizer.zero_grad() + pi_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.policy_net.parameters(), 0.5) + self.policy_optimizer.step() + + # soft update + for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): + target_param.data.copy_(target_param * (1 - args.tau) + param * args.tau) + + self.num_training += 1 + + def save(self): + torch.save(self.policy_net.state_dict(), './SAC_model/policy_net.pth') + torch.save(self.value_net.state_dict(), './SAC_model/value_net.pth') + torch.save(self.Q_net.state_dict(), './SAC_model/Q_net.pth') + print("====================================") + print("Model has been saved...") + print("====================================") + + def load(self): + torch.load(self.policy_net.state_dict(), './SAC_model/policy_net.pth') + torch.load(self.value_net.state_dict(), './SAC_model/value_net.pth') + torch.load(self.Q_net.state_dict(), './SAC_model/Q_net.pth') + print() + +def main(): + + agent = SAC() + if args.load: agent.load() + if args.render: env.render() + print("====================================") + print("Collection Experience...") + print("====================================") + + ep_r = 0 + for i in range(args.iteration): + state = env.reset() + for t in range(200): + action = agent.select_action(state) + next_state, reward, done, info = env.step(np.float32(action)) + ep_r += reward + if args.render: env.render() + agent.store(state, action, reward, next_state, done) + + if agent.num_transition >= args.capacity: + agent.update() + + state = next_state + if done or t == 199: + if i % 10 == 0: + print("Ep_i {}, the ep_r is {}, the t is {}".format(i, ep_r, t)) + break + if i % args.log_interval == 0: + agent.save() + agent.writer.add_scalar('ep_r', ep_r, global_step=i) + ep_r = 0 + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/Char09 SAC/SAC_BipedalWalker-v2.py b/Char09 SAC/SAC_BipedalWalker-v2.py new file mode 100644 index 0000000..c7fe83a --- /dev/null +++ b/Char09 SAC/SAC_BipedalWalker-v2.py @@ -0,0 +1,321 @@ +import argparse +from collections import namedtuple +from itertools import count +import pickle + +import os, random +import numpy as np + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from tensorboardX import SummaryWriter + + +''' +Implementation of soft actor critic, dual Q network version +Original paper: https://arxiv.org/abs/1801.01290 +Not the author's implementation ! +''' + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +parser = argparse.ArgumentParser() + +parser.add_argument("--env_name", default="BipedalWalker-v2") # OpenAI gym environment name +parser.add_argument('--tau', default=0.005, type=float) # target smoothing coefficient +parser.add_argument('--target_update_interval', default=1, type=int) +parser.add_argument('--gradient_steps', default=1, type=int) + +parser.add_argument('--learning_rate', default=3e-4, type=float) +parser.add_argument('--gamma', default=0.99, type=int) # discount gamma +parser.add_argument('--capacity', default=1000000, type=int) # replay buffer size +parser.add_argument('--iteration', default=100000, type=int) # num of games +parser.add_argument('--batch_size', default=128, type=int) # mini batch size +parser.add_argument('--seed', default=1, type=int) + +# optional parameters +parser.add_argument('--num_hidden_layers', default=2, type=int) +parser.add_argument('--num_hidden_units_per_layer', default=256, type=int) +parser.add_argument('--sample_frequency', default=256, type=int) +parser.add_argument('--activation', default='Relu', type=str) +parser.add_argument('--render', default=False, type=bool) # show UI or not +parser.add_argument('--log_interval', default=50, type=int) # +parser.add_argument('--load', default=False, type=bool) # load model +parser.add_argument('--render_interval', default=100, type=int) # after render_interval, the env.render() will work +args = parser.parse_args() + +class NormalizedActions(gym.ActionWrapper): + def _action(self, action): + low = self.action_space.low + high = self.action_space.high + + action = low + (action + 1.0) * 0.5 * (high - low) + action = np.clip(action, low, high) + + return action + + def _reverse_action(self, action): + low = self.action_space.low + high = self.action_space.high + + action = 2 * (action - low) / (high - low) - 1 + action = np.clip(action, low, high) + + return action + + +env = NormalizedActions(gym.make(args.env_name)) + +# Set seeds +env.seed(args.seed) +torch.manual_seed(args.seed) +np.random.seed(args.seed) + +state_dim = env.observation_space.shape[0] +action_dim = env.action_space.shape[0] +max_action = float(env.action_space.high[0]) +min_Val = torch.tensor(1e-7).float().to(device) + +class Replay_buffer(): + def __init__(self, capacity): + self.capacity = capacity + self.state_pool = torch.zeros(self.capacity, state_dim).float().to(device) + self.action_pool = torch.zeros(self.capacity, action_dim).float().to(device) + self.reward_pool = torch.zeros(self.capacity, 1).float().to(device) + self.next_state_pool = torch.zeros(self.capacity, state_dim).float().to(device) + self.done_pool = torch.zeros(self.capacity, 1).float().to(device) + self.num_transition = 0 + + def push(self, s, a, r, s_, d): + index = self.num_transition % self.capacity + s = torch.tensor(s).float().to(device) + a = torch.tensor(a).float().to(device) + r = torch.tensor(r).float().to(device) + s_ = torch.tensor(s_).float().to(device) + d = torch.tensor(d).float().to(device) + for pool, ele in zip([self.state_pool, self.action_pool, self.reward_pool, self.next_state_pool, self.done_pool], + [s, a, r, s_, d]): + pool[index] = ele + self.num_transition += 1 + + def sample(self, batch_size): + index = np.random.choice(range(self.capacity), batch_size, replace=False) + bn_s, bn_a, bn_r, bn_s_, bn_d = self.state_pool[index], self.action_pool[index], self.reward_pool[index],\ + self.next_state_pool[index], self.done_pool[index] + + return bn_s, bn_a, bn_r, bn_s_, bn_d + +class Actor(nn.Module): + def __init__(self, state_dim, action_dim=action_dim, min_log_std=-10, max_log_std=2): + super(Actor, self).__init__() + self.fc1 = nn.Linear(state_dim, 256) + self.fc2 = nn.Linear(256, 512) + self.mu_head = nn.Linear(512, action_dim) + self.log_std_head = nn.Linear(512, action_dim) + self.max_action = max_action + + self.min_log_std = min_log_std + self.max_log_std = max_log_std + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + mu = self.mu_head(x) + log_std_head = F.relu(self.log_std_head(x)) + log_std_head = torch.clamp(log_std_head, self.min_log_std, self.max_log_std) + return mu, log_std_head + + +class Critic(nn.Module): + def __init__(self, state_dim): + super(Critic, self).__init__() + self.fc1 = nn.Linear(state_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.fc3 = nn.Linear(256, 1) + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class Q(nn.Module): + def __init__(self, state_dim, action_dim): + super(Q, self).__init__() + self.fc1 = nn.Linear(state_dim + action_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.fc3 = nn.Linear(256, 1) + + def forward(self, s, a): + s = s.reshape(-1, state_dim) + a = a.reshape(-1, action_dim) + x = torch.cat((s, a), -1) # combination s and a + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class SAC(): + def __init__(self): + super(SAC, self).__init__() + + self.policy_net = Actor(state_dim).to(device) + self.value_net = Critic(state_dim).to(device) + self.Target_value_net = Critic(state_dim).to(device) + self.Q_net1 = Q(state_dim, action_dim).to(device) + self.Q_net2 = Q(state_dim, action_dim).to(device) + + self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) + self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) + self.Q1_optimizer = optim.Adam(self.Q_net1.parameters(), lr=args.learning_rate) + self.Q2_optimizer = optim.Adam(self.Q_net2.parameters(), lr=args.learning_rate) + + self.replay_buffer = Replay_buffer(args.capacity) + self.num_transition = 0 # pointer of replay buffer + self.num_training = 0 + self.writer = SummaryWriter('./exp-SAC_dual_Q_network') + + self.value_criterion = nn.MSELoss() + self.Q1_criterion = nn.MSELoss() + self.Q2_criterion = nn.MSELoss() + + for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): + target_param.data.copy_(param.data) + + os.makedirs('./SAC_model/', exist_ok=True) + + def select_action(self, state): + state = torch.FloatTensor(state).to(device) + mu, log_sigma = self.policy_net(state) + sigma = torch.exp(log_sigma) + dist = Normal(mu, sigma) + z = dist.sample() + action = torch.tanh(z).detach().cpu().numpy() + return action # return a scalar, float32 + + def evaluate(self, state): + batch_mu, batch_log_sigma = self.policy_net(state) + batch_sigma = torch.exp(batch_log_sigma) + dist = Normal(batch_mu, batch_sigma) + noise = Normal(0, 1) + + z = noise.sample() + action = torch.tanh(batch_mu + batch_sigma*z.to(device)) + log_prob = dist.log_prob(batch_mu + batch_sigma * z.to(device)) - torch.log(1 - action.pow(2) + min_Val) + return action, log_prob, z, batch_mu, batch_log_sigma + + def update(self): + if self.num_training % 500 == 0: + print("Training ... \t{} times ".format(self.num_training)) + + for _ in range(args.gradient_steps): + bn_s, bn_a, bn_r, bn_s_, bn_d = self.replay_buffer.sample(args.batch_size) + + target_value = self.Target_value_net(bn_s_) + next_q_value = bn_r + (1 - bn_d) * args.gamma * target_value + + excepted_value = self.value_net(bn_s) + excepted_Q1 = self.Q_net1(bn_s, bn_a) + excepted_Q2 = self.Q_net2(bn_s, bn_a) + sample_action, log_prob, z, batch_mu, batch_log_sigma = self.evaluate(bn_s) + excepted_new_Q = torch.min(self.Q_net1(bn_s, sample_action), self.Q_net2(bn_s, sample_action)) + next_value = excepted_new_Q - log_prob + + # !!!Note that the actions are sampled according to the current policy, + # instead of replay buffer. (From original paper) + V_loss = self.value_criterion(excepted_value, next_value.detach()).mean() # J_V + + # Dual Q net + Q1_loss = self.Q1_criterion(excepted_Q1, next_q_value.detach()).mean() # J_Q + Q2_loss = self.Q2_criterion(excepted_Q2, next_q_value.detach()).mean() + + pi_loss = (log_prob - excepted_new_Q).mean() # according to original paper + + self.writer.add_scalar('Loss/V_loss', V_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/Q1_loss', Q1_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/Q2_loss', Q2_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/policy_loss', pi_loss, global_step=self.num_training) + + # mini batch gradient descent + self.value_optimizer.zero_grad() + V_loss.backward(retain_graph=True) + nn.utils.clip_grad_norm_(self.value_net.parameters(), 0.5) + self.value_optimizer.step() + + self.Q1_optimizer.zero_grad() + Q1_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.Q_net1.parameters(), 0.5) + self.Q1_optimizer.step() + + self.Q2_optimizer.zero_grad() + Q2_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.Q_net2.parameters(), 0.5) + self.Q2_optimizer.step() + + self.policy_optimizer.zero_grad() + pi_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.policy_net.parameters(), 0.5) + self.policy_optimizer.step() + + # update target v net update + for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): + target_param.data.copy_(target_param * (1 - args.tau) + param * args.tau) + + self.num_training += 1 + + def save(self): + torch.save(self.policy_net.state_dict(), './SAC_model/policy_net.pth') + torch.save(self.value_net.state_dict(), './SAC_model/value_net.pth') + torch.save(self.Q_net1.state_dict(), './SAC_model/Q_net1.pth') + torch.save(self.Q_net2.state_dict(), './SAC_model/Q_net2.pth') + print("====================================") + print("Model has been saved...") + print("====================================") + + def load(self): + self.policy_net.load_state_dict(torch.load('./SAC_model/policy_net.pth')) + self.value_net.load_state_dict(torch.load( './SAC_model/value_net.pth')) + self.Q_net1.load_state_dict(torch.load('./SAC_model/Q_net1.pth')) + self.Q_net2.load_state_dict(torch.load('./SAC_model/Q_net2.pth')) + print("====================================") + print("model has been loaded...") + print("====================================") + + +def main(): + agent = SAC() + if args.load: agent.load() + print("====================================") + print("Collection Experience...") + print("====================================") + + ep_r = 0 + for i in range(args.iteration): + state = env.reset() + for t in range(200): + action = agent.select_action(state) + next_state, reward, done, info = env.step(np.float32(action)) + ep_r += reward + if args.render and i >= args.render_interval : env.render() + agent.replay_buffer.push(state, action, reward, next_state, done) + + state = next_state + if done: + if agent.replay_buffer.num_transition >= args.capacity: + agent.update() + if i > 100: + print("Ep_i \t{}, the ep_r is \t{}, the step is \t{}".format(i, ep_r, t)) + break + if i % args.log_interval == 0: + agent.save() + agent.writer.add_scalar('ep_r', ep_r, global_step=i) + ep_r = 0 + + +if __name__ == '__main__': + main() diff --git a/Char09 SAC/SAC_dual_Q_net.py b/Char09 SAC/SAC_dual_Q_net.py new file mode 100644 index 0000000..2a4134a --- /dev/null +++ b/Char09 SAC/SAC_dual_Q_net.py @@ -0,0 +1,312 @@ +import argparse +from collections import namedtuple +from itertools import count + +import os +import numpy as np + + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from tensorboardX import SummaryWriter + + +''' +Implementation of soft actor critic, dual Q network version +Original paper: https://arxiv.org/abs/1801.01290 +Not the author's implementation ! +''' + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +parser = argparse.ArgumentParser() + + +parser.add_argument("--env_name", default="Pendulum-v0") # OpenAI gym environment name +parser.add_argument('--tau', default=0.005, type=float) # target smoothing coefficient +parser.add_argument('--target_update_interval', default=1, type=int) +parser.add_argument('--gradient_steps', default=1, type=int) + + +parser.add_argument('--learning_rate', default=3e-4, type=int) +parser.add_argument('--gamma', default=0.99, type=int) # discount gamma +parser.add_argument('--capacity', default=10000, type=int) # replay buffer size +parser.add_argument('--iteration', default=100000, type=int) # num of games +parser.add_argument('--batch_size', default=128, type=int) # mini batch size +parser.add_argument('--seed', default=1, type=int) + +# optional parameters +parser.add_argument('--num_hidden_layers', default=2, type=int) +parser.add_argument('--num_hidden_units_per_layer', default=256, type=int) +parser.add_argument('--sample_frequency', default=256, type=int) +parser.add_argument('--activation', default='Relu', type=str) +parser.add_argument('--render', default=False, type=bool) # show UI or not +parser.add_argument('--log_interval', default=2000, type=int) # +parser.add_argument('--load', default=False, type=bool) # load model +args = parser.parse_args() + +class NormalizedActions(gym.ActionWrapper): + def _action(self, action): + low = self.action_space.low + high = self.action_space.high + + action = low + (action + 1.0) * 0.5 * (high - low) + action = np.clip(action, low, high) + + return action + + def _reverse_action(self, action): + low = self.action_space.low + high = self.action_space.high + + action = 2 * (action - low) / (high - low) - 1 + action = np.clip(action, low, high) + + return action + + +env = NormalizedActions(gym.make(args.env_name)) + +# Set seeds +env.seed(args.seed) +torch.manual_seed(args.seed) +np.random.seed(args.seed) + +state_dim = env.observation_space.shape[0] +action_dim = env.action_space.shape[0] +max_action = float(env.action_space.high[0]) +min_Val = torch.tensor(1e-7).float().to(device) +Transition = namedtuple('Transition', ['s', 'a', 'r', 's_', 'd']) + +class Actor(nn.Module): + def __init__(self, state_dim, min_log_std=-10, max_log_std=2): + super(Actor, self).__init__() + self.fc1 = nn.Linear(state_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.mu_head = nn.Linear(256, 1) + self.log_std_head = nn.Linear(256, 1) + self.max_action = max_action + + self.min_log_std = min_log_std + self.max_log_std = max_log_std + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + mu = self.mu_head(x) + log_std_head = F.relu(self.log_std_head(x)) + log_std_head = torch.clamp(log_std_head, self.min_log_std, self.max_log_std) + return mu, log_std_head + + +class Critic(nn.Module): + def __init__(self, state_dim): + super(Critic, self).__init__() + self.fc1 = nn.Linear(state_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.fc3 = nn.Linear(256, 1) + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class Q(nn.Module): + def __init__(self, state_dim, action_dim): + super(Q, self).__init__() + self.fc1 = nn.Linear(state_dim + action_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.fc3 = nn.Linear(256, 1) + + def forward(self, s, a): + s = s.reshape(-1, state_dim) + a = a.reshape(-1, action_dim) + x = torch.cat((s, a), -1) # combination s and a + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class SAC(): + def __init__(self): + super(SAC, self).__init__() + + self.policy_net = Actor(state_dim).to(device) + self.value_net = Critic(state_dim).to(device) + self.Target_value_net = Critic(state_dim).to(device) + self.Q_net1 = Q(state_dim, action_dim).to(device) + self.Q_net2 = Q(state_dim, action_dim).to(device) + + self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) + self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) + self.Q1_optimizer = optim.Adam(self.Q_net1.parameters(), lr=args.learning_rate) + self.Q2_optimizer = optim.Adam(self.Q_net2.parameters(), lr=args.learning_rate) + + self.replay_buffer = [Transition] * args.capacity + self.num_transition = 0 # pointer of replay buffer + self.num_training = 1 + self.writer = SummaryWriter('./exp-SAC_dual_Q_network') + + self.value_criterion = nn.MSELoss() + self.Q1_criterion = nn.MSELoss() + self.Q2_criterion = nn.MSELoss() + + for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): + target_param.data.copy_(param.data) + + os.makedirs('./SAC_model/', exist_ok=True) + + def select_action(self, state): + state = torch.FloatTensor(state).to(device) + mu, log_sigma = self.policy_net(state) + sigma = torch.exp(log_sigma) + dist = Normal(mu, sigma) + z = dist.sample() + action = torch.tanh(z).detach().cpu().numpy() + return action.item() # return a scalar, float32 + + def store(self, s, a, r, s_, d): + index = self.num_transition % args.capacity + transition = Transition(s, a, r, s_, d) + self.replay_buffer[index] = transition + self.num_transition += 1 + + def evaluate(self, state): + batch_mu, batch_log_sigma = self.policy_net(state) + batch_sigma = torch.exp(batch_log_sigma) + dist = Normal(batch_mu, batch_sigma) + noise = Normal(0, 1) + + z = noise.sample() + action = torch.tanh(batch_mu + batch_sigma*z.to(device)) + log_prob = dist.log_prob(batch_mu + batch_sigma * z.to(device)) - torch.log(1 - action.pow(2) + min_Val) + return action, log_prob, z, batch_mu, batch_log_sigma + + def update(self): + if self.num_training % 500 == 0: + print("Training ... {} times ".format(self.num_training)) + s = torch.tensor([t.s for t in self.replay_buffer]).float().to(device) + a = torch.tensor([t.a for t in self.replay_buffer]).to(device) + r = torch.tensor([t.r for t in self.replay_buffer]).to(device) + s_ = torch.tensor([t.s_ for t in self.replay_buffer]).float().to(device) + d = torch.tensor([t.d for t in self.replay_buffer]).float().to(device) + + for _ in range(args.gradient_steps): + #for index in BatchSampler(SubsetRandomSampler(range(args.capacity)), args.batch_size, False): + index = np.random.choice(range(args.capacity), args.batch_size, replace=False) + bn_s = s[index] + bn_a = a[index].reshape(-1, 1) + bn_r = r[index].reshape(-1, 1) + bn_s_ = s_[index] + bn_d = d[index].reshape(-1, 1) + + target_value = self.Target_value_net(bn_s_) + next_q_value = bn_r + (1 - bn_d) * args.gamma * target_value + + excepted_value = self.value_net(bn_s) + excepted_Q1 = self.Q_net1(bn_s, bn_a) + excepted_Q2 = self.Q_net2(bn_s, bn_a) + sample_action, log_prob, z, batch_mu, batch_log_sigma = self.evaluate(bn_s) + excepted_new_Q = torch.min(self.Q_net1(bn_s, sample_action), self.Q_net2(bn_s, sample_action)) + next_value = excepted_new_Q - log_prob + + # !!!Note that the actions are sampled according to the current policy, + # instead of replay buffer. (From original paper) + V_loss = self.value_criterion(excepted_value, next_value.detach()).mean() # J_V + + # Dual Q net + Q1_loss = self.Q1_criterion(excepted_Q1, next_q_value.detach()).mean() # J_Q + Q2_loss = self.Q2_criterion(excepted_Q2, next_q_value.detach()).mean() + + pi_loss = (log_prob - excepted_new_Q).mean() # according to original paper + + self.writer.add_scalar('Loss/V_loss', V_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/Q1_loss', Q1_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/Q2_loss', Q2_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/policy_loss', pi_loss, global_step=self.num_training) + + # mini batch gradient descent + self.value_optimizer.zero_grad() + V_loss.backward(retain_graph=True) + nn.utils.clip_grad_norm_(self.value_net.parameters(), 0.5) + self.value_optimizer.step() + + self.Q1_optimizer.zero_grad() + Q1_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.Q_net1.parameters(), 0.5) + self.Q1_optimizer.step() + + self.Q2_optimizer.zero_grad() + Q2_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.Q_net2.parameters(), 0.5) + self.Q2_optimizer.step() + + self.policy_optimizer.zero_grad() + pi_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.policy_net.parameters(), 0.5) + self.policy_optimizer.step() + + # update target v net update + for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): + target_param.data.copy_(target_param * (1 - args.tau) + param * args.tau) + + self.num_training += 1 + + def save(self): + torch.save(self.policy_net.state_dict(), './SAC_model/policy_net.pth') + torch.save(self.value_net.state_dict(), './SAC_model/value_net.pth') + torch.save(self.Q_net1.state_dict(), './SAC_model/Q_net1.pth') + torch.save(self.Q_net2.state_dict(), './SAC_model/Q_net2.pth') + print("====================================") + print("Model has been saved...") + print("====================================") + + def load(self): + self.policy_net.load_state_dict(torch.load('./SAC_model/policy_net.pth')) + self.value_net.load_state_dict(torch.load( './SAC_model/value_net.pth')) + self.Q_net1.load_state_dict(torch.load('./SAC_model/Q_net1.pth')) + self.Q_net2.load_state_dict(torch.load('./SAC_model/Q_net2.pth')) + print("model has been load") + + +def main(): + + agent = SAC() + if args.load: agent.load() + if args.render: env.render() + print("====================================") + print("Collection Experience...") + print("====================================") + + ep_r = 0 + for i in range(args.iteration): + state = env.reset() + for t in range(200): + action = agent.select_action(state) + next_state, reward, done, info = env.step(np.float32(action)) + ep_r += reward + if args.render: env.render() + agent.store(state, action, reward, next_state, done) + + if agent.num_transition >= args.capacity: + agent.update() + + state = next_state + if done or t == 199: + if i % 10 == 0: + print("Ep_i {}, the ep_r is {}, the t is {}".format(i, ep_r, t)) + break + if i % args.log_interval == 0: + agent.save() + agent.writer.add_scalar('ep_r', ep_r, global_step=i) + ep_r = 0 + + +if __name__ == '__main__': + main() diff --git a/Char09 SAC/SAC_ep_r_curve.png b/Char09 SAC/SAC_ep_r_curve.png new file mode 100644 index 0000000..81cfa8c Binary files /dev/null and b/Char09 SAC/SAC_ep_r_curve.png differ diff --git a/Char09 SAC/test_agent.py b/Char09 SAC/test_agent.py new file mode 100644 index 0000000..638fc67 --- /dev/null +++ b/Char09 SAC/test_agent.py @@ -0,0 +1,324 @@ +import argparse +from collections import namedtuple +from itertools import count +import pickle +import os +import numpy as np + + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from tensorboardX import SummaryWriter + + +''' +Implementation of soft actor critic, dual Q network version +Original paper: https://arxiv.org/abs/1801.01290 +Not the author's implementation ! +''' + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +parser = argparse.ArgumentParser() + + +parser.add_argument("--env_name", default="BipedalWalker-v2") # OpenAI gym environment name +parser.add_argument('--tau', default=0.005, type=float) # target smoothing coefficient +parser.add_argument('--target_update_interval', default=1, type=int) +parser.add_argument('--gradient_steps', default=1, type=int) +parser.add_argument('--mode', default='test', type=str) # test or train + +parser.add_argument('--learning_rate', default=3e-4, type=int) +parser.add_argument('--gamma', default=0.99, type=int) # discount gamma +parser.add_argument('--capacity', default=10000, type=int) # replay buffer size +parser.add_argument('--iteration', default=100000, type=int) # num of games +parser.add_argument('--batch_size', default=128, type=int) # mini batch size +parser.add_argument('--seed', default=1, type=int) + +# optional parameters +parser.add_argument('--num_hidden_layers', default=2, type=int) +parser.add_argument('--num_hidden_units_per_layer', default=256, type=int) +parser.add_argument('--sample_frequency', default=256, type=int) +parser.add_argument('--activation', default='Relu', type=str) +parser.add_argument('--render', default=False, type=bool) # show UI or not +parser.add_argument('--log_interval', default=50, type=int) # +parser.add_argument('--load', default=False, type=bool) # load model +parser.add_argument('--render_interval', default=100, type=int) # after render_interval, the env.render() will work +args = parser.parse_args() + +class NormalizedActions(gym.ActionWrapper): + def _action(self, action): + low = self.action_space.low + high = self.action_space.high + + action = low + (action + 1.0) * 0.5 * (high - low) + action = np.clip(action, low, high) + + return action + + def _reverse_action(self, action): + low = self.action_space.low + high = self.action_space.high + + action = 2 * (action - low) / (high - low) - 1 + action = np.clip(action, low, high) + + return action + + +env = NormalizedActions(gym.make(args.env_name)) + +# Set seeds +env.seed(args.seed) +torch.manual_seed(args.seed) +np.random.seed(args.seed) + +state_dim = env.observation_space.shape[0] +action_dim = env.action_space.shape[0] +max_action = float(env.action_space.high[0]) +min_Val = torch.tensor(1e-7).float().to(device) +Transition = namedtuple('Transition', ['s', 'a', 'r', 's_', 'd']) + +class Actor(nn.Module): + def __init__(self, state_dim, action_dim=action_dim, min_log_std=-10, max_log_std=2): + super(Actor, self).__init__() + self.fc1 = nn.Linear(state_dim, 256) + self.fc2 = nn.Linear(256, 512) + self.mu_head = nn.Linear(512, action_dim) + self.log_std_head = nn.Linear(512, action_dim) + self.max_action = max_action + + self.min_log_std = min_log_std + self.max_log_std = max_log_std + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + mu = self.mu_head(x) + log_std_head = F.relu(self.log_std_head(x)) + log_std_head = torch.clamp(log_std_head, self.min_log_std, self.max_log_std) + return mu, log_std_head + + +class Critic(nn.Module): + def __init__(self, state_dim): + super(Critic, self).__init__() + self.fc1 = nn.Linear(state_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.fc3 = nn.Linear(256, 1) + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class Q(nn.Module): + def __init__(self, state_dim, action_dim): + super(Q, self).__init__() + self.fc1 = nn.Linear(state_dim + action_dim, 256) + self.fc2 = nn.Linear(256, 256) + self.fc3 = nn.Linear(256, 1) + + def forward(self, s, a): + s = s.reshape(-1, state_dim) + a = a.reshape(-1, action_dim) + x = torch.cat((s, a), -1) # combination s and a + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x + + +class SAC(): + def __init__(self): + super(SAC, self).__init__() + + self.policy_net = Actor(state_dim).to(device) + self.value_net = Critic(state_dim).to(device) + self.Target_value_net = Critic(state_dim).to(device) + self.Q_net1 = Q(state_dim, action_dim).to(device) + self.Q_net2 = Q(state_dim, action_dim).to(device) + + self.policy_optimizer = optim.Adam(self.policy_net.parameters(), lr=args.learning_rate) + self.value_optimizer = optim.Adam(self.value_net.parameters(), lr=args.learning_rate) + self.Q1_optimizer = optim.Adam(self.Q_net1.parameters(), lr=args.learning_rate) + self.Q2_optimizer = optim.Adam(self.Q_net2.parameters(), lr=args.learning_rate) + + self.replay_buffer = [Transition] * args.capacity + self.num_transition = 0 # pointer of replay buffer + self.num_training = 0 + self.writer = SummaryWriter('./test_agent') + + self.value_criterion = nn.MSELoss() + self.Q1_criterion = nn.MSELoss() + self.Q2_criterion = nn.MSELoss() + + for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): + target_param.data.copy_(param.data) + + os.makedirs('./SAC_model/', exist_ok=True) + + def select_action(self, state): + state = torch.FloatTensor(state).to(device) + mu, log_sigma = self.policy_net(state) + sigma = torch.exp(log_sigma) + dist = Normal(mu, sigma) + z = dist.sample() + action = torch.tanh(z).detach().cpu().numpy() + return action # return a scalar, float32 + + def store(self, s, a, r, s_, d): + index = self.num_transition % args.capacity + transition = Transition(s, a, r, s_, d) + self.replay_buffer[index] = transition + self.num_transition += 1 + + def evaluate(self, state): + batch_mu, batch_log_sigma = self.policy_net(state) + batch_sigma = torch.exp(batch_log_sigma) + dist = Normal(batch_mu, batch_sigma) + noise = Normal(0, 1) + + z = noise.sample() + action = torch.tanh(batch_mu + batch_sigma*z.to(device)) + log_prob = dist.log_prob(batch_mu + batch_sigma * z.to(device)) - torch.log(1 - action.pow(2) + min_Val) + return action, log_prob, z, batch_mu, batch_log_sigma + + def update(self): + if self.num_training % 500 == 0: + print("Training ... \t{} times ".format(self.num_training)) + s = torch.tensor([t.s for t in self.replay_buffer]).float().to(device) + a = torch.tensor([t.a for t in self.replay_buffer]).to(device) + r = torch.tensor([t.r for t in self.replay_buffer]).to(device) + s_ = torch.tensor([t.s_ for t in self.replay_buffer]).float().to(device) + d = torch.tensor([t.d for t in self.replay_buffer]).float().to(device) + + for _ in range(args.gradient_steps): + #for index in BatchSampler(SubsetRandomSampler(range(args.capacity)), args.batch_size, False): + index = np.random.choice(range(args.capacity), args.batch_size, replace=False) + bn_s = s[index].reshape(-1, state_dim) + bn_a = a[index].reshape(-1, action_dim) + bn_r = r[index].reshape(-1, 1) + bn_s_ = s_[index].reshape(-1, state_dim) + bn_d = d[index].reshape(-1, 1) + + target_value = self.Target_value_net(bn_s_) + next_q_value = bn_r + (1 - bn_d) * args.gamma * target_value + + excepted_value = self.value_net(bn_s) + excepted_Q1 = self.Q_net1(bn_s, bn_a) + excepted_Q2 = self.Q_net2(bn_s, bn_a) + sample_action, log_prob, z, batch_mu, batch_log_sigma = self.evaluate(bn_s) + excepted_new_Q = torch.min(self.Q_net1(bn_s, sample_action), self.Q_net2(bn_s, sample_action)) + next_value = excepted_new_Q - log_prob + + # !!!Note that the actions are sampled according to the current policy, + # instead of replay buffer. (From original paper) + V_loss = self.value_criterion(excepted_value, next_value.detach()).mean() # J_V + + # Dual Q net + Q1_loss = self.Q1_criterion(excepted_Q1, next_q_value.detach()).mean() # J_Q + Q2_loss = self.Q2_criterion(excepted_Q2, next_q_value.detach()).mean() + + pi_loss = (log_prob - excepted_new_Q).mean() # according to original paper + + self.writer.add_scalar('Loss/V_loss', V_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/Q1_loss', Q1_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/Q2_loss', Q2_loss, global_step=self.num_training) + self.writer.add_scalar('Loss/policy_loss', pi_loss, global_step=self.num_training) + + # mini batch gradient descent + self.value_optimizer.zero_grad() + V_loss.backward(retain_graph=True) + nn.utils.clip_grad_norm_(self.value_net.parameters(), 0.5) + self.value_optimizer.step() + + self.Q1_optimizer.zero_grad() + Q1_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.Q_net1.parameters(), 0.5) + self.Q1_optimizer.step() + + self.Q2_optimizer.zero_grad() + Q2_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.Q_net2.parameters(), 0.5) + self.Q2_optimizer.step() + + self.policy_optimizer.zero_grad() + pi_loss.backward(retain_graph = True) + nn.utils.clip_grad_norm_(self.policy_net.parameters(), 0.5) + self.policy_optimizer.step() + + # update target v net update + for target_param, param in zip(self.Target_value_net.parameters(), self.value_net.parameters()): + target_param.data.copy_(target_param * (1 - args.tau) + param * args.tau) + + self.num_training += 1 + + def save(self): + torch.save(self.policy_net.state_dict(), './SAC_model/policy_net.pth') + torch.save(self.value_net.state_dict(), './SAC_model/value_net.pth') + torch.save(self.Q_net1.state_dict(), './SAC_model/Q_net1.pth') + torch.save(self.Q_net2.state_dict(), './SAC_model/Q_net1.pth') + print("====================================") + print("Model has been saved...") + print("====================================") + + def load(self): + self.policy_net.load_state_dict(torch.load('./SAC_model/policy_net.pth')) + self.value_net.load_state_dict(torch.load( './SAC_model/value_net.pth')) + self.Q_net1.load_state_dict(torch.load('./SAC_model/Q_net1.pth')) + self.Q_net2.load_state_dict(torch.load('./SAC_model/Q_net1.pth')) + print("model has been load") + + +def main(): + agent = SAC() + ep_r = 0 + if args.mode == 'test': + agent.load() + for i in range(args.iteration): + state = env.reset() + for t in count(): + action = agent.select_action(state) + next_state, reward, done, info = env.step(np.float32(action)) + ep_r += reward + env.render() + if done: + break + state = next_state + else: + print("====================================") + print("Collection Experience...") + print("====================================") + + + for i in range(args.iteration): + state = env.reset() + for t in range(200): + action = agent.select_action(state) + next_state, reward, done, info = env.step(np.float32(action)) + ep_r += reward + if args.render and i >= args.render_interval : env.render() + agent.store(state, action, reward, next_state, done) + + if agent.num_transition >= args.capacity: + agent.update() + + state = next_state + if done: + if i > 100: + print("Ep_i \t{}, the ep_r is \t{}, the step is \t{}".format(i, ep_r, t)) + break + if i % args.log_interval == 0: + agent.save() + agent.writer.add_scalar('ep_r', ep_r, global_step=i) + ep_r = 0 + + +if __name__ == '__main__': + main() diff --git a/Char10 TD3/Episode_reward_TD3_BipedakWalker.png b/Char10 TD3/Episode_reward_TD3_BipedakWalker.png new file mode 100644 index 0000000..2511fe3 Binary files /dev/null and b/Char10 TD3/Episode_reward_TD3_BipedakWalker.png differ diff --git a/Char10 TD3/TD3.py b/Char10 TD3/TD3.py new file mode 100644 index 0000000..6860929 --- /dev/null +++ b/Char10 TD3/TD3.py @@ -0,0 +1,309 @@ +import argparse +from collections import namedtuple +from itertools import count + +import os, sys, random +import numpy as np + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from tensorboardX import SummaryWriter + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +parser = argparse.ArgumentParser() + +parser.add_argument('--mode', default='train', type=str) # mode = 'train' or 'test' +parser.add_argument("--env_name", default="Pendulum-v0") # OpenAI gym environment name， BipedalWalker-v2 +parser.add_argument('--tau', default=0.005, type=float) # target smoothing coefficient +parser.add_argument('--target_update_interval', default=1, type=int) +parser.add_argument('--iteration', default=5, type=int) + +parser.add_argument('--learning_rate', default=3e-4, type=float) +parser.add_argument('--gamma', default=0.99, type=int) # discounted factor +parser.add_argument('--capacity', default=50000, type=int) # replay buffer size +parser.add_argument('--num_iteration', default=100000, type=int) # num of games +parser.add_argument('--batch_size', default=100, type=int) # mini batch size +parser.add_argument('--seed', default=1, type=int) + +# optional parameters +parser.add_argument('--num_hidden_layers', default=2, type=int) +parser.add_argument('--sample_frequency', default=256, type=int) +parser.add_argument('--activation', default='Relu', type=str) +parser.add_argument('--render', default=False, type=bool) # show UI or not +parser.add_argument('--log_interval', default=50, type=int) # +parser.add_argument('--load', default=False, type=bool) # load model +parser.add_argument('--render_interval', default=100, type=int) # after render_interval, the env.render() will work +parser.add_argument('--policy_noise', default=0.2, type=float) +parser.add_argument('--noise_clip', default=0.5, type=float) +parser.add_argument('--policy_delay', default=2, type=int) +parser.add_argument('--exploration_noise', default=0.1, type=float) +parser.add_argument('--max_episode', default=2000, type=int) +parser.add_argument('--print_log', default=5, type=int) +args = parser.parse_args() + + + +# Set seeds +# env.seed(args.seed) +# torch.manual_seed(args.seed) +# np.random.seed(args.seed) +device = 'cuda' if torch.cuda.is_available() else 'cpu' +script_name = os.path.basename(__file__) +env = gym.make(args.env_name) + +state_dim = env.observation_space.shape[0] +action_dim = env.action_space.shape[0] +max_action = float(env.action_space.high[0]) +min_Val = torch.tensor(1e-7).float().to(device) # min value + +directory = './exp' + script_name + args.env_name +'./' +''' +Implementation of TD3 with pytorch +Original paper: https://arxiv.org/abs/1802.09477 +Not the author's implementation ! +''' + +class Replay_buffer(): + ''' + Code based on: + https://github.com/openai/baselines/blob/master/baselines/deepq/replay_buffer.py + Expects tuples of (state, next_state, action, reward, done) + ''' + def __init__(self, max_size=args.capacity): + self.storage = [] + self.max_size = max_size + self.ptr = 0 + + def push(self, data): + if len(self.storage) == self.max_size: + self.storage[int(self.ptr)] = data + self.ptr = (self.ptr + 1) % self.max_size + else: + self.storage.append(data) + + def sample(self, batch_size): + ind = np.random.randint(0, len(self.storage), size=batch_size) + x, y, u, r, d = [], [], [], [], [] + + for i in ind: + X, Y, U, R, D = self.storage[i] + x.append(np.array(X, copy=False)) + y.append(np.array(Y, copy=False)) + u.append(np.array(U, copy=False)) + r.append(np.array(R, copy=False)) + d.append(np.array(D, copy=False)) + + return np.array(x), np.array(y), np.array(u), np.array(r).reshape(-1, 1), np.array(d).reshape(-1, 1) + + +class Actor(nn.Module): + + def __init__(self, state_dim, action_dim, max_action): + super(Actor, self).__init__() + + self.fc1 = nn.Linear(state_dim, 400) + self.fc2 = nn.Linear(400, 300) + self.fc3 = nn.Linear(300, action_dim) + + self.max_action = max_action + + def forward(self, state): + a = F.relu(self.fc1(state)) + a = F.relu(self.fc2(a)) + a = torch.tanh(self.fc3(a)) * self.max_action + return a + + +class Critic(nn.Module): + + def __init__(self, state_dim, action_dim): + super(Critic, self).__init__() + + self.fc1 = nn.Linear(state_dim + action_dim, 400) + self.fc2 = nn.Linear(400, 300) + self.fc3 = nn.Linear(300, 1) + + def forward(self, state, action): + state_action = torch.cat([state, action], 1) + + q = F.relu(self.fc1(state_action)) + q = F.relu(self.fc2(q)) + q = self.fc3(q) + return q + + +class TD3(): + def __init__(self, state_dim, action_dim, max_action): + + self.actor = Actor(state_dim, action_dim, max_action).to(device) + self.actor_target = Actor(state_dim, action_dim, max_action).to(device) + self.critic_1 = Critic(state_dim, action_dim).to(device) + self.critic_1_target = Critic(state_dim, action_dim).to(device) + self.critic_2 = Critic(state_dim, action_dim).to(device) + self.critic_2_target = Critic(state_dim, action_dim).to(device) + + self.actor_optimizer = optim.Adam(self.actor.parameters()) + self.critic_1_optimizer = optim.Adam(self.critic_1.parameters()) + self.critic_2_optimizer = optim.Adam(self.critic_2.parameters()) + + self.actor_target.load_state_dict(self.actor.state_dict()) + self.critic_1_target.load_state_dict(self.critic_1.state_dict()) + self.critic_2_target.load_state_dict(self.critic_2.state_dict()) + + self.max_action = max_action + self.memory = Replay_buffer(args.capacity) + self.writer = SummaryWriter(directory) + self.num_critic_update_iteration = 0 + self.num_actor_update_iteration = 0 + self.num_training = 0 + + def select_action(self, state): + state = torch.tensor(state.reshape(1, -1)).float().to(device) + return self.actor(state).cpu().data.numpy().flatten() + + def update(self, num_iteration): + + if self.num_training % 500 == 0: + print("====================================") + print("model has been trained for {} times...".format(self.num_training)) + print("====================================") + for i in range(num_iteration): + x, y, u, r, d = self.memory.sample(args.batch_size) + state = torch.FloatTensor(x).to(device) + action = torch.FloatTensor(u).to(device) + next_state = torch.FloatTensor(y).to(device) + done = torch.FloatTensor(d).to(device) + reward = torch.FloatTensor(r).to(device) + + # Select next action according to target policy: + noise = torch.ones_like(action).data.normal_(0, args.policy_noise).to(device) + noise = noise.clamp(-args.noise_clip, args.noise_clip) + next_action = (self.actor_target(next_state) + noise) + next_action = next_action.clamp(-self.max_action, self.max_action) + + # Compute target Q-value: + target_Q1 = self.critic_1_target(next_state, next_action) + target_Q2 = self.critic_2_target(next_state, next_action) + target_Q = torch.min(target_Q1, target_Q2) + target_Q = reward + ((1 - done) * args.gamma * target_Q).detach() + + # Optimize Critic 1: + current_Q1 = self.critic_1(state, action) + loss_Q1 = F.mse_loss(current_Q1, target_Q) + self.critic_1_optimizer.zero_grad() + loss_Q1.backward() + self.critic_1_optimizer.step() + self.writer.add_scalar('Loss/Q1_loss', loss_Q1, global_step=self.num_critic_update_iteration) + + # Optimize Critic 2: + current_Q2 = self.critic_2(state, action) + loss_Q2 = F.mse_loss(current_Q2, target_Q) + self.critic_2_optimizer.zero_grad() + loss_Q2.backward() + self.critic_2_optimizer.step() + self.writer.add_scalar('Loss/Q2_loss', loss_Q2, global_step=self.num_critic_update_iteration) + # Delayed policy updates: + if i % args.policy_delay == 0: + # Compute actor loss: + actor_loss = - self.critic_1(state, self.actor(state)).mean() + + # Optimize the actor + self.actor_optimizer.zero_grad() + actor_loss.backward() + self.actor_optimizer.step() + self.writer.add_scalar('Loss/actor_loss', actor_loss, global_step=self.num_actor_update_iteration) + for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()): + target_param.data.copy_(((1- args.tau) * target_param.data) + args.tau * param.data) + + for param, target_param in zip(self.critic_1.parameters(), self.critic_1_target.parameters()): + target_param.data.copy_(((1 - args.tau) * target_param.data) + args.tau * param.data) + + for param, target_param in zip(self.critic_2.parameters(), self.critic_2_target.parameters()): + target_param.data.copy_(((1 - args.tau) * target_param.data) + args.tau * param.data) + + self.num_actor_update_iteration += 1 + self.num_critic_update_iteration += 1 + self.num_training += 1 + + def save(self): + torch.save(self.actor.state_dict(), directory+'actor.pth') + torch.save(self.actor_target.state_dict(), directory+'actor_target.pth') + torch.save(self.critic_1.state_dict(), directory+'critic_1.pth') + torch.save(self.critic_1_target.state_dict(), directory+'critic_1_target.pth') + torch.save(self.critic_2.state_dict(), directory+'critic_2.pth') + torch.save(self.critic_2_target.state_dict(), directory+'critic_2_target.pth') + print("====================================") + print("Model has been saved...") + print("====================================") + + def load(self): + self.actor.load_state_dict(torch.load(directory + 'actor.pth')) + self.actor_target.load_state_dict(torch.load(directory + 'actor_target.pth')) + self.critic_1.load_state_dict(torch.load(directory + 'critic_1.pth')) + self.critic_1_target.load_state_dict(torch.load(directory + 'critic_1_target.pth')) + self.critic_2.load_state_dict(torch.load(directory + 'critic_2.pth')) + self.critic_2_target.load_state_dict(torch.load(directory + 'critic_2_target.pth')) + print("====================================") + print("model has been loaded...") + print("====================================") + + +def main(): + agent = TD3(state_dim, action_dim, max_action) + ep_r = 0 + + if args.mode == 'test': + agent.load() + for i in range(args.iteration): + state = env.reset() + for t in count(): + action = agent.select_action(state) + next_state, reward, done, info = env.step(np.float32(action)) + ep_r += reward + env.render() + if done or t ==2000 : + print("Ep_i \t{}, the ep_r is \t{:0.2f}, the step is \t{}".format(i, ep_r, t)) + break + state = next_state + + elif args.mode == 'train': + print("====================================") + print("Collection Experience...") + print("====================================") + if args.load: agent.load() + for i in range(args.num_iteration): + state = env.reset() + for t in range(2000): + + action = agent.select_action(state) + action = action + np.random.normal(0, args.exploration_noise, size=env.action_space.shape[0]) + action = action.clip(env.action_space.low, env.action_space.high) + next_state, reward, done, info = env.step(action) + ep_r += reward + if args.render and i >= args.render_interval : env.render() + agent.memory.push((state, next_state, action, reward, np.float(done))) + if i+1 % 10 == 0: + print('Episode {}, The memory size is {} '.format(i, len(agent.memory.storage))) + if len(agent.memory.storage) >= args.capacity-1: + agent.update(10) + + state = next_state + if done or t == args.max_episode -1: + agent.writer.add_scalar('ep_r', ep_r, global_step=i) + if i % args.print_log == 0: + print("Ep_i \t{}, the ep_r is \t{:0.2f}, the step is \t{}".format(i, ep_r, t)) + ep_r = 0 + break + + if i % args.log_interval == 0: + agent.save() + + else: + raise NameError("mode wrong!!!") + +if __name__ == '__main__': + main() diff --git a/Char10 TD3/TD3_BipedalWalker-v2.py b/Char10 TD3/TD3_BipedalWalker-v2.py new file mode 100644 index 0000000..4f771d1 --- /dev/null +++ b/Char10 TD3/TD3_BipedalWalker-v2.py @@ -0,0 +1,313 @@ +import argparse +from collections import namedtuple +from itertools import count + +import os, sys, random +import numpy as np + +import gym +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torch.distributions import Normal +from tensorboardX import SummaryWriter + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +parser = argparse.ArgumentParser() + +parser.add_argument('--mode', default='train', type=str) # mode = 'train' or 'test' +# OpenAI gym environment name, # ['BipedalWalker-v2', 'Pendulum-v0'] or any continuous environment +# Note that if you want test in another game, you should fine-tuning. +parser.add_argument("--env_name", default="BipedalWalker-v2") +parser.add_argument('--tau', default=0.005, type=float) # target smoothing coefficient +parser.add_argument('--target_update_interval', default=1, type=int) +parser.add_argument('--test_iteration', default=10, type=int) + +parser.add_argument('--learning_rate', default=3e-4, type=float) +parser.add_argument('--gamma', default=0.99, type=int) # discounted factor +parser.add_argument('--capacity', default=50000, type=int) # replay buffer size +parser.add_argument('--num_iteration', default=100000, type=int) # num of games +parser.add_argument('--batch_size', default=100, type=int) # mini batch size +parser.add_argument('--seed', default=False, type=bool) +parser.add_argument('--random_seed', default=9527, type=int) +# optional parameters +parser.add_argument('--num_hidden_layers', default=2, type=int) +parser.add_argument('--sample_frequency', default=256, type=int) +parser.add_argument('--render', default=False, type=bool) # show UI or not +parser.add_argument('--log_interval', default=50, type=int) # +parser.add_argument('--load', default=False, type=bool) # load model +parser.add_argument('--render_interval', default=100, type=int) # after render_interval, the env.render() will work +parser.add_argument('--policy_noise', default=0.2, type=float) +parser.add_argument('--noise_clip', default=0.5, type=float) +parser.add_argument('--policy_delay', default=2, type=int) +parser.add_argument('--exploration_noise', default=0.1, type=float) +parser.add_argument('--max_episode', default=2000, type=int) +parser.add_argument('--print_log', default=5, type=int) +args = parser.parse_args() + + + + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +script_name = os.path.basename(__file__) +env = gym.make(args.env_name) +if args.seed: + env.seed(args.random_seed) + torch.manual_seed(args.random_seed) + np.random.seed(args.random_seed) + +state_dim = env.observation_space.shape[0] +action_dim = env.action_space.shape[0] +max_action = float(env.action_space.high[0]) +min_Val = torch.tensor(1e-7).float().to(device) # min value + +directory = './exp' + script_name + args.env_name +'./' +''' +Implementation of TD3 with pytorch +Original paper: https://arxiv.org/abs/1802.09477 +Not the author's implementation ! +''' + +class Replay_buffer(): + ''' + Code based on: + https://github.com/openai/baselines/blob/master/baselines/deepq/replay_buffer.py + Expects tuples of (state, next_state, action, reward, done) + ''' + def __init__(self, max_size=args.capacity): + self.storage = [] + self.max_size = max_size + self.ptr = 0 + + def push(self, data): + if len(self.storage) == self.max_size: + self.storage[int(self.ptr)] = data + self.ptr = (self.ptr + 1) % self.max_size + else: + self.storage.append(data) + + def sample(self, batch_size): + ind = np.random.randint(0, len(self.storage), size=batch_size) + x, y, u, r, d = [], [], [], [], [] + + for i in ind: + X, Y, U, R, D = self.storage[i] + x.append(np.array(X, copy=False)) + y.append(np.array(Y, copy=False)) + u.append(np.array(U, copy=False)) + r.append(np.array(R, copy=False)) + d.append(np.array(D, copy=False)) + + return np.array(x), np.array(y), np.array(u), np.array(r).reshape(-1, 1), np.array(d).reshape(-1, 1) + + +class Actor(nn.Module): + + def __init__(self, state_dim, action_dim, max_action): + super(Actor, self).__init__() + + self.fc1 = nn.Linear(state_dim, 400) + self.fc2 = nn.Linear(400, 300) + self.fc3 = nn.Linear(300, action_dim) + + self.max_action = max_action + + def forward(self, state): + a = F.relu(self.fc1(state)) + a = F.relu(self.fc2(a)) + a = torch.tanh(self.fc3(a)) * self.max_action + return a + + +class Critic(nn.Module): + + def __init__(self, state_dim, action_dim): + super(Critic, self).__init__() + + self.fc1 = nn.Linear(state_dim + action_dim, 400) + self.fc2 = nn.Linear(400, 300) + self.fc3 = nn.Linear(300, 1) + + def forward(self, state, action): + state_action = torch.cat([state, action], 1) + + q = F.relu(self.fc1(state_action)) + q = F.relu(self.fc2(q)) + q = self.fc3(q) + return q + + +class TD3(): + def __init__(self, state_dim, action_dim, max_action): + + self.actor = Actor(state_dim, action_dim, max_action).to(device) + self.actor_target = Actor(state_dim, action_dim, max_action).to(device) + self.critic_1 = Critic(state_dim, action_dim).to(device) + self.critic_1_target = Critic(state_dim, action_dim).to(device) + self.critic_2 = Critic(state_dim, action_dim).to(device) + self.critic_2_target = Critic(state_dim, action_dim).to(device) + + self.actor_optimizer = optim.Adam(self.actor.parameters()) + self.critic_1_optimizer = optim.Adam(self.critic_1.parameters()) + self.critic_2_optimizer = optim.Adam(self.critic_2.parameters()) + + self.actor_target.load_state_dict(self.actor.state_dict()) + self.critic_1_target.load_state_dict(self.critic_1.state_dict()) + self.critic_2_target.load_state_dict(self.critic_2.state_dict()) + + self.max_action = max_action + self.memory = Replay_buffer(args.capacity) + self.writer = SummaryWriter(directory) + self.num_critic_update_iteration = 0 + self.num_actor_update_iteration = 0 + self.num_training = 0 + + def select_action(self, state): + state = torch.tensor(state.reshape(1, -1)).float().to(device) + return self.actor(state).cpu().data.numpy().flatten() + + def update(self, num_iteration): + + if self.num_training % 500 == 0: + print("====================================") + print("model has been trained for {} times...".format(self.num_training)) + print("====================================") + for i in range(num_iteration): + x, y, u, r, d = self.memory.sample(args.batch_size) + state = torch.FloatTensor(x).to(device) + action = torch.FloatTensor(u).to(device) + next_state = torch.FloatTensor(y).to(device) + done = torch.FloatTensor(d).to(device) + reward = torch.FloatTensor(r).to(device) + + # Select next action according to target policy: + noise = torch.ones_like(action).data.normal_(0, args.policy_noise).to(device) + noise = noise.clamp(-args.noise_clip, args.noise_clip) + next_action = (self.actor_target(next_state) + noise) + next_action = next_action.clamp(-self.max_action, self.max_action) + + # Compute target Q-value: + target_Q1 = self.critic_1_target(next_state, next_action) + target_Q2 = self.critic_2_target(next_state, next_action) + target_Q = torch.min(target_Q1, target_Q2) + target_Q = reward + ((1 - done) * args.gamma * target_Q).detach() + + # Optimize Critic 1: + current_Q1 = self.critic_1(state, action) + loss_Q1 = F.mse_loss(current_Q1, target_Q) + self.critic_1_optimizer.zero_grad() + loss_Q1.backward() + self.critic_1_optimizer.step() + self.writer.add_scalar('Loss/Q1_loss', loss_Q1, global_step=self.num_critic_update_iteration) + + # Optimize Critic 2: + current_Q2 = self.critic_2(state, action) + loss_Q2 = F.mse_loss(current_Q2, target_Q) + self.critic_2_optimizer.zero_grad() + loss_Q2.backward() + self.critic_2_optimizer.step() + self.writer.add_scalar('Loss/Q2_loss', loss_Q2, global_step=self.num_critic_update_iteration) + # Delayed policy updates: + if i % args.policy_delay == 0: + # Compute actor loss: + actor_loss = - self.critic_1(state, self.actor(state)).mean() + + # Optimize the actor + self.actor_optimizer.zero_grad() + actor_loss.backward() + self.actor_optimizer.step() + self.writer.add_scalar('Loss/actor_loss', actor_loss, global_step=self.num_actor_update_iteration) + for param, target_param in zip(self.actor.parameters(), self.actor_target.parameters()): + target_param.data.copy_(((1- args.tau) * target_param.data) + args.tau * param.data) + + for param, target_param in zip(self.critic_1.parameters(), self.critic_1_target.parameters()): + target_param.data.copy_(((1 - args.tau) * target_param.data) + args.tau * param.data) + + for param, target_param in zip(self.critic_2.parameters(), self.critic_2_target.parameters()): + target_param.data.copy_(((1 - args.tau) * target_param.data) + args.tau * param.data) + + self.num_actor_update_iteration += 1 + self.num_critic_update_iteration += 1 + self.num_training += 1 + + def save(self): + torch.save(self.actor.state_dict(), directory+'actor.pth') + torch.save(self.actor_target.state_dict(), directory+'actor_target.pth') + torch.save(self.critic_1.state_dict(), directory+'critic_1.pth') + torch.save(self.critic_1_target.state_dict(), directory+'critic_1_target.pth') + torch.save(self.critic_2.state_dict(), directory+'critic_2.pth') + torch.save(self.critic_2_target.state_dict(), directory+'critic_2_target.pth') + print("====================================") + print("Model has been saved...") + print("====================================") + + def load(self): + self.actor.load_state_dict(torch.load(directory + 'actor.pth')) + self.actor_target.load_state_dict(torch.load(directory + 'actor_target.pth')) + self.critic_1.load_state_dict(torch.load(directory + 'critic_1.pth')) + self.critic_1_target.load_state_dict(torch.load(directory + 'critic_1_target.pth')) + self.critic_2.load_state_dict(torch.load(directory + 'critic_2.pth')) + self.critic_2_target.load_state_dict(torch.load(directory + 'critic_2_target.pth')) + print("====================================") + print("model has been loaded...") + print("====================================") + + +def main(): + agent = TD3(state_dim, action_dim, max_action) + ep_r = 0 + + if args.mode == 'test': + agent.load() + for i in range(args.test_iteration): + state = env.reset() + for t in count(): + action = agent.select_action(state) + next_state, reward, done, info = env.step(np.float32(action)) + ep_r += reward + env.render() + if done or t ==2000 : + print("Ep_i \t{}, the ep_r is \t{:0.2f}, the step is \t{}".format(i, ep_r, t)) + ep_r = 0 + break + state = next_state + + + elif args.mode == 'train': + print("====================================") + print("Collection Experience...") + print("====================================") + if args.load: agent.load() + for i in range(args.num_iteration): + state = env.reset() + for t in range(2000): + + action = agent.select_action(state) + action = action + np.random.normal(0, args.exploration_noise, size=env.action_space.shape[0]) + action = action.clip(env.action_space.low, env.action_space.high) + next_state, reward, done, info = env.step(action) + ep_r += reward + if args.render and i >= args.render_interval : env.render() + agent.memory.push((state, next_state, action, reward, np.float(done))) + if i+1 % 10 == 0: + print('Episode {}, The memory size is {} '.format(i, len(agent.memory.storage))) + if len(agent.memory.storage) >= args.capacity-1: + agent.update(10) + + state = next_state + if done or t == args.max_episode -1: + agent.writer.add_scalar('ep_r', ep_r, global_step=i) + if i % args.print_log == 0: + print("Ep_i \t{}, the ep_r is \t{:0.2f}, the step is \t{}".format(i, ep_r, t)) + ep_r = 0 + break + + if i % args.log_interval == 0: + agent.save() + + else: + raise NameError("mode wrong!!!") + +if __name__ == '__main__': + main() diff --git a/Char10 TD3/TD3_Pendulum-v0.png b/Char10 TD3/TD3_Pendulum-v0.png new file mode 100644 index 0000000..1176cb3 Binary files /dev/null and b/Char10 TD3/TD3_Pendulum-v0.png differ diff --git a/Char10 TD3/expTD3.pyPendulum-v0./actor.pth b/Char10 TD3/expTD3.pyPendulum-v0./actor.pth new file mode 100644 index 0000000..af36bc9 Binary files /dev/null and b/Char10 TD3/expTD3.pyPendulum-v0./actor.pth differ diff --git a/Char10 TD3/expTD3.pyPendulum-v0./actor_target.pth b/Char10 TD3/expTD3.pyPendulum-v0./actor_target.pth new file mode 100644 index 0000000..447f749 Binary files /dev/null and b/Char10 TD3/expTD3.pyPendulum-v0./actor_target.pth differ diff --git a/Char10 TD3/expTD3.pyPendulum-v0./critic_1.pth b/Char10 TD3/expTD3.pyPendulum-v0./critic_1.pth new file mode 100644 index 0000000..ba10393 Binary files /dev/null and b/Char10 TD3/expTD3.pyPendulum-v0./critic_1.pth differ diff --git a/Char10 TD3/expTD3.pyPendulum-v0./critic_1_target.pth b/Char10 TD3/expTD3.pyPendulum-v0./critic_1_target.pth new file mode 100644 index 0000000..a5c288c Binary files /dev/null and b/Char10 TD3/expTD3.pyPendulum-v0./critic_1_target.pth differ diff --git a/Char10 TD3/expTD3.pyPendulum-v0./critic_2.pth b/Char10 TD3/expTD3.pyPendulum-v0./critic_2.pth new file mode 100644 index 0000000..ef0d2e3 Binary files /dev/null and b/Char10 TD3/expTD3.pyPendulum-v0./critic_2.pth differ diff --git a/Char10 TD3/expTD3.pyPendulum-v0./critic_2_target.pth b/Char10 TD3/expTD3.pyPendulum-v0./critic_2_target.pth new file mode 100644 index 0000000..e3b992f Binary files /dev/null and b/Char10 TD3/expTD3.pyPendulum-v0./critic_2_target.pth differ diff --git a/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./actor.pth b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./actor.pth new file mode 100644 index 0000000..0f52e13 Binary files /dev/null and b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./actor.pth differ diff --git a/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./actor_target.pth b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./actor_target.pth new file mode 100644 index 0000000..fdce1fd Binary files /dev/null and b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./actor_target.pth differ diff --git a/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_1.pth b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_1.pth new file mode 100644 index 0000000..f335da8 Binary files /dev/null and b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_1.pth differ diff --git a/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_1_target.pth b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_1_target.pth new file mode 100644 index 0000000..4bae066 Binary files /dev/null and b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_1_target.pth differ diff --git a/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_2.pth b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_2.pth new file mode 100644 index 0000000..821964e Binary files /dev/null and b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_2.pth differ diff --git a/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_2_target.pth b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_2_target.pth new file mode 100644 index 0000000..ee5575b Binary files /dev/null and b/Char10 TD3/expTD3_BipedalWalker-v2.pyBipedalWalker-v2./critic_2_target.pth differ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..300ec3e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Johnny He + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/More/Application in real world/README.md b/More/Application in real world/README.md new file mode 100644 index 0000000..58e8820 --- /dev/null +++ b/More/Application in real world/README.md @@ -0,0 +1,6 @@ +# This repo we introduce some application in real world + + +This [post](https://towardsdatascience.com/advanced-reinforcement-learning-6d769f529eb3) would give you an idea of the application of reinforcement learning in industry. + +Here I upload some papers about this topic. diff --git a/More/MARL/README.md b/More/MARL/README.md new file mode 100644 index 0000000..1dbf4f1 --- /dev/null +++ b/More/MARL/README.md @@ -0,0 +1,16 @@ +# Multi-agent reinforcement learing + +Multi-Agent Reinforcement Learning is a very interesting research area, which has strong connections with single-agent RL, multi-agent systems, game theory, evolutionary computation and optimization theory. + + +1. Learning to Communicate with Deep Multi-Agent Reinforcement Learning in PyTorch + +You can click [here](https://github.com/sweetice/learning-to-communicate-pytorch) to read code. + + + +2. MARL papers + +Paper list of multi-agent reinforcement learning(MARL) + +You can click [here](https://github.com/LantaoYu/MARL-Papers) to read the repo. diff --git a/More/plot.py b/More/plot.py new file mode 100644 index 0000000..c631bf1 --- /dev/null +++ b/More/plot.py @@ -0,0 +1,51 @@ +import pandas as pd +import numpy as np +import seaborn as sns +import matplotlib.pyplot as plt +import re +import os +sns.set(style='darkgrid') + +def get_info(filename): + filename = filename.replace('.npy', '') # remove .npy + algo, env, seed = re.split('_', filename) + seed = int(seed) + return algo, env, seed + + +def get_file_name(path='./'): + file_names = [] + for _, __, file_name in os.walk(path): + file_names += file_name + data_name = [f for f in file_names if '.npy' in f] + return data_name + +def exact_data(file_name, steps): + ''' + exact data from single .npy file + :param file_name: + :return: a Dataframe include time, seed, algo_name, avg_reward + ''' + avg_reward = np.load(file_name).reshape(-1, 1) + algo, env_name, seed = get_info(file_name) + df = pd.DataFrame(avg_reward) + df.columns = ['Average Return'] + df['Time Steps (1e6)'] = steps + df['Algorithm'] = algo + df['env'] = env_name + df['seed'] = seed + return df + + +if __name__ == '__main__': + file_names = get_file_name('./') + _, env_name, __ = get_info(file_names[0]) + df = pd.DataFrame([]) + steps = np.linspace(0, 1, 201) + for file in file_names: + data = exact_data(file, steps) + df = pd.concat([df, data], axis=0) + sns.lineplot(x='Time Steps (1e6)', y='Average Return', data=df, hue='Algorithm',ci=90) + plt.title(env_name) + plt.savefig(env_name + '.svg') + plt.show() diff --git a/More/readme.md b/More/readme.md new file mode 100644 index 0000000..3a81a9b --- /dev/null +++ b/More/readme.md @@ -0,0 +1,3 @@ +# More + +This folder give you more insights about RL. diff --git a/figures/test.png b/figures/test.png new file mode 100644 index 0000000..54335a8 Binary files /dev/null and b/figures/test.png differ diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..861437e --- /dev/null +++ b/readme.md @@ -0,0 +1,203 @@ +**Status:** Active (under active development, breaking changes may occur) + +This repository will implement the classic and state-of-the-art deep reinforcement learning algorithms. The aim of this repository is to provide clear pytorch code for people to learn the deep reinforcement learning algorithm. + +In the future, more state-of-the-art algorithms will be added and the existing codes will also be maintained. + +![demo](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/figures/grid.gif) + +## Requirements +- python <=3.6 +- tensorboardX +- gym >= 0.10 +- pytorch >= 0.4 + +**Note that tensorflow does not support python3.7** + +## Installation + +``` +pip install -r requirements.txt +``` + +If you fail: + +- Install gym + +``` +pip install gym +``` + + + +- Install the pytorch +```bash +please go to official webisite to install it: https://pytorch.org/ + +Recommend use Anaconda Virtual Environment to manage your packages + +``` + +- Install tensorboardX +```bash +pip install tensorboardX +pip install tensorflow==1.12 +``` + +- Test +``` +cd Char10\ TD3/ +python TD3_BipedalWalker-v2.py --mode test +``` + +You could see a bipedalwalker if you install successfully. + +BipedalWalker: + +![](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/figures/test.png) + +- 4. install openai-baselines (**Optional**) + +```bash +# clone the openai baselines +git clone https://github.com/openai/baselines.git +cd baselines +pip install -e . + +``` + +## DQN + +Here I uploaded two DQN models which is trianing CartPole-v0 and MountainCar-v0. + +### Tips for MountainCar-v0 + +This is a sparse binary reward task. Only when car reach the top of the mountain there is a none-zero reward. In genearal it may take 1e5 steps in stochastic policy. You can add a reward term, for example, to change to the current position of the Car is positively related. Of course, there is a more advanced approach that is inverse reinforcement learning. + +![value_loss](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/Char01%20DQN/DQN/pic/value_loss.jpg) +![step](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/Char01%20DQN/DQN/pic/finish_episode.jpg) +This is value loss for DQN, We can see that the loss increaded to 1e13, however, the network work well. Because the target_net and act_net are very different with the training process going on. The calculated loss cumulate large. The previous loss was small because the reward was very sparse, resulting in a small update of the two networks. + +### Papers Related to the DQN + + + 1. Playing Atari with Deep Reinforcement Learning [[arxiv]](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/1.dqn.ipynb) + 2. Deep Reinforcement Learning with Double Q-learning [[arxiv]](https://arxiv.org/abs/1509.06461) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/2.double%20dqn.ipynb) + 3. Dueling Network Architectures for Deep Reinforcement Learning [[arxiv]](https://arxiv.org/abs/1511.06581) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/3.dueling%20dqn.ipynb) + 4. Prioritized Experience Replay [[arxiv]](https://arxiv.org/abs/1511.05952) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/4.prioritized%20dqn.ipynb) + 5. Noisy Networks for Exploration [[arxiv]](https://arxiv.org/abs/1706.10295) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/5.noisy%20dqn.ipynb) + 6. A Distributional Perspective on Reinforcement Learning [[arxiv]](https://arxiv.org/pdf/1707.06887.pdf) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/6.categorical%20dqn.ipynb) + 7. Rainbow: Combining Improvements in Deep Reinforcement Learning [[arxiv]](https://arxiv.org/abs/1710.02298) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/7.rainbow%20dqn.ipynb) + 8. Distributional Reinforcement Learning with Quantile Regression [[arxiv]](https://arxiv.org/pdf/1710.10044.pdf) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/8.quantile%20regression%20dqn.ipynb) + 9. Hierarchical Deep Reinforcement Learning: Integrating Temporal Abstraction and Intrinsic Motivation [[arxiv]](https://arxiv.org/abs/1604.06057) [[code]](https://github.com/higgsfield/RL-Adventure/blob/master/9.hierarchical%20dqn.ipynb) + 10. Neural Episodic Control [[arxiv]](https://arxiv.org/pdf/1703.01988.pdf) [[code]](#) + + +## Policy Gradient + + +Use the following command to run a saved model + + +``` +python Run_Model.py +``` + + +Use the following command to train model + + +``` +python pytorch_MountainCar-v0.py +``` + + + +> policyNet.pkl + +This is a model that I have trained. + + +## Actor-Critic + +This is an algorithmic framework, and the classic REINFORCE method is stored under Actor-Critic. + +## DDPG +Episode reward in Pendulum-v0: + +![ep_r](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/Char05%20DDPG/DDPG_exp.jpg) + + +## PPO + +- Original paper: https://arxiv.org/abs/1707.06347 +- Openai Baselines blog post: https://blog.openai.com/openai-baselines-ppo/ + + +## A2C + +Advantage Policy Gradient, an paper in 2017 pointed out that the difference in performance between A2C and A3C is not obvious. + +The Asynchronous Advantage Actor Critic method (A3C) has been very influential since the paper was published. The algorithm combines a few key ideas: + +- An updating scheme that operates on fixed-length segments of experience (say, 20 timesteps) and uses these segments to compute estimators of the returns and advantage function. +- Architectures that share layers between the policy and value function. +- Asynchronous updates. + +## A3C + +Original paper: https://arxiv.org/abs/1602.01783 + +## SAC + +**This is not the implementation of the author of paper!!!** + +Episode reward in Pendulum-v0: + +![ep_r](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/Char09%20SAC/SAC_ep_r_curve.png) + +## TD3 + +**This is not the implementation of the author of paper!!!** + +Episode reward in Pendulum-v0: + +![ep_r](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/Char10%20TD3/TD3_Pendulum-v0.png) + +Episode reward in BipedalWalker-v2: +![ep_r](https://github.com/sweetice/Deep-reinforcement-learning-with-pytorch/blob/master/Char10%20TD3/Episode_reward_TD3_BipedakWalker.png) + +If you want to use the test your model: + +``` +python TD3_BipedalWalker-v2.py --mode test +``` + +## Papers Related to the Deep Reinforcement Learning +[01] [A Brief Survey of Deep Reinforcement Learning](https://arxiv.org/abs/1708.05866) +[02] [The Beta Policy for Continuous Control Reinforcement Learning](https://www.ri.cmu.edu/wp-content/uploads/2017/06/thesis-Chou.pdf) +[03] [Playing Atari with Deep Reinforcement Learning](https://www.cs.toronto.edu/~vmnih/docs/dqn.pdf) +[04] [Deep Reinforcement Learning with Double Q-learning](https://arxiv.org/abs/1509.06461) +[05] [Dueling Network Architectures for Deep Reinforcement Learning](https://arxiv.org/abs/1511.06581) +[06] [Continuous control with deep reinforcement learning](https://arxiv.org/abs/1509.02971) +[07] [Continuous Deep Q-Learning with Model-based Acceleration](https://arxiv.org/abs/1603.00748) +[08] [Asynchronous Methods for Deep Reinforcement Learning](https://arxiv.org/abs/1602.01783) +[09] [Trust Region Policy Optimization](https://arxiv.org/abs/1502.05477) +[10] [Proximal Policy Optimization Algorithms](https://arxiv.org/abs/1707.06347) +[11] [Scalable trust-region method for deep reinforcement learning using Kronecker-factored approximation](https://arxiv.org/abs/1708.05144) +[12] [High-Dimensional Continuous Control Using Generalized Advantage Estimation](https://arxiv.org/abs/1506.02438) +[13] [Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor](https://arxiv.org/abs/1801.01290) +[14] [Addressing Function Approximation Error in Actor-Critic Methods](https://arxiv.org/abs/1802.09477) + +## TO DO +- [x] DDPG +- [x] SAC +- [x] TD3 + + +# Best RL courses +- [OpenAI's spinning up](https://spinningup.openai.com/) +- [David Silver's course](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching.html) +- [Berkeley deep RL](http://rll.berkeley.edu/deeprlcourse/) +- [Practical RL](https://github.com/yandexdataschool/Practical_RL) +- [Deep Reinforcement Learning by Hung-yi Lee](https://www.youtube.com/playlist?list=PLJV_el3uVTsODxQFgzMzPLa16h6B8kWM_) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..156a1dd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +torch==1.0 +torchvision +tensorflow==1.12 +tensorboardX +gym +gym[atari]