wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

DDPG Pendulum

Browse Source
This commit is contained in:
Shangtong Zhang
2017-08-01 10:52:14 -06:00
parent 1be3b44999
commit 5116733f22
5 changed files with 69 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/DDPG_agent.py
+35 -54
View File
@@ -10,87 +10,67 @@ from utils import *
import pickle
class DDPGAgent:
def __init__(self,
task_fn,
actor_network_fn,
critic_network_fn,
actor_optimizer_fn,
critic_optimizer_fn,
replay_fn,
discount,
step_limit,
tau,
exploration_steps,
random_process_fn,
test_interval,
test_repetitions,
noise_decay_steps,
tag,
logger):
self.task = task_fn()
self.actor = actor_network_fn()
self.critic = critic_network_fn()
self.target_actor = actor_network_fn()
self.target_critic = critic_network_fn()
def __init__(self, config):
self.config = config
self.task = config.task_fn()
self.actor = config.actor_network_fn()
self.critic = config.critic_network_fn()
self.target_actor = config.actor_network_fn()
self.target_critic = config.critic_network_fn()
self.target_actor.load_state_dict(self.actor.state_dict())
self.target_critic.load_state_dict(self.critic.state_dict())
self.actor_opt = actor_optimizer_fn(self.actor.parameters())
self.critic_opt = critic_optimizer_fn(self.critic.parameters())
self.replay = replay_fn()
self.step_limit = step_limit
self.tau = tau
self.logger = logger
self.discount = discount
self.exploration_steps = exploration_steps
self.random_process = random_process_fn()
self.actor_opt = config.actor_optimizer_fn(self.actor.parameters())
self.critic_opt = config.critic_optimizer_fn(self.critic.parameters())
self.replay = config.replay_fn()
self.random_process = config.random_process_fn()
self.criterion = nn.MSELoss()
self.test_interval = test_interval
self.test_repetitions = test_repetitions
self.total_steps = 0
self.tag = tag
self.epsilon = 1.0
self.d_epsilon = 1.0 / noise_decay_steps
self.d_epsilon = 1.0 / config.noise_decay_interval
def soft_update(self, target, src):
for target_param, param in zip(target.parameters(), src.parameters()):
target_param.data.copy_(
target_param.data * (1.0 - self.tau) + param.data * self.tau
)
target_param.data.copy_(target_param.data * (1.0 - self.config.target_network_mix) +
param.data * self.config.target_network_mix)
def episode(self, deterministic=False):
self.random_process.reset_states()
state = self.task.reset()
state = self.config.state_shift_fn(state)
steps = 0
total_reward = 0.0
while not self.step_limit or steps < self.step_limit:
while not self.config or steps < self.config.max_episode_length:
action = self.actor.predict(np.stack([state])).flatten()
self.logger.histo_summary('action', action, self.total_steps)
self.config.logger.histo_summary('action', action, self.total_steps)
if not deterministic:
if self.total_steps < self.exploration_steps:
if self.total_steps < self.config.exploration_steps:
action = self.task.random_action()
else:
action += max(self.epsilon, 0) * self.random_process.sample()
self.logger.histo_summary('noised action', action, self.total_steps)
self.epsilon -= self.d_epsilon
self.config.logger.histo_summary('noised action', action, self.total_steps)
next_state, reward, done, info = self.task.step(action)
next_state = self.config.state_shift_fn(next_state)
self.config.logger.scalar_summary('reward', reward, self.total_steps)
total_reward += reward
reward = self.config.reward_shift_fn(reward)
if not deterministic:
self.replay.feed([state, action, reward, next_state, int(done)])
self.total_steps += 1
self.epsilon -= self.d_epsilon
steps += 1
total_reward += reward
state = next_state
if done:
break
if not deterministic and self.total_steps > self.exploration_steps:
if not deterministic and self.total_steps > self.config.exploration_steps:
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
q_next = self.target_critic.predict(next_states, self.target_actor.predict(next_states))
terminals = self.critic.to_torch_variable(terminals).unsqueeze(1)
rewards = self.critic.to_torch_variable(rewards).unsqueeze(1)
q_next = self.discount * q_next * (1 - terminals)
q_next = self.config.discount * q_next * (1 - terminals)
q_next.add_(rewards)
q_next = Variable(q_next.data)
q = self.critic.predict(states, actions)
@@ -126,20 +106,21 @@ class DDPGAgent:
reward = self.episode()
rewards.append(reward)
avg_reward = np.mean(rewards[-window_size:])
self.logger.info('episode %d, reward %f, avg reward %f, total steps %d' % (
self.config.logger.info('episode %d, reward %f, avg reward %f, total steps %d' % (
ep, reward, avg_reward, self.total_steps))
if self.test_interval and ep % self.test_interval == 0:
self.logger.info('Testing...')
self.save('data/%sddpg-model-%s.bin' % (self.tag, self.task.name))
if self.config.test_interval and ep % self.config.test_interval == 0:
self.config.logger.info('Testing...')
with open('data/%s-ddpg-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump(self.actor.state_dict(), f)
test_rewards = []
for _ in range(self.test_repetitions):
for _ in range(self.config.test_repetitions):
test_rewards.append(self.episode(True))
avg_reward = np.mean(test_rewards)
avg_test_rewards.append(avg_reward)
self.logger.info('Avg reward %f(%f)' % (
avg_reward, np.std(test_rewards) / np.sqrt(self.test_repetitions)))
with open('data/%sddpg-statistics-%s.bin' % (self.tag, self.task.name), 'wb') as f:
self.config.logger.info('Avg reward %f(%f)' % (
avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions)))
with open('data/%s-ddpg-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump({'rewards': rewards,
'test_rewards': avg_test_rewards}, f)
if avg_reward > self.task.success_threshold:
component/task.py
+4 -20
View File
@@ -80,22 +80,6 @@ class Pendulum(BasicTask):
name = 'Pendulum-v0'
success_threshold = -10
def __init__(self):
BasicTask.__init__(self)
self.env = gym.make(self.name)
self.env._max_episode_steps = sys.maxsize
self.action_dim = self.env.action_space.shape[0]
self.state_dim = self.env.observation_space.shape[0]
def step(self, action):
action = np.clip(action, -2, 2)
next_state, reward, done, info = self.env.step(action)
return next_state, reward, done, info
class MountainCarContinuous(BasicTask):
name = 'MountainCarContinuous-v0'
success_threshold = 90
def __init__(self):
BasicTask.__init__(self)
self.env = gym.make(self.name)
@@ -104,15 +88,15 @@ class MountainCarContinuous(BasicTask):
self.state_dim = self.env.observation_space.shape[0]
def normalize_state(self, state):
state = (state - self.env.unwrapped.low_state) / \
(self.env.unwrapped.high_state - self.env.unwrapped.low_state)
state = (state - self.env.observation_space.low) / \
(self.env.observation_space.high - self.env.observation_space.low)
state = state * 2 - 1
return state
def step(self, action):
action = np.clip(action, -1, 1)
action = np.clip(action, -2, 2)
next_state, reward, done, info = self.env.step(action)
return next_state, reward, done, info
return self.normalize_state(next_state), reward, done, info
class BipedalWalker(BasicTask):
name = 'BipedalWalker-v2'
main.py
+19 -20
View File
@@ -68,7 +68,6 @@ def a3c_pendulum():
config = Config()
config.task_fn = lambda: Pendulum()
config.reward_shift_fn = lambda reward: reward / 10
# config.task_fn = lambda: MountainCarContinuous()
task = config.task_fn()
config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001)
config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
@@ -184,25 +183,25 @@ def a3c_pixel_atari(name):
def ddpg_pendulum():
task_fn = lambda: Pendulum()
task = task_fn()
config = dict()
config['task_fn'] = task_fn
config['actor_network_fn'] = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh)
config['critic_network_fn'] = lambda: DDPGCriticNet(task.state_dim, task.action_dim)
config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4)
config['critic_optimizer_fn'] =\
config = Config()
config.task_fn = task_fn
config.actor_network_fn = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, 2)
config.critic_network_fn = lambda: DDPGCriticNet(task.state_dim, task.action_dim)
config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
config.critic_optimizer_fn =\
lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
config['replay_fn'] = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
config['discount'] = 0.99
config['step_limit'] = 200
config['tau'] = 0.001
config['exploration_steps'] = 100
config['random_process_fn'] = \
config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
config.discount = 0.99
config.max_episode_length = 200
config.target_network_mix = 0.001
config.exploration_steps = 100
config.noise_decay_interval = 10000
config.random_process_fn = \
lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
config['test_interval'] = 10
config['test_repetitions'] = 10
config['tag'] = ''
config['logger'] = Logger('./log', gym.logger)
agent = DDPGAgent(**config)
config.test_interval = 10
config.test_repetitions = 10
config.logger = Logger('./log', gym.logger)
agent = DDPGAgent(config)
agent.run()
def ddpg_bipedal_walker():
@@ -237,8 +236,9 @@ if __name__ == '__main__':
# dqn_cart_pole()
# async_cart_pole()
# a3c_cart_pole()
a3c_pendulum()
# a3c_pendulum()
# a3c_walker()
ddpg_pendulum()
# dqn_pixel_atari('PongNoFrameskip-v3')
# async_pixel_atari('PongNoFrameskip-v3')
@@ -248,5 +248,4 @@ if __name__ == '__main__':
# async_pixel_atari('BreakoutNoFrameskip-v3')
# a3c_pixel_atari('BreakoutNoFrameskip-v3')
# ddpg_pendulum()
# ddpg_bipedal_walker()
network/continuous_action_network.py
+5 -3
View File
@@ -47,13 +47,15 @@ class DDPGActorNet(nn.Module, BasicNet):
def __init__(self,
state_dim,
action_dim,
output_gate,
action_gate,
action_scale,
gpu=False):
super(DDPGActorNet, self).__init__()
self.layer1 = nn.Linear(state_dim, 400)
self.layer2 = nn.Linear(400, 300)
self.layer3 = nn.Linear(300, action_dim)
self.output_gate = output_gate
self.action_gate = action_gate
self.action_scale = action_scale
BasicNet.__init__(self, None, False, False)
self.init_weights()
@@ -76,7 +78,7 @@ class DDPGActorNet(nn.Module, BasicNet):
x = F.relu(self.layer1(x))
x = F.relu(self.layer2(x))
x = self.layer3(x)
# x = self.output_gate(self.layer3(x))
x = self.action_scale * self.action_gate(x)
return x
def predict(self, x, to_numpy=True):
utils/config.py
+6
View File
@@ -8,10 +8,14 @@ class Config:
def __init__(self):
self.task_fn = None
self.optimizer_fn = None
self.actor_optimizer_fn = None
self.critic_optimizer_fn = None
self.network_fn = None
self.actor_network_fn = None
self.critic_network_fn = None
self.policy_fn = None
self.replay_fn = None
self.random_process_fn = None
self.discount = 0.99
self.target_network_update_freq = 0
self.max_episode_length = 0
@@ -28,6 +32,8 @@ class Config:
self.gradient_clip = 40
self.entropy_weight = 0.01
self.gae_tau = 1.0
self.noise_decay_interval = 0
self.target_network_mix = 0.001
self.reward_shift_fn = lambda r: r
self.state_shift_fn = lambda s: s
self.action_shift_fn = lambda a: a