wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade to PyTorch v0.4

Browse Source
This commit is contained in:
Shangtong Zhang
2018-04-24 23:10:09 -06:00
parent fc7cfdf095
commit c7ceb876dd
13 changed files with 138 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/A2C_agent.py
+11 -11
View File
@@ -32,7 +32,7 @@ class A2CAgent(BaseAgent):
states = self.states
for _ in range(config.rollout_length):
prob, log_prob, value = self.network.predict(config.state_normalizer(states))
actions = [self.policy.sample(p) for p in prob.data.cpu().numpy()]
actions = [self.policy.sample(p) for p in prob.cpu().detach().numpy()]
next_states, rewards, terminals, _ = self.task.step(actions)
self.episode_rewards += rewards
rewards = config.reward_normalizer(rewards)
@@ -50,34 +50,34 @@ class A2CAgent(BaseAgent):
processed_rollout = [None] * (len(rollout) - 1)
advantages = self.network.tensor(np.zeros((config.num_workers, 1)))
returns = pending_value.data
returns = pending_value.detach()
for i in reversed(range(len(rollout) - 1)):
prob, log_prob, value, actions, rewards, terminals = rollout[i]
terminals = self.network.tensor(terminals).unsqueeze(1)
rewards = self.network.tensor(rewards).unsqueeze(1)
actions = self.network.tensor(actions, torch.LongTensor).unsqueeze(1)
actions = self.network.tensor(actions).unsqueeze(1).long()
next_value = rollout[i + 1][2]
returns = rewards + config.discount * terminals * returns
if not config.use_gae:
advantages = returns - value.data
advantages = returns - value.detach()
else:
td_error = rewards + config.discount * terminals * next_value.data - value.data
td_error = rewards + config.discount * terminals * next_value.detach() - value.detach()
advantages = advantages * config.gae_tau * config.discount * terminals + td_error
processed_rollout[i] = [prob, log_prob, value, actions, returns, advantages]
prob, log_prob, value, actions, returns, advantages = map(lambda x: torch.cat(x, dim=0), zip(*processed_rollout))
policy_loss = -log_prob.gather(1, Variable(actions)) * Variable(advantages)
policy_loss = -log_prob.gather(1, actions) * advantages
entropy_loss = torch.sum(prob * log_prob, dim=1, keepdim=True)
value_loss = 0.5 * (Variable(returns) - value).pow(2)
value_loss = 0.5 * (returns - value).pow(2)
self.policy_loss = np.mean(policy_loss.data.cpu().numpy())
self.entropy_loss = np.mean(entropy_loss.data.cpu().numpy())
self.value_loss = np.mean(value_loss.data.cpu().numpy())
self.policy_loss = np.mean(policy_loss.cpu().detach().numpy())
self.entropy_loss = np.mean(entropy_loss.cpu().detach().numpy())
self.value_loss = np.mean(value_loss.cpu().detach().numpy())
self.optimizer.zero_grad()
(policy_loss + config.entropy_weight * entropy_loss +
config.value_loss_weight * value_loss).mean().backward()
nn.utils.clip_grad_norm(self.network.parameters(), config.gradient_clip)
nn.utils.clip_grad_norm_(self.network.parameters(), config.gradient_clip)
self.optimizer.step()
self.evaluate(config.rollout_length)
agent/CategoricalDQN_agent.py
+11 -12
View File
@@ -34,8 +34,8 @@ class CategoricalDQNAgent(BaseAgent):
self.delta_atom = (config.categorical_v_max - config.categorical_v_min) / float(config.categorical_n_atoms - 1)
def evaluation_action(self, state):
value = self.network.predict(np.stack([self.config.state_normalizer(state)])).squeeze(0).data
value = (value * self.atoms).sum(-1).cpu().numpy().flatten()
value = self.network.predict(np.stack([self.config.state_normalizer(state)])).squeeze(0).detach()
value = (value * self.atoms).sum(-1).cpu().detach().numpy().flatten()
return np.argmax(value)
def episode(self, deterministic=False):
@@ -44,9 +44,9 @@ class CategoricalDQNAgent(BaseAgent):
total_reward = 0.0
steps = 0
while True:
value = self.network.predict(np.stack([self.config.state_normalizer(state)])).squeeze(0).data
value = self.network.predict(np.stack([self.config.state_normalizer(state)])).squeeze(0).detach()
# self.config.logger.histo_summary('prob', value, self.total_steps)
value = (value * self.atoms).sum(-1).cpu().numpy().flatten()
value = (value * self.atoms).sum(-1).cpu().detach().numpy().flatten()
# self.config.logger.histo_summary('q', value, self.total_steps)
if deterministic:
action = np.argmax(value)
@@ -68,13 +68,13 @@ class CategoricalDQNAgent(BaseAgent):
states, actions, rewards, next_states, terminals = experiences
states = self.config.state_normalizer(states)
next_states = self.config.state_normalizer(next_states)
prob_next = self.target_network.predict(next_states).data
prob_next = self.target_network.predict(next_states).detach()
q_next = (prob_next * self.atoms).sum(-1)
# self.config.logger.histo_summary('q next', q_next.cpu().numpy(), self.total_steps)
# self.config.logger.histo_summary('q next', q_next.cpu().detach().numpy(), self.total_steps)
_, a_next = torch.max(q_next, dim=1)
a_next = a_next.view(-1, 1, 1).expand(-1, -1, prob_next.size(2))
prob_next = prob_next.gather(1, a_next).squeeze(1)
# self.config.logger.histo_summary('prob next', prob_next.cpu().numpy(), self.total_steps)
# self.config.logger.histo_summary('prob next', prob_next.cpu().detach().numpy(), self.total_steps)
rewards = self.network.tensor(rewards)
terminals = self.network.tensor(terminals)
@@ -92,14 +92,13 @@ class CategoricalDQNAgent(BaseAgent):
target_prob[i].index_add_(0, u[i].long(), d_m_u[i])
prob = self.network.predict(states)
actions = self.network.tensor(actions, torch.LongTensor)
actions = self.network.tensor(actions).long()
actions = actions.view(-1, 1, 1).expand(-1, -1, prob.size(2))
prob = prob.gather(1, Variable(actions)).squeeze(1)
loss = -(Variable(target_prob) * prob.log()).sum(-1).mean()
# self.config.logger.scalar_summary('loss', loss.data.cpu().numpy().flatten(), self.total_steps)
prob = prob.gather(1, actions).squeeze(1)
loss = -(target_prob * prob.log()).sum(-1).mean()
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.network.parameters(), self.config.gradient_clip)
nn.utils.clip_grad_norm_(self.network.parameters(), self.config.gradient_clip)
self.optimizer.step()
self.evaluate()
agent/DDPG_agent.py
+8 -8
View File
@@ -35,8 +35,9 @@ class DDPGAgent(BaseAgent):
def soft_update(self, target, src):
for target_param, param in zip(target.parameters(), src.parameters()):
target_param.data.copy_(target_param.data * (1.0 - self.config.target_network_mix) +
param.data * self.config.target_network_mix)
target_param.detach_()
target_param.copy_(target_param * (1.0 - self.config.target_network_mix) +
param * self.config.target_network_mix)
def evaluation_action(self, state):
self.config.state_normalizer.set_read_only()
@@ -82,8 +83,8 @@ class DDPGAgent(BaseAgent):
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
q_next = target_critic.predict(next_states, target_actor.predict(next_states))
terminals = critic.variable(terminals).unsqueeze(1)
rewards = critic.variable(rewards).unsqueeze(1)
terminals = critic.tensor(terminals).unsqueeze(1)
rewards = critic.tensor(rewards).unsqueeze(1)
q_next = config.discount * q_next * (1 - terminals)
q_next.add_(rewards)
q_next = q_next.detach()
@@ -96,15 +97,14 @@ class DDPGAgent(BaseAgent):
self.critic_opt.step()
actions = actor.predict(states, False)
var_actions = Variable(actions.data, requires_grad=True)
var_actions = actions.detach().requires_grad_()
q = critic.predict(states, var_actions)
q.backward(critic.tensor(np.ones(q.size())))
actor.zero_grad()
self.actor_opt.zero_grad()
actions.backward(-var_actions.grad.data)
for param in actor.parameters():
param.grad.data.clamp(-config.gradient_clip, config.gradient_clip)
actions.backward(-var_actions.grad)
torch.nn.utils.clip_grad_value_(actor.parameters(), config.gradient_clip)
self.actor_opt.step()
self.soft_update(self.target_network, self.network)
agent/DQN_agent.py
+4 -4
View File
@@ -61,17 +61,17 @@ class DQNAgent(BaseAgent):
q_next = q_next.gather(1, best_actions.unsqueeze(1)).squeeze(1)
else:
q_next, _ = q_next.max(1)
terminals = self.network.variable(terminals)
rewards = self.network.variable(rewards)
terminals = self.network.tensor(terminals)
rewards = self.network.tensor(rewards)
q_next = self.config.discount * q_next * (1 - terminals)
q_next.add_(rewards)
actions = self.network.variable(actions, torch.LongTensor).unsqueeze(1)
actions = self.network.tensor(actions).unsqueeze(1).long()
q = self.network.predict(states, False)
q = q.gather(1, actions).squeeze(1)
loss = self.criterion(q, q_next)
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.network.parameters(), self.config.gradient_clip)
nn.utils.clip_grad_norm_(self.network.parameters(), self.config.gradient_clip)
self.optimizer.step()
self.evaluate()
agent/NStepDQN_agent.py
+6 -6
View File
@@ -36,7 +36,7 @@ class NStepDQNAgent(BaseAgent):
states = self.states
for _ in range(config.rollout_length):
q = self.network.predict(self.config.state_normalizer(states))
actions = [self.policy.sample(v) for v in q.data.cpu().numpy()]
actions = [self.policy.sample(v) for v in q.cpu().detach().numpy()]
next_states, rewards, terminals, _ = self.task.step(actions)
self.episode_rewards += rewards
rewards = config.reward_normalizer(rewards)
@@ -56,22 +56,22 @@ class NStepDQNAgent(BaseAgent):
self.states = states
processed_rollout = [None] * (len(rollout))
returns = self.target_network.predict(config.state_normalizer(states)).data
returns = self.target_network.predict(config.state_normalizer(states)).detach()
returns, _ = torch.max(returns, dim=1, keepdim=True)
for i in reversed(range(len(rollout))):
q, actions, rewards, terminals = rollout[i]
actions = self.network.tensor(actions, torch.LongTensor).unsqueeze(1)
q = q.gather(1, Variable(actions))
actions = self.network.tensor(actions).unsqueeze(1).long()
q = q.gather(1, actions)
terminals = self.network.tensor(terminals).unsqueeze(1)
rewards = self.network.tensor(rewards).unsqueeze(1)
returns = rewards + config.discount * terminals * returns
processed_rollout[i] = [q, returns]
q, returns= map(lambda x: torch.cat(x, dim=0), zip(*processed_rollout))
loss = 0.5 * (q - Variable(returns)).pow(2).mean()
loss = 0.5 * (q - returns).pow(2).mean()
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.network.parameters(), config.gradient_clip)
nn.utils.clip_grad_norm_(self.network.parameters(), config.gradient_clip)
self.optimizer.step()
self.evaluate(config.rollout_length)
agent/PPO_agent.py
+8 -10
View File
@@ -32,7 +32,7 @@ class PPOAgent(BaseAgent):
states = self.states
for _ in range(config.rollout_length):
actions, log_probs, _, values = self.network.predict(states)
next_states, rewards, terminals, _ = self.task.step(actions.data.cpu().numpy())
next_states, rewards, terminals, _ = self.task.step(actions.cpu().detach().numpy())
self.episode_rewards += rewards
rewards = config.reward_normalizer(rewards)
for i, terminal in enumerate(terminals):
@@ -49,33 +49,31 @@ class PPOAgent(BaseAgent):
processed_rollout = [None] * (len(rollout) - 1)
advantages = self.network.tensor(np.zeros((config.num_workers, 1)))
returns = pending_value.data
returns = pending_value.detach()
for i in reversed(range(len(rollout) - 1)):
states, value, actions, log_probs, rewards, terminals = rollout[i]
terminals = self.network.tensor(terminals).unsqueeze(1)
rewards = self.network.tensor(rewards).unsqueeze(1)
actions = self.network.variable(actions)
states = self.network.variable(states)
actions = self.network.tensor(actions)
states = self.network.tensor(states)
next_value = rollout[i + 1][1]
returns = rewards + config.discount * terminals * returns
if not config.use_gae:
advantages = returns - value.data
advantages = returns - value.detach()
else:
td_error = rewards + config.discount * terminals * next_value.data - value.data
td_error = rewards + config.discount * terminals * next_value.detach() - value.detach()
advantages = advantages * config.gae_tau * config.discount * terminals + td_error
processed_rollout[i] = [states, actions, log_probs, returns, advantages]
states, actions, log_probs_old, returns, advantages = map(lambda x: torch.cat(x, dim=0), zip(*processed_rollout))
advantages = (advantages - advantages.mean()) / advantages.std()
advantages = Variable(advantages)
returns = Variable(returns)
batcher = Batcher(states.size(0) // config.num_mini_batches, [np.arange(states.size(0))])
for _ in range(config.optimization_epochs):
batcher.shuffle()
while not batcher.end():
batch_indices = batcher.next_batch()[0]
batch_indices = self.network.variable(batch_indices, torch.LongTensor)
batch_indices = self.network.tensor(batch_indices).long()
sampled_states = states[batch_indices]
sampled_actions = actions[batch_indices]
sampled_log_probs_old = log_probs_old[batch_indices]
@@ -93,7 +91,7 @@ class PPOAgent(BaseAgent):
self.network.zero_grad()
(policy_loss + value_loss).backward()
nn.utils.clip_grad_norm(self.network.parameters(), config.gradient_clip)
nn.utils.clip_grad_norm_(self.network.parameters(), config.gradient_clip)
self.network.step()
steps = config.rollout_length * config.num_workers
agent/QuantileRegressionDQN_agent.py
+9 -9
View File
@@ -36,8 +36,8 @@ class QuantileRegressionDQNAgent(BaseAgent):
return 0.5 * x.pow(2) * cond + (x.abs() - 0.5) * (1 - cond)
def evaluation_action(self, state):
value = self.network.predict(np.stack([self.config.state_normalizer(state)])).squeeze(0).data
value = (value * self.quantile_weight).sum(-1).cpu().numpy().flatten()
value = self.network.predict(np.stack([self.config.state_normalizer(state)])).squeeze(0).detach()
value = (value * self.quantile_weight).sum(-1).cpu().detach().numpy().flatten()
return np.argmax(value)
def episode(self, deterministic=False):
@@ -46,8 +46,8 @@ class QuantileRegressionDQNAgent(BaseAgent):
total_reward = 0.0
steps = 0
while True:
value = self.network.predict(np.stack([self.config.state_normalizer(state)])).squeeze(0).data
value = (value * self.quantile_weight).sum(-1).cpu().numpy().flatten()
value = self.network.predict(np.stack([self.config.state_normalizer(state)])).squeeze(0).detach()
value = (value * self.quantile_weight).sum(-1).cpu().detach().numpy().flatten()
if deterministic:
action = np.argmax(value)
elif self.total_steps < self.config.exploration_steps:
@@ -69,7 +69,7 @@ class QuantileRegressionDQNAgent(BaseAgent):
states = self.config.state_normalizer(states)
next_states = self.config.state_normalizer(next_states)
quantiles_next = self.target_network.predict(next_states).data
quantiles_next = self.target_network.predict(next_states).detach()
q_next = (quantiles_next * self.quantile_weight).sum(-1)
_, a_next = torch.max(q_next, dim=1)
a_next = a_next.view(-1, 1, 1).expand(-1, -1, quantiles_next.size(2))
@@ -80,13 +80,13 @@ class QuantileRegressionDQNAgent(BaseAgent):
quantiles_next = rewards.view(-1, 1) + self.config.discount * (1 - terminals.view(-1, 1)) * quantiles_next
quantiles = self.network.predict(states)
actions = self.network.tensor(actions, torch.LongTensor)
actions = self.network.tensor(actions).long()
actions = actions.view(-1, 1, 1).expand(-1, -1, quantiles.size(2))
quantiles = quantiles.gather(1, Variable(actions)).squeeze(1)
quantiles = quantiles.gather(1, actions).squeeze(1)
quantiles_next = quantiles_next.t().unsqueeze(-1)
diff = Variable(quantiles_next) - quantiles
loss = self.huber(diff) * Variable(self.cumulative_density.view(1, -1) - (diff.data < 0).float()).abs()
diff = quantiles_next - quantiles
loss = self.huber(diff) * (self.cumulative_density.view(1, -1) - (diff.detach() < 0).float()).abs()
self.optimizer.zero_grad()
loss.mean(1).sum().backward()
main.py
+7 -5
View File
@@ -275,9 +275,9 @@ def dqn_ram_atari(name):
def ppo_continuous():
config = Config()
config.num_workers = 1
# task_fn = lambda log_dir: Pendulum(log_dir=log_dir)
task_fn = lambda log_dir: Pendulum(log_dir=log_dir)
# task_fn = lambda log_dir: Roboschool('RoboschoolInvertedPendulum-v1', log_dir=log_dir)
task_fn = lambda log_dir: Roboschool('RoboschoolAnt-v1', log_dir=log_dir)
# task_fn = lambda log_dir: Roboschool('RoboschoolAnt-v1', log_dir=log_dir)
# task_fn = lambda log_dir: Roboschool('RoboschoolReacher-v1', log_dir=log_dir)
# task_fn = lambda log_dir: Roboschool('RoboschoolHopper-v1', log_dir=log_dir)
# task_fn = lambda log_dir: DMControl('cartpole', 'balance', log_dir=log_dir)
@@ -307,15 +307,15 @@ def ppo_continuous():
def ddpg_continuous():
config = Config()
log_dir = get_default_log_dir(ddpg_continuous.__name__)
# config.task_fn = lambda: Pendulum(log_dir=log_dir)
config.task_fn = lambda: Pendulum(log_dir=log_dir)
# config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1', log_dir=log_dir)
# config.task_fn = lambda: Roboschool('RoboschoolReacher-v1', log_dir=log_dir)
config.task_fn = lambda: Roboschool('RoboschoolHopper-v1')
# config.task_fn = lambda: Roboschool('RoboschoolHopper-v1')
# config.task_fn = lambda: Roboschool('RoboschoolAnt-v1', log_dir=log_dir)
# config.task_fn = lambda: Roboschool('RoboschoolWalker2d-v1', log_dir=log_dir)
# config.task_fn = lambda: DMControl('cartpole', 'balance', log_dir=log_dir)
# config.task_fn = lambda: DMControl('finger', 'spin', log_dir=log_dir)
config.evaluation_env = Roboschool('RoboschoolHopper-v1', log_dir=log_dir)
# config.evaluation_env = Roboschool('RoboschoolHopper-v1', log_dir=log_dir)
config.actor_network_fn = lambda state_dim, action_dim: DeterministicActorNet(state_dim, action_dim)
config.critic_network_fn = lambda state_dim, action_dim: DeterministicCriticNet(state_dim, action_dim)
config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
@@ -374,6 +374,8 @@ if __name__ == '__main__':
mkdir('dataset')
mkdir('log')
os.system('export OMP_NUM_THREADS=1')
os.system('export MKL_NUM_THREADS=1')
torch.set_num_threads(1)
# logger.setLevel(logging.DEBUG)
logger.setLevel(logging.INFO)
model/action_conditional_video_prediction.py
+16 -15
View File
@@ -21,7 +21,7 @@ from utils import *
from tqdm import tqdm
from network import *
class Network(nn.Module, BasicNet):
class Network(nn.Module, BaseNet):
def __init__(self, num_actions, gpu=0):
super(Network, self).__init__()
@@ -46,17 +46,18 @@ class Network(nn.Module, BasicNet):
self.init_weights()
self.criterion = nn.MSELoss()
self.opt = torch.optim.Adam(self.parameters(), 1e-4)
self.set_gpu(gpu)
BasicNet.__init__(self, gpu)
def init_weights(self):
for layer in self.children():
if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.ConvTranspose2d):
nn.init.xavier_uniform(layer.weight.data)
nn.init.constant(layer.bias.data, 0)
nn.init.uniform(self.fc_encode.weight.data, -1, 1)
nn.init.uniform(self.fc_decode.weight.data, -1, 1)
nn.init.uniform(self.fc_action.weight.data, -0.1, 0.1)
nn.init.xavier_uniform_(layer.weight.data)
nn.init.constant_(layer.bias.data, 0)
nn.init.uniform_(self.fc_encode.weight.data, -1, 1)
nn.init.uniform_(self.fc_decode.weight.data, -1, 1)
nn.init.uniform_(self.fc_action.weight.data, -0.1, 0.1)
def forward(self, obs, action):
x = F.relu(self.conv1(obs))
@@ -78,9 +79,9 @@ class Network(nn.Module, BasicNet):
return x
def fit(self, x, a, y):
x = self.variable(x)
a = self.variable(a)
y = self.variable(y)
x = self.tensor(x)
a = self.tensor(a)
y = self.tensor(y)
y_ = self.forward(x, a)
loss = self.criterion(y_, y)
self.opt.zero_grad()
@@ -91,16 +92,16 @@ class Network(nn.Module, BasicNet):
return np.asscalar(loss.cpu().data.numpy())
def evaluate(self, x, a, y):
x = self.variable(x)
a = self.variable(a)
y = self.variable(y)
x = self.tensor(x)
a = self.tensor(a)
y = self.tensor(y)
y_ = self.forward(x, a)
loss = self.criterion(y_, y)
return np.asscalar(loss.cpu().data.numpy())
def predict(self, x, a):
x = self.variable(x)
a = self.variable(a)
x = self.tensor(x)
a = self.tensor(a)
return self.forward(x, a).cpu().data.numpy()
def load_episode(game, ep, num_actions, prefix):
network/network_bodies.py
+22 -22
View File
@@ -36,7 +36,7 @@ class TwoLayerFCBody(nn.Module):
y = self.gate(self.fc2(y))
return y
class DeterministicActorNet(nn.Module, BasicNet):
class DeterministicActorNet(nn.Module, BaseNet):
def __init__(self,
state_dim,
action_dim,
@@ -52,15 +52,15 @@ class DeterministicActorNet(nn.Module, BasicNet):
self.action_scale = action_scale
self.non_linear = non_linear
self.init_weights()
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def init_weights(self):
bound = 3e-3
nn.init.uniform(self.layer3.weight.data, -bound, bound)
nn.init.constant(self.layer3.bias.data, 0)
nn.init.uniform_(self.layer3.weight.data, -bound, bound)
nn.init.constant_(self.layer3.bias.data, 0)
def forward(self, x):
x = self.variable(x)
x = self.tensor(x)
x = self.non_linear(self.layer1(x))
x = self.non_linear(self.layer2(x))
x = self.layer3(x)
@@ -70,10 +70,10 @@ class DeterministicActorNet(nn.Module, BasicNet):
def predict(self, x, to_numpy=False):
y = self.forward(x)
if to_numpy:
y = y.cpu().data.numpy()
y = y.cpu().detach().numpy()
return y
class DeterministicCriticNet(nn.Module, BasicNet):
class DeterministicCriticNet(nn.Module, BaseNet):
def __init__(self,
state_dim,
action_dim,
@@ -85,16 +85,16 @@ class DeterministicCriticNet(nn.Module, BasicNet):
self.layer3 = nn.Linear(300, 1)
self.non_linear = non_linear
self.init_weights()
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def init_weights(self):
bound = 3e-3
nn.init.uniform(self.layer3.weight.data, -bound, bound)
nn.init.constant(self.layer3.bias.data, 0)
nn.init.uniform_(self.layer3.weight.data, -bound, bound)
nn.init.constant_(self.layer3.bias.data, 0)
def forward(self, x, action):
x = self.variable(x)
action = self.variable(action)
x = self.tensor(x)
action = self.tensor(action)
x = self.non_linear(self.layer1(x))
x = self.non_linear(self.layer2(torch.cat([x, action], dim=1)))
x = self.layer3(x)
@@ -103,7 +103,7 @@ class DeterministicCriticNet(nn.Module, BasicNet):
def predict(self, x, action):
return self.forward(x, action)
class GaussianActorNet(nn.Module, BasicNet):
class GaussianActorNet(nn.Module, BaseNet):
def __init__(self,
state_dim,
action_dim,
@@ -120,15 +120,15 @@ class GaussianActorNet(nn.Module, BasicNet):
self.non_linear = non_linear
self.init_weights()
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def init_weights(self):
bound = 3e-3
nn.init.uniform(self.fc_action.weight.data, -bound, bound)
nn.init.constant(self.fc_action.bias.data, 0)
nn.init.uniform_(self.fc_action.weight.data, -bound, bound)
nn.init.constant_(self.fc_action.bias.data, 0)
def forward(self, x):
x = self.variable(x)
x = self.tensor(x)
phi = self.non_linear(self.fc1(x))
phi = self.non_linear(self.fc2(phi))
mean = F.tanh(self.fc_action(phi))
@@ -139,7 +139,7 @@ class GaussianActorNet(nn.Module, BasicNet):
def predict(self, x):
return self.forward(x)
class GaussianCriticNet(nn.Module, BasicNet):
class GaussianCriticNet(nn.Module, BaseNet):
def __init__(self,
state_dim,
gpu=-1,
@@ -151,15 +151,15 @@ class GaussianCriticNet(nn.Module, BasicNet):
self.fc_value = nn.Linear(hidden_size, 1)
self.non_linear = non_linear
self.init_weights()
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def init_weights(self):
bound = 3e-3
nn.init.uniform(self.fc_value.weight.data, -bound, bound)
nn.init.constant(self.fc_value.bias.data, 0)
nn.init.uniform_(self.fc_value.weight.data, -bound, bound)
nn.init.constant_(self.fc_value.bias.data, 0)
def forward(self, x):
x = self.variable(x)
x = self.tensor(x)
phi = self.non_linear(self.fc1(x))
phi = self.non_linear(self.fc2(phi))
value = self.fc_value(phi)
network/network_heads.py
+20 -24
View File
@@ -6,89 +6,85 @@
from .network_utils import *
class VanillaNet(nn.Module, BasicNet):
class VanillaNet(nn.Module, BaseNet):
def __init__(self, output_dim, body, gpu=-1):
super(VanillaNet, self).__init__()
self.fc_head = layer_init(nn.Linear(body.feature_dim, output_dim))
self.body = body
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def predict(self, x, to_numpy=False):
phi = self.body(self.variable(x))
phi = self.body(self.tensor(x))
y = self.fc_head(phi)
if to_numpy:
y = y.cpu().data.numpy()
y = y.cpu().detach().numpy()
return y
class DuelingNet(nn.Module, BasicNet):
class DuelingNet(nn.Module, BaseNet):
def __init__(self, action_dim, body, gpu=-1):
super(DuelingNet, self).__init__()
self.fc_value = layer_init(nn.Linear(body.feature_dim, 1))
self.fc_advantage = layer_init(nn.Linear(body.feature_dim, action_dim))
self.body = body
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def predict(self, x, to_numpy=False):
phi = self.body(self.variable(x))
phi = self.body(self.tensor(x))
value = self.fc_value(phi)
advantange = self.fc_advantage(phi)
q = value.expand_as(advantange) + (advantange - advantange.mean(1, keepdim=True).expand_as(advantange))
if to_numpy:
return q.cpu().data.numpy()
return q.cpu().detach().numpy()
return q
class ActorCriticNet(nn.Module, BasicNet):
class ActorCriticNet(nn.Module, BaseNet):
def __init__(self, action_dim, body, gpu=-1):
super(ActorCriticNet, self).__init__()
self.fc_actor = layer_init(nn.Linear(body.feature_dim, action_dim))
self.fc_critic = layer_init(nn.Linear(body.feature_dim, 1))
self.body = body
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def predict(self, x, to_numpy=False):
phi = self.body(self.variable(x))
phi = self.body(self.tensor(x))
pre_prob = self.fc_actor(phi)
prob = F.softmax(pre_prob, dim=1)
log_prob = F.log_softmax(pre_prob, dim=1)
value = self.fc_critic(phi)
if to_numpy:
return prob.cpu().data.numpy()
return prob.cpu().detach().numpy()
return prob, log_prob, value
class CategoricalNet(nn.Module, BasicNet):
class CategoricalNet(nn.Module, BaseNet):
def __init__(self, action_dim, num_atoms, body, gpu=-1):
super(CategoricalNet, self).__init__()
self.fc_categorical = layer_init(nn.Linear(body.feature_dim, action_dim * num_atoms))
self.action_dim = action_dim
self.num_atoms = num_atoms
self.body = body
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def predict(self, x, to_numpy=False):
phi = self.body(self.variable(x))
phi = self.body(self.tensor(x))
pre_prob = self.fc_categorical(phi).view((-1, self.action_dim, self.num_atoms))
prob = F.softmax(pre_prob, dim=-1)
if to_numpy:
return prob.cpu().data.numpy()
return prob.cpu().detach().numpy()
return prob
class QuantileNet(nn.Module, BasicNet):
class QuantileNet(nn.Module, BaseNet):
def __init__(self, action_dim, num_quantiles, body, gpu=-1):
super(QuantileNet, self).__init__()
self.fc_quantiles = layer_init(nn.Linear(body.feature_dim, action_dim * num_quantiles))
self.action_dim = action_dim
self.num_quantiles = num_quantiles
self.body = body
BasicNet.__init__(self, gpu)
self.set_gpu(gpu)
def predict(self, x, to_numpy=False):
phi = self.body(self.variable(x))
phi = self.body(self.tensor(x))
quantiles = self.fc_quantiles(phi)
quantiles = quantiles.view((-1, self.action_dim, self.num_quantiles))
if to_numpy:
quantiles = quantiles.data.cpu().numpy()
quantiles = quantiles.cpu().detach().numpy()
return quantiles
network/network_utils.py
+15 -38
View File
@@ -5,37 +5,20 @@
#######################################################################
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
class BasicNet:
def __init__(self, gpu):
if not torch.cuda.is_available():
gpu = -1
self.gpu = gpu
if self.gpu >= 0:
self.cuda(self.gpu)
class BaseNet:
def set_gpu(self, gpu):
if gpu >= 0 and torch.cuda.is_available():
self.device = torch.device('gpu:%d' % (gpu))
else:
self.device = torch.device('cpu')
self.to(self.device)
def supported_dtype(self, x, torch_type):
if torch_type == torch.FloatTensor:
return np.asarray(x, dtype=np.float32)
if torch_type == torch.LongTensor:
return np.asarray(x, dtype=np.int64)
def variable(self, x, dtype=torch.FloatTensor):
if isinstance(x, Variable):
return x
x = dtype(torch.from_numpy(self.supported_dtype(x, dtype)))
if self.gpu >= 0:
x = x.cuda(self.gpu)
return Variable(x)
def tensor(self, x, dtype=torch.FloatTensor):
x = dtype(torch.from_numpy(self.supported_dtype(x, dtype)))
if self.gpu >= 0:
x = x.cuda(self.gpu)
def tensor(self, x):
x = torch.tensor(x, device=self.device, dtype=torch.float32)
return x
class DisjointActorCriticWrapper:
@@ -74,11 +57,8 @@ class GaussianActorCriticWrapper:
log_probs = torch.sum(log_probs, dim=1, keepdim=True)
return actions, log_probs, 0, values
def variable(self, x, dtype=torch.FloatTensor):
return self.actor.variable(x, dtype)
def tensor(self, x, dtype=torch.FloatTensor):
return self.actor.tensor(x, dtype)
def tensor(self, x):
return self.actor.tensor(x)
def zero_grad(self):
self.actor_opt.zero_grad()
@@ -112,11 +92,8 @@ class CategoricalActorCriticWrapper:
log_prob = dist.log_prob(action).unsqueeze(1)
return action, log_prob, entropy_loss.mean(0), value
def variable(self, x, dtype=torch.FloatTensor):
return self.network.variable(x, dtype)
def tensor(self, x, dtype=torch.FloatTensor):
return self.network.tensor(x, dtype)
def tensor(self, x):
return self.network.tensor(x)
def zero_grad(self):
self.opt.zero_grad()
@@ -134,6 +111,6 @@ class CategoricalActorCriticWrapper:
self.network.load_state_dict(state_dicts)
def layer_init(layer):
nn.init.orthogonal(layer.weight.data)
nn.init.constant(layer.bias.data, 0)
nn.init.orthogonal_(layer.weight.data)
nn.init.constant_(layer.bias.data, 0)
return layer
utils/tf_logger.py
+1 -1
View File
@@ -30,7 +30,7 @@ class Logger(object):
if isinstance(v, torch.autograd.Variable):
v = v.data
if isinstance(v, torch.FloatTensor):
v = v.cpu().numpy()
v = v.cpu().detach().numpy()
return v
def get_step(self, tag):