wassname/pytorch-soft-actor-critic
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-soft-actor-critic.git synced 2026-08-11 11:24:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

No need for list of rewards

Browse Source
This commit is contained in:
pranz24
2019-04-07 22:25:55 +05:30
parent 311491796a
commit ed1d0a55d2
2 changed files with 12 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+4 -8
View File
@@ -26,11 +26,11 @@ parser.add_argument('--automatic_entropy_tuning', type=bool, default=False, meta
help='Temperature parameter α automaically adjusted.')
parser.add_argument('--seed', type=int, default=456, metavar='N',
help='random seed (default: 456)')
parser.add_argument('--batch_size', type=int, default=128, metavar='N',
parser.add_argument('--batch_size', type=int, default=256, metavar='N',
help='batch size (default: 256)')
parser.add_argument('--num_steps', type=int, default=1000001, metavar='N',
help='maximum number of steps (default: 1000000)')
parser.add_argument('--hidden_size', type=int, default=300, metavar='N',
parser.add_argument('--hidden_size', type=int, default=256, metavar='N',
help='hidden size (default: 256)')
parser.add_argument('--updates_per_step', type=int, default=1, metavar='N',
help='model updates per simulator step (default: 1)')
@@ -59,8 +59,6 @@ writer = SummaryWriter(log_dir='runs/{}_VIREL_{}'.format(datetime.datetime.now()
memory = ReplayMemory(args.replay_size)
# Training Loop
rewards = []
test_rewards = []
total_numsteps = 0
updates = 0
@@ -104,8 +102,7 @@ for i_episode in itertools.count(1):
break
writer.add_scalar('reward/train', episode_reward, i_episode)
rewards.append(episode_reward)
print("Episode: {}, total numsteps: {}, episode steps: {}, reward: {}".format(i_episode, total_numsteps, episode_steps, np.round(rewards[-1],2)))
print("Episode: {}, total numsteps: {}, episode steps: {}, reward: {}".format(i_episode, total_numsteps, episode_steps, round(episode_reward, 2)))
if i_episode % 10 == 0 and args.eval == True:
state = env.reset()
@@ -123,9 +120,8 @@ for i_episode in itertools.count(1):
writer.add_scalar('reward/test', episode_reward, i_episode)
test_rewards.append(episode_reward)
print("----------------------------------------")
print("Test Episode: {}, reward: {}".format(i_episode, test_rewards[-1]))
print("Test Episode: {}, reward: {}".format(i_episode, round(episode_reward, 2)))
print("----------------------------------------")
env.close()
sac.py
+8 -8
View File
@@ -61,14 +61,6 @@ class SAC(object):
qf1_loss = F.mse_loss(qf1, next_q_value) # JQ = 𝔼(st,at)~D[0.5(Q1(st,at) - r(st,at) - γ(𝔼st+1~p[V(st+1)]))^2]
qf2_loss = F.mse_loss(qf2, next_q_value) # JQ = 𝔼(st,at)~D[0.5(Q1(st,at) - r(st,at) - γ(𝔼st+1~p[V(st+1)]))^2]
self.critic_optim.zero_grad()
qf1_loss.backward()
self.critic_optim.step()
self.critic_optim.zero_grad()
qf2_loss.backward()
self.critic_optim.step()
pi, log_pi, mean, log_std = self.policy.sample(state_batch)
qf1_pi, qf2_pi = self.critic(state_batch, pi)
@@ -89,6 +81,14 @@ class SAC(object):
vf_loss.backward()
self.value_optim.step()
self.critic_optim.zero_grad()
qf1_loss.backward()
self.critic_optim.step()
self.critic_optim.zero_grad()
qf2_loss.backward()
self.critic_optim.step()
self.policy_optim.zero_grad()
policy_loss.backward()
self.policy_optim.step()