From ed1d0a55d2c9c98cc78bcce9cdcce1a8dea42a36 Mon Sep 17 00:00:00 2001 From: pranz24 Date: Sun, 7 Apr 2019 22:25:55 +0530 Subject: [PATCH] No need for list of rewards --- main.py | 12 ++++-------- sac.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/main.py b/main.py index ad48aa7..c20feb3 100644 --- a/main.py +++ b/main.py @@ -26,11 +26,11 @@ parser.add_argument('--automatic_entropy_tuning', type=bool, default=False, meta help='Temperature parameter α automaically adjusted.') parser.add_argument('--seed', type=int, default=456, metavar='N', help='random seed (default: 456)') -parser.add_argument('--batch_size', type=int, default=128, metavar='N', +parser.add_argument('--batch_size', type=int, default=256, metavar='N', help='batch size (default: 256)') parser.add_argument('--num_steps', type=int, default=1000001, metavar='N', help='maximum number of steps (default: 1000000)') -parser.add_argument('--hidden_size', type=int, default=300, metavar='N', +parser.add_argument('--hidden_size', type=int, default=256, metavar='N', help='hidden size (default: 256)') parser.add_argument('--updates_per_step', type=int, default=1, metavar='N', help='model updates per simulator step (default: 1)') @@ -59,8 +59,6 @@ writer = SummaryWriter(log_dir='runs/{}_VIREL_{}'.format(datetime.datetime.now() memory = ReplayMemory(args.replay_size) # Training Loop -rewards = [] -test_rewards = [] total_numsteps = 0 updates = 0 @@ -104,8 +102,7 @@ for i_episode in itertools.count(1): break writer.add_scalar('reward/train', episode_reward, i_episode) - rewards.append(episode_reward) - print("Episode: {}, total numsteps: {}, episode steps: {}, reward: {}".format(i_episode, total_numsteps, episode_steps, np.round(rewards[-1],2))) + print("Episode: {}, total numsteps: {}, episode steps: {}, reward: {}".format(i_episode, total_numsteps, episode_steps, round(episode_reward, 2))) if i_episode % 10 == 0 and args.eval == True: state = env.reset() @@ -123,9 +120,8 @@ for i_episode in itertools.count(1): writer.add_scalar('reward/test', episode_reward, i_episode) - test_rewards.append(episode_reward) print("----------------------------------------") - print("Test Episode: {}, reward: {}".format(i_episode, test_rewards[-1])) + print("Test Episode: {}, reward: {}".format(i_episode, round(episode_reward, 2))) print("----------------------------------------") env.close() diff --git a/sac.py b/sac.py index ab9416f..98e0a68 100644 --- a/sac.py +++ b/sac.py @@ -61,14 +61,6 @@ class SAC(object): qf1_loss = F.mse_loss(qf1, next_q_value) # JQ = 𝔼(st,at)~D[0.5(Q1(st,at) - r(st,at) - γ(𝔼st+1~p[V(st+1)]))^2] qf2_loss = F.mse_loss(qf2, next_q_value) # JQ = 𝔼(st,at)~D[0.5(Q1(st,at) - r(st,at) - γ(𝔼st+1~p[V(st+1)]))^2] - self.critic_optim.zero_grad() - qf1_loss.backward() - self.critic_optim.step() - - self.critic_optim.zero_grad() - qf2_loss.backward() - self.critic_optim.step() - pi, log_pi, mean, log_std = self.policy.sample(state_batch) qf1_pi, qf2_pi = self.critic(state_batch, pi) @@ -89,6 +81,14 @@ class SAC(object): vf_loss.backward() self.value_optim.step() + self.critic_optim.zero_grad() + qf1_loss.backward() + self.critic_optim.step() + + self.critic_optim.zero_grad() + qf2_loss.backward() + self.critic_optim.step() + self.policy_optim.zero_grad() policy_loss.backward() self.policy_optim.step()